MFV r350898, r351075: 8423 8199 7432 Implement large_dnode pool feature

[FreeBSD/FreeBSD.git] / contrib / processor-trace / libipt / src / pt_block_decoder.c

/*
 * Copyright (c) 2016-2018, Intel Corporation
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *  * Neither the name of Intel Corporation nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "pt_block_decoder.h"
#include "pt_block_cache.h"
#include "pt_section.h"
#include "pt_image.h"
#include "pt_insn.h"
#include "pt_config.h"
#include "pt_asid.h"
#include "pt_compiler.h"

#include "intel-pt.h"

#include <string.h>
#include <stdlib.h>


static int pt_blk_proceed_trailing_event(struct pt_block_decoder *,
                                         struct pt_block *);


static int pt_blk_status(const struct pt_block_decoder *decoder, int flags)
{
        int status;

        if (!decoder)
                return -pte_internal;

        status = decoder->status;

        /* Indicate whether tracing is disabled or enabled.
         *
         * This duplicates the indication in struct pt_insn and covers the case
         * where we indicate the status after synchronizing.
         */
        if (!decoder->enabled)
                flags |= pts_ip_suppressed;

        /* Forward end-of-trace indications.
         *
         * Postpone it as long as we're still processing events, though.
         */
        if ((status & pts_eos) && !decoder->process_event)
                flags |= pts_eos;

        return flags;
}

static void pt_blk_reset(struct pt_block_decoder *decoder)
{
        if (!decoder)
                return;

        decoder->mode = ptem_unknown;
        decoder->ip = 0ull;
        decoder->status = 0;
        decoder->enabled = 0;
        decoder->process_event = 0;
        decoder->speculative = 0;
        decoder->process_insn = 0;
        decoder->bound_paging = 0;
        decoder->bound_vmcs = 0;
        decoder->bound_ptwrite = 0;

        memset(&decoder->event, 0, sizeof(decoder->event));
        pt_retstack_init(&decoder->retstack);
        pt_asid_init(&decoder->asid);
}

/* Initialize the query decoder flags based on our flags. */

static int pt_blk_init_qry_flags(struct pt_conf_flags *qflags,
                                 const struct pt_conf_flags *flags)
{
        if (!qflags || !flags)
                return -pte_internal;

        memset(qflags, 0, sizeof(*qflags));

        return 0;
}

int pt_blk_decoder_init(struct pt_block_decoder *decoder,
                        const struct pt_config *uconfig)
{
        struct pt_config config;
        int errcode;

        if (!decoder)
                return -pte_internal;

        errcode = pt_config_from_user(&config, uconfig);
        if (errcode < 0)
                return errcode;

        /* The user supplied decoder flags. */
        decoder->flags = config.flags;

        /* Set the flags we need for the query decoder we use. */
        errcode = pt_blk_init_qry_flags(&config.flags, &decoder->flags);
        if (errcode < 0)
                return errcode;

        errcode = pt_qry_decoder_init(&decoder->query, &config);
        if (errcode < 0)
                return errcode;

        pt_image_init(&decoder->default_image, NULL);
        decoder->image = &decoder->default_image;

        errcode = pt_msec_cache_init(&decoder->scache);
        if (errcode < 0)
                return errcode;

        pt_blk_reset(decoder);

        return 0;
}

void pt_blk_decoder_fini(struct pt_block_decoder *decoder)
{
        if (!decoder)
                return;

        pt_msec_cache_fini(&decoder->scache);
        pt_image_fini(&decoder->default_image);
        pt_qry_decoder_fini(&decoder->query);
}

struct pt_block_decoder *
pt_blk_alloc_decoder(const struct pt_config *config)
{
        struct pt_block_decoder *decoder;
        int errcode;

        decoder = malloc(sizeof(*decoder));
        if (!decoder)
                return NULL;

        errcode = pt_blk_decoder_init(decoder, config);
        if (errcode < 0) {
                free(decoder);
                return NULL;
        }

        return decoder;
}

void pt_blk_free_decoder(struct pt_block_decoder *decoder)
{
        if (!decoder)
                return;

        pt_blk_decoder_fini(decoder);
        free(decoder);
}

/* Maybe synthesize a tick event.
 *
 * If we're not already processing events, check the current time against the
 * last event's time.  If it changed, synthesize a tick event with the new time.
 *
 * Returns zero if no tick event has been created.
 * Returns a positive integer if a tick event has been created.
 * Returns a negative error code otherwise.
 */
static int pt_blk_tick(struct pt_block_decoder *decoder, uint64_t ip)
{
        struct pt_event *ev;
        uint64_t tsc;
        uint32_t lost_mtc, lost_cyc;
        int errcode;

        if (!decoder)
                return -pte_internal;

        /* We're not generating tick events if tracing is disabled. */
        if (!decoder->enabled)
                return -pte_internal;

        /* Events already provide a timestamp so there is no need to synthesize
         * an artificial tick event.  There's no room, either, since this would
         * overwrite the in-progress event.
         *
         * In rare cases where we need to proceed to an event location using
         * trace this may cause us to miss a timing update if the event is not
         * forwarded to the user.
         *
         * The only case I can come up with at the moment is a MODE.EXEC binding
         * to the TIP IP of a far branch.
         */
        if (decoder->process_event)
                return 0;

        errcode = pt_qry_time(&decoder->query, &tsc, &lost_mtc, &lost_cyc);
        if (errcode < 0) {
                /* If we don't have wall-clock time, we use relative time. */
                if (errcode != -pte_no_time)
                        return errcode;
        }

        ev = &decoder->event;

        /* We're done if time has not changed since the last event. */
        if (tsc == ev->tsc)
                return 0;

        /* Time has changed so we create a new tick event. */
        memset(ev, 0, sizeof(*ev));
        ev->type = ptev_tick;
        ev->variant.tick.ip = ip;

        /* Indicate if we have wall-clock time or only relative time. */
        if (errcode != -pte_no_time)
                ev->has_tsc = 1;
        ev->tsc = tsc;
        ev->lost_mtc = lost_mtc;
        ev->lost_cyc = lost_cyc;

        /* We now have an event to process. */
        decoder->process_event = 1;

        return 1;
}

/* Query an indirect branch.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_indirect_branch(struct pt_block_decoder *decoder,
                                  uint64_t *ip)
{
        uint64_t evip;
        int status, errcode;

        if (!decoder)
                return -pte_internal;

        evip = decoder->ip;

        status = pt_qry_indirect_branch(&decoder->query, ip);
        if (status < 0)
                return status;

        if (decoder->flags.variant.block.enable_tick_events) {
                errcode = pt_blk_tick(decoder, evip);
                if (errcode < 0)
                        return errcode;
        }

        return status;
}

/* Query a conditional branch.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_cond_branch(struct pt_block_decoder *decoder, int *taken)
{
        int status, errcode;

        if (!decoder)
                return -pte_internal;

        status = pt_qry_cond_branch(&decoder->query, taken);
        if (status < 0)
                return status;

        if (decoder->flags.variant.block.enable_tick_events) {
                errcode = pt_blk_tick(decoder, decoder->ip);
                if (errcode < 0)
                        return errcode;
        }

        return status;
}

static int pt_blk_start(struct pt_block_decoder *decoder, int status)
{
        if (!decoder)
                return -pte_internal;

        if (status < 0)
                return status;

        decoder->status = status;
        if (!(status & pts_ip_suppressed))
                decoder->enabled = 1;

        /* We will always have an event.
         *
         * If we synchronized onto an empty PSB+, tracing is disabled and we'll
         * process events until the enabled event.
         *
         * If tracing is enabled, PSB+ must at least provide the execution mode,
         * which we're going to forward to the user.
         */
        return pt_blk_proceed_trailing_event(decoder, NULL);
}

static int pt_blk_sync_reset(struct pt_block_decoder *decoder)
{
        if (!decoder)
                return -pte_internal;

        pt_blk_reset(decoder);

        return 0;
}

int pt_blk_sync_forward(struct pt_block_decoder *decoder)
{
        int errcode, status;

        if (!decoder)
                return -pte_invalid;

        errcode = pt_blk_sync_reset(decoder);
        if (errcode < 0)
                return errcode;

        status = pt_qry_sync_forward(&decoder->query, &decoder->ip);

        return pt_blk_start(decoder, status);
}

int pt_blk_sync_backward(struct pt_block_decoder *decoder)
{
        int errcode, status;

        if (!decoder)
                return -pte_invalid;

        errcode = pt_blk_sync_reset(decoder);
        if (errcode < 0)
                return errcode;

        status = pt_qry_sync_backward(&decoder->query, &decoder->ip);

        return pt_blk_start(decoder, status);
}

int pt_blk_sync_set(struct pt_block_decoder *decoder, uint64_t offset)
{
        int errcode, status;

        if (!decoder)
                return -pte_invalid;

        errcode = pt_blk_sync_reset(decoder);
        if (errcode < 0)
                return errcode;

        status = pt_qry_sync_set(&decoder->query, &decoder->ip, offset);

        return pt_blk_start(decoder, status);
}

int pt_blk_get_offset(const struct pt_block_decoder *decoder, uint64_t *offset)
{
        if (!decoder)
                return -pte_invalid;

        return pt_qry_get_offset(&decoder->query, offset);
}

int pt_blk_get_sync_offset(const struct pt_block_decoder *decoder,
                           uint64_t *offset)
{
        if (!decoder)
                return -pte_invalid;

        return pt_qry_get_sync_offset(&decoder->query, offset);
}

struct pt_image *pt_blk_get_image(struct pt_block_decoder *decoder)
{
        if (!decoder)
                return NULL;

        return decoder->image;
}

int pt_blk_set_image(struct pt_block_decoder *decoder, struct pt_image *image)
{
        if (!decoder)
                return -pte_invalid;

        if (!image)
                image = &decoder->default_image;

        decoder->image = image;
        return 0;
}

const struct pt_config *
pt_blk_get_config(const struct pt_block_decoder *decoder)
{
        if (!decoder)
                return NULL;

        return pt_qry_get_config(&decoder->query);
}

int pt_blk_time(struct pt_block_decoder *decoder, uint64_t *time,
                uint32_t *lost_mtc, uint32_t *lost_cyc)
{
        if (!decoder || !time)
                return -pte_invalid;

        return pt_qry_time(&decoder->query, time, lost_mtc, lost_cyc);
}

int pt_blk_core_bus_ratio(struct pt_block_decoder *decoder, uint32_t *cbr)
{
        if (!decoder || !cbr)
                return -pte_invalid;

        return pt_qry_core_bus_ratio(&decoder->query, cbr);
}

int pt_blk_asid(const struct pt_block_decoder *decoder, struct pt_asid *asid,
                size_t size)
{
        if (!decoder || !asid)
                return -pte_invalid;

        return pt_asid_to_user(asid, &decoder->asid, size);
}

/* Fetch the next pending event.
 *
 * Checks for pending events.  If an event is pending, fetches it (if not
 * already in process).
 *
 * Returns zero if no event is pending.
 * Returns a positive integer if an event is pending or in process.
 * Returns a negative error code otherwise.
 */
static inline int pt_blk_fetch_event(struct pt_block_decoder *decoder)
{
        int status;

        if (!decoder)
                return -pte_internal;

        if (decoder->process_event)
                return 1;

        if (!(decoder->status & pts_event_pending))
                return 0;

        status = pt_qry_event(&decoder->query, &decoder->event,
                              sizeof(decoder->event));
        if (status < 0)
                return status;

        decoder->process_event = 1;
        decoder->status = status;

        return 1;
}

static inline int pt_blk_block_is_empty(const struct pt_block *block)
{
        if (!block)
                return 1;

        return !block->ninsn;
}

static inline int block_to_user(struct pt_block *ublock, size_t size,
                                const struct pt_block *block)
{
        if (!ublock || !block)
                return -pte_internal;

        if (ublock == block)
                return 0;

        /* Zero out any unknown bytes. */
        if (sizeof(*block) < size) {
                memset(ublock + sizeof(*block), 0, size - sizeof(*block));

                size = sizeof(*block);
        }

        memcpy(ublock, block, size);

        return 0;
}

static int pt_insn_false(const struct pt_insn *insn,
                         const struct pt_insn_ext *iext)
{
        (void) insn;
        (void) iext;

        return 0;
}

/* Determine the next IP using trace.
 *
 * Tries to determine the IP of the next instruction using trace and provides it
 * in @pip.
 *
 * Not requiring trace to determine the IP is treated as an internal error.
 *
 * Does not update the return compression stack for indirect calls.  This is
 * expected to have been done, already, when trying to determine the next IP
 * without using trace.
 *
 * Does not update @decoder->status.  The caller is expected to do that.
 *
 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
 * code otherwise.
 * Returns -pte_internal if @pip, @decoder, @insn, or @iext are NULL.
 * Returns -pte_internal if no trace is required.
 */
static int pt_blk_next_ip(uint64_t *pip, struct pt_block_decoder *decoder,
                          const struct pt_insn *insn,
                          const struct pt_insn_ext *iext)
{
        int status, errcode;

        if (!pip || !decoder || !insn || !iext)
                return -pte_internal;

        /* We handle non-taken conditional branches, and compressed returns
         * directly in the switch.
         *
         * All kinds of branches are handled below the switch.
         */
        switch (insn->iclass) {
        case ptic_cond_jump: {
                uint64_t ip;
                int taken;

                status = pt_blk_cond_branch(decoder, &taken);
                if (status < 0)
                        return status;

                ip = insn->ip + insn->size;
                if (taken)
                        ip += iext->variant.branch.displacement;

                *pip = ip;
                return status;
        }

        case ptic_return: {
                int taken;

                /* Check for a compressed return. */
                status = pt_blk_cond_branch(decoder, &taken);
                if (status < 0) {
                        if (status != -pte_bad_query)
                                return status;

                        break;
                }

                /* A compressed return is indicated by a taken conditional
                 * branch.
                 */
                if (!taken)
                        return -pte_bad_retcomp;

                errcode = pt_retstack_pop(&decoder->retstack, pip);
                if (errcode < 0)
                        return errcode;

                return status;
        }

        case ptic_jump:
        case ptic_call:
                /* A direct jump or call wouldn't require trace. */
                if (iext->variant.branch.is_direct)
                        return -pte_internal;

                break;

        case ptic_far_call:
        case ptic_far_return:
        case ptic_far_jump:
                break;

        case ptic_ptwrite:
        case ptic_other:
                return -pte_internal;

        case ptic_error:
                return -pte_bad_insn;
        }

        /* Process an indirect branch.
         *
         * This covers indirect jumps and calls, non-compressed returns, and all
         * flavors of far transfers.
         */
        return pt_blk_indirect_branch(decoder, pip);
}

/* Proceed to the next IP using trace.
 *
 * We failed to proceed without trace.  This ends the current block.  Now use
 * trace to do one final step to determine the start IP of the next block.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_proceed_with_trace(struct pt_block_decoder *decoder,
                                     const struct pt_insn *insn,
                                     const struct pt_insn_ext *iext)
{
        int status;

        if (!decoder)
                return -pte_internal;

        status = pt_blk_next_ip(&decoder->ip, decoder, insn, iext);
        if (status < 0)
                return status;

        /* Preserve the query decoder's response which indicates upcoming
         * events.
         */
        decoder->status = status;

        /* We do need an IP in order to proceed. */
        if (status & pts_ip_suppressed)
                return -pte_noip;

        return 0;
}

/* Decode one instruction in a known section.
 *
 * Decode the instruction at @insn->ip in @msec assuming execution mode
 * @insn->mode.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_decode_in_section(struct pt_insn *insn,
                                    struct pt_insn_ext *iext,
                                    const struct pt_mapped_section *msec)
{
        int status;

        if (!insn || !iext)
                return -pte_internal;

        /* We know that @ip is contained in @section.
         *
         * Note that we need to translate @ip into a section offset.
         */
        status = pt_msec_read(msec, insn->raw, sizeof(insn->raw), insn->ip);
        if (status < 0)
                return status;

        /* We initialize @insn->size to the maximal possible size.  It will be
         * set to the actual size during instruction decode.
         */
        insn->size = (uint8_t) status;

        return pt_ild_decode(insn, iext);
}

/* Update the return-address stack if @insn is a near call.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static inline int pt_blk_log_call(struct pt_block_decoder *decoder,
                                  const struct pt_insn *insn,
                                  const struct pt_insn_ext *iext)
{
        if (!decoder || !insn || !iext)
                return -pte_internal;

        if (insn->iclass != ptic_call)
                return 0;

        /* Ignore direct calls to the next instruction that are used for
         * position independent code.
         */
        if (iext->variant.branch.is_direct &&
            !iext->variant.branch.displacement)
                return 0;

        return pt_retstack_push(&decoder->retstack, insn->ip + insn->size);
}

/* Proceed by one instruction.
 *
 * Tries to decode the instruction at @decoder->ip and, on success, adds it to
 * @block and provides it in @pinsn and @piext.
 *
 * The instruction will not be added if:
 *
 *   - the memory could not be read:  return error
 *   - it could not be decoded:       return error
 *   - @block is already full:        return zero
 *   - @block would switch sections:  return zero
 *
 * Returns a positive integer if the instruction was added.
 * Returns zero if the instruction didn't fit into @block.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_one_insn(struct pt_block_decoder *decoder,
                                   struct pt_block *block,
                                   struct pt_insn *pinsn,
                                   struct pt_insn_ext *piext)
{
        struct pt_insn_ext iext;
        struct pt_insn insn;
        uint16_t ninsn;
        int status;

        if (!decoder || !block || !pinsn || !piext)
                return -pte_internal;

        /* There's nothing to do if there is no room in @block. */
        ninsn = block->ninsn + 1;
        if (!ninsn)
                return 0;

        /* The truncated instruction must be last. */
        if (block->truncated)
                return 0;

        memset(&insn, 0, sizeof(insn));
        memset(&iext, 0, sizeof(iext));

        insn.mode = decoder->mode;
        insn.ip = decoder->ip;

        status = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
        if (status < 0)
                return status;

        /* We do not switch sections inside a block. */
        if (insn.isid != block->isid) {
                if (!pt_blk_block_is_empty(block))
                        return 0;

                block->isid = insn.isid;
        }

        /* If we couldn't read @insn's memory in one chunk from @insn.isid, we
         * provide the memory in @block.
         */
        if (insn.truncated) {
                memcpy(block->raw, insn.raw, insn.size);
                block->size = insn.size;
                block->truncated = 1;
        }

        /* Log calls' return addresses for return compression. */
        status = pt_blk_log_call(decoder, &insn, &iext);
        if (status < 0)
                return status;

        /* We have a new instruction. */
        block->iclass = insn.iclass;
        block->end_ip = insn.ip;
        block->ninsn = ninsn;

        *pinsn = insn;
        *piext = iext;

        return 1;
}


/* Proceed to a particular type of instruction without using trace.
 *
 * Proceed until we reach an instruction for which @predicate returns a positive
 * integer or until:
 *
 *   - @predicate returns an error:  return error
 *   - @block is full:               return zero
 *   - @block would switch sections: return zero
 *   - we would need trace:          return -pte_bad_query
 *
 * Provide the last instruction that was reached in @insn and @iext.
 *
 * Update @decoder->ip to point to the last IP that was reached.  If we fail due
 * to lack of trace or if we reach a desired instruction, this is @insn->ip;
 * otherwise this is the next instruction's IP.
 *
 * Returns a positive integer if a suitable instruction was reached.
 * Returns zero if no such instruction was reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_to_insn(struct pt_block_decoder *decoder,
                                  struct pt_block *block,
                                  struct pt_insn *insn,
                                  struct pt_insn_ext *iext,
                                  int (*predicate)(const struct pt_insn *,
                                                   const struct pt_insn_ext *))
{
        int status;

        if (!decoder || !insn || !predicate)
                return -pte_internal;

        for (;;) {
                status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
                if (status <= 0)
                        return status;

                /* We're done if this instruction matches the spec (positive
                 * status) or we run into an error (negative status).
                 */
                status = predicate(insn, iext);
                if (status != 0)
                        return status;

                /* Let's see if we can proceed to the next IP without trace. */
                status = pt_insn_next_ip(&decoder->ip, insn, iext);
                if (status < 0)
                        return status;

                /* End the block if the user asked us to.
                 *
                 * We only need to take care about direct near branches.
                 * Indirect and far branches require trace and will naturally
                 * end a block.
                 */
                if ((decoder->flags.variant.block.end_on_call &&
                     (insn->iclass == ptic_call)) ||
                    (decoder->flags.variant.block.end_on_jump &&
                     (insn->iclass == ptic_jump)))
                        return 0;
        }
}

/* Proceed to a particular IP without using trace.
 *
 * Proceed until we reach @ip or until:
 *
 *   - @block is full:               return zero
 *   - @block would switch sections: return zero
 *   - we would need trace:          return -pte_bad_query
 *
 * Provide the last instruction that was reached in @insn and @iext.  If we
 * reached @ip, this is the instruction preceding it.
 *
 * Update @decoder->ip to point to the last IP that was reached.  If we fail due
 * to lack of trace, this is @insn->ip; otherwise this is the next instruction's
 * IP.
 *
 * Returns a positive integer if @ip was reached.
 * Returns zero if no such instruction was reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_to_ip(struct pt_block_decoder *decoder,
                                struct pt_block *block, struct pt_insn *insn,
                                struct pt_insn_ext *iext, uint64_t ip)
{
        int status;

        if (!decoder || !insn)
                return -pte_internal;

        for (;;) {
                /* We're done when we reach @ip.  We may not even have to decode
                 * a single instruction in some cases.
                 */
                if (decoder->ip == ip)
                        return 1;

                status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
                if (status <= 0)
                        return status;

                /* Let's see if we can proceed to the next IP without trace. */
                status = pt_insn_next_ip(&decoder->ip, insn, iext);
                if (status < 0)
                        return status;

                /* End the block if the user asked us to.
                 *
                 * We only need to take care about direct near branches.
                 * Indirect and far branches require trace and will naturally
                 * end a block.
                 *
                 * The call at the end of the block may have reached @ip; make
                 * sure to indicate that.
                 */
                if ((decoder->flags.variant.block.end_on_call &&
                     (insn->iclass == ptic_call)) ||
                    (decoder->flags.variant.block.end_on_jump &&
                     (insn->iclass == ptic_jump))) {
                        return (decoder->ip == ip ? 1 : 0);
                }
        }
}

/* Proceed to a particular IP with trace, if necessary.
 *
 * Proceed until we reach @ip or until:
 *
 *   - @block is full:               return zero
 *   - @block would switch sections: return zero
 *   - we need trace:                return zero
 *
 * Update @decoder->ip to point to the last IP that was reached.
 *
 * A return of zero ends @block.
 *
 * Returns a positive integer if @ip was reached.
 * Returns zero if no such instruction was reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_to_ip_with_trace(struct pt_block_decoder *decoder,
                                           struct pt_block *block,
                                           uint64_t ip)
{
        struct pt_insn_ext iext;
        struct pt_insn insn;
        int status;

        /* Try to reach @ip without trace.
         *
         * We're also OK if @block overflowed or we switched sections and we
         * have to try again in the next iteration.
         */
        status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext, ip);
        if (status != -pte_bad_query)
                return status;

        /* Needing trace is not an error.  We use trace to determine the next
         * start IP and end the block.
         */
        return pt_blk_proceed_with_trace(decoder, &insn, &iext);
}

static int pt_insn_skl014(const struct pt_insn *insn,
                          const struct pt_insn_ext *iext)
{
        if (!insn || !iext)
                return 0;

        switch (insn->iclass) {
        default:
                return 0;

        case ptic_call:
        case ptic_jump:
                return iext->variant.branch.is_direct;

        case ptic_other:
                return pt_insn_changes_cr3(insn, iext);
        }
}

/* Proceed to the location of a synchronous disabled event with suppressed IP
 * considering SKL014.
 *
 * We have a (synchronous) disabled event pending.  Proceed to the event
 * location and indicate whether we were able to reach it.
 *
 * With SKL014 a TIP.PGD with suppressed IP may also be generated by a direct
 * unconditional branch that clears FilterEn by jumping out of a filter region
 * or into a TraceStop region.  Use the filter configuration to determine the
 * exact branch the event binds to.
 *
 * The last instruction that was reached is stored in @insn/@iext.
 *
 * Returns a positive integer if the event location was reached.
 * Returns zero if the event location was not reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_skl014(struct pt_block_decoder *decoder,
                                 struct pt_block *block, struct pt_insn *insn,
                                 struct pt_insn_ext *iext)
{
        const struct pt_conf_addr_filter *addr_filter;
        int status;

        if (!decoder || !block || !insn || !iext)
                return -pte_internal;

        addr_filter = &decoder->query.config.addr_filter;
        for (;;) {
                uint64_t ip;

                status = pt_blk_proceed_to_insn(decoder, block, insn, iext,
                                                pt_insn_skl014);
                if (status <= 0)
                        break;

                /* The erratum doesn't apply if we can bind the event to a
                 * CR3-changing instruction.
                 */
                if (pt_insn_changes_cr3(insn, iext))
                        break;

                /* Check the filter against the branch target. */
                status = pt_insn_next_ip(&ip, insn, iext);
                if (status < 0)
                        break;

                status = pt_filter_addr_check(addr_filter, ip);
                if (status <= 0) {
                        /* We need to flip the indication.
                         *
                         * We reached the event location when @ip lies inside a
                         * tracing-disabled region.
                         */
                        if (!status)
                                status = 1;

                        break;
                }

                /* This is not the correct instruction.  Proceed past it and try
                 * again.
                 */
                decoder->ip = ip;

                /* End the block if the user asked us to.
                 *
                 * We only need to take care about direct near branches.
                 * Indirect and far branches require trace and will naturally
                 * end a block.
                 */
                if ((decoder->flags.variant.block.end_on_call &&
                    (insn->iclass == ptic_call)) ||
                    (decoder->flags.variant.block.end_on_jump &&
                    (insn->iclass == ptic_jump)))
                        break;
        }

        return status;
}

/* Proceed to the event location for a disabled event.
 *
 * We have a (synchronous) disabled event pending.  Proceed to the event
 * location and indicate whether we were able to reach it.
 *
 * The last instruction that was reached is stored in @insn/@iext.
 *
 * Returns a positive integer if the event location was reached.
 * Returns zero if the event location was not reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_to_disabled(struct pt_block_decoder *decoder,
                                      struct pt_block *block,
                                      struct pt_insn *insn,
                                      struct pt_insn_ext *iext,
                                      const struct pt_event *ev)
{
        if (!decoder || !block || !ev)
                return -pte_internal;

        if (ev->ip_suppressed) {
                /* Due to SKL014 the TIP.PGD payload may be suppressed also for
                 * direct branches.
                 *
                 * If we don't have a filter configuration we assume that no
                 * address filters were used and the erratum does not apply.
                 *
                 * We might otherwise disable tracing too early.
                 */
                if (decoder->query.config.addr_filter.config.addr_cfg &&
                    decoder->query.config.errata.skl014)
                        return pt_blk_proceed_skl014(decoder, block, insn,
                                                     iext);

                /* A synchronous disabled event also binds to far branches and
                 * CPL-changing instructions.  Both would require trace,
                 * however, and are thus implicitly handled by erroring out.
                 *
                 * The would-require-trace error is handled by our caller.
                 */
                return pt_blk_proceed_to_insn(decoder, block, insn, iext,
                                              pt_insn_changes_cr3);
        } else
                return pt_blk_proceed_to_ip(decoder, block, insn, iext,
                                            ev->variant.disabled.ip);
}

/* Set the expected resume address for a synchronous disable.
 *
 * On a synchronous disable, @decoder->ip still points to the instruction to
 * which the event bound.  That's not where we expect tracing to resume.
 *
 * For calls, a fair assumption is that tracing resumes after returning from the
 * called function.  For other types of instructions, we simply don't know.
 *
 * Returns zero on success, a negative pt_error_code otherwise.
 */
static int pt_blk_set_disable_resume_ip(struct pt_block_decoder *decoder,
                                        const struct pt_insn *insn)
{
        if (!decoder || !insn)
                return -pte_internal;

        switch (insn->iclass) {
        case ptic_call:
        case ptic_far_call:
                decoder->ip = insn->ip + insn->size;
                break;

        default:
                decoder->ip = 0ull;
                break;
        }

        return 0;
}

/* Proceed to the event location for an async paging event.
 *
 * We have an async paging event pending.  Proceed to the event location and
 * indicate whether we were able to reach it.  Needing trace in order to proceed
 * is not an error in this case but ends the block.
 *
 * Returns a positive integer if the event location was reached.
 * Returns zero if the event location was not reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_to_async_paging(struct pt_block_decoder *decoder,
                                          struct pt_block *block,
                                          const struct pt_event *ev)
{
        int status;

        if (!decoder || !ev)
                return -pte_internal;

        /* Apply the event immediately if we don't have an IP. */
        if (ev->ip_suppressed)
                return 1;

        status = pt_blk_proceed_to_ip_with_trace(decoder, block,
                                                 ev->variant.async_paging.ip);
        if (status < 0)
                return status;

        /* We may have reached the IP. */
        return (decoder->ip == ev->variant.async_paging.ip ? 1 : 0);
}

/* Proceed to the event location for an async vmcs event.
 *
 * We have an async vmcs event pending.  Proceed to the event location and
 * indicate whether we were able to reach it.  Needing trace in order to proceed
 * is not an error in this case but ends the block.
 *
 * Returns a positive integer if the event location was reached.
 * Returns zero if the event location was not reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_to_async_vmcs(struct pt_block_decoder *decoder,
                                        struct pt_block *block,
                                        const struct pt_event *ev)
{
        int status;

        if (!decoder || !ev)
                return -pte_internal;

        /* Apply the event immediately if we don't have an IP. */
        if (ev->ip_suppressed)
                return 1;

        status = pt_blk_proceed_to_ip_with_trace(decoder, block,
                                                 ev->variant.async_vmcs.ip);
        if (status < 0)
                return status;

        /* We may have reached the IP. */
        return (decoder->ip == ev->variant.async_vmcs.ip ? 1 : 0);
}

/* Proceed to the event location for an exec mode event.
 *
 * We have an exec mode event pending.  Proceed to the event location and
 * indicate whether we were able to reach it.  Needing trace in order to proceed
 * is not an error in this case but ends the block.
 *
 * Returns a positive integer if the event location was reached.
 * Returns zero if the event location was not reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_to_exec_mode(struct pt_block_decoder *decoder,
                                       struct pt_block *block,
                                       const struct pt_event *ev)
{
        int status;

        if (!decoder || !ev)
                return -pte_internal;

        /* Apply the event immediately if we don't have an IP. */
        if (ev->ip_suppressed)
                return 1;

        status = pt_blk_proceed_to_ip_with_trace(decoder, block,
                                                 ev->variant.exec_mode.ip);
        if (status < 0)
                return status;

        /* We may have reached the IP. */
        return (decoder->ip == ev->variant.exec_mode.ip ? 1 : 0);
}

/* Proceed to the event location for a ptwrite event.
 *
 * We have a ptwrite event pending.  Proceed to the event location and indicate
 * whether we were able to reach it.
 *
 * In case of the event binding to a ptwrite instruction, we pass beyond that
 * instruction and update the event to provide the instruction's IP.
 *
 * In the case of the event binding to an IP provided in the event, we move
 * beyond the instruction at that IP.
 *
 * Returns a positive integer if the event location was reached.
 * Returns zero if the event location was not reached.
 * Returns a negative error code otherwise.
 */
static int pt_blk_proceed_to_ptwrite(struct pt_block_decoder *decoder,
                                     struct pt_block *block,
                                     struct pt_insn *insn,
                                     struct pt_insn_ext *iext,
                                     struct pt_event *ev)
{
        int status;

        if (!insn || !ev)
                return -pte_internal;

        /* If we don't have an IP, the event binds to the next PTWRITE
         * instruction.
         *
         * If we have an IP it still binds to the next PTWRITE instruction but
         * now the IP tells us where that instruction is.  This makes most sense
         * when tracing is disabled and we don't have any other means of finding
         * the PTWRITE instruction.  We nevertheless distinguish the two cases,
         * here.
         *
         * In both cases, we move beyond the PTWRITE instruction, so it will be
         * the last instruction in the current block and @decoder->ip will point
         * to the instruction following it.
         */
        if (ev->ip_suppressed) {
                status = pt_blk_proceed_to_insn(decoder, block, insn, iext,
                                                pt_insn_is_ptwrite);
                if (status <= 0)
                        return status;

                /* We now know the IP of the PTWRITE instruction corresponding
                 * to this event.  Fill it in to make it more convenient for the
                 * user to process the event.
                 */
                ev->variant.ptwrite.ip = insn->ip;
                ev->ip_suppressed = 0;
        } else {
                status = pt_blk_proceed_to_ip(decoder, block, insn, iext,
                                              ev->variant.ptwrite.ip);
                if (status <= 0)
                        return status;

                /* We reached the PTWRITE instruction and @decoder->ip points to
                 * it; @insn/@iext still contain the preceding instruction.
                 *
                 * Proceed beyond the PTWRITE to account for it.  Note that we
                 * may still overflow the block, which would cause us to
                 * postpone both instruction and event to the next block.
                 */
                status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
                if (status <= 0)
                        return status;
        }

        return 1;
}

/* Try to work around erratum SKD022.
 *
 * If we get an asynchronous disable on VMLAUNCH or VMRESUME, the FUP that
 * caused the disable to be asynchronous might have been bogous.
 *
 * Returns a positive integer if the erratum has been handled.
 * Returns zero if the erratum does not apply.
 * Returns a negative error code otherwise.
 */
static int pt_blk_handle_erratum_skd022(struct pt_block_decoder *decoder,
                                        struct pt_event *ev)
{
        struct pt_insn_ext iext;
        struct pt_insn insn;
        int errcode;

        if (!decoder || !ev)
                return -pte_internal;

        insn.mode = decoder->mode;
        insn.ip = ev->variant.async_disabled.at;

        errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
        if (errcode < 0)
                return 0;

        switch (iext.iclass) {
        default:
                /* The erratum does not apply. */
                return 0;

        case PTI_INST_VMLAUNCH:
        case PTI_INST_VMRESUME:
                /* The erratum may apply.  We can't be sure without a lot more
                 * analysis.  Let's assume it does.
                 *
                 * We turn the async disable into a sync disable.  Our caller
                 * will restart event processing.
                 */
                ev->type = ptev_disabled;
                ev->variant.disabled.ip = ev->variant.async_disabled.ip;

                return 1;
        }
}

/* Postpone proceeding past @insn/@iext and indicate a pending event.
 *
 * There may be further events pending on @insn/@iext.  Postpone proceeding past
 * @insn/@iext until we processed all events that bind to it.
 *
 * Returns a non-negative pt_status_flag bit-vector indicating a pending event
 * on success, a negative pt_error_code otherwise.
 */
static int pt_blk_postpone_insn(struct pt_block_decoder *decoder,
                                const struct pt_insn *insn,
                                const struct pt_insn_ext *iext)
{
        if (!decoder || !insn || !iext)
                return -pte_internal;

        /* Only one can be active. */
        if (decoder->process_insn)
                return -pte_internal;

        decoder->process_insn = 1;
        decoder->insn = *insn;
        decoder->iext = *iext;

        return pt_blk_status(decoder, pts_event_pending);
}

/* Remove any postponed instruction from @decoder.
 *
 * Returns zero on success, a negative pt_error_code otherwise.
 */
static int pt_blk_clear_postponed_insn(struct pt_block_decoder *decoder)
{
        if (!decoder)
                return -pte_internal;

        decoder->process_insn = 0;
        decoder->bound_paging = 0;
        decoder->bound_vmcs = 0;
        decoder->bound_ptwrite = 0;

        return 0;
}

/* Proceed past a postponed instruction.
 *
 * If an instruction has been postponed in @decoder, proceed past it.
 *
 * Returns zero on success, a negative pt_error_code otherwise.
 */
static int pt_blk_proceed_postponed_insn(struct pt_block_decoder *decoder)
{
        int status;

        if (!decoder)
                return -pte_internal;

        /* There's nothing to do if we have no postponed instruction. */
        if (!decoder->process_insn)
                return 0;

        /* There's nothing to do if tracing got disabled. */
        if (!decoder->enabled)
                return pt_blk_clear_postponed_insn(decoder);

        status = pt_insn_next_ip(&decoder->ip, &decoder->insn, &decoder->iext);
        if (status < 0) {
                if (status != -pte_bad_query)
                        return status;

                status = pt_blk_proceed_with_trace(decoder, &decoder->insn,
                                                   &decoder->iext);
                if (status < 0)
                        return status;
        }

        return pt_blk_clear_postponed_insn(decoder);
}

/* Proceed to the next event.
 *
 * We have an event pending.  Proceed to the event location and indicate the
 * event to the user.
 *
 * On our way to the event location we may also be forced to postpone the event
 * to the next block, e.g. if we overflow the number of instructions in the
 * block or if we need trace in order to reach the event location.
 *
 * If we're not able to reach the event location, we return zero.  This is what
 * pt_blk_status() would return since:
 *
 *   - we suppress pts_eos as long as we're processing events
 *   - we do not set pts_ip_suppressed since tracing must be enabled
 *
 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
 * code otherwise.
 */
static int pt_blk_proceed_event(struct pt_block_decoder *decoder,
                                struct pt_block *block)
{
        struct pt_insn_ext iext;
        struct pt_insn insn;
        struct pt_event *ev;
        int status;

        if (!decoder || !decoder->process_event || !block)
                return -pte_internal;

        ev = &decoder->event;
        switch (ev->type) {
        case ptev_enabled:
                break;

        case ptev_disabled:
                status = pt_blk_proceed_to_disabled(decoder, block, &insn,
                                                    &iext, ev);
                if (status <= 0) {
                        /* A synchronous disable event also binds to the next
                         * indirect or conditional branch, i.e. to any branch
                         * that would have required trace.
                         */
                        if (status != -pte_bad_query)
                                return status;

                        status = pt_blk_set_disable_resume_ip(decoder, &insn);
                        if (status < 0)
                                return status;
                }

                break;

        case ptev_async_disabled:
                status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
                                              ev->variant.async_disabled.at);
                if (status <= 0)
                        return status;

                if (decoder->query.config.errata.skd022) {
                        status = pt_blk_handle_erratum_skd022(decoder, ev);
                        if (status != 0) {
                                if (status < 0)
                                        return status;

                                /* If the erratum hits, we modify the event.
                                 * Try again.
                                 */
                                return pt_blk_proceed_event(decoder, block);
                        }
                }

                break;

        case ptev_async_branch:
                status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
                                              ev->variant.async_branch.from);
                if (status <= 0)
                        return status;

                break;

        case ptev_paging:
                if (!decoder->enabled)
                        break;

                status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
                                                pt_insn_binds_to_pip);
                if (status <= 0)
                        return status;

                /* We bound a paging event.  Make sure we do not bind further
                 * paging events to this instruction.
                 */
                decoder->bound_paging = 1;

                return pt_blk_postpone_insn(decoder, &insn, &iext);

        case ptev_async_paging:
                status = pt_blk_proceed_to_async_paging(decoder, block, ev);
                if (status <= 0)
                        return status;

                break;

        case ptev_vmcs:
                if (!decoder->enabled)
                        break;

                status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
                                                pt_insn_binds_to_vmcs);
                if (status <= 0)
                        return status;

                /* We bound a vmcs event.  Make sure we do not bind further vmcs
                 * events to this instruction.
                 */
                decoder->bound_vmcs = 1;

                return pt_blk_postpone_insn(decoder, &insn, &iext);

        case ptev_async_vmcs:
                status = pt_blk_proceed_to_async_vmcs(decoder, block, ev);
                if (status <= 0)
                        return status;

                break;

        case ptev_overflow:
                break;

        case ptev_exec_mode:
                status = pt_blk_proceed_to_exec_mode(decoder, block, ev);
                if (status <= 0)
                        return status;

                break;

        case ptev_tsx:
                if (ev->ip_suppressed)
                        break;

                status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
                                              ev->variant.tsx.ip);
                if (status <= 0)
                        return status;

                break;

        case ptev_stop:
                break;

        case ptev_exstop:
                if (!decoder->enabled || ev->ip_suppressed)
                        break;

                status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
                                              ev->variant.exstop.ip);
                if (status <= 0)
                        return status;

                break;

        case ptev_mwait:
                if (!decoder->enabled || ev->ip_suppressed)
                        break;

                status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
                                              ev->variant.mwait.ip);
                if (status <= 0)
                        return status;

                break;

        case ptev_pwre:
        case ptev_pwrx:
                break;

        case ptev_ptwrite:
                if (!decoder->enabled)
                        break;

                status = pt_blk_proceed_to_ptwrite(decoder, block, &insn,
                                                   &iext, ev);
                if (status <= 0)
                        return status;

                /* We bound a ptwrite event.  Make sure we do not bind further
                 * ptwrite events to this instruction.
                 */
                decoder->bound_ptwrite = 1;

                return pt_blk_postpone_insn(decoder, &insn, &iext);

        case ptev_tick:
        case ptev_cbr:
        case ptev_mnt:
                break;
        }

        return pt_blk_status(decoder, pts_event_pending);
}

/* Proceed to the next decision point without using the block cache.
 *
 * Tracing is enabled and we don't have an event pending.  Proceed as far as
 * we get without trace.  Stop when we either:
 *
 *   - need trace in order to continue
 *   - overflow the max number of instructions in a block
 *
 * We actually proceed one instruction further to get the start IP for the next
 * block.  This only updates @decoder's internal state, though.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_proceed_no_event_uncached(struct pt_block_decoder *decoder,
                                            struct pt_block *block)
{
        struct pt_insn_ext iext;
        struct pt_insn insn;
        int status;

        if (!decoder || !block)
                return -pte_internal;

        /* This is overly conservative, really.  We shouldn't get a bad-query
         * status unless we decoded at least one instruction successfully.
         */
        memset(&insn, 0, sizeof(insn));
        memset(&iext, 0, sizeof(iext));

        /* Proceed as far as we get without trace. */
        status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
                                        pt_insn_false);
        if (status < 0) {
                if (status != -pte_bad_query)
                        return status;

                return pt_blk_proceed_with_trace(decoder, &insn, &iext);
        }

        return 0;
}

/* Check if @ip is contained in @section loaded at @laddr.
 *
 * Returns non-zero if it is.
 * Returns zero if it isn't or of @section is NULL.
 */
static inline int pt_blk_is_in_section(const struct pt_mapped_section *msec,
                                       uint64_t ip)
{
        uint64_t begin, end;

        begin = pt_msec_begin(msec);
        end = pt_msec_end(msec);

        return (begin <= ip && ip < end);
}

/* Insert a trampoline block cache entry.
 *
 * Add a trampoline block cache entry at @ip to continue at @nip, where @nip
 * must be the next instruction after @ip.
 *
 * Both @ip and @nip must be section-relative
 *
 * Returns zero on success, a negative error code otherwise.
 */
static inline int pt_blk_add_trampoline(struct pt_block_cache *bcache,
                                        uint64_t ip, uint64_t nip,
                                        enum pt_exec_mode mode)
{
        struct pt_bcache_entry bce;
        int64_t disp;

        /* The displacement from @ip to @nip for the trampoline. */
        disp = (int64_t) (nip - ip);

        memset(&bce, 0, sizeof(bce));
        bce.displacement = (int32_t) disp;
        bce.ninsn = 1;
        bce.mode = mode;
        bce.qualifier = ptbq_again;

        /* If we can't reach @nip without overflowing the displacement field, we
         * have to stop and re-decode the instruction at @ip.
         */
        if ((int64_t) bce.displacement != disp) {

                memset(&bce, 0, sizeof(bce));
                bce.ninsn = 1;
                bce.mode = mode;
                bce.qualifier = ptbq_decode;
        }

        return pt_bcache_add(bcache, ip, bce);
}

/* Insert a decode block cache entry.
 *
 * Add a decode block cache entry at @ioff.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static inline int pt_blk_add_decode(struct pt_block_cache *bcache,
                                    uint64_t ioff, enum pt_exec_mode mode)
{
        struct pt_bcache_entry bce;

        memset(&bce, 0, sizeof(bce));
        bce.ninsn = 1;
        bce.mode = mode;
        bce.qualifier = ptbq_decode;

        return pt_bcache_add(bcache, ioff, bce);
}

enum {
        /* The maximum number of steps when filling the block cache. */
        bcache_fill_steps       = 0x400
};

/* Proceed to the next instruction and fill the block cache for @decoder->ip.
 *
 * Tracing is enabled and we don't have an event pending.  The current IP is not
 * yet cached.
 *
 * Proceed one instruction without using the block cache, then try to proceed
 * further using the block cache.
 *
 * On our way back, add a block cache entry for the IP before proceeding.  Note
 * that the recursion is bounded by @steps and ultimately by the maximum number
 * of instructions in a block.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int
pt_blk_proceed_no_event_fill_cache(struct pt_block_decoder *decoder,
                                   struct pt_block *block,
                                   struct pt_block_cache *bcache,
                                   const struct pt_mapped_section *msec,
                                   size_t steps)
{
        struct pt_bcache_entry bce;
        struct pt_insn_ext iext;
        struct pt_insn insn;
        uint64_t nip, dip;
        int64_t disp, ioff, noff;
        int status;

        if (!decoder || !steps)
                return -pte_internal;

        /* Proceed one instruction by decoding and examining it.
         *
         * Note that we also return on a status of zero that indicates that the
         * instruction didn't fit into @block.
         */
        status = pt_blk_proceed_one_insn(decoder, block, &insn, &iext);
        if (status <= 0)
                return status;

        ioff = pt_msec_unmap(msec, insn.ip);

        /* Let's see if we can proceed to the next IP without trace.
         *
         * If we can't, this is certainly a decision point.
         */
        status = pt_insn_next_ip(&decoder->ip, &insn, &iext);
        if (status < 0) {
                if (status != -pte_bad_query)
                        return status;

                memset(&bce, 0, sizeof(bce));
                bce.ninsn = 1;
                bce.mode = insn.mode;
                bce.isize = insn.size;

                /* Clear the instruction size in case of overflows. */
                if ((uint8_t) bce.isize != insn.size)
                        bce.isize = 0;

                switch (insn.iclass) {
                case ptic_ptwrite:
                case ptic_error:
                case ptic_other:
                        return -pte_internal;

                case ptic_jump:
                        /* A direct jump doesn't require trace. */
                        if (iext.variant.branch.is_direct)
                                return -pte_internal;

                        bce.qualifier = ptbq_indirect;
                        break;

                case ptic_call:
                        /* A direct call doesn't require trace. */
                        if (iext.variant.branch.is_direct)
                                return -pte_internal;

                        bce.qualifier = ptbq_ind_call;
                        break;

                case ptic_return:
                        bce.qualifier = ptbq_return;
                        break;

                case ptic_cond_jump:
                        bce.qualifier = ptbq_cond;
                        break;

                case ptic_far_call:
                case ptic_far_return:
                case ptic_far_jump:
                        bce.qualifier = ptbq_indirect;
                        break;
                }

                /* If the block was truncated, we have to decode its last
                 * instruction each time.
                 *
                 * We could have skipped the above switch and size assignment in
                 * this case but this is already a slow and hopefully infrequent
                 * path.
                 */
                if (block->truncated)
                        bce.qualifier = ptbq_decode;

                status = pt_bcache_add(bcache, ioff, bce);
                if (status < 0)
                        return status;

                return pt_blk_proceed_with_trace(decoder, &insn, &iext);
        }

        /* The next instruction's IP. */
        nip = decoder->ip;
        noff = pt_msec_unmap(msec, nip);

        /* Even if we were able to proceed without trace, we might have to stop
         * here for various reasons:
         *
         *   - at near direct calls to update the return-address stack
         *
         *     We are forced to re-decode @insn to get the branch displacement.
         *
         *     Even though it is constant, we don't cache it to avoid increasing
         *     the size of a cache entry.  Note that the displacement field is
         *     zero for this entry and we might be tempted to use it - but other
         *     entries that point to this decision point will have non-zero
         *     displacement.
         *
         *     We could proceed after a near direct call but we migh as well
         *     postpone it to the next iteration.  Make sure to end the block if
         *     @decoder->flags.variant.block.end_on_call is set, though.
         *
         *   - at near direct backwards jumps to detect section splits
         *
         *     In case the current section is split underneath us, we must take
         *     care to detect that split.
         *
         *     There is one corner case where the split is in the middle of a
         *     linear sequence of instructions that branches back into the
         *     originating section.
         *
         *     Calls, indirect branches, and far branches are already covered
         *     since they either require trace or already require us to stop
         *     (i.e. near direct calls) for other reasons.  That leaves near
         *     direct backward jumps.
         *
         *     Instead of the decode stop at the jump instruction we're using we
         *     could have made sure that other block cache entries that extend
         *     this one insert a trampoline to the jump's entry.  This would
         *     have been a bit more complicated.
         *
         *   - if we switched sections
         *
         *     This ends a block just like a branch that requires trace.
         *
         *     We need to re-decode @insn in order to determine the start IP of
         *     the next block.
         *
         *   - if the block is truncated
         *
         *     We need to read the last instruction's memory from multiple
         *     sections and provide it to the user.
         *
         *     We could still use the block cache but then we'd have to handle
         *     this case for each qualifier.  Truncation is hopefully rare and
         *     having to read the memory for the instruction from multiple
         *     sections is already slow.  Let's rather keep things simple and
         *     route it through the decode flow, where we already have
         *     everything in place.
         */
        switch (insn.iclass) {
        case ptic_call:
                return pt_blk_add_decode(bcache, ioff, insn.mode);

        case ptic_jump:
                /* An indirect branch requires trace and should have been
                 * handled above.
                 */
                if (!iext.variant.branch.is_direct)
                        return -pte_internal;

                if (iext.variant.branch.displacement < 0 ||
                    decoder->flags.variant.block.end_on_jump)
                        return pt_blk_add_decode(bcache, ioff, insn.mode);

                fallthrough;
        default:
                if (!pt_blk_is_in_section(msec, nip) || block->truncated)
                        return pt_blk_add_decode(bcache, ioff, insn.mode);

                break;
        }

        /* We proceeded one instruction.  Let's see if we have a cache entry for
         * the next instruction.
         */
        status = pt_bcache_lookup(&bce, bcache, noff);
        if (status < 0)
                return status;

        /* If we don't have a valid cache entry, yet, fill the cache some more.
         *
         * On our way back, we add a cache entry for this instruction based on
         * the cache entry of the succeeding instruction.
         */
        if (!pt_bce_is_valid(bce)) {
                /* If we exceeded the maximum number of allowed steps, we insert
                 * a trampoline to the next instruction.
                 *
                 * The next time we encounter the same code, we will use the
                 * trampoline to jump directly to where we left off this time
                 * and continue from there.
                 */
                steps -= 1;
                if (!steps)
                        return pt_blk_add_trampoline(bcache, ioff, noff,
                                                     insn.mode);

                status = pt_blk_proceed_no_event_fill_cache(decoder, block,
                                                            bcache, msec,
                                                            steps);
                if (status < 0)
                        return status;

                /* Let's see if we have more luck this time. */
                status = pt_bcache_lookup(&bce, bcache, noff);
                if (status < 0)
                        return status;

                /* If we still don't have a valid cache entry, we're done.  Most
                 * likely, @block overflowed and we couldn't proceed past the
                 * next instruction.
                 */
                if (!pt_bce_is_valid(bce))
                        return 0;
        }

        /* We must not have switched execution modes.
         *
         * This would require an event and we're on the no-event flow.
         */
        if (pt_bce_exec_mode(bce) != insn.mode)
                return -pte_internal;

        /* The decision point IP and the displacement from @insn.ip. */
        dip = nip + bce.displacement;
        disp = (int64_t) (dip - insn.ip);

        /* We may have switched sections if the section was split.  See
         * pt_blk_proceed_no_event_cached() for a more elaborate comment.
         *
         * We're not adding a block cache entry since this won't apply to the
         * original section which may be shared with other decoders.
         *
         * We will instead take the slow path until the end of the section.
         */
        if (!pt_blk_is_in_section(msec, dip))
                return 0;

        /* Let's try to reach @nip's decision point from @insn.ip.
         *
         * There are two fields that may overflow: @bce.ninsn and
         * @bce.displacement.
         */
        bce.ninsn += 1;
        bce.displacement = (int32_t) disp;

        /* If none of them overflowed, we're done.
         *
         * If one or both overflowed, let's try to insert a trampoline, i.e. we
         * try to reach @dip via a ptbq_again entry to @nip.
         */
        if (!bce.ninsn || ((int64_t) bce.displacement != disp))
                return pt_blk_add_trampoline(bcache, ioff, noff, insn.mode);

        /* We're done.  Add the cache entry.
         *
         * There's a chance that other decoders updated the cache entry in the
         * meantime.  They should have come to the same conclusion as we,
         * though, and the cache entries should be identical.
         *
         * Cache updates are atomic so even if the two versions were not
         * identical, we wouldn't care because they are both correct.
         */
        return pt_bcache_add(bcache, ioff, bce);
}

/* Proceed at a potentially truncated instruction.
 *
 * We were not able to decode the instruction at @decoder->ip in @decoder's
 * cached section.  This is typically caused by not having enough bytes.
 *
 * Try to decode the instruction again using the entire image.  If this succeeds
 * we expect to end up with an instruction that was truncated in the section it
 * started.  We provide the full instruction in this case and end the block.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_proceed_truncated(struct pt_block_decoder *decoder,
                                    struct pt_block *block)
{
        struct pt_insn_ext iext;
        struct pt_insn insn;
        int errcode;

        if (!decoder || !block)
                return -pte_internal;

        memset(&iext, 0, sizeof(iext));
        memset(&insn, 0, sizeof(insn));

        insn.mode = decoder->mode;
        insn.ip = decoder->ip;

        errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
        if (errcode < 0)
                return errcode;

        /* We shouldn't use this function if the instruction isn't truncated. */
        if (!insn.truncated)
                return -pte_internal;

        /* Provide the instruction in the block.  This ends the block. */
        memcpy(block->raw, insn.raw, insn.size);
        block->iclass = insn.iclass;
        block->size = insn.size;
        block->truncated = 1;

        /* Log calls' return addresses for return compression. */
        errcode = pt_blk_log_call(decoder, &insn, &iext);
        if (errcode < 0)
                return errcode;

        /* Let's see if we can proceed to the next IP without trace.
         *
         * The truncated instruction ends the block but we still need to get the
         * next block's start IP.
         */
        errcode = pt_insn_next_ip(&decoder->ip, &insn, &iext);
        if (errcode < 0) {
                if (errcode != -pte_bad_query)
                        return errcode;

                return pt_blk_proceed_with_trace(decoder, &insn, &iext);
        }

        return 0;
}

/* Proceed to the next decision point using the block cache.
 *
 * Tracing is enabled and we don't have an event pending.  We already set
 * @block's isid.  All reads are done within @msec as we're not switching
 * sections between blocks.
 *
 * Proceed as far as we get without trace.  Stop when we either:
 *
 *   - need trace in order to continue
 *   - overflow the max number of instructions in a block
 *
 * We actually proceed one instruction further to get the start IP for the next
 * block.  This only updates @decoder's internal state, though.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_proceed_no_event_cached(struct pt_block_decoder *decoder,
                                          struct pt_block *block,
                                          struct pt_block_cache *bcache,
                                          const struct pt_mapped_section *msec)
{
        struct pt_bcache_entry bce;
        uint16_t binsn, ninsn;
        uint64_t offset, nip;
        int status;

        if (!decoder || !block)
                return -pte_internal;

        offset = pt_msec_unmap(msec, decoder->ip);
        status = pt_bcache_lookup(&bce, bcache, offset);
        if (status < 0)
                return status;

        /* If we don't find a valid cache entry, fill the cache. */
        if (!pt_bce_is_valid(bce))
                return pt_blk_proceed_no_event_fill_cache(decoder, block,
                                                          bcache, msec,
                                                          bcache_fill_steps);

        /* If we switched sections, the origianl section must have been split
         * underneath us.  A split preserves the block cache of the original
         * section.
         *
         * Crossing sections requires ending the block so we can indicate the
         * proper isid for the entire block.
         *
         * Plus there's the chance that the new section that caused the original
         * section to split changed instructions.
         *
         * This check will also cover changes to a linear sequence of code we
         * would otherwise have jumped over as long as the start and end are in
         * different sub-sections.
         *
         * Since we stop on every (backwards) branch (through an artificial stop
         * in the case of a near direct backward branch) we will detect all
         * section splits.
         *
         * Switch to the slow path until we reach the end of this section.
         */
        nip = decoder->ip + bce.displacement;
        if (!pt_blk_is_in_section(msec, nip))
                return pt_blk_proceed_no_event_uncached(decoder, block);

        /* We have a valid cache entry.  Let's first check if the way to the
         * decision point still fits into @block.
         *
         * If it doesn't, we end the block without filling it as much as we
         * could since this would require us to switch to the slow path.
         *
         * On the next iteration, we will start with an empty block, which is
         * guaranteed to have enough room for at least one block cache entry.
         */
        binsn = block->ninsn;
        ninsn = binsn + (uint16_t) bce.ninsn;
        if (ninsn < binsn)
                return 0;

        /* Jump ahead to the decision point and proceed from there.
         *
         * We're not switching execution modes so even if @block already has an
         * execution mode, it will be the one we're going to set.
         */
        decoder->ip = nip;

        /* We don't know the instruction class so we should be setting it to
         * ptic_error.  Since we will be able to fill it back in later in most
         * cases, we move the clearing to the switch cases that don't.
         */
        block->end_ip = nip;
        block->ninsn = ninsn;
        block->mode = pt_bce_exec_mode(bce);


        switch (pt_bce_qualifier(bce)) {
        case ptbq_again:
                /* We're not able to reach the actual decision point due to
                 * overflows so we inserted a trampoline.
                 *
                 * We don't know the instruction and it is not guaranteed that
                 * we will proceed further (e.g. if @block overflowed).  Let's
                 * clear any previously stored instruction class which has
                 * become invalid when we updated @block->ninsn.
                 */
                block->iclass = ptic_error;

                return pt_blk_proceed_no_event_cached(decoder, block, bcache,
                                                      msec);

        case ptbq_cond:
                /* We're at a conditional branch. */
                block->iclass = ptic_cond_jump;

                /* Let's first check whether we know the size of the
                 * instruction.  If we do, we might get away without decoding
                 * the instruction.
                 *
                 * If we don't know the size we might as well do the full decode
                 * and proceed-with-trace flow we do for ptbq_decode.
                 */
                if (bce.isize) {
                        uint64_t ip;
                        int taken;

                        /* If the branch is not taken, we don't need to decode
                         * the instruction at @decoder->ip.
                         *
                         * If it is taken, we have to implement everything here.
                         * We can't use the normal decode and proceed-with-trace
                         * flow since we already consumed the TNT bit.
                         */
                        status = pt_blk_cond_branch(decoder, &taken);
                        if (status < 0)
                                return status;

                        /* Preserve the query decoder's response which indicates
                         * upcoming events.
                         */
                        decoder->status = status;

                        ip = decoder->ip;
                        if (taken) {
                                struct pt_insn_ext iext;
                                struct pt_insn insn;

                                memset(&iext, 0, sizeof(iext));
                                memset(&insn, 0, sizeof(insn));

                                insn.mode = pt_bce_exec_mode(bce);
                                insn.ip = ip;

                                status = pt_blk_decode_in_section(&insn, &iext,
                                                                  msec);
                                if (status < 0)
                                        return status;

                                ip += iext.variant.branch.displacement;
                        }

                        decoder->ip = ip + bce.isize;
                        break;
                }

                fallthrough;
        case ptbq_decode: {
                struct pt_insn_ext iext;
                struct pt_insn insn;

                /* We need to decode the instruction at @decoder->ip and decide
                 * what to do based on that.
                 *
                 * We already accounted for the instruction so we can't just
                 * call pt_blk_proceed_one_insn().
                 */

                memset(&iext, 0, sizeof(iext));
                memset(&insn, 0, sizeof(insn));

                insn.mode = pt_bce_exec_mode(bce);
                insn.ip = decoder->ip;

                status = pt_blk_decode_in_section(&insn, &iext, msec);
                if (status < 0) {
                        if (status != -pte_bad_insn)
                                return status;

                        return pt_blk_proceed_truncated(decoder, block);
                }

                /* We just decoded @insn so we know the instruction class. */
                block->iclass = insn.iclass;

                /* Log calls' return addresses for return compression. */
                status = pt_blk_log_call(decoder, &insn, &iext);
                if (status < 0)
                        return status;

                /* Let's see if we can proceed to the next IP without trace.
                 *
                 * Note that we also stop due to displacement overflows or to
                 * maintain the return-address stack for near direct calls.
                 */
                status = pt_insn_next_ip(&decoder->ip, &insn, &iext);
                if (status < 0) {
                        if (status != -pte_bad_query)
                                return status;

                        /* We can't, so let's proceed with trace, which
                         * completes the block.
                         */
                        return pt_blk_proceed_with_trace(decoder, &insn, &iext);
                }

                /* End the block if the user asked us to.
                 *
                 * We only need to take care about direct near branches.
                 * Indirect and far branches require trace and will naturally
                 * end a block.
                 */
                if ((decoder->flags.variant.block.end_on_call &&
                     (insn.iclass == ptic_call)) ||
                    (decoder->flags.variant.block.end_on_jump &&
                     (insn.iclass == ptic_jump)))
                        break;

                /* If we can proceed without trace and we stay in @msec we may
                 * proceed further.
                 *
                 * We're done if we switch sections, though.
                 */
                if (!pt_blk_is_in_section(msec, decoder->ip))
                        break;

                return pt_blk_proceed_no_event_cached(decoder, block, bcache,
                                                      msec);
        }

        case ptbq_ind_call: {
                uint64_t ip;

                /* We're at a near indirect call. */
                block->iclass = ptic_call;

                /* We need to update the return-address stack and query the
                 * destination IP.
                 */
                ip = decoder->ip;

                /* If we already know the size of the instruction, we don't need
                 * to re-decode it.
                 */
                if (bce.isize)
                        ip += bce.isize;
                else {
                        struct pt_insn_ext iext;
                        struct pt_insn insn;

                        memset(&iext, 0, sizeof(iext));
                        memset(&insn, 0, sizeof(insn));

                        insn.mode = pt_bce_exec_mode(bce);
                        insn.ip = ip;

                        status = pt_blk_decode_in_section(&insn, &iext, msec);
                        if (status < 0)
                                return status;

                        ip += insn.size;
                }

                status = pt_retstack_push(&decoder->retstack, ip);
                if (status < 0)
                        return status;

                status = pt_blk_indirect_branch(decoder, &decoder->ip);
                if (status < 0)
                        return status;

                /* Preserve the query decoder's response which indicates
                 * upcoming events.
                 */
                decoder->status = status;
                break;
        }

        case ptbq_return: {
                int taken;

                /* We're at a near return. */
                block->iclass = ptic_return;

                /* Check for a compressed return. */
                status = pt_blk_cond_branch(decoder, &taken);
                if (status < 0) {
                        if (status != -pte_bad_query)
                                return status;

                        /* The return is not compressed.  We need another query
                         * to determine the destination IP.
                         */
                        status = pt_blk_indirect_branch(decoder, &decoder->ip);
                        if (status < 0)
                                return status;

                        /* Preserve the query decoder's response which indicates
                         * upcoming events.
                         */
                        decoder->status = status;
                        break;
                }

                /* Preserve the query decoder's response which indicates
                 * upcoming events.
                 */
                decoder->status = status;

                /* A compressed return is indicated by a taken conditional
                 * branch.
                 */
                if (!taken)
                        return -pte_bad_retcomp;

                return pt_retstack_pop(&decoder->retstack, &decoder->ip);
        }

        case ptbq_indirect:
                /* We're at an indirect jump or far transfer.
                 *
                 * We don't know the exact instruction class and there's no
                 * reason to decode the instruction for any other purpose.
                 *
                 * Indicate that we don't know the instruction class and leave
                 * it to our caller to decode the instruction if needed.
                 */
                block->iclass = ptic_error;

                /* This is neither a near call nor return so we don't need to
                 * touch the return-address stack.
                 *
                 * Just query the destination IP.
                 */
                status = pt_blk_indirect_branch(decoder, &decoder->ip);
                if (status < 0)
                        return status;

                /* Preserve the query decoder's response which indicates
                 * upcoming events.
                 */
                decoder->status = status;
                break;
        }

        return 0;
}

static int pt_blk_msec_fill(struct pt_block_decoder *decoder,
                            const struct pt_mapped_section **pmsec)
{
        const struct pt_mapped_section *msec;
        struct pt_section *section;
        int isid, errcode;

        if (!decoder || !pmsec)
                return -pte_internal;

        isid = pt_msec_cache_fill(&decoder->scache, &msec,  decoder->image,
                                  &decoder->asid, decoder->ip);
        if (isid < 0)
                return isid;

        section = pt_msec_section(msec);
        if (!section)
                return -pte_internal;

        *pmsec = msec;

        errcode = pt_section_request_bcache(section);
        if (errcode < 0)
                return errcode;

        return isid;
}

static inline int pt_blk_msec_lookup(struct pt_block_decoder *decoder,
                                     const struct pt_mapped_section **pmsec)
{
        int isid;

        if (!decoder)
                return -pte_internal;

        isid = pt_msec_cache_read(&decoder->scache, pmsec, decoder->image,
                                  decoder->ip);
        if (isid < 0) {
                if (isid != -pte_nomap)
                        return isid;

                return pt_blk_msec_fill(decoder, pmsec);
        }

        return isid;
}

/* Proceed to the next decision point - try using the cache.
 *
 * Tracing is enabled and we don't have an event pending.  Proceed as far as
 * we get without trace.  Stop when we either:
 *
 *   - need trace in order to continue
 *   - overflow the max number of instructions in a block
 *
 * We actually proceed one instruction further to get the start IP for the next
 * block.  This only updates @decoder's internal state, though.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_proceed_no_event(struct pt_block_decoder *decoder,
                                   struct pt_block *block)
{
        const struct pt_mapped_section *msec;
        struct pt_block_cache *bcache;
        struct pt_section *section;
        int isid;

        if (!decoder || !block)
                return -pte_internal;

        isid = pt_blk_msec_lookup(decoder, &msec);
        if (isid < 0) {
                if (isid != -pte_nomap)
                        return isid;

                /* Even if there is no such section in the image, we may still
                 * read the memory via the callback function.
                 */
                return pt_blk_proceed_no_event_uncached(decoder, block);
        }

        /* We do not switch sections inside a block. */
        if (isid != block->isid) {
                if (!pt_blk_block_is_empty(block))
                        return 0;

                block->isid = isid;
        }

        section = pt_msec_section(msec);
        if (!section)
                return -pte_internal;

        bcache = pt_section_bcache(section);
        if (!bcache)
                return pt_blk_proceed_no_event_uncached(decoder, block);

        return pt_blk_proceed_no_event_cached(decoder, block, bcache, msec);
}

/* Proceed to the next event or decision point.
 *
 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
 * code otherwise.
 */
static int pt_blk_proceed(struct pt_block_decoder *decoder,
                          struct pt_block *block)
{
        int status;

        status = pt_blk_fetch_event(decoder);
        if (status != 0) {
                if (status < 0)
                        return status;

                return pt_blk_proceed_event(decoder, block);
        }

        /* If tracing is disabled we should either be out of trace or we should
         * have taken the event flow above.
         */
        if (!decoder->enabled) {
                if (decoder->status & pts_eos)
                        return -pte_eos;

                return -pte_no_enable;
        }

        status = pt_blk_proceed_no_event(decoder, block);
        if (status < 0)
                return status;

        return pt_blk_proceed_trailing_event(decoder, block);
}

enum {
        /* The maximum number of steps to take when determining whether the
         * event location can be reached.
         */
        bdm64_max_steps = 0x100
};

/* Try to work around erratum BDM64.
 *
 * If we got a transaction abort immediately following a branch that produced
 * trace, the trace for that branch might have been corrupted.
 *
 * Returns a positive integer if the erratum was handled.
 * Returns zero if the erratum does not seem to apply.
 * Returns a negative error code otherwise.
 */
static int pt_blk_handle_erratum_bdm64(struct pt_block_decoder *decoder,
                                       const struct pt_block *block,
                                       const struct pt_event *ev)
{
        struct pt_insn_ext iext;
        struct pt_insn insn;
        int status;

        if (!decoder || !block || !ev)
                return -pte_internal;

        /* This only affects aborts. */
        if (!ev->variant.tsx.aborted)
                return 0;

        /* This only affects branches that require trace.
         *
         * If the erratum hits, that branch ended the current block and brought
         * us to the trailing event flow.
         */
        if (pt_blk_block_is_empty(block))
                return 0;

        insn.mode = block->mode;
        insn.ip = block->end_ip;

        status = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
        if (status < 0)
                return 0;

        if (!pt_insn_is_branch(&insn, &iext))
                return 0;

        /* Let's check if we can reach the event location from here.
         *
         * If we can, let's assume the erratum did not hit.  We might still be
         * wrong but we're not able to tell.
         */
        status = pt_insn_range_is_contiguous(decoder->ip, ev->variant.tsx.ip,
                                             decoder->mode, decoder->image,
                                             &decoder->asid, bdm64_max_steps);
        if (status > 0)
                return status;

        /* We can't reach the event location.  This could either mean that we
         * stopped too early (and status is zero) or that the erratum hit.
         *
         * We assume the latter and pretend that the previous branch brought us
         * to the event location, instead.
         */
        decoder->ip = ev->variant.tsx.ip;

        return 1;
}

/* Check whether a trailing TSX event should be postponed.
 *
 * This involves handling erratum BDM64.
 *
 * Returns a positive integer if the event is to be postponed.
 * Returns zero if the event should be processed.
 * Returns a negative error code otherwise.
 */
static inline int pt_blk_postpone_trailing_tsx(struct pt_block_decoder *decoder,
                                               struct pt_block *block,
                                               const struct pt_event *ev)
{
        int status;

        if (!decoder || !ev)
                return -pte_internal;

        if (ev->ip_suppressed)
                return 0;

        if (block && decoder->query.config.errata.bdm64) {
                status = pt_blk_handle_erratum_bdm64(decoder, block, ev);
                if (status < 0)
                        return 1;
        }

        if (decoder->ip != ev->variant.tsx.ip)
                return 1;

        return 0;
}

/* Proceed with events that bind to the current decoder IP.
 *
 * This function is used in the following scenarios:
 *
 *   - we just synchronized onto the trace stream
 *   - we ended a block and proceeded to the next IP
 *   - we processed an event that was indicated by this function
 *
 * Check if there is an event at the current IP that needs to be indicated to
 * the user.
 *
 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
 * code otherwise.
 */
static int pt_blk_proceed_trailing_event(struct pt_block_decoder *decoder,
                                         struct pt_block *block)
{
        struct pt_event *ev;
        int status;

        if (!decoder)
                return -pte_internal;

        status = pt_blk_fetch_event(decoder);
        if (status <= 0) {
                if (status < 0)
                        return status;

                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                return pt_blk_status(decoder, 0);
        }

        ev = &decoder->event;
        switch (ev->type) {
        case ptev_disabled:
                /* Synchronous disable events are normally indicated on the
                 * event flow.
                 */
                if (!decoder->process_insn)
                        break;

                /* A sync disable may bind to a CR3 changing instruction. */
                if (ev->ip_suppressed &&
                    pt_insn_changes_cr3(&decoder->insn, &decoder->iext))
                        return pt_blk_status(decoder, pts_event_pending);

                /* Or it binds to the next branch that would require trace.
                 *
                 * Try to complete processing the current instruction by
                 * proceeding past it.  If that fails because it would require
                 * trace, we can apply the disabled event.
                 */
                status = pt_insn_next_ip(&decoder->ip, &decoder->insn,
                                         &decoder->iext);
                if (status < 0) {
                        if (status != -pte_bad_query)
                                return status;

                        status = pt_blk_set_disable_resume_ip(decoder,
                                                              &decoder->insn);
                        if (status < 0)
                                return status;

                        return pt_blk_status(decoder, pts_event_pending);
                }

                /* We proceeded past the current instruction. */
                status = pt_blk_clear_postponed_insn(decoder);
                if (status < 0)
                        return status;

                /* This might have brought us to the disable IP. */
                if (!ev->ip_suppressed &&
                    decoder->ip == ev->variant.disabled.ip)
                        return pt_blk_status(decoder, pts_event_pending);

                break;

        case ptev_enabled:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_async_disabled:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                if (decoder->ip != ev->variant.async_disabled.at)
                        break;

                if (decoder->query.config.errata.skd022) {
                        status = pt_blk_handle_erratum_skd022(decoder, ev);
                        if (status != 0) {
                                if (status < 0)
                                        return status;

                                /* If the erratum applies, the event is modified
                                 * to a synchronous disable event that will be
                                 * processed on the next pt_blk_proceed_event()
                                 * call.  We're done.
                                 */
                                break;
                        }
                }

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_async_branch:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                if (decoder->ip != ev->variant.async_branch.from)
                        break;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_paging:
                /* We apply the event immediately if we're not tracing. */
                if (!decoder->enabled)
                        return pt_blk_status(decoder, pts_event_pending);

                /* Synchronous paging events are normally indicated on the event
                 * flow, unless they bind to the same instruction as a previous
                 * event.
                 *
                 * We bind at most one paging event to an instruction, though.
                 */
                if (!decoder->process_insn || decoder->bound_paging)
                        break;

                /* We're done if we're not binding to the currently postponed
                 * instruction.  We will process the event on the normal event
                 * flow in the next iteration.
                 */
                if (!pt_insn_binds_to_pip(&decoder->insn, &decoder->iext))
                        break;

                /* We bound a paging event.  Make sure we do not bind further
                 * paging events to this instruction.
                 */
                decoder->bound_paging = 1;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_async_paging:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                if (!ev->ip_suppressed &&
                    decoder->ip != ev->variant.async_paging.ip)
                        break;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_vmcs:
                /* We apply the event immediately if we're not tracing. */
                if (!decoder->enabled)
                        return pt_blk_status(decoder, pts_event_pending);

                /* Synchronous vmcs events are normally indicated on the event
                 * flow, unless they bind to the same instruction as a previous
                 * event.
                 *
                 * We bind at most one vmcs event to an instruction, though.
                 */
                if (!decoder->process_insn || decoder->bound_vmcs)
                        break;

                /* We're done if we're not binding to the currently postponed
                 * instruction.  We will process the event on the normal event
                 * flow in the next iteration.
                 */
                if (!pt_insn_binds_to_vmcs(&decoder->insn, &decoder->iext))
                        break;

                /* We bound a vmcs event.  Make sure we do not bind further vmcs
                 * events to this instruction.
                 */
                decoder->bound_vmcs = 1;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_async_vmcs:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                if (!ev->ip_suppressed &&
                    decoder->ip != ev->variant.async_vmcs.ip)
                        break;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_overflow:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_exec_mode:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                if (!ev->ip_suppressed &&
                    decoder->ip != ev->variant.exec_mode.ip)
                        break;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_tsx:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                status = pt_blk_postpone_trailing_tsx(decoder, block, ev);
                if (status != 0) {
                        if (status < 0)
                                return status;

                        break;
                }

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_stop:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_exstop:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                if (!ev->ip_suppressed && decoder->enabled &&
                    decoder->ip != ev->variant.exstop.ip)
                        break;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_mwait:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                if (!ev->ip_suppressed && decoder->enabled &&
                    decoder->ip != ev->variant.mwait.ip)
                        break;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_pwre:
        case ptev_pwrx:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_ptwrite:
                /* We apply the event immediately if we're not tracing. */
                if (!decoder->enabled)
                        return pt_blk_status(decoder, pts_event_pending);

                /* Ptwrite events are normally indicated on the event flow,
                 * unless they bind to the same instruction as a previous event.
                 *
                 * We bind at most one ptwrite event to an instruction, though.
                 */
                if (!decoder->process_insn || decoder->bound_ptwrite)
                        break;

                /* We're done if we're not binding to the currently postponed
                 * instruction.  We will process the event on the normal event
                 * flow in the next iteration.
                 */
                if (!ev->ip_suppressed ||
                    !pt_insn_is_ptwrite(&decoder->insn, &decoder->iext))
                        break;

                /* We bound a ptwrite event.  Make sure we do not bind further
                 * ptwrite events to this instruction.
                 */
                decoder->bound_ptwrite = 1;

                return pt_blk_status(decoder, pts_event_pending);

        case ptev_tick:
        case ptev_cbr:
        case ptev_mnt:
                /* This event does not bind to an instruction. */
                status = pt_blk_proceed_postponed_insn(decoder);
                if (status < 0)
                        return status;

                return pt_blk_status(decoder, pts_event_pending);
        }

        /* No further events.  Proceed past any postponed instruction. */
        status = pt_blk_proceed_postponed_insn(decoder);
        if (status < 0)
                return status;

        return pt_blk_status(decoder, 0);
}

int pt_blk_next(struct pt_block_decoder *decoder, struct pt_block *ublock,
                size_t size)
{
        struct pt_block block, *pblock;
        int errcode, status;

        if (!decoder || !ublock)
                return -pte_invalid;

        pblock = size == sizeof(block) ? ublock : &block;

        /* Zero-initialize the block in case of error returns. */
        memset(pblock, 0, sizeof(*pblock));

        /* Fill in a few things from the current decode state.
         *
         * This reflects the state of the last pt_blk_next() or pt_blk_start()
         * call.  Note that, unless we stop with tracing disabled, we proceed
         * already to the start IP of the next block.
         *
         * Some of the state may later be overwritten as we process events.
         */
        pblock->ip = decoder->ip;
        pblock->mode = decoder->mode;
        if (decoder->speculative)
                pblock->speculative = 1;

        /* Proceed one block. */
        status = pt_blk_proceed(decoder, pblock);

        errcode = block_to_user(ublock, size, pblock);
        if (errcode < 0)
                return errcode;

        return status;
}

/* Process an enabled event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_enabled(struct pt_block_decoder *decoder,
                                  const struct pt_event *ev)
{
        if (!decoder || !ev)
                return -pte_internal;

        /* This event can't be a status update. */
        if (ev->status_update)
                return -pte_bad_context;

        /* We must have an IP in order to start decoding. */
        if (ev->ip_suppressed)
                return -pte_noip;

        /* We must currently be disabled. */
        if (decoder->enabled)
                return -pte_bad_context;

        decoder->ip = ev->variant.enabled.ip;
        decoder->enabled = 1;
        decoder->process_event = 0;

        return 0;
}

/* Process a disabled event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_disabled(struct pt_block_decoder *decoder,
                                   const struct pt_event *ev)
{
        if (!decoder || !ev)
                return -pte_internal;

        /* This event can't be a status update. */
        if (ev->status_update)
                return -pte_bad_context;

        /* We must currently be enabled. */
        if (!decoder->enabled)
                return -pte_bad_context;

        /* We preserve @decoder->ip.  This is where we expect tracing to resume
         * and we'll indicate that on the subsequent enabled event if tracing
         * actually does resume from there.
         */
        decoder->enabled = 0;
        decoder->process_event = 0;

        return 0;
}

/* Process an asynchronous branch event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_async_branch(struct pt_block_decoder *decoder,
                                       const struct pt_event *ev)
{
        if (!decoder || !ev)
                return -pte_internal;

        /* This event can't be a status update. */
        if (ev->status_update)
                return -pte_bad_context;

        /* We must currently be enabled. */
        if (!decoder->enabled)
                return -pte_bad_context;

        /* Jump to the branch destination.  We will continue from there in the
         * next iteration.
         */
        decoder->ip = ev->variant.async_branch.to;
        decoder->process_event = 0;

        return 0;
}

/* Process a paging event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_paging(struct pt_block_decoder *decoder,
                                 const struct pt_event *ev)
{
        uint64_t cr3;
        int errcode;

        if (!decoder || !ev)
                return -pte_internal;

        cr3 = ev->variant.paging.cr3;
        if (decoder->asid.cr3 != cr3) {
                errcode = pt_msec_cache_invalidate(&decoder->scache);
                if (errcode < 0)
                        return errcode;

                decoder->asid.cr3 = cr3;
        }

        decoder->process_event = 0;

        return 0;
}

/* Process a vmcs event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_vmcs(struct pt_block_decoder *decoder,
                               const struct pt_event *ev)
{
        uint64_t vmcs;
        int errcode;

        if (!decoder || !ev)
                return -pte_internal;

        vmcs = ev->variant.vmcs.base;
        if (decoder->asid.vmcs != vmcs) {
                errcode = pt_msec_cache_invalidate(&decoder->scache);
                if (errcode < 0)
                        return errcode;

                decoder->asid.vmcs = vmcs;
        }

        decoder->process_event = 0;

        return 0;
}

/* Process an overflow event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_overflow(struct pt_block_decoder *decoder,
                                   const struct pt_event *ev)
{
        if (!decoder || !ev)
                return -pte_internal;

        /* This event can't be a status update. */
        if (ev->status_update)
                return -pte_bad_context;

        /* If the IP is suppressed, the overflow resolved while tracing was
         * disabled.  Otherwise it resolved while tracing was enabled.
         */
        if (ev->ip_suppressed) {
                /* Tracing is disabled.  It doesn't make sense to preserve the
                 * previous IP.  This will just be misleading.  Even if tracing
                 * had been disabled before, as well, we might have missed the
                 * re-enable in the overflow.
                 */
                decoder->enabled = 0;
                decoder->ip = 0ull;
        } else {
                /* Tracing is enabled and we're at the IP at which the overflow
                 * resolved.
                 */
                decoder->enabled = 1;
                decoder->ip = ev->variant.overflow.ip;
        }

        /* We don't know the TSX state.  Let's assume we execute normally.
         *
         * We also don't know the execution mode.  Let's keep what we have
         * in case we don't get an update before we have to decode the next
         * instruction.
         */
        decoder->speculative = 0;
        decoder->process_event = 0;

        return 0;
}

/* Process an exec mode event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_exec_mode(struct pt_block_decoder *decoder,
                                    const struct pt_event *ev)
{
        enum pt_exec_mode mode;

        if (!decoder || !ev)
                return -pte_internal;

        /* Use status update events to diagnose inconsistencies. */
        mode = ev->variant.exec_mode.mode;
        if (ev->status_update && decoder->enabled &&
            decoder->mode != ptem_unknown && decoder->mode != mode)
                return -pte_bad_status_update;

        decoder->mode = mode;
        decoder->process_event = 0;

        return 0;
}

/* Process a tsx event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_tsx(struct pt_block_decoder *decoder,
                              const struct pt_event *ev)
{
        if (!decoder || !ev)
                return -pte_internal;

        decoder->speculative = ev->variant.tsx.speculative;
        decoder->process_event = 0;

        return 0;
}

/* Process a stop event.
 *
 * Returns zero on success, a negative error code otherwise.
 */
static int pt_blk_process_stop(struct pt_block_decoder *decoder,
                               const struct pt_event *ev)
{
        if (!decoder || !ev)
                return -pte_internal;

        /* This event can't be a status update. */
        if (ev->status_update)
                return -pte_bad_context;

        /* Tracing is always disabled before it is stopped. */
        if (decoder->enabled)
                return -pte_bad_context;

        decoder->process_event = 0;

        return 0;
}

int pt_blk_event(struct pt_block_decoder *decoder, struct pt_event *uevent,
                 size_t size)
{
        struct pt_event *ev;
        int status;

        if (!decoder || !uevent)
                return -pte_invalid;

        /* We must currently process an event. */
        if (!decoder->process_event)
                return -pte_bad_query;

        ev = &decoder->event;
        switch (ev->type) {
        case ptev_enabled:
                /* Indicate that tracing resumes from the IP at which tracing
                 * had been disabled before (with some special treatment for
                 * calls).
                 */
                if (ev->variant.enabled.ip == decoder->ip)
                        ev->variant.enabled.resumed = 1;

                status = pt_blk_process_enabled(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_async_disabled:
                if (decoder->ip != ev->variant.async_disabled.at)
                        return -pte_bad_query;

                fallthrough;
        case ptev_disabled:

                status = pt_blk_process_disabled(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_async_branch:
                if (decoder->ip != ev->variant.async_branch.from)
                        return -pte_bad_query;

                status = pt_blk_process_async_branch(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_async_paging:
                if (!ev->ip_suppressed &&
                    decoder->ip != ev->variant.async_paging.ip)
                        return -pte_bad_query;

                fallthrough;
        case ptev_paging:
                status = pt_blk_process_paging(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_async_vmcs:
                if (!ev->ip_suppressed &&
                    decoder->ip != ev->variant.async_vmcs.ip)
                        return -pte_bad_query;

                fallthrough;
        case ptev_vmcs:
                status = pt_blk_process_vmcs(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_overflow:
                status = pt_blk_process_overflow(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_exec_mode:
                if (!ev->ip_suppressed &&
                    decoder->ip != ev->variant.exec_mode.ip)
                        return -pte_bad_query;

                status = pt_blk_process_exec_mode(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_tsx:
                if (!ev->ip_suppressed && decoder->ip != ev->variant.tsx.ip)
                        return -pte_bad_query;

                status = pt_blk_process_tsx(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_stop:
                status = pt_blk_process_stop(decoder, ev);
                if (status < 0)
                        return status;

                break;

        case ptev_exstop:
                if (!ev->ip_suppressed && decoder->enabled &&
                    decoder->ip != ev->variant.exstop.ip)
                        return -pte_bad_query;

                decoder->process_event = 0;
                break;

        case ptev_mwait:
                if (!ev->ip_suppressed && decoder->enabled &&
                    decoder->ip != ev->variant.mwait.ip)
                        return -pte_bad_query;

                decoder->process_event = 0;
                break;

        case ptev_pwre:
        case ptev_pwrx:
        case ptev_ptwrite:
        case ptev_tick:
        case ptev_cbr:
        case ptev_mnt:
                decoder->process_event = 0;
                break;
        }

        /* Copy the event to the user.  Make sure we're not writing beyond the
         * memory provided by the user.
         *
         * We might truncate details of an event but only for those events the
         * user can't know about, anyway.
         */
        if (sizeof(*ev) < size)
                size = sizeof(*ev);

        memcpy(uevent, ev, size);

        /* Indicate further events. */
        return pt_blk_proceed_trailing_event(decoder, NULL);
}