2 * Copyright (c) 2016-2018, Intel Corporation
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright notice,
8 * this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright notice,
10 * this list of conditions and the following disclaimer in the documentation
11 * and/or other materials provided with the distribution.
12 * * Neither the name of Intel Corporation nor the names of its contributors
13 * may be used to endorse or promote products derived from this software
14 * without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
29 #include "pt_block_decoder.h"
30 #include "pt_block_cache.h"
31 #include "pt_section.h"
34 #include "pt_config.h"
36 #include "pt_compiler.h"
44 static int pt_blk_proceed_trailing_event(struct pt_block_decoder *,
48 static int pt_blk_status(const struct pt_block_decoder *decoder, int flags)
55 status = decoder->status;
57 /* Indicate whether tracing is disabled or enabled.
59 * This duplicates the indication in struct pt_insn and covers the case
60 * where we indicate the status after synchronizing.
62 if (!decoder->enabled)
63 flags |= pts_ip_suppressed;
65 /* Forward end-of-trace indications.
67 * Postpone it as long as we're still processing events, though.
69 if ((status & pts_eos) && !decoder->process_event)
75 static void pt_blk_reset(struct pt_block_decoder *decoder)
80 decoder->mode = ptem_unknown;
84 decoder->process_event = 0;
85 decoder->speculative = 0;
86 decoder->process_insn = 0;
87 decoder->bound_paging = 0;
88 decoder->bound_vmcs = 0;
89 decoder->bound_ptwrite = 0;
91 memset(&decoder->event, 0, sizeof(decoder->event));
92 pt_retstack_init(&decoder->retstack);
93 pt_asid_init(&decoder->asid);
96 /* Initialize the query decoder flags based on our flags. */
98 static int pt_blk_init_qry_flags(struct pt_conf_flags *qflags,
99 const struct pt_conf_flags *flags)
101 if (!qflags || !flags)
102 return -pte_internal;
104 memset(qflags, 0, sizeof(*qflags));
109 int pt_blk_decoder_init(struct pt_block_decoder *decoder,
110 const struct pt_config *uconfig)
112 struct pt_config config;
116 return -pte_internal;
118 errcode = pt_config_from_user(&config, uconfig);
122 /* The user supplied decoder flags. */
123 decoder->flags = config.flags;
125 /* Set the flags we need for the query decoder we use. */
126 errcode = pt_blk_init_qry_flags(&config.flags, &decoder->flags);
130 errcode = pt_qry_decoder_init(&decoder->query, &config);
134 pt_image_init(&decoder->default_image, NULL);
135 decoder->image = &decoder->default_image;
137 errcode = pt_msec_cache_init(&decoder->scache);
141 pt_blk_reset(decoder);
146 void pt_blk_decoder_fini(struct pt_block_decoder *decoder)
151 pt_msec_cache_fini(&decoder->scache);
152 pt_image_fini(&decoder->default_image);
153 pt_qry_decoder_fini(&decoder->query);
156 struct pt_block_decoder *
157 pt_blk_alloc_decoder(const struct pt_config *config)
159 struct pt_block_decoder *decoder;
162 decoder = malloc(sizeof(*decoder));
166 errcode = pt_blk_decoder_init(decoder, config);
175 void pt_blk_free_decoder(struct pt_block_decoder *decoder)
180 pt_blk_decoder_fini(decoder);
184 /* Maybe synthesize a tick event.
186 * If we're not already processing events, check the current time against the
187 * last event's time. If it changed, synthesize a tick event with the new time.
189 * Returns zero if no tick event has been created.
190 * Returns a positive integer if a tick event has been created.
191 * Returns a negative error code otherwise.
193 static int pt_blk_tick(struct pt_block_decoder *decoder, uint64_t ip)
197 uint32_t lost_mtc, lost_cyc;
201 return -pte_internal;
203 /* We're not generating tick events if tracing is disabled. */
204 if (!decoder->enabled)
205 return -pte_internal;
207 /* Events already provide a timestamp so there is no need to synthesize
208 * an artificial tick event. There's no room, either, since this would
209 * overwrite the in-progress event.
211 * In rare cases where we need to proceed to an event location using
212 * trace this may cause us to miss a timing update if the event is not
213 * forwarded to the user.
215 * The only case I can come up with at the moment is a MODE.EXEC binding
216 * to the TIP IP of a far branch.
218 if (decoder->process_event)
221 errcode = pt_qry_time(&decoder->query, &tsc, &lost_mtc, &lost_cyc);
223 /* If we don't have wall-clock time, we use relative time. */
224 if (errcode != -pte_no_time)
228 ev = &decoder->event;
230 /* We're done if time has not changed since the last event. */
234 /* Time has changed so we create a new tick event. */
235 memset(ev, 0, sizeof(*ev));
236 ev->type = ptev_tick;
237 ev->variant.tick.ip = ip;
239 /* Indicate if we have wall-clock time or only relative time. */
240 if (errcode != -pte_no_time)
243 ev->lost_mtc = lost_mtc;
244 ev->lost_cyc = lost_cyc;
246 /* We now have an event to process. */
247 decoder->process_event = 1;
252 /* Query an indirect branch.
254 * Returns zero on success, a negative error code otherwise.
256 static int pt_blk_indirect_branch(struct pt_block_decoder *decoder,
263 return -pte_internal;
267 status = pt_qry_indirect_branch(&decoder->query, ip);
271 if (decoder->flags.variant.block.enable_tick_events) {
272 errcode = pt_blk_tick(decoder, evip);
280 /* Query a conditional branch.
282 * Returns zero on success, a negative error code otherwise.
284 static int pt_blk_cond_branch(struct pt_block_decoder *decoder, int *taken)
289 return -pte_internal;
291 status = pt_qry_cond_branch(&decoder->query, taken);
295 if (decoder->flags.variant.block.enable_tick_events) {
296 errcode = pt_blk_tick(decoder, decoder->ip);
304 static int pt_blk_start(struct pt_block_decoder *decoder, int status)
307 return -pte_internal;
312 decoder->status = status;
313 if (!(status & pts_ip_suppressed))
314 decoder->enabled = 1;
316 /* We will always have an event.
318 * If we synchronized onto an empty PSB+, tracing is disabled and we'll
319 * process events until the enabled event.
321 * If tracing is enabled, PSB+ must at least provide the execution mode,
322 * which we're going to forward to the user.
324 return pt_blk_proceed_trailing_event(decoder, NULL);
327 static int pt_blk_sync_reset(struct pt_block_decoder *decoder)
330 return -pte_internal;
332 pt_blk_reset(decoder);
337 int pt_blk_sync_forward(struct pt_block_decoder *decoder)
344 errcode = pt_blk_sync_reset(decoder);
348 status = pt_qry_sync_forward(&decoder->query, &decoder->ip);
350 return pt_blk_start(decoder, status);
353 int pt_blk_sync_backward(struct pt_block_decoder *decoder)
360 errcode = pt_blk_sync_reset(decoder);
364 status = pt_qry_sync_backward(&decoder->query, &decoder->ip);
366 return pt_blk_start(decoder, status);
369 int pt_blk_sync_set(struct pt_block_decoder *decoder, uint64_t offset)
376 errcode = pt_blk_sync_reset(decoder);
380 status = pt_qry_sync_set(&decoder->query, &decoder->ip, offset);
382 return pt_blk_start(decoder, status);
385 int pt_blk_get_offset(const struct pt_block_decoder *decoder, uint64_t *offset)
390 return pt_qry_get_offset(&decoder->query, offset);
393 int pt_blk_get_sync_offset(const struct pt_block_decoder *decoder,
399 return pt_qry_get_sync_offset(&decoder->query, offset);
402 struct pt_image *pt_blk_get_image(struct pt_block_decoder *decoder)
407 return decoder->image;
410 int pt_blk_set_image(struct pt_block_decoder *decoder, struct pt_image *image)
416 image = &decoder->default_image;
418 decoder->image = image;
422 const struct pt_config *
423 pt_blk_get_config(const struct pt_block_decoder *decoder)
428 return pt_qry_get_config(&decoder->query);
431 int pt_blk_time(struct pt_block_decoder *decoder, uint64_t *time,
432 uint32_t *lost_mtc, uint32_t *lost_cyc)
434 if (!decoder || !time)
437 return pt_qry_time(&decoder->query, time, lost_mtc, lost_cyc);
440 int pt_blk_core_bus_ratio(struct pt_block_decoder *decoder, uint32_t *cbr)
442 if (!decoder || !cbr)
445 return pt_qry_core_bus_ratio(&decoder->query, cbr);
448 int pt_blk_asid(const struct pt_block_decoder *decoder, struct pt_asid *asid,
451 if (!decoder || !asid)
454 return pt_asid_to_user(asid, &decoder->asid, size);
457 /* Fetch the next pending event.
459 * Checks for pending events. If an event is pending, fetches it (if not
460 * already in process).
462 * Returns zero if no event is pending.
463 * Returns a positive integer if an event is pending or in process.
464 * Returns a negative error code otherwise.
466 static inline int pt_blk_fetch_event(struct pt_block_decoder *decoder)
471 return -pte_internal;
473 if (decoder->process_event)
476 if (!(decoder->status & pts_event_pending))
479 status = pt_qry_event(&decoder->query, &decoder->event,
480 sizeof(decoder->event));
484 decoder->process_event = 1;
485 decoder->status = status;
490 static inline int pt_blk_block_is_empty(const struct pt_block *block)
495 return !block->ninsn;
498 static inline int block_to_user(struct pt_block *ublock, size_t size,
499 const struct pt_block *block)
501 if (!ublock || !block)
502 return -pte_internal;
507 /* Zero out any unknown bytes. */
508 if (sizeof(*block) < size) {
509 memset(ublock + sizeof(*block), 0, size - sizeof(*block));
511 size = sizeof(*block);
514 memcpy(ublock, block, size);
519 static int pt_insn_false(const struct pt_insn *insn,
520 const struct pt_insn_ext *iext)
528 /* Determine the next IP using trace.
530 * Tries to determine the IP of the next instruction using trace and provides it
533 * Not requiring trace to determine the IP is treated as an internal error.
535 * Does not update the return compression stack for indirect calls. This is
536 * expected to have been done, already, when trying to determine the next IP
537 * without using trace.
539 * Does not update @decoder->status. The caller is expected to do that.
541 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
543 * Returns -pte_internal if @pip, @decoder, @insn, or @iext are NULL.
544 * Returns -pte_internal if no trace is required.
546 static int pt_blk_next_ip(uint64_t *pip, struct pt_block_decoder *decoder,
547 const struct pt_insn *insn,
548 const struct pt_insn_ext *iext)
552 if (!pip || !decoder || !insn || !iext)
553 return -pte_internal;
555 /* We handle non-taken conditional branches, and compressed returns
556 * directly in the switch.
558 * All kinds of branches are handled below the switch.
560 switch (insn->iclass) {
561 case ptic_cond_jump: {
565 status = pt_blk_cond_branch(decoder, &taken);
569 ip = insn->ip + insn->size;
571 ip += iext->variant.branch.displacement;
580 /* Check for a compressed return. */
581 status = pt_blk_cond_branch(decoder, &taken);
583 if (status != -pte_bad_query)
589 /* A compressed return is indicated by a taken conditional
593 return -pte_bad_retcomp;
595 errcode = pt_retstack_pop(&decoder->retstack, pip);
604 /* A direct jump or call wouldn't require trace. */
605 if (iext->variant.branch.is_direct)
606 return -pte_internal;
611 case ptic_far_return:
617 return -pte_internal;
620 return -pte_bad_insn;
623 /* Process an indirect branch.
625 * This covers indirect jumps and calls, non-compressed returns, and all
626 * flavors of far transfers.
628 return pt_blk_indirect_branch(decoder, pip);
631 /* Proceed to the next IP using trace.
633 * We failed to proceed without trace. This ends the current block. Now use
634 * trace to do one final step to determine the start IP of the next block.
636 * Returns zero on success, a negative error code otherwise.
638 static int pt_blk_proceed_with_trace(struct pt_block_decoder *decoder,
639 const struct pt_insn *insn,
640 const struct pt_insn_ext *iext)
645 return -pte_internal;
647 status = pt_blk_next_ip(&decoder->ip, decoder, insn, iext);
651 /* Preserve the query decoder's response which indicates upcoming
654 decoder->status = status;
656 /* We do need an IP in order to proceed. */
657 if (status & pts_ip_suppressed)
663 /* Decode one instruction in a known section.
665 * Decode the instruction at @insn->ip in @msec assuming execution mode
668 * Returns zero on success, a negative error code otherwise.
670 static int pt_blk_decode_in_section(struct pt_insn *insn,
671 struct pt_insn_ext *iext,
672 const struct pt_mapped_section *msec)
677 return -pte_internal;
679 /* We know that @ip is contained in @section.
681 * Note that we need to translate @ip into a section offset.
683 status = pt_msec_read(msec, insn->raw, sizeof(insn->raw), insn->ip);
687 /* We initialize @insn->size to the maximal possible size. It will be
688 * set to the actual size during instruction decode.
690 insn->size = (uint8_t) status;
692 return pt_ild_decode(insn, iext);
695 /* Update the return-address stack if @insn is a near call.
697 * Returns zero on success, a negative error code otherwise.
699 static inline int pt_blk_log_call(struct pt_block_decoder *decoder,
700 const struct pt_insn *insn,
701 const struct pt_insn_ext *iext)
703 if (!decoder || !insn || !iext)
704 return -pte_internal;
706 if (insn->iclass != ptic_call)
709 /* Ignore direct calls to the next instruction that are used for
710 * position independent code.
712 if (iext->variant.branch.is_direct &&
713 !iext->variant.branch.displacement)
716 return pt_retstack_push(&decoder->retstack, insn->ip + insn->size);
719 /* Proceed by one instruction.
721 * Tries to decode the instruction at @decoder->ip and, on success, adds it to
722 * @block and provides it in @pinsn and @piext.
724 * The instruction will not be added if:
726 * - the memory could not be read: return error
727 * - it could not be decoded: return error
728 * - @block is already full: return zero
729 * - @block would switch sections: return zero
731 * Returns a positive integer if the instruction was added.
732 * Returns zero if the instruction didn't fit into @block.
733 * Returns a negative error code otherwise.
735 static int pt_blk_proceed_one_insn(struct pt_block_decoder *decoder,
736 struct pt_block *block,
737 struct pt_insn *pinsn,
738 struct pt_insn_ext *piext)
740 struct pt_insn_ext iext;
745 if (!decoder || !block || !pinsn || !piext)
746 return -pte_internal;
748 /* There's nothing to do if there is no room in @block. */
749 ninsn = block->ninsn + 1;
753 /* The truncated instruction must be last. */
754 if (block->truncated)
757 memset(&insn, 0, sizeof(insn));
758 memset(&iext, 0, sizeof(iext));
760 insn.mode = decoder->mode;
761 insn.ip = decoder->ip;
763 status = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
767 /* We do not switch sections inside a block. */
768 if (insn.isid != block->isid) {
769 if (!pt_blk_block_is_empty(block))
772 block->isid = insn.isid;
775 /* If we couldn't read @insn's memory in one chunk from @insn.isid, we
776 * provide the memory in @block.
778 if (insn.truncated) {
779 memcpy(block->raw, insn.raw, insn.size);
780 block->size = insn.size;
781 block->truncated = 1;
784 /* Log calls' return addresses for return compression. */
785 status = pt_blk_log_call(decoder, &insn, &iext);
789 /* We have a new instruction. */
790 block->iclass = insn.iclass;
791 block->end_ip = insn.ip;
792 block->ninsn = ninsn;
801 /* Proceed to a particular type of instruction without using trace.
803 * Proceed until we reach an instruction for which @predicate returns a positive
806 * - @predicate returns an error: return error
807 * - @block is full: return zero
808 * - @block would switch sections: return zero
809 * - we would need trace: return -pte_bad_query
811 * Provide the last instruction that was reached in @insn and @iext.
813 * Update @decoder->ip to point to the last IP that was reached. If we fail due
814 * to lack of trace or if we reach a desired instruction, this is @insn->ip;
815 * otherwise this is the next instruction's IP.
817 * Returns a positive integer if a suitable instruction was reached.
818 * Returns zero if no such instruction was reached.
819 * Returns a negative error code otherwise.
821 static int pt_blk_proceed_to_insn(struct pt_block_decoder *decoder,
822 struct pt_block *block,
823 struct pt_insn *insn,
824 struct pt_insn_ext *iext,
825 int (*predicate)(const struct pt_insn *,
826 const struct pt_insn_ext *))
830 if (!decoder || !insn || !predicate)
831 return -pte_internal;
834 status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
838 /* We're done if this instruction matches the spec (positive
839 * status) or we run into an error (negative status).
841 status = predicate(insn, iext);
845 /* Let's see if we can proceed to the next IP without trace. */
846 status = pt_insn_next_ip(&decoder->ip, insn, iext);
850 /* End the block if the user asked us to.
852 * We only need to take care about direct near branches.
853 * Indirect and far branches require trace and will naturally
856 if ((decoder->flags.variant.block.end_on_call &&
857 (insn->iclass == ptic_call)) ||
858 (decoder->flags.variant.block.end_on_jump &&
859 (insn->iclass == ptic_jump)))
864 /* Proceed to a particular IP without using trace.
866 * Proceed until we reach @ip or until:
868 * - @block is full: return zero
869 * - @block would switch sections: return zero
870 * - we would need trace: return -pte_bad_query
872 * Provide the last instruction that was reached in @insn and @iext. If we
873 * reached @ip, this is the instruction preceding it.
875 * Update @decoder->ip to point to the last IP that was reached. If we fail due
876 * to lack of trace, this is @insn->ip; otherwise this is the next instruction's
879 * Returns a positive integer if @ip was reached.
880 * Returns zero if no such instruction was reached.
881 * Returns a negative error code otherwise.
883 static int pt_blk_proceed_to_ip(struct pt_block_decoder *decoder,
884 struct pt_block *block, struct pt_insn *insn,
885 struct pt_insn_ext *iext, uint64_t ip)
889 if (!decoder || !insn)
890 return -pte_internal;
893 /* We're done when we reach @ip. We may not even have to decode
894 * a single instruction in some cases.
896 if (decoder->ip == ip)
899 status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
903 /* Let's see if we can proceed to the next IP without trace. */
904 status = pt_insn_next_ip(&decoder->ip, insn, iext);
908 /* End the block if the user asked us to.
910 * We only need to take care about direct near branches.
911 * Indirect and far branches require trace and will naturally
914 * The call at the end of the block may have reached @ip; make
915 * sure to indicate that.
917 if ((decoder->flags.variant.block.end_on_call &&
918 (insn->iclass == ptic_call)) ||
919 (decoder->flags.variant.block.end_on_jump &&
920 (insn->iclass == ptic_jump))) {
921 return (decoder->ip == ip ? 1 : 0);
926 /* Proceed to a particular IP with trace, if necessary.
928 * Proceed until we reach @ip or until:
930 * - @block is full: return zero
931 * - @block would switch sections: return zero
932 * - we need trace: return zero
934 * Update @decoder->ip to point to the last IP that was reached.
936 * A return of zero ends @block.
938 * Returns a positive integer if @ip was reached.
939 * Returns zero if no such instruction was reached.
940 * Returns a negative error code otherwise.
942 static int pt_blk_proceed_to_ip_with_trace(struct pt_block_decoder *decoder,
943 struct pt_block *block,
946 struct pt_insn_ext iext;
950 /* Try to reach @ip without trace.
952 * We're also OK if @block overflowed or we switched sections and we
953 * have to try again in the next iteration.
955 status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext, ip);
956 if (status != -pte_bad_query)
959 /* Needing trace is not an error. We use trace to determine the next
960 * start IP and end the block.
962 return pt_blk_proceed_with_trace(decoder, &insn, &iext);
965 static int pt_insn_skl014(const struct pt_insn *insn,
966 const struct pt_insn_ext *iext)
971 switch (insn->iclass) {
977 return iext->variant.branch.is_direct;
980 return pt_insn_changes_cr3(insn, iext);
984 /* Proceed to the location of a synchronous disabled event with suppressed IP
985 * considering SKL014.
987 * We have a (synchronous) disabled event pending. Proceed to the event
988 * location and indicate whether we were able to reach it.
990 * With SKL014 a TIP.PGD with suppressed IP may also be generated by a direct
991 * unconditional branch that clears FilterEn by jumping out of a filter region
992 * or into a TraceStop region. Use the filter configuration to determine the
993 * exact branch the event binds to.
995 * The last instruction that was reached is stored in @insn/@iext.
997 * Returns a positive integer if the event location was reached.
998 * Returns zero if the event location was not reached.
999 * Returns a negative error code otherwise.
1001 static int pt_blk_proceed_skl014(struct pt_block_decoder *decoder,
1002 struct pt_block *block, struct pt_insn *insn,
1003 struct pt_insn_ext *iext)
1005 const struct pt_conf_addr_filter *addr_filter;
1008 if (!decoder || !block || !insn || !iext)
1009 return -pte_internal;
1011 addr_filter = &decoder->query.config.addr_filter;
1015 status = pt_blk_proceed_to_insn(decoder, block, insn, iext,
1020 /* The erratum doesn't apply if we can bind the event to a
1021 * CR3-changing instruction.
1023 if (pt_insn_changes_cr3(insn, iext))
1026 /* Check the filter against the branch target. */
1027 status = pt_insn_next_ip(&ip, insn, iext);
1031 status = pt_filter_addr_check(addr_filter, ip);
1033 /* We need to flip the indication.
1035 * We reached the event location when @ip lies inside a
1036 * tracing-disabled region.
1044 /* This is not the correct instruction. Proceed past it and try
1049 /* End the block if the user asked us to.
1051 * We only need to take care about direct near branches.
1052 * Indirect and far branches require trace and will naturally
1055 if ((decoder->flags.variant.block.end_on_call &&
1056 (insn->iclass == ptic_call)) ||
1057 (decoder->flags.variant.block.end_on_jump &&
1058 (insn->iclass == ptic_jump)))
1065 /* Proceed to the event location for a disabled event.
1067 * We have a (synchronous) disabled event pending. Proceed to the event
1068 * location and indicate whether we were able to reach it.
1070 * The last instruction that was reached is stored in @insn/@iext.
1072 * Returns a positive integer if the event location was reached.
1073 * Returns zero if the event location was not reached.
1074 * Returns a negative error code otherwise.
1076 static int pt_blk_proceed_to_disabled(struct pt_block_decoder *decoder,
1077 struct pt_block *block,
1078 struct pt_insn *insn,
1079 struct pt_insn_ext *iext,
1080 const struct pt_event *ev)
1082 if (!decoder || !block || !ev)
1083 return -pte_internal;
1085 if (ev->ip_suppressed) {
1086 /* Due to SKL014 the TIP.PGD payload may be suppressed also for
1089 * If we don't have a filter configuration we assume that no
1090 * address filters were used and the erratum does not apply.
1092 * We might otherwise disable tracing too early.
1094 if (decoder->query.config.addr_filter.config.addr_cfg &&
1095 decoder->query.config.errata.skl014)
1096 return pt_blk_proceed_skl014(decoder, block, insn,
1099 /* A synchronous disabled event also binds to far branches and
1100 * CPL-changing instructions. Both would require trace,
1101 * however, and are thus implicitly handled by erroring out.
1103 * The would-require-trace error is handled by our caller.
1105 return pt_blk_proceed_to_insn(decoder, block, insn, iext,
1106 pt_insn_changes_cr3);
1108 return pt_blk_proceed_to_ip(decoder, block, insn, iext,
1109 ev->variant.disabled.ip);
1112 /* Set the expected resume address for a synchronous disable.
1114 * On a synchronous disable, @decoder->ip still points to the instruction to
1115 * which the event bound. That's not where we expect tracing to resume.
1117 * For calls, a fair assumption is that tracing resumes after returning from the
1118 * called function. For other types of instructions, we simply don't know.
1120 * Returns zero on success, a negative pt_error_code otherwise.
1122 static int pt_blk_set_disable_resume_ip(struct pt_block_decoder *decoder,
1123 const struct pt_insn *insn)
1125 if (!decoder || !insn)
1126 return -pte_internal;
1128 switch (insn->iclass) {
1131 decoder->ip = insn->ip + insn->size;
1142 /* Proceed to the event location for an async paging event.
1144 * We have an async paging event pending. Proceed to the event location and
1145 * indicate whether we were able to reach it. Needing trace in order to proceed
1146 * is not an error in this case but ends the block.
1148 * Returns a positive integer if the event location was reached.
1149 * Returns zero if the event location was not reached.
1150 * Returns a negative error code otherwise.
1152 static int pt_blk_proceed_to_async_paging(struct pt_block_decoder *decoder,
1153 struct pt_block *block,
1154 const struct pt_event *ev)
1158 if (!decoder || !ev)
1159 return -pte_internal;
1161 /* Apply the event immediately if we don't have an IP. */
1162 if (ev->ip_suppressed)
1165 status = pt_blk_proceed_to_ip_with_trace(decoder, block,
1166 ev->variant.async_paging.ip);
1170 /* We may have reached the IP. */
1171 return (decoder->ip == ev->variant.async_paging.ip ? 1 : 0);
1174 /* Proceed to the event location for an async vmcs event.
1176 * We have an async vmcs event pending. Proceed to the event location and
1177 * indicate whether we were able to reach it. Needing trace in order to proceed
1178 * is not an error in this case but ends the block.
1180 * Returns a positive integer if the event location was reached.
1181 * Returns zero if the event location was not reached.
1182 * Returns a negative error code otherwise.
1184 static int pt_blk_proceed_to_async_vmcs(struct pt_block_decoder *decoder,
1185 struct pt_block *block,
1186 const struct pt_event *ev)
1190 if (!decoder || !ev)
1191 return -pte_internal;
1193 /* Apply the event immediately if we don't have an IP. */
1194 if (ev->ip_suppressed)
1197 status = pt_blk_proceed_to_ip_with_trace(decoder, block,
1198 ev->variant.async_vmcs.ip);
1202 /* We may have reached the IP. */
1203 return (decoder->ip == ev->variant.async_vmcs.ip ? 1 : 0);
1206 /* Proceed to the event location for an exec mode event.
1208 * We have an exec mode event pending. Proceed to the event location and
1209 * indicate whether we were able to reach it. Needing trace in order to proceed
1210 * is not an error in this case but ends the block.
1212 * Returns a positive integer if the event location was reached.
1213 * Returns zero if the event location was not reached.
1214 * Returns a negative error code otherwise.
1216 static int pt_blk_proceed_to_exec_mode(struct pt_block_decoder *decoder,
1217 struct pt_block *block,
1218 const struct pt_event *ev)
1222 if (!decoder || !ev)
1223 return -pte_internal;
1225 /* Apply the event immediately if we don't have an IP. */
1226 if (ev->ip_suppressed)
1229 status = pt_blk_proceed_to_ip_with_trace(decoder, block,
1230 ev->variant.exec_mode.ip);
1234 /* We may have reached the IP. */
1235 return (decoder->ip == ev->variant.exec_mode.ip ? 1 : 0);
1238 /* Proceed to the event location for a ptwrite event.
1240 * We have a ptwrite event pending. Proceed to the event location and indicate
1241 * whether we were able to reach it.
1243 * In case of the event binding to a ptwrite instruction, we pass beyond that
1244 * instruction and update the event to provide the instruction's IP.
1246 * In the case of the event binding to an IP provided in the event, we move
1247 * beyond the instruction at that IP.
1249 * Returns a positive integer if the event location was reached.
1250 * Returns zero if the event location was not reached.
1251 * Returns a negative error code otherwise.
1253 static int pt_blk_proceed_to_ptwrite(struct pt_block_decoder *decoder,
1254 struct pt_block *block,
1255 struct pt_insn *insn,
1256 struct pt_insn_ext *iext,
1257 struct pt_event *ev)
1262 return -pte_internal;
1264 /* If we don't have an IP, the event binds to the next PTWRITE
1267 * If we have an IP it still binds to the next PTWRITE instruction but
1268 * now the IP tells us where that instruction is. This makes most sense
1269 * when tracing is disabled and we don't have any other means of finding
1270 * the PTWRITE instruction. We nevertheless distinguish the two cases,
1273 * In both cases, we move beyond the PTWRITE instruction, so it will be
1274 * the last instruction in the current block and @decoder->ip will point
1275 * to the instruction following it.
1277 if (ev->ip_suppressed) {
1278 status = pt_blk_proceed_to_insn(decoder, block, insn, iext,
1279 pt_insn_is_ptwrite);
1283 /* We now know the IP of the PTWRITE instruction corresponding
1284 * to this event. Fill it in to make it more convenient for the
1285 * user to process the event.
1287 ev->variant.ptwrite.ip = insn->ip;
1288 ev->ip_suppressed = 0;
1290 status = pt_blk_proceed_to_ip(decoder, block, insn, iext,
1291 ev->variant.ptwrite.ip);
1295 /* We reached the PTWRITE instruction and @decoder->ip points to
1296 * it; @insn/@iext still contain the preceding instruction.
1298 * Proceed beyond the PTWRITE to account for it. Note that we
1299 * may still overflow the block, which would cause us to
1300 * postpone both instruction and event to the next block.
1302 status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
1310 /* Try to work around erratum SKD022.
1312 * If we get an asynchronous disable on VMLAUNCH or VMRESUME, the FUP that
1313 * caused the disable to be asynchronous might have been bogous.
1315 * Returns a positive integer if the erratum has been handled.
1316 * Returns zero if the erratum does not apply.
1317 * Returns a negative error code otherwise.
1319 static int pt_blk_handle_erratum_skd022(struct pt_block_decoder *decoder,
1320 struct pt_event *ev)
1322 struct pt_insn_ext iext;
1323 struct pt_insn insn;
1326 if (!decoder || !ev)
1327 return -pte_internal;
1329 insn.mode = decoder->mode;
1330 insn.ip = ev->variant.async_disabled.at;
1332 errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
1336 switch (iext.iclass) {
1338 /* The erratum does not apply. */
1341 case PTI_INST_VMLAUNCH:
1342 case PTI_INST_VMRESUME:
1343 /* The erratum may apply. We can't be sure without a lot more
1344 * analysis. Let's assume it does.
1346 * We turn the async disable into a sync disable. Our caller
1347 * will restart event processing.
1349 ev->type = ptev_disabled;
1350 ev->variant.disabled.ip = ev->variant.async_disabled.ip;
1356 /* Postpone proceeding past @insn/@iext and indicate a pending event.
1358 * There may be further events pending on @insn/@iext. Postpone proceeding past
1359 * @insn/@iext until we processed all events that bind to it.
1361 * Returns a non-negative pt_status_flag bit-vector indicating a pending event
1362 * on success, a negative pt_error_code otherwise.
1364 static int pt_blk_postpone_insn(struct pt_block_decoder *decoder,
1365 const struct pt_insn *insn,
1366 const struct pt_insn_ext *iext)
1368 if (!decoder || !insn || !iext)
1369 return -pte_internal;
1371 /* Only one can be active. */
1372 if (decoder->process_insn)
1373 return -pte_internal;
1375 decoder->process_insn = 1;
1376 decoder->insn = *insn;
1377 decoder->iext = *iext;
1379 return pt_blk_status(decoder, pts_event_pending);
1382 /* Remove any postponed instruction from @decoder.
1384 * Returns zero on success, a negative pt_error_code otherwise.
1386 static int pt_blk_clear_postponed_insn(struct pt_block_decoder *decoder)
1389 return -pte_internal;
1391 decoder->process_insn = 0;
1392 decoder->bound_paging = 0;
1393 decoder->bound_vmcs = 0;
1394 decoder->bound_ptwrite = 0;
1399 /* Proceed past a postponed instruction.
1401 * If an instruction has been postponed in @decoder, proceed past it.
1403 * Returns zero on success, a negative pt_error_code otherwise.
1405 static int pt_blk_proceed_postponed_insn(struct pt_block_decoder *decoder)
1410 return -pte_internal;
1412 /* There's nothing to do if we have no postponed instruction. */
1413 if (!decoder->process_insn)
1416 /* There's nothing to do if tracing got disabled. */
1417 if (!decoder->enabled)
1418 return pt_blk_clear_postponed_insn(decoder);
1420 status = pt_insn_next_ip(&decoder->ip, &decoder->insn, &decoder->iext);
1422 if (status != -pte_bad_query)
1425 status = pt_blk_proceed_with_trace(decoder, &decoder->insn,
1431 return pt_blk_clear_postponed_insn(decoder);
1434 /* Proceed to the next event.
1436 * We have an event pending. Proceed to the event location and indicate the
1437 * event to the user.
1439 * On our way to the event location we may also be forced to postpone the event
1440 * to the next block, e.g. if we overflow the number of instructions in the
1441 * block or if we need trace in order to reach the event location.
1443 * If we're not able to reach the event location, we return zero. This is what
1444 * pt_blk_status() would return since:
1446 * - we suppress pts_eos as long as we're processing events
1447 * - we do not set pts_ip_suppressed since tracing must be enabled
1449 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
1452 static int pt_blk_proceed_event(struct pt_block_decoder *decoder,
1453 struct pt_block *block)
1455 struct pt_insn_ext iext;
1456 struct pt_insn insn;
1457 struct pt_event *ev;
1460 if (!decoder || !decoder->process_event || !block)
1461 return -pte_internal;
1463 ev = &decoder->event;
1469 status = pt_blk_proceed_to_disabled(decoder, block, &insn,
1472 /* A synchronous disable event also binds to the next
1473 * indirect or conditional branch, i.e. to any branch
1474 * that would have required trace.
1476 if (status != -pte_bad_query)
1479 status = pt_blk_set_disable_resume_ip(decoder, &insn);
1486 case ptev_async_disabled:
1487 status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
1488 ev->variant.async_disabled.at);
1492 if (decoder->query.config.errata.skd022) {
1493 status = pt_blk_handle_erratum_skd022(decoder, ev);
1498 /* If the erratum hits, we modify the event.
1501 return pt_blk_proceed_event(decoder, block);
1507 case ptev_async_branch:
1508 status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
1509 ev->variant.async_branch.from);
1516 if (!decoder->enabled)
1519 status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
1520 pt_insn_binds_to_pip);
1524 /* We bound a paging event. Make sure we do not bind further
1525 * paging events to this instruction.
1527 decoder->bound_paging = 1;
1529 return pt_blk_postpone_insn(decoder, &insn, &iext);
1531 case ptev_async_paging:
1532 status = pt_blk_proceed_to_async_paging(decoder, block, ev);
1539 if (!decoder->enabled)
1542 status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
1543 pt_insn_binds_to_vmcs);
1547 /* We bound a vmcs event. Make sure we do not bind further vmcs
1548 * events to this instruction.
1550 decoder->bound_vmcs = 1;
1552 return pt_blk_postpone_insn(decoder, &insn, &iext);
1554 case ptev_async_vmcs:
1555 status = pt_blk_proceed_to_async_vmcs(decoder, block, ev);
1564 case ptev_exec_mode:
1565 status = pt_blk_proceed_to_exec_mode(decoder, block, ev);
1572 if (ev->ip_suppressed)
1575 status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
1576 ev->variant.tsx.ip);
1586 if (!decoder->enabled || ev->ip_suppressed)
1589 status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
1590 ev->variant.exstop.ip);
1597 if (!decoder->enabled || ev->ip_suppressed)
1600 status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
1601 ev->variant.mwait.ip);
1612 if (!decoder->enabled)
1615 status = pt_blk_proceed_to_ptwrite(decoder, block, &insn,
1620 /* We bound a ptwrite event. Make sure we do not bind further
1621 * ptwrite events to this instruction.
1623 decoder->bound_ptwrite = 1;
1625 return pt_blk_postpone_insn(decoder, &insn, &iext);
1633 return pt_blk_status(decoder, pts_event_pending);
1636 /* Proceed to the next decision point without using the block cache.
1638 * Tracing is enabled and we don't have an event pending. Proceed as far as
1639 * we get without trace. Stop when we either:
1641 * - need trace in order to continue
1642 * - overflow the max number of instructions in a block
1644 * We actually proceed one instruction further to get the start IP for the next
1645 * block. This only updates @decoder's internal state, though.
1647 * Returns zero on success, a negative error code otherwise.
1649 static int pt_blk_proceed_no_event_uncached(struct pt_block_decoder *decoder,
1650 struct pt_block *block)
1652 struct pt_insn_ext iext;
1653 struct pt_insn insn;
1656 if (!decoder || !block)
1657 return -pte_internal;
1659 /* This is overly conservative, really. We shouldn't get a bad-query
1660 * status unless we decoded at least one instruction successfully.
1662 memset(&insn, 0, sizeof(insn));
1663 memset(&iext, 0, sizeof(iext));
1665 /* Proceed as far as we get without trace. */
1666 status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
1669 if (status != -pte_bad_query)
1672 return pt_blk_proceed_with_trace(decoder, &insn, &iext);
1678 /* Check if @ip is contained in @section loaded at @laddr.
1680 * Returns non-zero if it is.
1681 * Returns zero if it isn't or of @section is NULL.
1683 static inline int pt_blk_is_in_section(const struct pt_mapped_section *msec,
1686 uint64_t begin, end;
1688 begin = pt_msec_begin(msec);
1689 end = pt_msec_end(msec);
1691 return (begin <= ip && ip < end);
1694 /* Insert a trampoline block cache entry.
1696 * Add a trampoline block cache entry at @ip to continue at @nip, where @nip
1697 * must be the next instruction after @ip.
1699 * Both @ip and @nip must be section-relative
1701 * Returns zero on success, a negative error code otherwise.
1703 static inline int pt_blk_add_trampoline(struct pt_block_cache *bcache,
1704 uint64_t ip, uint64_t nip,
1705 enum pt_exec_mode mode)
1707 struct pt_bcache_entry bce;
1710 /* The displacement from @ip to @nip for the trampoline. */
1711 disp = (int64_t) (nip - ip);
1713 memset(&bce, 0, sizeof(bce));
1714 bce.displacement = (int32_t) disp;
1717 bce.qualifier = ptbq_again;
1719 /* If we can't reach @nip without overflowing the displacement field, we
1720 * have to stop and re-decode the instruction at @ip.
1722 if ((int64_t) bce.displacement != disp) {
1724 memset(&bce, 0, sizeof(bce));
1727 bce.qualifier = ptbq_decode;
1730 return pt_bcache_add(bcache, ip, bce);
1733 /* Insert a decode block cache entry.
1735 * Add a decode block cache entry at @ioff.
1737 * Returns zero on success, a negative error code otherwise.
1739 static inline int pt_blk_add_decode(struct pt_block_cache *bcache,
1740 uint64_t ioff, enum pt_exec_mode mode)
1742 struct pt_bcache_entry bce;
1744 memset(&bce, 0, sizeof(bce));
1747 bce.qualifier = ptbq_decode;
1749 return pt_bcache_add(bcache, ioff, bce);
1753 /* The maximum number of steps when filling the block cache. */
1754 bcache_fill_steps = 0x400
1757 /* Proceed to the next instruction and fill the block cache for @decoder->ip.
1759 * Tracing is enabled and we don't have an event pending. The current IP is not
1762 * Proceed one instruction without using the block cache, then try to proceed
1763 * further using the block cache.
1765 * On our way back, add a block cache entry for the IP before proceeding. Note
1766 * that the recursion is bounded by @steps and ultimately by the maximum number
1767 * of instructions in a block.
1769 * Returns zero on success, a negative error code otherwise.
1772 pt_blk_proceed_no_event_fill_cache(struct pt_block_decoder *decoder,
1773 struct pt_block *block,
1774 struct pt_block_cache *bcache,
1775 const struct pt_mapped_section *msec,
1778 struct pt_bcache_entry bce;
1779 struct pt_insn_ext iext;
1780 struct pt_insn insn;
1782 int64_t disp, ioff, noff;
1785 if (!decoder || !steps)
1786 return -pte_internal;
1788 /* Proceed one instruction by decoding and examining it.
1790 * Note that we also return on a status of zero that indicates that the
1791 * instruction didn't fit into @block.
1793 status = pt_blk_proceed_one_insn(decoder, block, &insn, &iext);
1797 ioff = pt_msec_unmap(msec, insn.ip);
1799 /* Let's see if we can proceed to the next IP without trace.
1801 * If we can't, this is certainly a decision point.
1803 status = pt_insn_next_ip(&decoder->ip, &insn, &iext);
1805 if (status != -pte_bad_query)
1808 memset(&bce, 0, sizeof(bce));
1810 bce.mode = insn.mode;
1811 bce.isize = insn.size;
1813 /* Clear the instruction size in case of overflows. */
1814 if ((uint8_t) bce.isize != insn.size)
1817 switch (insn.iclass) {
1821 return -pte_internal;
1824 /* A direct jump doesn't require trace. */
1825 if (iext.variant.branch.is_direct)
1826 return -pte_internal;
1828 bce.qualifier = ptbq_indirect;
1832 /* A direct call doesn't require trace. */
1833 if (iext.variant.branch.is_direct)
1834 return -pte_internal;
1836 bce.qualifier = ptbq_ind_call;
1840 bce.qualifier = ptbq_return;
1843 case ptic_cond_jump:
1844 bce.qualifier = ptbq_cond;
1848 case ptic_far_return:
1850 bce.qualifier = ptbq_indirect;
1854 /* If the block was truncated, we have to decode its last
1855 * instruction each time.
1857 * We could have skipped the above switch and size assignment in
1858 * this case but this is already a slow and hopefully infrequent
1861 if (block->truncated)
1862 bce.qualifier = ptbq_decode;
1864 status = pt_bcache_add(bcache, ioff, bce);
1868 return pt_blk_proceed_with_trace(decoder, &insn, &iext);
1871 /* The next instruction's IP. */
1873 noff = pt_msec_unmap(msec, nip);
1875 /* Even if we were able to proceed without trace, we might have to stop
1876 * here for various reasons:
1878 * - at near direct calls to update the return-address stack
1880 * We are forced to re-decode @insn to get the branch displacement.
1882 * Even though it is constant, we don't cache it to avoid increasing
1883 * the size of a cache entry. Note that the displacement field is
1884 * zero for this entry and we might be tempted to use it - but other
1885 * entries that point to this decision point will have non-zero
1888 * We could proceed after a near direct call but we migh as well
1889 * postpone it to the next iteration. Make sure to end the block if
1890 * @decoder->flags.variant.block.end_on_call is set, though.
1892 * - at near direct backwards jumps to detect section splits
1894 * In case the current section is split underneath us, we must take
1895 * care to detect that split.
1897 * There is one corner case where the split is in the middle of a
1898 * linear sequence of instructions that branches back into the
1899 * originating section.
1901 * Calls, indirect branches, and far branches are already covered
1902 * since they either require trace or already require us to stop
1903 * (i.e. near direct calls) for other reasons. That leaves near
1904 * direct backward jumps.
1906 * Instead of the decode stop at the jump instruction we're using we
1907 * could have made sure that other block cache entries that extend
1908 * this one insert a trampoline to the jump's entry. This would
1909 * have been a bit more complicated.
1911 * - if we switched sections
1913 * This ends a block just like a branch that requires trace.
1915 * We need to re-decode @insn in order to determine the start IP of
1918 * - if the block is truncated
1920 * We need to read the last instruction's memory from multiple
1921 * sections and provide it to the user.
1923 * We could still use the block cache but then we'd have to handle
1924 * this case for each qualifier. Truncation is hopefully rare and
1925 * having to read the memory for the instruction from multiple
1926 * sections is already slow. Let's rather keep things simple and
1927 * route it through the decode flow, where we already have
1928 * everything in place.
1930 switch (insn.iclass) {
1932 return pt_blk_add_decode(bcache, ioff, insn.mode);
1935 /* An indirect branch requires trace and should have been
1938 if (!iext.variant.branch.is_direct)
1939 return -pte_internal;
1941 if (iext.variant.branch.displacement < 0 ||
1942 decoder->flags.variant.block.end_on_jump)
1943 return pt_blk_add_decode(bcache, ioff, insn.mode);
1947 if (!pt_blk_is_in_section(msec, nip) || block->truncated)
1948 return pt_blk_add_decode(bcache, ioff, insn.mode);
1953 /* We proceeded one instruction. Let's see if we have a cache entry for
1954 * the next instruction.
1956 status = pt_bcache_lookup(&bce, bcache, noff);
1960 /* If we don't have a valid cache entry, yet, fill the cache some more.
1962 * On our way back, we add a cache entry for this instruction based on
1963 * the cache entry of the succeeding instruction.
1965 if (!pt_bce_is_valid(bce)) {
1966 /* If we exceeded the maximum number of allowed steps, we insert
1967 * a trampoline to the next instruction.
1969 * The next time we encounter the same code, we will use the
1970 * trampoline to jump directly to where we left off this time
1971 * and continue from there.
1975 return pt_blk_add_trampoline(bcache, ioff, noff,
1978 status = pt_blk_proceed_no_event_fill_cache(decoder, block,
1984 /* Let's see if we have more luck this time. */
1985 status = pt_bcache_lookup(&bce, bcache, noff);
1989 /* If we still don't have a valid cache entry, we're done. Most
1990 * likely, @block overflowed and we couldn't proceed past the
1993 if (!pt_bce_is_valid(bce))
1997 /* We must not have switched execution modes.
1999 * This would require an event and we're on the no-event flow.
2001 if (pt_bce_exec_mode(bce) != insn.mode)
2002 return -pte_internal;
2004 /* The decision point IP and the displacement from @insn.ip. */
2005 dip = nip + bce.displacement;
2006 disp = (int64_t) (dip - insn.ip);
2008 /* We may have switched sections if the section was split. See
2009 * pt_blk_proceed_no_event_cached() for a more elaborate comment.
2011 * We're not adding a block cache entry since this won't apply to the
2012 * original section which may be shared with other decoders.
2014 * We will instead take the slow path until the end of the section.
2016 if (!pt_blk_is_in_section(msec, dip))
2019 /* Let's try to reach @nip's decision point from @insn.ip.
2021 * There are two fields that may overflow: @bce.ninsn and
2022 * @bce.displacement.
2025 bce.displacement = (int32_t) disp;
2027 /* If none of them overflowed, we're done.
2029 * If one or both overflowed, let's try to insert a trampoline, i.e. we
2030 * try to reach @dip via a ptbq_again entry to @nip.
2032 if (!bce.ninsn || ((int64_t) bce.displacement != disp))
2033 return pt_blk_add_trampoline(bcache, ioff, noff, insn.mode);
2035 /* We're done. Add the cache entry.
2037 * There's a chance that other decoders updated the cache entry in the
2038 * meantime. They should have come to the same conclusion as we,
2039 * though, and the cache entries should be identical.
2041 * Cache updates are atomic so even if the two versions were not
2042 * identical, we wouldn't care because they are both correct.
2044 return pt_bcache_add(bcache, ioff, bce);
2047 /* Proceed at a potentially truncated instruction.
2049 * We were not able to decode the instruction at @decoder->ip in @decoder's
2050 * cached section. This is typically caused by not having enough bytes.
2052 * Try to decode the instruction again using the entire image. If this succeeds
2053 * we expect to end up with an instruction that was truncated in the section it
2054 * started. We provide the full instruction in this case and end the block.
2056 * Returns zero on success, a negative error code otherwise.
2058 static int pt_blk_proceed_truncated(struct pt_block_decoder *decoder,
2059 struct pt_block *block)
2061 struct pt_insn_ext iext;
2062 struct pt_insn insn;
2065 if (!decoder || !block)
2066 return -pte_internal;
2068 memset(&iext, 0, sizeof(iext));
2069 memset(&insn, 0, sizeof(insn));
2071 insn.mode = decoder->mode;
2072 insn.ip = decoder->ip;
2074 errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
2078 /* We shouldn't use this function if the instruction isn't truncated. */
2079 if (!insn.truncated)
2080 return -pte_internal;
2082 /* Provide the instruction in the block. This ends the block. */
2083 memcpy(block->raw, insn.raw, insn.size);
2084 block->iclass = insn.iclass;
2085 block->size = insn.size;
2086 block->truncated = 1;
2088 /* Log calls' return addresses for return compression. */
2089 errcode = pt_blk_log_call(decoder, &insn, &iext);
2093 /* Let's see if we can proceed to the next IP without trace.
2095 * The truncated instruction ends the block but we still need to get the
2096 * next block's start IP.
2098 errcode = pt_insn_next_ip(&decoder->ip, &insn, &iext);
2100 if (errcode != -pte_bad_query)
2103 return pt_blk_proceed_with_trace(decoder, &insn, &iext);
2109 /* Proceed to the next decision point using the block cache.
2111 * Tracing is enabled and we don't have an event pending. We already set
2112 * @block's isid. All reads are done within @msec as we're not switching
2113 * sections between blocks.
2115 * Proceed as far as we get without trace. Stop when we either:
2117 * - need trace in order to continue
2118 * - overflow the max number of instructions in a block
2120 * We actually proceed one instruction further to get the start IP for the next
2121 * block. This only updates @decoder's internal state, though.
2123 * Returns zero on success, a negative error code otherwise.
2125 static int pt_blk_proceed_no_event_cached(struct pt_block_decoder *decoder,
2126 struct pt_block *block,
2127 struct pt_block_cache *bcache,
2128 const struct pt_mapped_section *msec)
2130 struct pt_bcache_entry bce;
2131 uint16_t binsn, ninsn;
2132 uint64_t offset, nip;
2135 if (!decoder || !block)
2136 return -pte_internal;
2138 offset = pt_msec_unmap(msec, decoder->ip);
2139 status = pt_bcache_lookup(&bce, bcache, offset);
2143 /* If we don't find a valid cache entry, fill the cache. */
2144 if (!pt_bce_is_valid(bce))
2145 return pt_blk_proceed_no_event_fill_cache(decoder, block,
2149 /* If we switched sections, the origianl section must have been split
2150 * underneath us. A split preserves the block cache of the original
2153 * Crossing sections requires ending the block so we can indicate the
2154 * proper isid for the entire block.
2156 * Plus there's the chance that the new section that caused the original
2157 * section to split changed instructions.
2159 * This check will also cover changes to a linear sequence of code we
2160 * would otherwise have jumped over as long as the start and end are in
2161 * different sub-sections.
2163 * Since we stop on every (backwards) branch (through an artificial stop
2164 * in the case of a near direct backward branch) we will detect all
2167 * Switch to the slow path until we reach the end of this section.
2169 nip = decoder->ip + bce.displacement;
2170 if (!pt_blk_is_in_section(msec, nip))
2171 return pt_blk_proceed_no_event_uncached(decoder, block);
2173 /* We have a valid cache entry. Let's first check if the way to the
2174 * decision point still fits into @block.
2176 * If it doesn't, we end the block without filling it as much as we
2177 * could since this would require us to switch to the slow path.
2179 * On the next iteration, we will start with an empty block, which is
2180 * guaranteed to have enough room for at least one block cache entry.
2182 binsn = block->ninsn;
2183 ninsn = binsn + (uint16_t) bce.ninsn;
2187 /* Jump ahead to the decision point and proceed from there.
2189 * We're not switching execution modes so even if @block already has an
2190 * execution mode, it will be the one we're going to set.
2194 /* We don't know the instruction class so we should be setting it to
2195 * ptic_error. Since we will be able to fill it back in later in most
2196 * cases, we move the clearing to the switch cases that don't.
2198 block->end_ip = nip;
2199 block->ninsn = ninsn;
2200 block->mode = pt_bce_exec_mode(bce);
2203 switch (pt_bce_qualifier(bce)) {
2205 /* We're not able to reach the actual decision point due to
2206 * overflows so we inserted a trampoline.
2208 * We don't know the instruction and it is not guaranteed that
2209 * we will proceed further (e.g. if @block overflowed). Let's
2210 * clear any previously stored instruction class which has
2211 * become invalid when we updated @block->ninsn.
2213 block->iclass = ptic_error;
2215 return pt_blk_proceed_no_event_cached(decoder, block, bcache,
2219 /* We're at a conditional branch. */
2220 block->iclass = ptic_cond_jump;
2222 /* Let's first check whether we know the size of the
2223 * instruction. If we do, we might get away without decoding
2226 * If we don't know the size we might as well do the full decode
2227 * and proceed-with-trace flow we do for ptbq_decode.
2233 /* If the branch is not taken, we don't need to decode
2234 * the instruction at @decoder->ip.
2236 * If it is taken, we have to implement everything here.
2237 * We can't use the normal decode and proceed-with-trace
2238 * flow since we already consumed the TNT bit.
2240 status = pt_blk_cond_branch(decoder, &taken);
2244 /* Preserve the query decoder's response which indicates
2247 decoder->status = status;
2251 struct pt_insn_ext iext;
2252 struct pt_insn insn;
2254 memset(&iext, 0, sizeof(iext));
2255 memset(&insn, 0, sizeof(insn));
2257 insn.mode = pt_bce_exec_mode(bce);
2260 status = pt_blk_decode_in_section(&insn, &iext,
2265 ip += iext.variant.branch.displacement;
2268 decoder->ip = ip + bce.isize;
2274 struct pt_insn_ext iext;
2275 struct pt_insn insn;
2277 /* We need to decode the instruction at @decoder->ip and decide
2278 * what to do based on that.
2280 * We already accounted for the instruction so we can't just
2281 * call pt_blk_proceed_one_insn().
2284 memset(&iext, 0, sizeof(iext));
2285 memset(&insn, 0, sizeof(insn));
2287 insn.mode = pt_bce_exec_mode(bce);
2288 insn.ip = decoder->ip;
2290 status = pt_blk_decode_in_section(&insn, &iext, msec);
2292 if (status != -pte_bad_insn)
2295 return pt_blk_proceed_truncated(decoder, block);
2298 /* We just decoded @insn so we know the instruction class. */
2299 block->iclass = insn.iclass;
2301 /* Log calls' return addresses for return compression. */
2302 status = pt_blk_log_call(decoder, &insn, &iext);
2306 /* Let's see if we can proceed to the next IP without trace.
2308 * Note that we also stop due to displacement overflows or to
2309 * maintain the return-address stack for near direct calls.
2311 status = pt_insn_next_ip(&decoder->ip, &insn, &iext);
2313 if (status != -pte_bad_query)
2316 /* We can't, so let's proceed with trace, which
2317 * completes the block.
2319 return pt_blk_proceed_with_trace(decoder, &insn, &iext);
2322 /* End the block if the user asked us to.
2324 * We only need to take care about direct near branches.
2325 * Indirect and far branches require trace and will naturally
2328 if ((decoder->flags.variant.block.end_on_call &&
2329 (insn.iclass == ptic_call)) ||
2330 (decoder->flags.variant.block.end_on_jump &&
2331 (insn.iclass == ptic_jump)))
2334 /* If we can proceed without trace and we stay in @msec we may
2337 * We're done if we switch sections, though.
2339 if (!pt_blk_is_in_section(msec, decoder->ip))
2342 return pt_blk_proceed_no_event_cached(decoder, block, bcache,
2346 case ptbq_ind_call: {
2349 /* We're at a near indirect call. */
2350 block->iclass = ptic_call;
2352 /* We need to update the return-address stack and query the
2357 /* If we already know the size of the instruction, we don't need
2363 struct pt_insn_ext iext;
2364 struct pt_insn insn;
2366 memset(&iext, 0, sizeof(iext));
2367 memset(&insn, 0, sizeof(insn));
2369 insn.mode = pt_bce_exec_mode(bce);
2372 status = pt_blk_decode_in_section(&insn, &iext, msec);
2379 status = pt_retstack_push(&decoder->retstack, ip);
2383 status = pt_blk_indirect_branch(decoder, &decoder->ip);
2387 /* Preserve the query decoder's response which indicates
2390 decoder->status = status;
2397 /* We're at a near return. */
2398 block->iclass = ptic_return;
2400 /* Check for a compressed return. */
2401 status = pt_blk_cond_branch(decoder, &taken);
2403 if (status != -pte_bad_query)
2406 /* The return is not compressed. We need another query
2407 * to determine the destination IP.
2409 status = pt_blk_indirect_branch(decoder, &decoder->ip);
2413 /* Preserve the query decoder's response which indicates
2416 decoder->status = status;
2420 /* Preserve the query decoder's response which indicates
2423 decoder->status = status;
2425 /* A compressed return is indicated by a taken conditional
2429 return -pte_bad_retcomp;
2431 return pt_retstack_pop(&decoder->retstack, &decoder->ip);
2435 /* We're at an indirect jump or far transfer.
2437 * We don't know the exact instruction class and there's no
2438 * reason to decode the instruction for any other purpose.
2440 * Indicate that we don't know the instruction class and leave
2441 * it to our caller to decode the instruction if needed.
2443 block->iclass = ptic_error;
2445 /* This is neither a near call nor return so we don't need to
2446 * touch the return-address stack.
2448 * Just query the destination IP.
2450 status = pt_blk_indirect_branch(decoder, &decoder->ip);
2454 /* Preserve the query decoder's response which indicates
2457 decoder->status = status;
2464 static int pt_blk_msec_fill(struct pt_block_decoder *decoder,
2465 const struct pt_mapped_section **pmsec)
2467 const struct pt_mapped_section *msec;
2468 struct pt_section *section;
2471 if (!decoder || !pmsec)
2472 return -pte_internal;
2474 isid = pt_msec_cache_fill(&decoder->scache, &msec, decoder->image,
2475 &decoder->asid, decoder->ip);
2479 section = pt_msec_section(msec);
2481 return -pte_internal;
2485 errcode = pt_section_request_bcache(section);
2492 static inline int pt_blk_msec_lookup(struct pt_block_decoder *decoder,
2493 const struct pt_mapped_section **pmsec)
2498 return -pte_internal;
2500 isid = pt_msec_cache_read(&decoder->scache, pmsec, decoder->image,
2503 if (isid != -pte_nomap)
2506 return pt_blk_msec_fill(decoder, pmsec);
2512 /* Proceed to the next decision point - try using the cache.
2514 * Tracing is enabled and we don't have an event pending. Proceed as far as
2515 * we get without trace. Stop when we either:
2517 * - need trace in order to continue
2518 * - overflow the max number of instructions in a block
2520 * We actually proceed one instruction further to get the start IP for the next
2521 * block. This only updates @decoder's internal state, though.
2523 * Returns zero on success, a negative error code otherwise.
2525 static int pt_blk_proceed_no_event(struct pt_block_decoder *decoder,
2526 struct pt_block *block)
2528 const struct pt_mapped_section *msec;
2529 struct pt_block_cache *bcache;
2530 struct pt_section *section;
2533 if (!decoder || !block)
2534 return -pte_internal;
2536 isid = pt_blk_msec_lookup(decoder, &msec);
2538 if (isid != -pte_nomap)
2541 /* Even if there is no such section in the image, we may still
2542 * read the memory via the callback function.
2544 return pt_blk_proceed_no_event_uncached(decoder, block);
2547 /* We do not switch sections inside a block. */
2548 if (isid != block->isid) {
2549 if (!pt_blk_block_is_empty(block))
2555 section = pt_msec_section(msec);
2557 return -pte_internal;
2559 bcache = pt_section_bcache(section);
2561 return pt_blk_proceed_no_event_uncached(decoder, block);
2563 return pt_blk_proceed_no_event_cached(decoder, block, bcache, msec);
2566 /* Proceed to the next event or decision point.
2568 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
2571 static int pt_blk_proceed(struct pt_block_decoder *decoder,
2572 struct pt_block *block)
2576 status = pt_blk_fetch_event(decoder);
2581 return pt_blk_proceed_event(decoder, block);
2584 /* If tracing is disabled we should either be out of trace or we should
2585 * have taken the event flow above.
2587 if (!decoder->enabled) {
2588 if (decoder->status & pts_eos)
2591 return -pte_no_enable;
2594 status = pt_blk_proceed_no_event(decoder, block);
2598 return pt_blk_proceed_trailing_event(decoder, block);
2602 /* The maximum number of steps to take when determining whether the
2603 * event location can be reached.
2605 bdm64_max_steps = 0x100
2608 /* Try to work around erratum BDM64.
2610 * If we got a transaction abort immediately following a branch that produced
2611 * trace, the trace for that branch might have been corrupted.
2613 * Returns a positive integer if the erratum was handled.
2614 * Returns zero if the erratum does not seem to apply.
2615 * Returns a negative error code otherwise.
2617 static int pt_blk_handle_erratum_bdm64(struct pt_block_decoder *decoder,
2618 const struct pt_block *block,
2619 const struct pt_event *ev)
2621 struct pt_insn_ext iext;
2622 struct pt_insn insn;
2625 if (!decoder || !block || !ev)
2626 return -pte_internal;
2628 /* This only affects aborts. */
2629 if (!ev->variant.tsx.aborted)
2632 /* This only affects branches that require trace.
2634 * If the erratum hits, that branch ended the current block and brought
2635 * us to the trailing event flow.
2637 if (pt_blk_block_is_empty(block))
2640 insn.mode = block->mode;
2641 insn.ip = block->end_ip;
2643 status = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
2647 if (!pt_insn_is_branch(&insn, &iext))
2650 /* Let's check if we can reach the event location from here.
2652 * If we can, let's assume the erratum did not hit. We might still be
2653 * wrong but we're not able to tell.
2655 status = pt_insn_range_is_contiguous(decoder->ip, ev->variant.tsx.ip,
2656 decoder->mode, decoder->image,
2657 &decoder->asid, bdm64_max_steps);
2661 /* We can't reach the event location. This could either mean that we
2662 * stopped too early (and status is zero) or that the erratum hit.
2664 * We assume the latter and pretend that the previous branch brought us
2665 * to the event location, instead.
2667 decoder->ip = ev->variant.tsx.ip;
2672 /* Check whether a trailing TSX event should be postponed.
2674 * This involves handling erratum BDM64.
2676 * Returns a positive integer if the event is to be postponed.
2677 * Returns zero if the event should be processed.
2678 * Returns a negative error code otherwise.
2680 static inline int pt_blk_postpone_trailing_tsx(struct pt_block_decoder *decoder,
2681 struct pt_block *block,
2682 const struct pt_event *ev)
2686 if (!decoder || !ev)
2687 return -pte_internal;
2689 if (ev->ip_suppressed)
2692 if (block && decoder->query.config.errata.bdm64) {
2693 status = pt_blk_handle_erratum_bdm64(decoder, block, ev);
2698 if (decoder->ip != ev->variant.tsx.ip)
2704 /* Proceed with events that bind to the current decoder IP.
2706 * This function is used in the following scenarios:
2708 * - we just synchronized onto the trace stream
2709 * - we ended a block and proceeded to the next IP
2710 * - we processed an event that was indicated by this function
2712 * Check if there is an event at the current IP that needs to be indicated to
2715 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
2718 static int pt_blk_proceed_trailing_event(struct pt_block_decoder *decoder,
2719 struct pt_block *block)
2721 struct pt_event *ev;
2725 return -pte_internal;
2727 status = pt_blk_fetch_event(decoder);
2732 status = pt_blk_proceed_postponed_insn(decoder);
2736 return pt_blk_status(decoder, 0);
2739 ev = &decoder->event;
2742 /* Synchronous disable events are normally indicated on the
2745 if (!decoder->process_insn)
2748 /* A sync disable may bind to a CR3 changing instruction. */
2749 if (ev->ip_suppressed &&
2750 pt_insn_changes_cr3(&decoder->insn, &decoder->iext))
2751 return pt_blk_status(decoder, pts_event_pending);
2753 /* Or it binds to the next branch that would require trace.
2755 * Try to complete processing the current instruction by
2756 * proceeding past it. If that fails because it would require
2757 * trace, we can apply the disabled event.
2759 status = pt_insn_next_ip(&decoder->ip, &decoder->insn,
2762 if (status != -pte_bad_query)
2765 status = pt_blk_set_disable_resume_ip(decoder,
2770 return pt_blk_status(decoder, pts_event_pending);
2773 /* We proceeded past the current instruction. */
2774 status = pt_blk_clear_postponed_insn(decoder);
2778 /* This might have brought us to the disable IP. */
2779 if (!ev->ip_suppressed &&
2780 decoder->ip == ev->variant.disabled.ip)
2781 return pt_blk_status(decoder, pts_event_pending);
2786 /* This event does not bind to an instruction. */
2787 status = pt_blk_proceed_postponed_insn(decoder);
2791 return pt_blk_status(decoder, pts_event_pending);
2793 case ptev_async_disabled:
2794 /* This event does not bind to an instruction. */
2795 status = pt_blk_proceed_postponed_insn(decoder);
2799 if (decoder->ip != ev->variant.async_disabled.at)
2802 if (decoder->query.config.errata.skd022) {
2803 status = pt_blk_handle_erratum_skd022(decoder, ev);
2808 /* If the erratum applies, the event is modified
2809 * to a synchronous disable event that will be
2810 * processed on the next pt_blk_proceed_event()
2817 return pt_blk_status(decoder, pts_event_pending);
2819 case ptev_async_branch:
2820 /* This event does not bind to an instruction. */
2821 status = pt_blk_proceed_postponed_insn(decoder);
2825 if (decoder->ip != ev->variant.async_branch.from)
2828 return pt_blk_status(decoder, pts_event_pending);
2831 /* We apply the event immediately if we're not tracing. */
2832 if (!decoder->enabled)
2833 return pt_blk_status(decoder, pts_event_pending);
2835 /* Synchronous paging events are normally indicated on the event
2836 * flow, unless they bind to the same instruction as a previous
2839 * We bind at most one paging event to an instruction, though.
2841 if (!decoder->process_insn || decoder->bound_paging)
2844 /* We're done if we're not binding to the currently postponed
2845 * instruction. We will process the event on the normal event
2846 * flow in the next iteration.
2848 if (!pt_insn_binds_to_pip(&decoder->insn, &decoder->iext))
2851 /* We bound a paging event. Make sure we do not bind further
2852 * paging events to this instruction.
2854 decoder->bound_paging = 1;
2856 return pt_blk_status(decoder, pts_event_pending);
2858 case ptev_async_paging:
2859 /* This event does not bind to an instruction. */
2860 status = pt_blk_proceed_postponed_insn(decoder);
2864 if (!ev->ip_suppressed &&
2865 decoder->ip != ev->variant.async_paging.ip)
2868 return pt_blk_status(decoder, pts_event_pending);
2871 /* We apply the event immediately if we're not tracing. */
2872 if (!decoder->enabled)
2873 return pt_blk_status(decoder, pts_event_pending);
2875 /* Synchronous vmcs events are normally indicated on the event
2876 * flow, unless they bind to the same instruction as a previous
2879 * We bind at most one vmcs event to an instruction, though.
2881 if (!decoder->process_insn || decoder->bound_vmcs)
2884 /* We're done if we're not binding to the currently postponed
2885 * instruction. We will process the event on the normal event
2886 * flow in the next iteration.
2888 if (!pt_insn_binds_to_vmcs(&decoder->insn, &decoder->iext))
2891 /* We bound a vmcs event. Make sure we do not bind further vmcs
2892 * events to this instruction.
2894 decoder->bound_vmcs = 1;
2896 return pt_blk_status(decoder, pts_event_pending);
2898 case ptev_async_vmcs:
2899 /* This event does not bind to an instruction. */
2900 status = pt_blk_proceed_postponed_insn(decoder);
2904 if (!ev->ip_suppressed &&
2905 decoder->ip != ev->variant.async_vmcs.ip)
2908 return pt_blk_status(decoder, pts_event_pending);
2911 /* This event does not bind to an instruction. */
2912 status = pt_blk_proceed_postponed_insn(decoder);
2916 return pt_blk_status(decoder, pts_event_pending);
2918 case ptev_exec_mode:
2919 /* This event does not bind to an instruction. */
2920 status = pt_blk_proceed_postponed_insn(decoder);
2924 if (!ev->ip_suppressed &&
2925 decoder->ip != ev->variant.exec_mode.ip)
2928 return pt_blk_status(decoder, pts_event_pending);
2931 /* This event does not bind to an instruction. */
2932 status = pt_blk_proceed_postponed_insn(decoder);
2936 status = pt_blk_postpone_trailing_tsx(decoder, block, ev);
2944 return pt_blk_status(decoder, pts_event_pending);
2947 /* This event does not bind to an instruction. */
2948 status = pt_blk_proceed_postponed_insn(decoder);
2952 return pt_blk_status(decoder, pts_event_pending);
2955 /* This event does not bind to an instruction. */
2956 status = pt_blk_proceed_postponed_insn(decoder);
2960 if (!ev->ip_suppressed && decoder->enabled &&
2961 decoder->ip != ev->variant.exstop.ip)
2964 return pt_blk_status(decoder, pts_event_pending);
2967 /* This event does not bind to an instruction. */
2968 status = pt_blk_proceed_postponed_insn(decoder);
2972 if (!ev->ip_suppressed && decoder->enabled &&
2973 decoder->ip != ev->variant.mwait.ip)
2976 return pt_blk_status(decoder, pts_event_pending);
2980 /* This event does not bind to an instruction. */
2981 status = pt_blk_proceed_postponed_insn(decoder);
2985 return pt_blk_status(decoder, pts_event_pending);
2988 /* We apply the event immediately if we're not tracing. */
2989 if (!decoder->enabled)
2990 return pt_blk_status(decoder, pts_event_pending);
2992 /* Ptwrite events are normally indicated on the event flow,
2993 * unless they bind to the same instruction as a previous event.
2995 * We bind at most one ptwrite event to an instruction, though.
2997 if (!decoder->process_insn || decoder->bound_ptwrite)
3000 /* We're done if we're not binding to the currently postponed
3001 * instruction. We will process the event on the normal event
3002 * flow in the next iteration.
3004 if (!ev->ip_suppressed ||
3005 !pt_insn_is_ptwrite(&decoder->insn, &decoder->iext))
3008 /* We bound a ptwrite event. Make sure we do not bind further
3009 * ptwrite events to this instruction.
3011 decoder->bound_ptwrite = 1;
3013 return pt_blk_status(decoder, pts_event_pending);
3018 /* This event does not bind to an instruction. */
3019 status = pt_blk_proceed_postponed_insn(decoder);
3023 return pt_blk_status(decoder, pts_event_pending);
3026 /* No further events. Proceed past any postponed instruction. */
3027 status = pt_blk_proceed_postponed_insn(decoder);
3031 return pt_blk_status(decoder, 0);
3034 int pt_blk_next(struct pt_block_decoder *decoder, struct pt_block *ublock,
3037 struct pt_block block, *pblock;
3038 int errcode, status;
3040 if (!decoder || !ublock)
3041 return -pte_invalid;
3043 pblock = size == sizeof(block) ? ublock : █
3045 /* Zero-initialize the block in case of error returns. */
3046 memset(pblock, 0, sizeof(*pblock));
3048 /* Fill in a few things from the current decode state.
3050 * This reflects the state of the last pt_blk_next() or pt_blk_start()
3051 * call. Note that, unless we stop with tracing disabled, we proceed
3052 * already to the start IP of the next block.
3054 * Some of the state may later be overwritten as we process events.
3056 pblock->ip = decoder->ip;
3057 pblock->mode = decoder->mode;
3058 if (decoder->speculative)
3059 pblock->speculative = 1;
3061 /* Proceed one block. */
3062 status = pt_blk_proceed(decoder, pblock);
3064 errcode = block_to_user(ublock, size, pblock);
3071 /* Process an enabled event.
3073 * Returns zero on success, a negative error code otherwise.
3075 static int pt_blk_process_enabled(struct pt_block_decoder *decoder,
3076 const struct pt_event *ev)
3078 if (!decoder || !ev)
3079 return -pte_internal;
3081 /* This event can't be a status update. */
3082 if (ev->status_update)
3083 return -pte_bad_context;
3085 /* We must have an IP in order to start decoding. */
3086 if (ev->ip_suppressed)
3089 /* We must currently be disabled. */
3090 if (decoder->enabled)
3091 return -pte_bad_context;
3093 decoder->ip = ev->variant.enabled.ip;
3094 decoder->enabled = 1;
3095 decoder->process_event = 0;
3100 /* Process a disabled event.
3102 * Returns zero on success, a negative error code otherwise.
3104 static int pt_blk_process_disabled(struct pt_block_decoder *decoder,
3105 const struct pt_event *ev)
3107 if (!decoder || !ev)
3108 return -pte_internal;
3110 /* This event can't be a status update. */
3111 if (ev->status_update)
3112 return -pte_bad_context;
3114 /* We must currently be enabled. */
3115 if (!decoder->enabled)
3116 return -pte_bad_context;
3118 /* We preserve @decoder->ip. This is where we expect tracing to resume
3119 * and we'll indicate that on the subsequent enabled event if tracing
3120 * actually does resume from there.
3122 decoder->enabled = 0;
3123 decoder->process_event = 0;
3128 /* Process an asynchronous branch event.
3130 * Returns zero on success, a negative error code otherwise.
3132 static int pt_blk_process_async_branch(struct pt_block_decoder *decoder,
3133 const struct pt_event *ev)
3135 if (!decoder || !ev)
3136 return -pte_internal;
3138 /* This event can't be a status update. */
3139 if (ev->status_update)
3140 return -pte_bad_context;
3142 /* We must currently be enabled. */
3143 if (!decoder->enabled)
3144 return -pte_bad_context;
3146 /* Jump to the branch destination. We will continue from there in the
3149 decoder->ip = ev->variant.async_branch.to;
3150 decoder->process_event = 0;
3155 /* Process a paging event.
3157 * Returns zero on success, a negative error code otherwise.
3159 static int pt_blk_process_paging(struct pt_block_decoder *decoder,
3160 const struct pt_event *ev)
3165 if (!decoder || !ev)
3166 return -pte_internal;
3168 cr3 = ev->variant.paging.cr3;
3169 if (decoder->asid.cr3 != cr3) {
3170 errcode = pt_msec_cache_invalidate(&decoder->scache);
3174 decoder->asid.cr3 = cr3;
3177 decoder->process_event = 0;
3182 /* Process a vmcs event.
3184 * Returns zero on success, a negative error code otherwise.
3186 static int pt_blk_process_vmcs(struct pt_block_decoder *decoder,
3187 const struct pt_event *ev)
3192 if (!decoder || !ev)
3193 return -pte_internal;
3195 vmcs = ev->variant.vmcs.base;
3196 if (decoder->asid.vmcs != vmcs) {
3197 errcode = pt_msec_cache_invalidate(&decoder->scache);
3201 decoder->asid.vmcs = vmcs;
3204 decoder->process_event = 0;
3209 /* Process an overflow event.
3211 * Returns zero on success, a negative error code otherwise.
3213 static int pt_blk_process_overflow(struct pt_block_decoder *decoder,
3214 const struct pt_event *ev)
3216 if (!decoder || !ev)
3217 return -pte_internal;
3219 /* This event can't be a status update. */
3220 if (ev->status_update)
3221 return -pte_bad_context;
3223 /* If the IP is suppressed, the overflow resolved while tracing was
3224 * disabled. Otherwise it resolved while tracing was enabled.
3226 if (ev->ip_suppressed) {
3227 /* Tracing is disabled. It doesn't make sense to preserve the
3228 * previous IP. This will just be misleading. Even if tracing
3229 * had been disabled before, as well, we might have missed the
3230 * re-enable in the overflow.
3232 decoder->enabled = 0;
3235 /* Tracing is enabled and we're at the IP at which the overflow
3238 decoder->enabled = 1;
3239 decoder->ip = ev->variant.overflow.ip;
3242 /* We don't know the TSX state. Let's assume we execute normally.
3244 * We also don't know the execution mode. Let's keep what we have
3245 * in case we don't get an update before we have to decode the next
3248 decoder->speculative = 0;
3249 decoder->process_event = 0;
3254 /* Process an exec mode event.
3256 * Returns zero on success, a negative error code otherwise.
3258 static int pt_blk_process_exec_mode(struct pt_block_decoder *decoder,
3259 const struct pt_event *ev)
3261 enum pt_exec_mode mode;
3263 if (!decoder || !ev)
3264 return -pte_internal;
3266 /* Use status update events to diagnose inconsistencies. */
3267 mode = ev->variant.exec_mode.mode;
3268 if (ev->status_update && decoder->enabled &&
3269 decoder->mode != ptem_unknown && decoder->mode != mode)
3270 return -pte_bad_status_update;
3272 decoder->mode = mode;
3273 decoder->process_event = 0;
3278 /* Process a tsx event.
3280 * Returns zero on success, a negative error code otherwise.
3282 static int pt_blk_process_tsx(struct pt_block_decoder *decoder,
3283 const struct pt_event *ev)
3285 if (!decoder || !ev)
3286 return -pte_internal;
3288 decoder->speculative = ev->variant.tsx.speculative;
3289 decoder->process_event = 0;
3294 /* Process a stop event.
3296 * Returns zero on success, a negative error code otherwise.
3298 static int pt_blk_process_stop(struct pt_block_decoder *decoder,
3299 const struct pt_event *ev)
3301 if (!decoder || !ev)
3302 return -pte_internal;
3304 /* This event can't be a status update. */
3305 if (ev->status_update)
3306 return -pte_bad_context;
3308 /* Tracing is always disabled before it is stopped. */
3309 if (decoder->enabled)
3310 return -pte_bad_context;
3312 decoder->process_event = 0;
3317 int pt_blk_event(struct pt_block_decoder *decoder, struct pt_event *uevent,
3320 struct pt_event *ev;
3323 if (!decoder || !uevent)
3324 return -pte_invalid;
3326 /* We must currently process an event. */
3327 if (!decoder->process_event)
3328 return -pte_bad_query;
3330 ev = &decoder->event;
3333 /* Indicate that tracing resumes from the IP at which tracing
3334 * had been disabled before (with some special treatment for
3337 if (ev->variant.enabled.ip == decoder->ip)
3338 ev->variant.enabled.resumed = 1;
3340 status = pt_blk_process_enabled(decoder, ev);
3346 case ptev_async_disabled:
3347 if (decoder->ip != ev->variant.async_disabled.at)
3348 return -pte_bad_query;
3353 status = pt_blk_process_disabled(decoder, ev);
3359 case ptev_async_branch:
3360 if (decoder->ip != ev->variant.async_branch.from)
3361 return -pte_bad_query;
3363 status = pt_blk_process_async_branch(decoder, ev);
3369 case ptev_async_paging:
3370 if (!ev->ip_suppressed &&
3371 decoder->ip != ev->variant.async_paging.ip)
3372 return -pte_bad_query;
3376 status = pt_blk_process_paging(decoder, ev);
3382 case ptev_async_vmcs:
3383 if (!ev->ip_suppressed &&
3384 decoder->ip != ev->variant.async_vmcs.ip)
3385 return -pte_bad_query;
3389 status = pt_blk_process_vmcs(decoder, ev);
3396 status = pt_blk_process_overflow(decoder, ev);
3402 case ptev_exec_mode:
3403 if (!ev->ip_suppressed &&
3404 decoder->ip != ev->variant.exec_mode.ip)
3405 return -pte_bad_query;
3407 status = pt_blk_process_exec_mode(decoder, ev);
3414 if (!ev->ip_suppressed && decoder->ip != ev->variant.tsx.ip)
3415 return -pte_bad_query;
3417 status = pt_blk_process_tsx(decoder, ev);
3424 status = pt_blk_process_stop(decoder, ev);
3431 if (!ev->ip_suppressed && decoder->enabled &&
3432 decoder->ip != ev->variant.exstop.ip)
3433 return -pte_bad_query;
3435 decoder->process_event = 0;
3439 if (!ev->ip_suppressed && decoder->enabled &&
3440 decoder->ip != ev->variant.mwait.ip)
3441 return -pte_bad_query;
3443 decoder->process_event = 0;
3452 decoder->process_event = 0;
3456 /* Copy the event to the user. Make sure we're not writing beyond the
3457 * memory provided by the user.
3459 * We might truncate details of an event but only for those events the
3460 * user can't know about, anyway.
3462 if (sizeof(*ev) < size)
3465 memcpy(uevent, ev, size);
3467 /* Indicate further events. */
3468 return pt_blk_proceed_trailing_event(decoder, NULL);