2 * Copyright (c) 2013-2018, Intel Corporation
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright notice,
8 * this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright notice,
10 * this list of conditions and the following disclaimer in the documentation
11 * and/or other materials provided with the distribution.
12 * * Neither the name of Intel Corporation nor the names of its contributors
13 * may be used to endorse or promote products derived from this software
14 * without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
29 #include "pt_insn_decoder.h"
31 #include "pt_config.h"
33 #include "pt_compiler.h"
41 static int pt_insn_check_ip_event(struct pt_insn_decoder *,
42 const struct pt_insn *,
43 const struct pt_insn_ext *);
46 static void pt_insn_reset(struct pt_insn_decoder *decoder)
51 decoder->mode = ptem_unknown;
55 decoder->process_event = 0;
56 decoder->speculative = 0;
57 decoder->process_insn = 0;
58 decoder->bound_paging = 0;
59 decoder->bound_vmcs = 0;
60 decoder->bound_ptwrite = 0;
62 pt_retstack_init(&decoder->retstack);
63 pt_asid_init(&decoder->asid);
66 static int pt_insn_status(const struct pt_insn_decoder *decoder, int flags)
73 status = decoder->status;
75 /* Indicate whether tracing is disabled or enabled.
77 * This duplicates the indication in struct pt_insn and covers the case
78 * where we indicate the status after synchronizing.
80 if (!decoder->enabled)
81 flags |= pts_ip_suppressed;
83 /* Forward end-of-trace indications.
85 * Postpone it as long as we're still processing events, though.
87 if ((status & pts_eos) && !decoder->process_event)
93 /* Initialize the query decoder flags based on our flags. */
95 static int pt_insn_init_qry_flags(struct pt_conf_flags *qflags,
96 const struct pt_conf_flags *flags)
98 if (!qflags || !flags)
101 memset(qflags, 0, sizeof(*qflags));
106 int pt_insn_decoder_init(struct pt_insn_decoder *decoder,
107 const struct pt_config *uconfig)
109 struct pt_config config;
113 return -pte_internal;
115 errcode = pt_config_from_user(&config, uconfig);
119 /* The user supplied decoder flags. */
120 decoder->flags = config.flags;
122 /* Set the flags we need for the query decoder we use. */
123 errcode = pt_insn_init_qry_flags(&config.flags, &decoder->flags);
127 errcode = pt_qry_decoder_init(&decoder->query, &config);
131 pt_image_init(&decoder->default_image, NULL);
132 decoder->image = &decoder->default_image;
134 errcode = pt_msec_cache_init(&decoder->scache);
138 pt_insn_reset(decoder);
143 void pt_insn_decoder_fini(struct pt_insn_decoder *decoder)
148 pt_msec_cache_fini(&decoder->scache);
149 pt_image_fini(&decoder->default_image);
150 pt_qry_decoder_fini(&decoder->query);
153 struct pt_insn_decoder *pt_insn_alloc_decoder(const struct pt_config *config)
155 struct pt_insn_decoder *decoder;
158 decoder = malloc(sizeof(*decoder));
162 errcode = pt_insn_decoder_init(decoder, config);
171 void pt_insn_free_decoder(struct pt_insn_decoder *decoder)
176 pt_insn_decoder_fini(decoder);
180 /* Maybe synthesize a tick event.
182 * If we're not already processing events, check the current time against the
183 * last event's time. If it changed, synthesize a tick event with the new time.
185 * Returns zero if no tick event has been created.
186 * Returns a positive integer if a tick event has been created.
187 * Returns a negative error code otherwise.
189 static int pt_insn_tick(struct pt_insn_decoder *decoder, uint64_t ip)
193 uint32_t lost_mtc, lost_cyc;
197 return -pte_internal;
199 /* We're not generating tick events if tracing is disabled. */
200 if (!decoder->enabled)
201 return -pte_internal;
203 /* Events already provide a timestamp so there is no need to synthesize
204 * an artificial tick event. There's no room, either, since this would
205 * overwrite the in-progress event.
207 * In rare cases where we need to proceed to an event location using
208 * trace this may cause us to miss a timing update if the event is not
209 * forwarded to the user.
211 * The only case I can come up with at the moment is a MODE.EXEC binding
212 * to the TIP IP of a far branch.
214 if (decoder->process_event)
217 errcode = pt_qry_time(&decoder->query, &tsc, &lost_mtc, &lost_cyc);
219 /* If we don't have wall-clock time, we use relative time. */
220 if (errcode != -pte_no_time)
224 ev = &decoder->event;
226 /* We're done if time has not changed since the last event. */
230 /* Time has changed so we create a new tick event. */
231 memset(ev, 0, sizeof(*ev));
232 ev->type = ptev_tick;
233 ev->variant.tick.ip = ip;
235 /* Indicate if we have wall-clock time or only relative time. */
236 if (errcode != -pte_no_time)
239 ev->lost_mtc = lost_mtc;
240 ev->lost_cyc = lost_cyc;
242 /* We now have an event to process. */
243 decoder->process_event = 1;
248 /* Query an indirect branch.
250 * Returns zero on success, a negative error code otherwise.
252 static int pt_insn_indirect_branch(struct pt_insn_decoder *decoder,
259 return -pte_internal;
263 status = pt_qry_indirect_branch(&decoder->query, ip);
267 if (decoder->flags.variant.insn.enable_tick_events) {
268 errcode = pt_insn_tick(decoder, evip);
276 /* Query a conditional branch.
278 * Returns zero on success, a negative error code otherwise.
280 static int pt_insn_cond_branch(struct pt_insn_decoder *decoder, int *taken)
285 return -pte_internal;
287 status = pt_qry_cond_branch(&decoder->query, taken);
291 if (decoder->flags.variant.insn.enable_tick_events) {
292 errcode = pt_insn_tick(decoder, decoder->ip);
300 static int pt_insn_start(struct pt_insn_decoder *decoder, int status)
303 return -pte_internal;
308 decoder->status = status;
310 if (!(status & pts_ip_suppressed))
311 decoder->enabled = 1;
313 /* Process any initial events.
315 * Some events are processed after proceeding to the next IP in order to
316 * indicate things like tracing disable or trace stop in the preceding
317 * instruction. Those events will be processed without such an
318 * indication before decoding the current instruction.
320 * We do this already here so we can indicate user-events that precede
321 * the first instruction.
323 return pt_insn_check_ip_event(decoder, NULL, NULL);
326 int pt_insn_sync_forward(struct pt_insn_decoder *decoder)
333 pt_insn_reset(decoder);
335 status = pt_qry_sync_forward(&decoder->query, &decoder->ip);
337 return pt_insn_start(decoder, status);
340 int pt_insn_sync_backward(struct pt_insn_decoder *decoder)
347 pt_insn_reset(decoder);
349 status = pt_qry_sync_backward(&decoder->query, &decoder->ip);
351 return pt_insn_start(decoder, status);
354 int pt_insn_sync_set(struct pt_insn_decoder *decoder, uint64_t offset)
361 pt_insn_reset(decoder);
363 status = pt_qry_sync_set(&decoder->query, &decoder->ip, offset);
365 return pt_insn_start(decoder, status);
368 int pt_insn_get_offset(const struct pt_insn_decoder *decoder, uint64_t *offset)
373 return pt_qry_get_offset(&decoder->query, offset);
376 int pt_insn_get_sync_offset(const struct pt_insn_decoder *decoder,
382 return pt_qry_get_sync_offset(&decoder->query, offset);
385 struct pt_image *pt_insn_get_image(struct pt_insn_decoder *decoder)
390 return decoder->image;
393 int pt_insn_set_image(struct pt_insn_decoder *decoder,
394 struct pt_image *image)
400 image = &decoder->default_image;
402 decoder->image = image;
406 const struct pt_config *
407 pt_insn_get_config(const struct pt_insn_decoder *decoder)
412 return pt_qry_get_config(&decoder->query);
415 int pt_insn_time(struct pt_insn_decoder *decoder, uint64_t *time,
416 uint32_t *lost_mtc, uint32_t *lost_cyc)
418 if (!decoder || !time)
421 return pt_qry_time(&decoder->query, time, lost_mtc, lost_cyc);
424 int pt_insn_core_bus_ratio(struct pt_insn_decoder *decoder, uint32_t *cbr)
426 if (!decoder || !cbr)
429 return pt_qry_core_bus_ratio(&decoder->query, cbr);
432 int pt_insn_asid(const struct pt_insn_decoder *decoder, struct pt_asid *asid,
435 if (!decoder || !asid)
438 return pt_asid_to_user(asid, &decoder->asid, size);
441 static inline int event_pending(struct pt_insn_decoder *decoder)
448 if (decoder->process_event)
451 status = decoder->status;
452 if (!(status & pts_event_pending))
455 status = pt_qry_event(&decoder->query, &decoder->event,
456 sizeof(decoder->event));
460 decoder->process_event = 1;
461 decoder->status = status;
465 static int check_erratum_skd022(struct pt_insn_decoder *decoder)
467 struct pt_insn_ext iext;
472 return -pte_internal;
474 insn.mode = decoder->mode;
475 insn.ip = decoder->ip;
477 errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
481 switch (iext.iclass) {
485 case PTI_INST_VMLAUNCH:
486 case PTI_INST_VMRESUME:
491 static inline int handle_erratum_skd022(struct pt_insn_decoder *decoder)
498 return -pte_internal;
500 errcode = check_erratum_skd022(decoder);
504 /* We turn the async disable into a sync disable. It will be processed
505 * after decoding the instruction.
507 ev = &decoder->event;
509 ip = ev->variant.async_disabled.ip;
511 ev->type = ptev_disabled;
512 ev->variant.disabled.ip = ip;
517 static int pt_insn_proceed(struct pt_insn_decoder *decoder,
518 const struct pt_insn *insn,
519 const struct pt_insn_ext *iext)
521 if (!decoder || !insn || !iext)
522 return -pte_internal;
524 /* Branch displacements apply to the next instruction. */
525 decoder->ip += insn->size;
527 /* We handle non-branches, non-taken conditional branches, and
528 * compressed returns directly in the switch and do some pre-work for
531 * All kinds of branches are handled below the switch.
533 switch (insn->iclass) {
538 case ptic_cond_jump: {
541 status = pt_insn_cond_branch(decoder, &taken);
545 decoder->status = status;
553 /* Log the call for return compression.
555 * Unless this is a call to the next instruction as is used
556 * for position independent code.
558 if (iext->variant.branch.displacement ||
559 !iext->variant.branch.is_direct)
560 pt_retstack_push(&decoder->retstack, decoder->ip);
567 /* Check for a compressed return. */
568 status = pt_insn_cond_branch(decoder, &taken);
570 decoder->status = status;
572 /* A compressed return is indicated by a taken
573 * conditional branch.
576 return -pte_bad_retcomp;
578 return pt_retstack_pop(&decoder->retstack,
587 case ptic_far_return:
592 return -pte_bad_insn;
595 /* Process a direct or indirect branch.
597 * This combines calls, uncompressed returns, taken conditional jumps,
598 * and all flavors of far transfers.
600 if (iext->variant.branch.is_direct)
601 decoder->ip += iext->variant.branch.displacement;
605 status = pt_insn_indirect_branch(decoder, &decoder->ip);
610 decoder->status = status;
612 /* We do need an IP to proceed. */
613 if (status & pts_ip_suppressed)
620 static int pt_insn_at_skl014(const struct pt_event *ev,
621 const struct pt_insn *insn,
622 const struct pt_insn_ext *iext,
623 const struct pt_config *config)
628 if (!ev || !insn || !iext || !config)
629 return -pte_internal;
631 if (!ev->ip_suppressed)
634 switch (insn->iclass) {
637 /* The erratum only applies to unconditional direct branches. */
638 if (!iext->variant.branch.is_direct)
641 /* Check the filter against the branch target. */
644 ip += iext->variant.branch.displacement;
646 status = pt_filter_addr_check(&config->addr_filter, ip);
662 static int pt_insn_at_disabled_event(const struct pt_event *ev,
663 const struct pt_insn *insn,
664 const struct pt_insn_ext *iext,
665 const struct pt_config *config)
667 if (!ev || !insn || !iext || !config)
668 return -pte_internal;
670 if (ev->ip_suppressed) {
671 if (pt_insn_is_far_branch(insn, iext) ||
672 pt_insn_changes_cpl(insn, iext) ||
673 pt_insn_changes_cr3(insn, iext))
676 /* If we don't have a filter configuration we assume that no
677 * address filters were used and the erratum does not apply.
679 * We might otherwise disable tracing too early.
681 if (config->addr_filter.config.addr_cfg &&
682 config->errata.skl014 &&
683 pt_insn_at_skl014(ev, insn, iext, config))
686 switch (insn->iclass) {
693 /* If we got an IP with the disabled event, we may
694 * ignore direct branches that go to a different IP.
696 if (iext->variant.branch.is_direct) {
701 ip += iext->variant.branch.displacement;
703 if (ip != ev->variant.disabled.ip)
710 case ptic_far_return:
716 return -pte_bad_insn;
723 /* Postpone proceeding past @insn/@iext and indicate a pending event.
725 * There may be further events pending on @insn/@iext. Postpone proceeding past
726 * @insn/@iext until we processed all events that bind to it.
728 * Returns a non-negative pt_status_flag bit-vector indicating a pending event
729 * on success, a negative pt_error_code otherwise.
731 static int pt_insn_postpone(struct pt_insn_decoder *decoder,
732 const struct pt_insn *insn,
733 const struct pt_insn_ext *iext)
735 if (!decoder || !insn || !iext)
736 return -pte_internal;
738 if (!decoder->process_insn) {
739 decoder->process_insn = 1;
740 decoder->insn = *insn;
741 decoder->iext = *iext;
744 return pt_insn_status(decoder, pts_event_pending);
747 /* Remove any postponed instruction from @decoder.
749 * Returns zero on success, a negative pt_error_code otherwise.
751 static int pt_insn_clear_postponed(struct pt_insn_decoder *decoder)
754 return -pte_internal;
756 decoder->process_insn = 0;
757 decoder->bound_paging = 0;
758 decoder->bound_vmcs = 0;
759 decoder->bound_ptwrite = 0;
764 /* Proceed past a postponed instruction.
766 * Returns zero on success, a negative pt_error_code otherwise.
768 static int pt_insn_proceed_postponed(struct pt_insn_decoder *decoder)
773 return -pte_internal;
775 if (!decoder->process_insn)
776 return -pte_internal;
778 /* There's nothing to do if tracing got disabled. */
779 if (!decoder->enabled)
780 return pt_insn_clear_postponed(decoder);
782 status = pt_insn_proceed(decoder, &decoder->insn, &decoder->iext);
786 return pt_insn_clear_postponed(decoder);
789 /* Check for events that bind to instruction.
791 * Check whether an event is pending that binds to @insn/@iext, and, if that is
792 * the case, proceed past @insn/@iext and indicate the event by setting
795 * If that is not the case, we return zero. This is what pt_insn_status() would
798 * - we suppress pts_eos as long as we're processing events
799 * - we do not set pts_ip_suppressed since tracing must be enabled
801 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
804 static int pt_insn_check_insn_event(struct pt_insn_decoder *decoder,
805 const struct pt_insn *insn,
806 const struct pt_insn_ext *iext)
812 return -pte_internal;
814 status = event_pending(decoder);
818 ev = &decoder->event;
822 case ptev_async_paging:
823 case ptev_async_vmcs:
824 case ptev_async_disabled:
825 case ptev_async_branch:
836 /* We're only interested in events that bind to instructions. */
840 status = pt_insn_at_disabled_event(ev, insn, iext,
841 &decoder->query.config);
845 /* We're at a synchronous disable event location.
847 * Let's determine the IP at which we expect tracing to resume.
849 status = pt_insn_next_ip(&decoder->ip, insn, iext);
851 /* We don't know the IP on error. */
854 /* For indirect calls, assume that we return to the next
857 * We only check the instruction class, not the
858 * is_direct property, since direct calls would have
859 * been handled by pt_insn_nex_ip() or would have
860 * provoked a different error.
862 if (status != -pte_bad_query)
865 switch (insn->iclass) {
868 decoder->ip = insn->ip + insn->size;
879 /* We bind at most one paging event to an instruction. */
880 if (decoder->bound_paging)
883 if (!pt_insn_binds_to_pip(insn, iext))
886 /* We bound a paging event. Make sure we do not bind further
887 * paging events to this instruction.
889 decoder->bound_paging = 1;
891 return pt_insn_postpone(decoder, insn, iext);
894 /* We bind at most one vmcs event to an instruction. */
895 if (decoder->bound_vmcs)
898 if (!pt_insn_binds_to_vmcs(insn, iext))
901 /* We bound a vmcs event. Make sure we do not bind further vmcs
902 * events to this instruction.
904 decoder->bound_vmcs = 1;
906 return pt_insn_postpone(decoder, insn, iext);
909 /* We bind at most one ptwrite event to an instruction. */
910 if (decoder->bound_ptwrite)
913 if (ev->ip_suppressed) {
914 if (!pt_insn_is_ptwrite(insn, iext))
917 /* Fill in the event IP. Our users will need them to
918 * make sense of the PTWRITE payload.
920 ev->variant.ptwrite.ip = decoder->ip;
921 ev->ip_suppressed = 0;
923 /* The ptwrite event contains the IP of the ptwrite
924 * instruction (CLIP) unlike most events that contain
925 * the IP of the first instruction that did not complete
928 * It's easier to handle this case here, as well.
930 if (decoder->ip != ev->variant.ptwrite.ip)
934 /* We bound a ptwrite event. Make sure we do not bind further
935 * ptwrite events to this instruction.
937 decoder->bound_ptwrite = 1;
939 return pt_insn_postpone(decoder, insn, iext);
942 return pt_insn_status(decoder, pts_event_pending);
946 /* The maximum number of steps to take when determining whether the
947 * event location can be reached.
949 bdm64_max_steps = 0x100
952 /* Try to work around erratum BDM64.
954 * If we got a transaction abort immediately following a branch that produced
955 * trace, the trace for that branch might have been corrupted.
957 * Returns a positive integer if the erratum was handled.
958 * Returns zero if the erratum does not seem to apply.
959 * Returns a negative error code otherwise.
961 static int handle_erratum_bdm64(struct pt_insn_decoder *decoder,
962 const struct pt_event *ev,
963 const struct pt_insn *insn,
964 const struct pt_insn_ext *iext)
968 if (!decoder || !ev || !insn || !iext)
969 return -pte_internal;
971 /* This only affects aborts. */
972 if (!ev->variant.tsx.aborted)
975 /* This only affects branches. */
976 if (!pt_insn_is_branch(insn, iext))
979 /* Let's check if we can reach the event location from here.
981 * If we can, let's assume the erratum did not hit. We might still be
982 * wrong but we're not able to tell.
984 status = pt_insn_range_is_contiguous(decoder->ip, ev->variant.tsx.ip,
985 decoder->mode, decoder->image,
986 &decoder->asid, bdm64_max_steps);
990 /* We can't reach the event location. This could either mean that we
991 * stopped too early (and status is zero) or that the erratum hit.
993 * We assume the latter and pretend that the previous branch brought us
994 * to the event location, instead.
996 decoder->ip = ev->variant.tsx.ip;
1001 /* Check whether a peek TSX event should be postponed.
1003 * This involves handling erratum BDM64.
1005 * Returns a positive integer if the event is to be postponed.
1006 * Returns zero if the event should be processed.
1007 * Returns a negative error code otherwise.
1009 static inline int pt_insn_postpone_tsx(struct pt_insn_decoder *decoder,
1010 const struct pt_insn *insn,
1011 const struct pt_insn_ext *iext,
1012 const struct pt_event *ev)
1016 if (!decoder || !ev)
1017 return -pte_internal;
1019 if (ev->ip_suppressed)
1022 if (insn && iext && decoder->query.config.errata.bdm64) {
1023 status = handle_erratum_bdm64(decoder, ev, insn, iext);
1028 if (decoder->ip != ev->variant.tsx.ip)
1034 /* Check for events that bind to an IP.
1036 * Check whether an event is pending that binds to @decoder->ip, and, if that is
1037 * the case, indicate the event by setting pt_pts_event_pending.
1039 * Returns a non-negative pt_status_flag bit-vector on success, a negative error
1042 static int pt_insn_check_ip_event(struct pt_insn_decoder *decoder,
1043 const struct pt_insn *insn,
1044 const struct pt_insn_ext *iext)
1046 struct pt_event *ev;
1050 return -pte_internal;
1052 status = event_pending(decoder);
1057 return pt_insn_status(decoder, 0);
1060 ev = &decoder->event;
1066 return pt_insn_status(decoder, pts_event_pending);
1068 case ptev_async_disabled:
1069 if (ev->variant.async_disabled.at != decoder->ip)
1072 if (decoder->query.config.errata.skd022) {
1075 errcode = handle_erratum_skd022(decoder);
1080 /* If the erratum applies, we postpone the
1081 * modified event to the next call to
1088 return pt_insn_status(decoder, pts_event_pending);
1091 status = pt_insn_postpone_tsx(decoder, insn, iext, ev);
1099 return pt_insn_status(decoder, pts_event_pending);
1101 case ptev_async_branch:
1102 if (ev->variant.async_branch.from != decoder->ip)
1105 return pt_insn_status(decoder, pts_event_pending);
1108 return pt_insn_status(decoder, pts_event_pending);
1110 case ptev_exec_mode:
1111 if (!ev->ip_suppressed &&
1112 ev->variant.exec_mode.ip != decoder->ip)
1115 return pt_insn_status(decoder, pts_event_pending);
1118 if (decoder->enabled)
1121 return pt_insn_status(decoder, pts_event_pending);
1123 case ptev_async_paging:
1124 if (!ev->ip_suppressed &&
1125 ev->variant.async_paging.ip != decoder->ip)
1128 return pt_insn_status(decoder, pts_event_pending);
1131 if (decoder->enabled)
1134 return pt_insn_status(decoder, pts_event_pending);
1136 case ptev_async_vmcs:
1137 if (!ev->ip_suppressed &&
1138 ev->variant.async_vmcs.ip != decoder->ip)
1141 return pt_insn_status(decoder, pts_event_pending);
1144 return pt_insn_status(decoder, pts_event_pending);
1147 if (!ev->ip_suppressed && decoder->enabled &&
1148 decoder->ip != ev->variant.exstop.ip)
1151 return pt_insn_status(decoder, pts_event_pending);
1154 if (!ev->ip_suppressed && decoder->enabled &&
1155 decoder->ip != ev->variant.mwait.ip)
1158 return pt_insn_status(decoder, pts_event_pending);
1162 return pt_insn_status(decoder, pts_event_pending);
1165 /* Any event binding to the current PTWRITE instruction is
1166 * handled in pt_insn_check_insn_event().
1168 * Any subsequent ptwrite event binds to a different instruction
1169 * and must wait until the next iteration - as long as tracing
1172 * When tracing is disabled, we forward all ptwrite events
1173 * immediately to the user.
1175 if (decoder->enabled)
1178 return pt_insn_status(decoder, pts_event_pending);
1183 return pt_insn_status(decoder, pts_event_pending);
1186 return pt_insn_status(decoder, 0);
1189 static inline int insn_to_user(struct pt_insn *uinsn, size_t size,
1190 const struct pt_insn *insn)
1192 if (!uinsn || !insn)
1193 return -pte_internal;
1198 /* Zero out any unknown bytes. */
1199 if (sizeof(*insn) < size) {
1200 memset(uinsn + sizeof(*insn), 0, size - sizeof(*insn));
1202 size = sizeof(*insn);
1205 memcpy(uinsn, insn, size);
1210 static int pt_insn_decode_cached(struct pt_insn_decoder *decoder,
1211 const struct pt_mapped_section *msec,
1212 struct pt_insn *insn, struct pt_insn_ext *iext)
1216 if (!decoder || !insn || !iext)
1217 return -pte_internal;
1219 /* Try reading the memory containing @insn from the cached section. If
1220 * that fails, if we don't have a cached section, or if decode fails
1221 * later on, fall back to decoding @insn from @decoder->image.
1223 * The latter will also handle truncated instructions that cross section
1228 return pt_insn_decode(insn, iext, decoder->image,
1231 status = pt_msec_read(msec, insn->raw, sizeof(insn->raw), insn->ip);
1233 if (status != -pte_nomap)
1236 return pt_insn_decode(insn, iext, decoder->image,
1240 /* We initialize @insn->size to the maximal possible size. It will be
1241 * set to the actual size during instruction decode.
1243 insn->size = (uint8_t) status;
1245 status = pt_ild_decode(insn, iext);
1247 if (status != -pte_bad_insn)
1250 return pt_insn_decode(insn, iext, decoder->image,
1257 static int pt_insn_msec_lookup(struct pt_insn_decoder *decoder,
1258 const struct pt_mapped_section **pmsec)
1260 struct pt_msec_cache *scache;
1261 struct pt_image *image;
1265 if (!decoder || !pmsec)
1266 return -pte_internal;
1268 scache = &decoder->scache;
1269 image = decoder->image;
1272 isid = pt_msec_cache_read(scache, pmsec, image, ip);
1274 if (isid != -pte_nomap)
1277 return pt_msec_cache_fill(scache, pmsec, image,
1278 &decoder->asid, ip);
1284 int pt_insn_next(struct pt_insn_decoder *decoder, struct pt_insn *uinsn,
1287 const struct pt_mapped_section *msec;
1288 struct pt_insn_ext iext;
1289 struct pt_insn insn, *pinsn;
1292 if (!uinsn || !decoder)
1293 return -pte_invalid;
1295 /* Tracing must be enabled.
1297 * If it isn't we should be processing events until we either run out of
1298 * trace or process a tracing enabled event.
1300 if (!decoder->enabled) {
1301 if (decoder->status & pts_eos)
1304 return -pte_no_enable;
1307 pinsn = size == sizeof(insn) ? uinsn : &insn;
1309 /* Zero-initialize the instruction in case of error returns. */
1310 memset(pinsn, 0, sizeof(*pinsn));
1312 /* Fill in a few things from the current decode state.
1314 * This reflects the state of the last pt_insn_next(), pt_insn_event()
1315 * or pt_insn_start() call.
1317 if (decoder->speculative)
1318 pinsn->speculative = 1;
1319 pinsn->ip = decoder->ip;
1320 pinsn->mode = decoder->mode;
1322 isid = pt_insn_msec_lookup(decoder, &msec);
1324 if (isid != -pte_nomap)
1330 /* We set an incorrect isid if @msec is NULL. This will be corrected
1331 * when we read the memory from the image later on.
1335 status = pt_insn_decode_cached(decoder, msec, pinsn, &iext);
1337 /* Provide the incomplete instruction - the IP and mode fields
1338 * are valid and may help diagnose the error.
1340 (void) insn_to_user(uinsn, size, pinsn);
1344 /* Provide the decoded instruction to the user. It won't change during
1347 status = insn_to_user(uinsn, size, pinsn);
1351 /* Check for events that bind to the current instruction.
1353 * If an event is indicated, we're done.
1355 status = pt_insn_check_insn_event(decoder, pinsn, &iext);
1360 if (status & pts_event_pending)
1364 /* Determine the next instruction's IP. */
1365 status = pt_insn_proceed(decoder, pinsn, &iext);
1369 /* Indicate events that bind to the new IP.
1371 * Although we only look at the IP for binding events, we pass the
1372 * decoded instruction in order to handle errata.
1374 return pt_insn_check_ip_event(decoder, pinsn, &iext);
1377 static int pt_insn_process_enabled(struct pt_insn_decoder *decoder)
1379 struct pt_event *ev;
1382 return -pte_internal;
1384 ev = &decoder->event;
1386 /* This event can't be a status update. */
1387 if (ev->status_update)
1388 return -pte_bad_context;
1390 /* We must have an IP in order to start decoding. */
1391 if (ev->ip_suppressed)
1394 /* We must currently be disabled. */
1395 if (decoder->enabled)
1396 return -pte_bad_context;
1398 decoder->ip = ev->variant.enabled.ip;
1399 decoder->enabled = 1;
1404 static int pt_insn_process_disabled(struct pt_insn_decoder *decoder)
1406 struct pt_event *ev;
1409 return -pte_internal;
1411 ev = &decoder->event;
1413 /* This event can't be a status update. */
1414 if (ev->status_update)
1415 return -pte_bad_context;
1417 /* We must currently be enabled. */
1418 if (!decoder->enabled)
1419 return -pte_bad_context;
1421 /* We preserve @decoder->ip. This is where we expect tracing to resume
1422 * and we'll indicate that on the subsequent enabled event if tracing
1423 * actually does resume from there.
1425 decoder->enabled = 0;
1430 static int pt_insn_process_async_branch(struct pt_insn_decoder *decoder)
1432 struct pt_event *ev;
1435 return -pte_internal;
1437 ev = &decoder->event;
1439 /* This event can't be a status update. */
1440 if (ev->status_update)
1441 return -pte_bad_context;
1443 /* Tracing must be enabled in order to make sense of the event. */
1444 if (!decoder->enabled)
1445 return -pte_bad_context;
1447 decoder->ip = ev->variant.async_branch.to;
1452 static int pt_insn_process_paging(struct pt_insn_decoder *decoder)
1458 return -pte_internal;
1460 cr3 = decoder->event.variant.paging.cr3;
1461 if (decoder->asid.cr3 != cr3) {
1462 errcode = pt_msec_cache_invalidate(&decoder->scache);
1466 decoder->asid.cr3 = cr3;
1472 static int pt_insn_process_overflow(struct pt_insn_decoder *decoder)
1474 struct pt_event *ev;
1477 return -pte_internal;
1479 ev = &decoder->event;
1481 /* This event can't be a status update. */
1482 if (ev->status_update)
1483 return -pte_bad_context;
1485 /* If the IP is suppressed, the overflow resolved while tracing was
1486 * disabled. Otherwise it resolved while tracing was enabled.
1488 if (ev->ip_suppressed) {
1489 /* Tracing is disabled.
1491 * It doesn't make sense to preserve the previous IP. This will
1492 * just be misleading. Even if tracing had been disabled
1493 * before, as well, we might have missed the re-enable in the
1496 decoder->enabled = 0;
1499 /* Tracing is enabled and we're at the IP at which the overflow
1502 decoder->ip = ev->variant.overflow.ip;
1503 decoder->enabled = 1;
1506 /* We don't know the TSX state. Let's assume we execute normally.
1508 * We also don't know the execution mode. Let's keep what we have
1509 * in case we don't get an update before we have to decode the next
1512 decoder->speculative = 0;
1517 static int pt_insn_process_exec_mode(struct pt_insn_decoder *decoder)
1519 enum pt_exec_mode mode;
1520 struct pt_event *ev;
1523 return -pte_internal;
1525 ev = &decoder->event;
1526 mode = ev->variant.exec_mode.mode;
1528 /* Use status update events to diagnose inconsistencies. */
1529 if (ev->status_update && decoder->enabled &&
1530 decoder->mode != ptem_unknown && decoder->mode != mode)
1531 return -pte_bad_status_update;
1533 decoder->mode = mode;
1538 static int pt_insn_process_tsx(struct pt_insn_decoder *decoder)
1541 return -pte_internal;
1543 decoder->speculative = decoder->event.variant.tsx.speculative;
1548 static int pt_insn_process_stop(struct pt_insn_decoder *decoder)
1550 struct pt_event *ev;
1553 return -pte_internal;
1555 ev = &decoder->event;
1557 /* This event can't be a status update. */
1558 if (ev->status_update)
1559 return -pte_bad_context;
1561 /* Tracing is always disabled before it is stopped. */
1562 if (decoder->enabled)
1563 return -pte_bad_context;
1568 static int pt_insn_process_vmcs(struct pt_insn_decoder *decoder)
1574 return -pte_internal;
1576 vmcs = decoder->event.variant.vmcs.base;
1577 if (decoder->asid.vmcs != vmcs) {
1578 errcode = pt_msec_cache_invalidate(&decoder->scache);
1582 decoder->asid.vmcs = vmcs;
1588 int pt_insn_event(struct pt_insn_decoder *decoder, struct pt_event *uevent,
1591 struct pt_event *ev;
1594 if (!decoder || !uevent)
1595 return -pte_invalid;
1597 /* We must currently process an event. */
1598 if (!decoder->process_event)
1599 return -pte_bad_query;
1601 ev = &decoder->event;
1604 /* This is not a user event.
1606 * We either indicated it wrongly or the user called
1607 * pt_insn_event() without a pts_event_pending indication.
1609 return -pte_bad_query;
1612 /* Indicate that tracing resumes from the IP at which tracing
1613 * had been disabled before (with some special treatment for
1616 if (decoder->ip == ev->variant.enabled.ip)
1617 ev->variant.enabled.resumed = 1;
1619 status = pt_insn_process_enabled(decoder);
1625 case ptev_async_disabled:
1626 if (!ev->ip_suppressed &&
1627 decoder->ip != ev->variant.async_disabled.at)
1628 return -pte_bad_query;
1632 status = pt_insn_process_disabled(decoder);
1638 case ptev_async_branch:
1639 if (decoder->ip != ev->variant.async_branch.from)
1640 return -pte_bad_query;
1642 status = pt_insn_process_async_branch(decoder);
1648 case ptev_async_paging:
1649 if (!ev->ip_suppressed &&
1650 decoder->ip != ev->variant.async_paging.ip)
1651 return -pte_bad_query;
1655 status = pt_insn_process_paging(decoder);
1661 case ptev_async_vmcs:
1662 if (!ev->ip_suppressed &&
1663 decoder->ip != ev->variant.async_vmcs.ip)
1664 return -pte_bad_query;
1668 status = pt_insn_process_vmcs(decoder);
1675 status = pt_insn_process_overflow(decoder);
1681 case ptev_exec_mode:
1682 status = pt_insn_process_exec_mode(decoder);
1689 status = pt_insn_process_tsx(decoder);
1696 status = pt_insn_process_stop(decoder);
1703 if (!ev->ip_suppressed && decoder->enabled &&
1704 decoder->ip != ev->variant.exstop.ip)
1705 return -pte_bad_query;
1710 if (!ev->ip_suppressed && decoder->enabled &&
1711 decoder->ip != ev->variant.mwait.ip)
1712 return -pte_bad_query;
1725 /* Copy the event to the user. Make sure we're not writing beyond the
1726 * memory provided by the user.
1728 * We might truncate details of an event but only for those events the
1729 * user can't know about, anyway.
1731 if (sizeof(*ev) < size)
1734 memcpy(uevent, ev, size);
1736 /* This completes processing of the current event. */
1737 decoder->process_event = 0;
1739 /* If we just handled an instruction event, check for further events
1740 * that bind to this instruction.
1742 * If we don't have further events, proceed beyond the instruction so we
1743 * can check for IP events, as well.
1745 if (decoder->process_insn) {
1746 status = pt_insn_check_insn_event(decoder, &decoder->insn,
1753 if (status & pts_event_pending)
1757 /* Proceed to the next instruction. */
1758 status = pt_insn_proceed_postponed(decoder);
1763 /* Indicate further events that bind to the same IP. */
1764 return pt_insn_check_ip_event(decoder, NULL, NULL);