2 * Copyright (c) 2014-2019, Intel Corporation
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright notice,
8 * this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright notice,
10 * this list of conditions and the following disclaimer in the documentation
11 * and/or other materials provided with the distribution.
12 * * Neither the name of Intel Corporation nor the names of its contributors
13 * may be used to endorse or promote products derived from this software
14 * without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
30 #include "pt_opcodes.h"
38 void pt_time_init(struct pt_time *time)
43 memset(time, 0, sizeof(*time));
46 int pt_time_query_tsc(uint64_t *tsc, uint32_t *lost_mtc,
47 uint32_t *lost_cyc, const struct pt_time *time)
55 *lost_mtc = time->lost_mtc;
57 *lost_cyc = time->lost_cyc;
65 int pt_time_query_cbr(uint32_t *cbr, const struct pt_time *time)
78 /* Compute the distance between two CTC sources.
80 * We adjust a single wrap-around but fail if the distance is bigger than that.
82 * Returns zero on success, a negative error code otherwise.
84 static int pt_time_ctc_delta(uint32_t *ctc_delta, uint32_t ctc,
85 uint32_t last_ctc, const struct pt_config *config)
87 if (!config || !ctc_delta)
90 /* Correct a single wrap-around. If we lost enough MTCs to wrap
91 * around twice, timing will be wrong until the next TSC.
94 ctc += 1u << (config->mtc_freq + pt_pl_mtc_bit_size);
96 /* Since we only store the CTC between TMA/MTC or MTC/TMC a
97 * single correction should suffice.
100 return -pte_bad_packet;
103 *ctc_delta = ctc - last_ctc;
107 /* Translate CTC into the same unit as the FastCounter by multiplying with P.
109 * Returns zero on success, a negative error code otherwise.
111 static int pt_time_ctc_fc(uint64_t *fc, uint64_t ctc,
112 const struct pt_config *config)
117 return -pte_internal;
119 eax = config->cpuid_0x15_eax;
120 ebx = config->cpuid_0x15_ebx;
122 /* Neither multiply nor divide by zero. */
124 return -pte_bad_config;
126 *fc = (ctc * ebx) / eax;
130 int pt_time_update_tsc(struct pt_time *time,
131 const struct pt_packet_tsc *packet,
132 const struct pt_config *config)
136 if (!time || !packet)
137 return -pte_internal;
142 time->tsc = time->base = packet->tsc;
146 /* We got the full time; we recover from previous losses. */
153 int pt_time_update_cbr(struct pt_time *time,
154 const struct pt_packet_cbr *packet,
155 const struct pt_config *config)
161 if (!time || !packet)
162 return -pte_internal;
166 return -pte_bad_packet;
174 int pt_time_update_tma(struct pt_time *time,
175 const struct pt_packet_tma *packet,
176 const struct pt_config *config)
178 uint32_t ctc, mtc_freq, mtc_hi, ctc_mask;
181 if (!time || !packet || !config)
182 return -pte_internal;
184 /* Without a TSC something is seriously wrong. */
186 return -pte_bad_context;
188 /* We shouldn't have more than one TMA per TSC. */
190 return -pte_bad_context;
192 /* We're ignoring MTC between TSC and TMA. */
194 return -pte_internal;
199 mtc_freq = config->mtc_freq;
200 mtc_hi = mtc_freq + pt_pl_mtc_bit_size;
202 /* A mask for the relevant CTC bits ignoring high-order bits that are
203 * not provided by MTC.
205 ctc_mask = (1u << mtc_hi) - 1u;
211 /* If the MTC frequency is low enough that TMA provides the full CTC
212 * value, we can use the TMA as an MTC.
214 * If it isn't, we will estimate the preceding MTC based on the CTC bits
215 * the TMA provides at the next MTC. We forget about the previous MTC
218 * If no MTC packets are dropped around TMA, we will estimate the
219 * forgotten value again at the next MTC.
221 * If MTC packets are dropped, we can't really tell where in this
222 * extended MTC period the TSC occurred. The estimation will place it
223 * right before the next MTC.
225 if (mtc_hi <= pt_pl_tma_ctc_bit_size)
228 /* In both cases, we store the TMA's CTC bits until the next MTC. */
229 time->ctc = time->ctc_cyc = ctc & ctc_mask;
234 int pt_time_update_mtc(struct pt_time *time,
235 const struct pt_packet_mtc *packet,
236 const struct pt_config *config)
238 uint32_t last_ctc, ctc, ctc_delta;
241 int errcode, have_tsc, have_tma, have_mtc;
243 if (!time || !packet || !config)
244 return -pte_internal;
246 have_tsc = time->have_tsc;
247 have_tma = time->have_tma;
248 have_mtc = time->have_mtc;
250 /* We ignore MTCs between TSC and TMA to avoid apparent CTC overflows.
252 * Later MTCs will ensure that no time is lost - provided TMA provides
253 * enough bits. If TMA doesn't provide any of the MTC bits we may place
254 * the TSC into the wrong MTC period.
256 if (have_tsc && !have_tma)
260 last_ctc = time->ctc;
261 mtc_freq = config->mtc_freq;
263 ctc = (uint32_t) packet->ctc << mtc_freq;
265 /* Store our CTC value if we have or would have reset FC. */
266 if (time->fc || time->lost_cyc || !have_mtc)
269 /* Prepare for the next packet in case we error out below. */
274 /* We recover from previous CYC losses. */
277 /* Avoid a big jump when we see the first MTC with an arbitrary CTC
281 uint32_t ctc_lo, ctc_hi;
283 /* If we have not seen a TMA, we ignore this first MTC.
285 * We have no idea where in this MTC period tracing started.
286 * We could lose an entire MTC period or just a tiny fraction.
288 * On the other hand, if we assumed a previous MTC value, we
289 * might make just the same error.
294 /* The TMA's CTC value didn't provide enough bits - otherwise,
295 * we would have treated the TMA as an MTC.
297 if (last_ctc & ~(uint32_t) pt_pl_tma_ctc_mask)
298 return -pte_internal;
300 /* Split this MTC's CTC value into low and high parts with
301 * respect to the bits provided by TMA.
303 ctc_lo = ctc & (uint32_t) pt_pl_tma_ctc_mask;
304 ctc_hi = ctc & ~(uint32_t) pt_pl_tma_ctc_mask;
306 /* We estimate the high-order CTC bits that are not provided by
307 * TMA based on the CTC bits provided by this MTC.
309 * We assume that no MTC packets were dropped around TMA. If
310 * there are, we might place the TSC into the wrong MTC period
311 * depending on how many CTC bits TMA provides and how many MTC
312 * packets were dropped.
314 * Note that we may underflow which results in more bits to be
315 * set than MTC packets may provide. Drop those extra bits.
317 if (ctc_lo < last_ctc) {
318 ctc_hi -= 1u << pt_pl_tma_ctc_bit_size;
319 ctc_hi &= (uint32_t) pt_pl_mtc_mask << mtc_freq;
325 errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
331 errcode = pt_time_ctc_fc(&tsc, ctc_delta, config);
336 time->tsc = time->base = base;
341 /* Adjust a CYC packet's payload spanning multiple MTC periods.
343 * CYC packets measure the Fast Counter since the last CYC(-eligible) packet.
344 * Depending on the CYC threshold, we may not get a CYC for each MTC, so a CYC
345 * period may overlap with or even span multiple MTC periods.
347 * We can't do much about the overlap case without examining all packets in
348 * the respective periods. We leave this as expected imprecision.
350 * If we find a CYC packet to span multiple MTC packets, though, we try to
351 * approximate the portion for the current MTC period by subtracting the
352 * estimated portion for previous MTC periods using calibration information.
354 * We only consider MTC. For the first CYC after TSC, the corresponding TMA
355 * will contain the Fast Counter at TSC.
357 * Returns zero on success, a negative error code otherwise.
359 static int pt_time_adjust_cyc(uint64_t *cyc, const struct pt_time *time,
360 const struct pt_config *config, uint64_t fcr)
362 uint32_t last_ctc, ctc, ctc_delta;
363 uint64_t fc, total_cyc, old_cyc;
366 if (!time || !config || !fcr)
367 return -pte_internal;
369 last_ctc = time->ctc_cyc;
372 /* There is nothing to do if this is the current MTC period. */
376 /* Calibration computes
378 * fc = (ctc_delta * cpuid[0x15].ebx) / cpuid[0x15].eax.
379 * fcr = (fc << pt_tcal_fcr_shr) / cyc
381 * So cyc = (fc << pt_tcal_fcr_shr) / fcr.
384 errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
388 errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
392 old_cyc = (fc << pt_tcal_fcr_shr) / fcr;
395 /* Make sure we don't wrap around. If we would, attribute the entire
396 * CYC payload to any previous MTC period.
398 * We lost an unknown portion of the CYC payload for the current MTC
399 * period, but it's usually better to run too slow than too fast.
401 if (total_cyc < old_cyc)
404 *cyc = total_cyc - old_cyc;
408 int pt_time_update_cyc(struct pt_time *time,
409 const struct pt_packet_cyc *packet,
410 const struct pt_config *config, uint64_t fcr)
414 if (!time || !packet || !config)
415 return -pte_internal;
427 errcode = pt_time_adjust_cyc(&cyc, time, config, fcr);
432 fc += (cyc * fcr) >> pt_tcal_fcr_shr;
435 time->tsc = time->base + fc;
440 void pt_tcal_init(struct pt_time_cal *tcal)
445 memset(tcal, 0, sizeof(*tcal));
447 tcal->min_fcr = UINT64_MAX;
450 static int pt_tcal_have_fcr(const struct pt_time_cal *tcal)
455 return (tcal->min_fcr <= tcal->max_fcr);
458 int pt_tcal_fcr(uint64_t *fcr, const struct pt_time_cal *tcal)
461 return -pte_internal;
463 if (!pt_tcal_have_fcr(tcal))
471 int pt_tcal_set_fcr(struct pt_time_cal *tcal, uint64_t fcr)
474 return -pte_internal;
478 if (fcr < tcal->min_fcr)
481 if (fcr > tcal->max_fcr)
487 int pt_tcal_update_psb(struct pt_time_cal *tcal,
488 const struct pt_config *config)
490 if (!tcal || !config)
491 return -pte_internal;
493 if (config->errata.skl168)
494 tcal->check_skl168 = 1;
499 int pt_tcal_update_tsc(struct pt_time_cal *tcal,
500 const struct pt_packet_tsc *packet,
501 const struct pt_config *config)
505 if (!tcal || !packet)
506 return -pte_internal;
508 /* A TSC outside of PSB+ may indicate loss of time. We do not use it
509 * for calibration. We store the TSC value for calibration at the next
510 * TSC in PSB+, though.
512 tcal->tsc = packet->tsc;
513 tcal->cyc_tsc = 0ull;
518 int pt_tcal_header_tsc(struct pt_time_cal *tcal,
519 const struct pt_packet_tsc *packet,
520 const struct pt_config *config)
522 uint64_t tsc, last_tsc, tsc_delta, cyc, fcr;
526 if (!tcal || !packet)
527 return -pte_internal;
529 last_tsc = tcal->tsc;
535 tcal->cyc_tsc = 0ull;
537 if (!last_tsc || !cyc)
540 /* Prefer MTC over TSC for calibration. */
544 /* Correct a single wrap-around. */
545 if (tsc < last_tsc) {
546 tsc += 1ull << pt_pl_tsc_bit_size;
549 return -pte_bad_packet;
552 tsc_delta = tsc - last_tsc;
554 /* We shift the nominator to improve rounding precision.
556 * Since we're only collecting the CYCs between two TSC, we shouldn't
557 * overflow. Let's rather fail than overflow.
559 if (tsc_delta & ~(~0ull >> pt_tcal_fcr_shr))
560 return -pte_internal;
562 fcr = (tsc_delta << pt_tcal_fcr_shr) / cyc;
564 return pt_tcal_set_fcr(tcal, fcr);
567 int pt_tcal_update_cbr(struct pt_time_cal *tcal,
568 const struct pt_packet_cbr *packet,
569 const struct pt_config *config)
571 /* A CBR outside of PSB+ indicates a frequency change. Reset our
576 return pt_tcal_header_cbr(tcal, packet, config);
579 int pt_tcal_header_cbr(struct pt_time_cal *tcal,
580 const struct pt_packet_cbr *packet,
581 const struct pt_config *config)
583 uint64_t cbr, p1, fcr;
585 if (!tcal || !packet || !config)
586 return -pte_internal;
588 p1 = config->nom_freq;
592 /* If we know the nominal frequency, we can use it for calibration. */
595 return -pte_bad_packet;
597 fcr = (p1 << pt_tcal_fcr_shr) / cbr;
599 return pt_tcal_set_fcr(tcal, fcr);
602 int pt_tcal_update_tma(struct pt_time_cal *tcal,
603 const struct pt_packet_tma *packet,
604 const struct pt_config *config)
614 int pt_tcal_update_mtc(struct pt_time_cal *tcal,
615 const struct pt_packet_mtc *packet,
616 const struct pt_config *config)
618 uint32_t last_ctc, ctc, ctc_delta, have_mtc, check_skl168;
619 uint64_t cyc, fc, fcr;
622 if (!tcal || !packet || !config)
623 return -pte_internal;
625 last_ctc = tcal->ctc;
626 have_mtc = tcal->have_mtc;
628 check_skl168 = tcal->check_skl168;
630 /* This only affects the first MTC after PSB. */
631 tcal->check_skl168 = 0;
633 ctc = (uint32_t) packet->ctc << config->mtc_freq;
635 /* We need at least two MTC (including this). */
637 tcal->cyc_mtc = 0ull;
644 /* Without any cycles, we can't calibrate. Try again at the next
645 * MTC and distribute the cycles over the combined MTC period.
650 /* Prepare for the next packet in case we error out below. */
652 tcal->cyc_mtc = 0ull;
655 /* Let's pretend we will fail. We'll correct it at the end. */
658 errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
662 errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
666 /* We shift the nominator to improve rounding precision.
668 * Since we're only collecting the CYCs between two MTC, we shouldn't
669 * overflow. Let's rather fail than overflow.
671 if (fc & ~(~0ull >> pt_tcal_fcr_shr))
672 return -pte_internal;
674 fcr = (fc << pt_tcal_fcr_shr) / cyc;
676 /* SKL168: Intel(R) PT CYC Packets Can be Dropped When Immediately
679 * We skip this MTC if we lost one or more MTC since the last PSB or if
680 * it looks like we lost a wrap CYC packet.
682 * This is not an error but we count that MTC as lost.
685 /* If we lost one or more MTC, the case is clear. */
686 if ((1u << config->mtc_freq) < ctc_delta)
689 /* The case is less clear for a lost wrap CYC packet since we do
690 * have some variation in the number of cycles.
692 * The CYC counter wraps on the affected processors every 4096
693 * cycles. For low MTC frequencies (high values), losing one
694 * may not be noticeable.
696 * We restrict the workaround to higher MTC frequencies (lower
699 * We also need a previous FCR so we know how many cycles to
702 if ((config->mtc_freq < 10) && pt_tcal_have_fcr(tcal)) {
705 /* We choose a slightly lower adjustment to account for
706 * some normal variation.
708 dfc = (tcal->fcr * (cyc + 0xf00)) >> pt_tcal_fcr_shr;
710 /* If we didn't drop a wrap CYC, @dfc should be way
711 * bigger than @fc. If it isn't, we assume that the
719 errcode = pt_tcal_set_fcr(tcal, fcr);
723 /* We updated the FCR. This recovers from previous MTC losses. */
729 int pt_tcal_update_cyc(struct pt_time_cal *tcal,
730 const struct pt_packet_cyc *packet,
731 const struct pt_config *config)
737 if (!tcal || !packet)
738 return -pte_internal;
741 tcal->cyc_mtc += cyc;
742 tcal->cyc_tsc += cyc;
747 int pt_tcal_update_ovf(struct pt_time_cal *tcal,
748 const struct pt_config *config)
750 if (!tcal || !config)
751 return -pte_internal;
754 tcal->cyc_tsc = 0ull;
755 tcal->cyc_mtc = 0ull;