2 * Copyright (c) 2014-2018, Intel Corporation
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright notice,
8 * this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright notice,
10 * this list of conditions and the following disclaimer in the documentation
11 * and/or other materials provided with the distribution.
12 * * Neither the name of Intel Corporation nor the names of its contributors
13 * may be used to endorse or promote products derived from this software
14 * without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
30 #include "pt_opcodes.h"
38 void pt_time_init(struct pt_time *time)
43 memset(time, 0, sizeof(*time));
46 int pt_time_query_tsc(uint64_t *tsc, uint32_t *lost_mtc,
47 uint32_t *lost_cyc, const struct pt_time *time)
55 *lost_mtc = time->lost_mtc;
57 *lost_cyc = time->lost_cyc;
65 int pt_time_query_cbr(uint32_t *cbr, const struct pt_time *time)
78 /* Compute the distance between two CTC sources.
80 * We adjust a single wrap-around but fail if the distance is bigger than that.
82 * Returns zero on success, a negative error code otherwise.
84 static int pt_time_ctc_delta(uint32_t *ctc_delta, uint32_t ctc,
85 uint32_t last_ctc, const struct pt_config *config)
87 if (!config || !ctc_delta)
90 /* Correct a single wrap-around. If we lost enough MTCs to wrap
91 * around twice, timing will be wrong until the next TSC.
94 ctc += 1u << (config->mtc_freq + pt_pl_mtc_bit_size);
96 /* Since we only store the CTC between TMA/MTC or MTC/TMC a
97 * single correction should suffice.
100 return -pte_bad_packet;
103 *ctc_delta = ctc - last_ctc;
107 /* Translate CTC into the same unit as the FastCounter by multiplying with P.
109 * Returns zero on success, a negative error code otherwise.
111 static int pt_time_ctc_fc(uint64_t *fc, uint64_t ctc,
112 const struct pt_config *config)
117 return -pte_internal;
119 eax = config->cpuid_0x15_eax;
120 ebx = config->cpuid_0x15_ebx;
122 /* Neither multiply nor divide by zero. */
124 return -pte_bad_config;
126 *fc = (ctc * ebx) / eax;
130 int pt_time_update_tsc(struct pt_time *time,
131 const struct pt_packet_tsc *packet,
132 const struct pt_config *config)
136 if (!time || !packet)
137 return -pte_internal;
142 time->tsc = time->base = packet->tsc;
146 /* We got the full time; we recover from previous losses. */
153 int pt_time_update_cbr(struct pt_time *time,
154 const struct pt_packet_cbr *packet,
155 const struct pt_config *config)
159 if (!time || !packet)
160 return -pte_internal;
163 time->cbr = packet->ratio;
168 int pt_time_update_tma(struct pt_time *time,
169 const struct pt_packet_tma *packet,
170 const struct pt_config *config)
172 uint32_t ctc, mtc_freq, mtc_hi, ctc_mask;
175 if (!time || !packet || !config)
176 return -pte_internal;
178 /* Without a TSC something is seriously wrong. */
180 return -pte_bad_context;
182 /* We shouldn't have more than one TMA per TSC. */
184 return -pte_bad_context;
186 /* We're ignoring MTC between TSC and TMA. */
188 return -pte_internal;
193 mtc_freq = config->mtc_freq;
194 mtc_hi = mtc_freq + pt_pl_mtc_bit_size;
196 /* A mask for the relevant CTC bits ignoring high-order bits that are
197 * not provided by MTC.
199 ctc_mask = (1u << mtc_hi) - 1u;
205 /* If the MTC frequency is low enough that TMA provides the full CTC
206 * value, we can use the TMA as an MTC.
208 * If it isn't, we will estimate the preceding MTC based on the CTC bits
209 * the TMA provides at the next MTC. We forget about the previous MTC
212 * If no MTC packets are dropped around TMA, we will estimate the
213 * forgotten value again at the next MTC.
215 * If MTC packets are dropped, we can't really tell where in this
216 * extended MTC period the TSC occurred. The estimation will place it
217 * right before the next MTC.
219 if (mtc_hi <= pt_pl_tma_ctc_bit_size)
222 /* In both cases, we store the TMA's CTC bits until the next MTC. */
223 time->ctc = time->ctc_cyc = ctc & ctc_mask;
228 int pt_time_update_mtc(struct pt_time *time,
229 const struct pt_packet_mtc *packet,
230 const struct pt_config *config)
232 uint32_t last_ctc, ctc, ctc_delta;
235 int errcode, have_tsc, have_tma, have_mtc;
237 if (!time || !packet || !config)
238 return -pte_internal;
240 have_tsc = time->have_tsc;
241 have_tma = time->have_tma;
242 have_mtc = time->have_mtc;
244 /* We ignore MTCs between TSC and TMA to avoid apparent CTC overflows.
246 * Later MTCs will ensure that no time is lost - provided TMA provides
247 * enough bits. If TMA doesn't provide any of the MTC bits we may place
248 * the TSC into the wrong MTC period.
250 if (have_tsc && !have_tma)
254 last_ctc = time->ctc;
255 mtc_freq = config->mtc_freq;
257 ctc = packet->ctc << mtc_freq;
259 /* Store our CTC value if we have or would have reset FC. */
260 if (time->fc || time->lost_cyc || !have_mtc)
263 /* Prepare for the next packet in case we error out below. */
268 /* We recover from previous CYC losses. */
271 /* Avoid a big jump when we see the first MTC with an arbitrary CTC
275 uint32_t ctc_lo, ctc_hi;
277 /* If we have not seen a TMA, we ignore this first MTC.
279 * We have no idea where in this MTC period tracing started.
280 * We could lose an entire MTC period or just a tiny fraction.
282 * On the other hand, if we assumed a previous MTC value, we
283 * might make just the same error.
288 /* The TMA's CTC value didn't provide enough bits - otherwise,
289 * we would have treated the TMA as an MTC.
291 if (last_ctc & ~pt_pl_tma_ctc_mask)
292 return -pte_internal;
294 /* Split this MTC's CTC value into low and high parts with
295 * respect to the bits provided by TMA.
297 ctc_lo = ctc & pt_pl_tma_ctc_mask;
298 ctc_hi = ctc & ~pt_pl_tma_ctc_mask;
300 /* We estimate the high-order CTC bits that are not provided by
301 * TMA based on the CTC bits provided by this MTC.
303 * We assume that no MTC packets were dropped around TMA. If
304 * there are, we might place the TSC into the wrong MTC period
305 * depending on how many CTC bits TMA provides and how many MTC
306 * packets were dropped.
308 * Note that we may underflow which results in more bits to be
309 * set than MTC packets may provide. Drop those extra bits.
311 if (ctc_lo < last_ctc) {
312 ctc_hi -= 1u << pt_pl_tma_ctc_bit_size;
313 ctc_hi &= pt_pl_mtc_mask << mtc_freq;
319 errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
325 errcode = pt_time_ctc_fc(&tsc, ctc_delta, config);
330 time->tsc = time->base = base;
335 /* Adjust a CYC packet's payload spanning multiple MTC periods.
337 * CYC packets measure the Fast Counter since the last CYC(-eligible) packet.
338 * Depending on the CYC threshold, we may not get a CYC for each MTC, so a CYC
339 * period may overlap with or even span multiple MTC periods.
341 * We can't do much about the overlap case without examining all packets in
342 * the respective periods. We leave this as expected imprecision.
344 * If we find a CYC packet to span multiple MTC packets, though, we try to
345 * approximate the portion for the current MTC period by subtracting the
346 * estimated portion for previous MTC periods using calibration information.
348 * We only consider MTC. For the first CYC after TSC, the corresponding TMA
349 * will contain the Fast Counter at TSC.
351 * Returns zero on success, a negative error code otherwise.
353 static int pt_time_adjust_cyc(uint64_t *cyc, const struct pt_time *time,
354 const struct pt_config *config, uint64_t fcr)
356 uint32_t last_ctc, ctc, ctc_delta;
357 uint64_t fc, total_cyc, old_cyc;
360 if (!time || !config || !fcr)
361 return -pte_internal;
363 last_ctc = time->ctc_cyc;
366 /* There is nothing to do if this is the current MTC period. */
370 /* Calibration computes
372 * fc = (ctc_delta * cpuid[0x15].ebx) / cpuid[0x15].eax.
373 * fcr = (fc << pt_tcal_fcr_shr) / cyc
375 * So cyc = (fc << pt_tcal_fcr_shr) / fcr.
378 errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
382 errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
386 old_cyc = (fc << pt_tcal_fcr_shr) / fcr;
389 /* Make sure we don't wrap around. If we would, attribute the entire
390 * CYC payload to any previous MTC period.
392 * We lost an unknown portion of the CYC payload for the current MTC
393 * period, but it's usually better to run too slow than too fast.
395 if (total_cyc < old_cyc)
398 *cyc = total_cyc - old_cyc;
402 int pt_time_update_cyc(struct pt_time *time,
403 const struct pt_packet_cyc *packet,
404 const struct pt_config *config, uint64_t fcr)
408 if (!time || !packet || !config)
409 return -pte_internal;
421 errcode = pt_time_adjust_cyc(&cyc, time, config, fcr);
426 fc += (cyc * fcr) >> pt_tcal_fcr_shr;
429 time->tsc = time->base + fc;
434 void pt_tcal_init(struct pt_time_cal *tcal)
439 memset(tcal, 0, sizeof(*tcal));
441 tcal->min_fcr = UINT64_MAX;
444 static int pt_tcal_have_fcr(const struct pt_time_cal *tcal)
449 return (tcal->min_fcr <= tcal->max_fcr);
452 int pt_tcal_fcr(uint64_t *fcr, const struct pt_time_cal *tcal)
455 return -pte_internal;
457 if (!pt_tcal_have_fcr(tcal))
465 int pt_tcal_set_fcr(struct pt_time_cal *tcal, uint64_t fcr)
468 return -pte_internal;
472 if (fcr < tcal->min_fcr)
475 if (fcr > tcal->max_fcr)
481 int pt_tcal_update_tsc(struct pt_time_cal *tcal,
482 const struct pt_packet_tsc *packet,
483 const struct pt_config *config)
487 if (!tcal || !packet)
488 return -pte_internal;
490 /* A TSC outside of PSB+ may indicate loss of time. We do not use it
491 * for calibration. We store the TSC value for calibration at the next
492 * TSC in PSB+, though.
494 tcal->tsc = packet->tsc;
495 tcal->cyc_tsc = 0ull;
500 int pt_tcal_header_tsc(struct pt_time_cal *tcal,
501 const struct pt_packet_tsc *packet,
502 const struct pt_config *config)
504 uint64_t tsc, last_tsc, tsc_delta, cyc, fcr;
508 if (!tcal || !packet)
509 return -pte_internal;
511 last_tsc = tcal->tsc;
517 tcal->cyc_tsc = 0ull;
519 if (!last_tsc || !cyc)
522 /* Correct a single wrap-around. */
523 if (tsc < last_tsc) {
524 tsc += 1ull << pt_pl_tsc_bit_size;
527 return -pte_bad_packet;
530 tsc_delta = tsc - last_tsc;
532 /* We shift the nominator to improve rounding precision.
534 * Since we're only collecting the CYCs between two TSC, we shouldn't
535 * overflow. Let's rather fail than overflow.
537 if (tsc_delta & ~(~0ull >> pt_tcal_fcr_shr))
538 return -pte_internal;
540 fcr = (tsc_delta << pt_tcal_fcr_shr) / cyc;
542 return pt_tcal_set_fcr(tcal, fcr);
545 int pt_tcal_update_cbr(struct pt_time_cal *tcal,
546 const struct pt_packet_cbr *packet,
547 const struct pt_config *config)
549 /* A CBR outside of PSB+ indicates a frequency change. Reset our
554 return pt_tcal_header_cbr(tcal, packet, config);
557 int pt_tcal_header_cbr(struct pt_time_cal *tcal,
558 const struct pt_packet_cbr *packet,
559 const struct pt_config *config)
561 uint64_t cbr, p1, fcr;
563 if (!tcal || !packet || !config)
564 return -pte_internal;
566 p1 = config->nom_freq;
570 /* If we know the nominal frequency, we can use it for calibration. */
573 fcr = (p1 << pt_tcal_fcr_shr) / cbr;
575 return pt_tcal_set_fcr(tcal, fcr);
578 int pt_tcal_update_tma(struct pt_time_cal *tcal,
579 const struct pt_packet_tma *packet,
580 const struct pt_config *config)
590 int pt_tcal_update_mtc(struct pt_time_cal *tcal,
591 const struct pt_packet_mtc *packet,
592 const struct pt_config *config)
594 uint32_t last_ctc, ctc, ctc_delta, have_mtc;
595 uint64_t cyc, fc, fcr;
598 if (!tcal || !packet || !config)
599 return -pte_internal;
601 last_ctc = tcal->ctc;
602 have_mtc = tcal->have_mtc;
605 ctc = packet->ctc << config->mtc_freq;
607 /* We need at least two MTC (including this). */
609 tcal->cyc_mtc = 0ull;
616 /* Without any cycles, we can't calibrate. Try again at the next
617 * MTC and distribute the cycles over the combined MTC period.
622 /* Prepare for the next packet in case we error out below. */
624 tcal->cyc_mtc = 0ull;
627 /* Let's pretend we will fail. We'll correct it at the end. */
630 errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
634 errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
638 /* We shift the nominator to improve rounding precision.
640 * Since we're only collecting the CYCs between two MTC, we shouldn't
641 * overflow. Let's rather fail than overflow.
643 if (fc & ~(~0ull >> pt_tcal_fcr_shr))
644 return -pte_internal;
646 fcr = (fc << pt_tcal_fcr_shr) / cyc;
648 errcode = pt_tcal_set_fcr(tcal, fcr);
652 /* We updated the FCR. This recovers from previous MTC losses. */
658 int pt_tcal_update_cyc(struct pt_time_cal *tcal,
659 const struct pt_packet_cyc *packet,
660 const struct pt_config *config)
666 if (!tcal || !packet)
667 return -pte_internal;
670 tcal->cyc_mtc += cyc;
671 tcal->cyc_tsc += cyc;