2 * Copyright (c) 2009 Adrian Chadd
3 * Copyright (c) 2012 Spectra Logic Corporation
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * \file dev/xen/timer/timer.c
31 * \brief A timer driver for the Xen hypervisor's PV clock.
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
37 #include <sys/param.h>
38 #include <sys/systm.h>
40 #include <sys/kernel.h>
41 #include <sys/module.h>
43 #include <sys/timetc.h>
44 #include <sys/timeet.h>
46 #include <sys/limits.h>
47 #include <sys/clock.h>
49 #include <xen/xen-os.h>
50 #include <xen/features.h>
51 #include <xen/xen_intr.h>
52 #include <xen/hypervisor.h>
53 #include <xen/interface/io/xenbus.h>
54 #include <xen/interface/vcpu.h>
56 #include <machine/cpu.h>
57 #include <machine/cpufunc.h>
58 #include <machine/clock.h>
59 #include <machine/_inttypes.h>
60 #include <machine/smp.h>
64 static devclass_t xentimer_devclass;
66 #define NSEC_IN_SEC 1000000000ULL
67 #define NSEC_IN_USEC 1000ULL
68 /* 18446744073 = int(2^64 / NSEC_IN_SC) = 1 ns in 64-bit fractions */
69 #define FRAC_IN_NSEC 18446744073LL
71 /* Xen timers may fire up to 100us off */
72 #define XENTIMER_MIN_PERIOD_IN_NSEC 100*NSEC_IN_USEC
73 #define XENCLOCK_RESOLUTION 10000000
75 #define ETIME 62 /* Xen "bad time" error */
77 #define XENTIMER_QUALITY 950
79 struct xentimer_pcpu_data {
81 uint64_t last_processed;
85 DPCPU_DEFINE(struct xentimer_pcpu_data, xentimer_pcpu);
87 DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
89 struct xentimer_softc {
91 struct timecounter tc;
95 /* Last time; this guarantees a monotonically increasing clock. */
96 volatile uint64_t xen_timer_last_time = 0;
99 xentimer_identify(driver_t *driver, device_t parent)
104 /* Handle all Xen PV timers in one device instance. */
105 if (devclass_get_device(xentimer_devclass, 0))
108 BUS_ADD_CHILD(parent, 0, "xen_et", 0);
112 xentimer_probe(device_t dev)
114 KASSERT((xen_domain()), ("Trying to use Xen timer on bare metal"));
116 * In order to attach, this driver requires the following:
117 * - Vector callback support by the hypervisor, in order to deliver
118 * timer interrupts to the correct CPU for CPUs other than 0.
119 * - Access to the hypervisor shared info page, in order to look up
120 * each VCPU's timer information and the Xen wallclock time.
121 * - The hypervisor must say its PV clock is "safe" to use.
122 * - The hypervisor must support VCPUOP hypercalls.
123 * - The maximum number of CPUs supported by FreeBSD must not exceed
124 * the number of VCPUs supported by the hypervisor.
126 #define XTREQUIRES(condition, reason...) \
127 if (!(condition)) { \
128 device_printf(dev, ## reason); \
129 device_detach(dev); \
133 if (xen_hvm_domain()) {
134 XTREQUIRES(xen_vector_callback_enabled,
135 "vector callbacks unavailable\n");
136 XTREQUIRES(xen_feature(XENFEAT_hvm_safe_pvclock),
137 "HVM safe pvclock unavailable\n");
139 XTREQUIRES(HYPERVISOR_shared_info != NULL,
140 "shared info page unavailable\n");
141 XTREQUIRES(HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, 0, NULL) == 0,
142 "VCPUOPs interface unavailable\n");
144 device_set_desc(dev, "Xen PV Clock");
145 return (BUS_PROBE_NOWILDCARD);
149 * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction,
150 * yielding a 64-bit result.
152 static inline uint64_t
153 scale_delta(uint64_t delta, uint32_t mul_frac, int shift)
162 #if defined(__i386__)
167 * For i386, the formula looks like:
169 * lower = (mul_frac * (delta & UINT_MAX)) >> 32
170 * upper = mul_frac * (delta >> 32)
171 * product = lower + upper
181 : "=A" (product), "=r" (tmp1), "=r" (tmp2)
182 : "a" ((uint32_t)delta), "1" ((uint32_t)(delta >> 32)),
185 #elif defined(__amd64__)
190 "mulq %[mul_frac] ; shrd $32, %[hi], %[lo]"
191 : [lo]"=a" (product), [hi]"=d" (tmp)
192 : "0" (delta), [mul_frac]"rm"((uint64_t)mul_frac));
195 #error "xentimer: unsupported architecture"
202 get_nsec_offset(struct vcpu_time_info *tinfo)
205 return (scale_delta(rdtsc() - tinfo->tsc_timestamp,
206 tinfo->tsc_to_system_mul, tinfo->tsc_shift));
210 * Read the current hypervisor system uptime value from Xen.
211 * See <xen/interface/xen.h> for a description of how this works.
214 xen_fetch_vcpu_tinfo(struct vcpu_time_info *dst, struct vcpu_time_info *src)
218 dst->version = src->version;
220 dst->tsc_timestamp = src->tsc_timestamp;
221 dst->system_time = src->system_time;
222 dst->tsc_to_system_mul = src->tsc_to_system_mul;
223 dst->tsc_shift = src->tsc_shift;
225 } while ((src->version & 1) | (dst->version ^ src->version));
227 return (dst->version);
231 * \brief Get the current time, in nanoseconds, since the hypervisor booted.
233 * \note This function returns the current CPU's idea of this value, unless
234 * it happens to be less than another CPU's previously determined value.
237 xen_fetch_vcpu_time(void)
239 struct vcpu_time_info dst;
240 struct vcpu_time_info *src;
241 uint32_t pre_version;
243 volatile uint64_t last;
244 struct vcpu_info *vcpu = DPCPU_GET(vcpu_info);
250 pre_version = xen_fetch_vcpu_tinfo(&dst, src);
252 now = dst.system_time + get_nsec_offset(&dst);
254 } while (pre_version != src->version);
257 * Enforce a monotonically increasing clock time across all
258 * VCPUs. If our time is too old, use the last time and return.
259 * Otherwise, try to update the last time.
262 last = xen_timer_last_time;
267 } while (!atomic_cmpset_64(&xen_timer_last_time, last, now));
275 xentimer_get_timecount(struct timecounter *tc)
278 return ((uint32_t)xen_fetch_vcpu_time() & UINT_MAX);
282 * \brief Fetch the hypervisor boot time, known as the "Xen wallclock".
284 * \param ts Timespec to store the current stable value.
285 * \param version Pointer to store the corresponding wallclock version.
287 * \note This value is updated when Domain-0 shifts its clock to follow
288 * clock drift, e.g. as detected by NTP.
291 xen_fetch_wallclock(struct timespec *ts)
293 shared_info_t *src = HYPERVISOR_shared_info;
294 uint32_t version = 0;
297 version = src->wc_version;
299 ts->tv_sec = src->wc_sec;
300 ts->tv_nsec = src->wc_nsec;
302 } while ((src->wc_version & 1) | (version ^ src->wc_version));
306 xen_fetch_uptime(struct timespec *ts)
308 uint64_t uptime = xen_fetch_vcpu_time();
309 ts->tv_sec = uptime / NSEC_IN_SEC;
310 ts->tv_nsec = uptime % NSEC_IN_SEC;
314 xentimer_settime(device_t dev __unused, struct timespec *ts)
317 * Don't return EINVAL here; just silently fail if the domain isn't
318 * privileged enough to set the TOD.
324 * \brief Return current time according to the Xen Hypervisor wallclock.
326 * \param dev Xentimer device.
327 * \param ts Pointer to store the wallclock time.
329 * \note The Xen time structures document the hypervisor start time and the
330 * uptime-since-hypervisor-start (in nsec.) They need to be combined
331 * in order to calculate a TOD clock.
334 xentimer_gettime(device_t dev, struct timespec *ts)
336 struct timespec u_ts;
339 xen_fetch_wallclock(ts);
340 xen_fetch_uptime(&u_ts);
341 timespecadd(ts, &u_ts);
347 * \brief Handle a timer interrupt for the Xen PV timer driver.
349 * \param arg Xen timer driver softc that is expecting the interrupt.
352 xentimer_intr(void *arg)
354 struct xentimer_softc *sc = (struct xentimer_softc *)arg;
355 struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu);
357 pcpu->last_processed = xen_fetch_vcpu_time();
358 if (pcpu->timer != 0 && sc->et.et_active)
359 sc->et.et_event_cb(&sc->et, sc->et.et_arg);
361 return (FILTER_HANDLED);
365 xentimer_vcpu_start_timer(int vcpu, uint64_t next_time)
367 struct vcpu_set_singleshot_timer single;
369 single.timeout_abs_ns = next_time;
370 single.flags = VCPU_SSHOTTMR_future;
371 return (HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, vcpu, &single));
375 xentimer_vcpu_stop_timer(int vcpu)
378 return (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, vcpu, NULL));
382 * \brief Set the next oneshot time for the current CPU.
384 * \param et Xen timer driver event timer to schedule on.
385 * \param first Delta to the next time to schedule the interrupt for.
386 * \param period Not used.
388 * \note See eventtimers(9) for more information.
394 xentimer_et_start(struct eventtimer *et,
395 sbintime_t first, sbintime_t period)
397 int error = 0, i = 0;
398 struct xentimer_softc *sc = et->et_priv;
399 int cpu = PCPU_GET(vcpu_id);
400 struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu);
401 uint64_t first_in_ns, next_time;
403 /* See sbttots() for this formula. */
404 first_in_ns = (((first >> 32) * NSEC_IN_SEC) +
405 (((uint64_t)NSEC_IN_SEC * (uint32_t)first) >> 32));
408 * Retry any timer scheduling failures, where the hypervisor
409 * returns -ETIME. Sometimes even a 100us timer period isn't large
410 * enough, but larger period instances are relatively uncommon.
412 * XXX Remove the panics once et_start() and its consumers are
413 * equipped to deal with start failures.
417 panic("can't schedule timer");
418 next_time = xen_fetch_vcpu_time() + first_in_ns;
419 error = xentimer_vcpu_start_timer(cpu, next_time);
420 } while (error == -ETIME);
423 panic("%s: Error %d setting singleshot timer to %"PRIu64"\n",
424 device_get_nameunit(sc->dev), error, next_time);
426 pcpu->timer = next_time;
431 * \brief Cancel the event timer's currently running timer, if any.
434 xentimer_et_stop(struct eventtimer *et)
436 int cpu = PCPU_GET(vcpu_id);
437 struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu);
440 return (xentimer_vcpu_stop_timer(cpu));
444 * \brief Attach a Xen PV timer driver instance.
446 * \param dev Bus device object to attach.
452 xentimer_attach(device_t dev)
454 struct xentimer_softc *sc = device_get_softc(dev);
459 /* Bind an event channel to a VIRQ on each VCPU. */
461 struct xentimer_pcpu_data *pcpu;
463 pcpu = DPCPU_ID_PTR(i, xentimer_pcpu);
464 error = HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, i, NULL);
466 device_printf(dev, "Error disabling Xen periodic timer "
471 error = xen_intr_bind_virq(dev, VIRQ_TIMER, i, xentimer_intr,
472 NULL, sc, INTR_TYPE_CLK, &pcpu->irq_handle);
474 device_printf(dev, "Error %d binding VIRQ_TIMER "
475 "to VCPU %d\n", error, i);
478 xen_intr_describe(pcpu->irq_handle, "c%d", i);
481 /* Register the event timer. */
482 sc->et.et_name = "XENTIMER";
483 sc->et.et_quality = XENTIMER_QUALITY;
484 sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU;
485 sc->et.et_frequency = NSEC_IN_SEC;
486 /* See tstosbt() for this formula */
487 sc->et.et_min_period = (XENTIMER_MIN_PERIOD_IN_NSEC *
488 (((uint64_t)1 << 63) / 500000000) >> 32);
489 sc->et.et_max_period = ((sbintime_t)4 << 32);
490 sc->et.et_start = xentimer_et_start;
491 sc->et.et_stop = xentimer_et_stop;
493 et_register(&sc->et);
495 /* Register the timecounter. */
496 sc->tc.tc_name = "XENTIMER";
497 sc->tc.tc_quality = XENTIMER_QUALITY;
498 sc->tc.tc_flags = TC_FLAGS_SUSPEND_SAFE;
500 * The underlying resolution is in nanoseconds, since the timer info
501 * scales TSC frequencies using a fraction that represents time in
502 * terms of nanoseconds.
504 sc->tc.tc_frequency = NSEC_IN_SEC;
505 sc->tc.tc_counter_mask = ~0u;
506 sc->tc.tc_get_timecount = xentimer_get_timecount;
510 /* Register the Hypervisor wall clock */
511 clock_register(dev, XENCLOCK_RESOLUTION);
517 xentimer_detach(device_t dev)
520 /* Implement Xen PV clock teardown - XXX see hpet_detach ? */
522 * 1. need to deregister timecounter
523 * 2. need to deregister event timer
524 * 3. need to deregister virtual IRQ event channels
530 xentimer_percpu_resume(void *arg)
532 device_t dev = (device_t) arg;
533 struct xentimer_softc *sc = device_get_softc(dev);
535 xentimer_et_start(&sc->et, sc->et.et_min_period, 0);
539 xentimer_resume(device_t dev)
544 /* Disable the periodic timer */
546 error = HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, i, NULL);
549 "Error disabling Xen periodic timer on CPU %d\n",
555 /* Reset the last uptime value */
556 xen_timer_last_time = 0;
558 /* Reset the RTC clock */
559 inittodr(time_second);
561 /* Kick the timers on all CPUs */
562 smp_rendezvous(NULL, xentimer_percpu_resume, NULL, dev);
565 device_printf(dev, "resumed operation after suspension\n");
571 xentimer_suspend(device_t dev)
576 static device_method_t xentimer_methods[] = {
577 DEVMETHOD(device_identify, xentimer_identify),
578 DEVMETHOD(device_probe, xentimer_probe),
579 DEVMETHOD(device_attach, xentimer_attach),
580 DEVMETHOD(device_detach, xentimer_detach),
581 DEVMETHOD(device_suspend, xentimer_suspend),
582 DEVMETHOD(device_resume, xentimer_resume),
583 /* clock interface */
584 DEVMETHOD(clock_gettime, xentimer_gettime),
585 DEVMETHOD(clock_settime, xentimer_settime),
589 static driver_t xentimer_driver = {
592 sizeof(struct xentimer_softc),
595 DRIVER_MODULE(xentimer, nexus, xentimer_driver, xentimer_devclass, 0, 0);
596 MODULE_DEPEND(xentimer, nexus, 1, 1, 1);