2 * Copyright (c) 2013 Gleb Smirnoff <glebius@FreeBSD.org>
3 * Copyright (c) 2010 Juniper Networks, Inc.
4 * Copyright (c) 2009 Robert N. M. Watson
5 * Copyright (c) 2009 Bjoern A. Zeeb <bz@FreeBSD.org>
6 * Copyright (c) 2008 Yahoo!, Inc.
9 * Written by: John Baldwin <jhb@FreeBSD.org>
11 * This software was developed by Robert N. M. Watson under contract
12 * to Juniper Networks, Inc.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 * 3. Neither the name of the author nor the names of any co-contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 #include <sys/cdefs.h>
40 __FBSDID("$FreeBSD$");
42 #include <sys/param.h>
44 #include <sys/sysctl.h>
49 #include "kvm_private.h"
51 static struct nlist kvm_pcpu_nl[] = {
52 { .n_name = "_cpuid_to_pcpu" },
53 { .n_name = "_mp_maxcpus" },
54 { .n_name = "_mp_ncpus" },
57 #define NL_CPUID_TO_PCPU 0
58 #define NL_MP_MAXCPUS 1
62 * Kernel per-CPU data state. We cache this stuff on the first
65 * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the
66 * consumer has multiple handles in flight to differently configured
69 static void **pcpu_data;
74 _kvm_pcpu_init(kvm_t *kd)
80 if (kvm_nlist(kd, kvm_pcpu_nl) < 0)
82 if (kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value == 0) {
83 _kvm_err(kd, kd->program, "unable to find cpuid_to_pcpu");
86 if (kvm_pcpu_nl[NL_MP_MAXCPUS].n_value == 0) {
87 _kvm_err(kd, kd->program, "unable to find mp_maxcpus");
90 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_MAXCPUS].n_value, &max,
91 sizeof(max)) != sizeof(max)) {
92 _kvm_err(kd, kd->program, "cannot read mp_maxcpus");
95 if (kvm_pcpu_nl[NL_MP_NCPUS].n_value == 0) {
96 _kvm_err(kd, kd->program, "unable to find mp_ncpus");
99 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_NCPUS].n_value, &mp_ncpus,
100 sizeof(mp_ncpus)) != sizeof(mp_ncpus)) {
101 _kvm_err(kd, kd->program, "cannot read mp_ncpus");
104 len = max * sizeof(void *);
107 _kvm_err(kd, kd->program, "out of memory");
110 if (kvm_read(kd, kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value, data, len) !=
112 _kvm_err(kd, kd->program, "cannot read cpuid_to_pcpu array");
122 _kvm_pcpu_clear(void)
131 kvm_getpcpu(kvm_t *kd, int cpu)
141 if (_kvm_pcpu_init(kd) < 0)
144 if (cpu >= maxcpu || pcpu_data[cpu] == NULL)
147 buf = malloc(sizeof(struct pcpu));
149 _kvm_err(kd, kd->program, "out of memory");
152 if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf,
153 sizeof(struct pcpu)) != sizeof(struct pcpu)) {
154 _kvm_err(kd, kd->program, "unable to read per-CPU data");
162 kvm_getmaxcpu(kvm_t *kd)
171 if (_kvm_pcpu_init(kd) < 0)
177 kvm_getncpus(kvm_t *kd)
181 if (_kvm_pcpu_init(kd) < 0)
187 _kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error)
190 if (!kd->dpcpu_initialized) {
192 _kvm_err(kd, kd->program, "%s: not initialized",
196 if (cpu >= kd->dpcpu_maxcpus) {
198 _kvm_err(kd, kd->program, "%s: CPU %u too big",
202 if (kd->dpcpu_off[cpu] == 0) {
204 _kvm_err(kd, kd->program, "%s: CPU %u not found",
208 kd->dpcpu_curcpu = cpu;
209 kd->dpcpu_curoff = kd->dpcpu_off[cpu];
214 * Set up libkvm to handle dynamic per-CPU memory.
217 _kvm_dpcpu_init(kvm_t *kd)
219 struct nlist nl[] = {
220 #define NLIST_START_SET_PCPU 0
221 { .n_name = "___start_" DPCPU_SETNAME },
222 #define NLIST_STOP_SET_PCPU 1
223 { .n_name = "___stop_" DPCPU_SETNAME },
224 #define NLIST_DPCPU_OFF 2
225 { .n_name = "_dpcpu_off" },
226 #define NLIST_MP_MAXCPUS 3
227 { .n_name = "_mp_maxcpus" },
230 uintptr_t *dpcpu_off_buf;
235 * Locate and cache locations of important symbols using the internal
236 * version of _kvm_nlist, turning off initialization to avoid
237 * recursion in case of unresolveable symbols.
239 if (_kvm_nlist(kd, nl, 0) != 0)
241 if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus,
242 sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus))
244 len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf);
245 dpcpu_off_buf = malloc(len);
246 if (dpcpu_off_buf == NULL)
248 if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) !=
253 kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value;
254 kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value;
255 kd->dpcpu_maxcpus = dpcpu_maxcpus;
256 kd->dpcpu_off = dpcpu_off_buf;
257 kd->dpcpu_initialized = 1;
258 (void)_kvm_dpcpu_setcpu(kd, 0, 0);
263 * Check whether the dpcpu module has been initialized sucessfully or not,
264 * initialize it if permitted.
267 _kvm_dpcpu_initialized(kvm_t *kd, int intialize)
270 if (kd->dpcpu_initialized || !intialize)
271 return (kd->dpcpu_initialized);
273 (void)_kvm_dpcpu_init(kd);
275 return (kd->dpcpu_initialized);
279 * Check whether the value is within the dpcpu symbol range and only if so
280 * adjust the offset relative to the current offset.
283 _kvm_dpcpu_validaddr(kvm_t *kd, uintptr_t value)
289 if (!kd->dpcpu_initialized)
292 if (value < kd->dpcpu_start || value >= kd->dpcpu_stop)
295 return (kd->dpcpu_curoff + value);
299 kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu)
303 if (!kd->dpcpu_initialized) {
304 ret = _kvm_dpcpu_init(kd);
306 _kvm_err(kd, kd->program, "%s: init failed",
312 return (_kvm_dpcpu_setcpu(kd, cpu, 1));
316 * Obtain a per-CPU copy for given cpu from UMA_ZONE_PCPU allocation.
319 kvm_read_zpcpu(kvm_t *kd, u_long base, void *buf, size_t size, int cpu)
322 return (kvm_read(kd, (uintptr_t)(base + sizeof(struct pcpu) * cpu),
327 * Fetch value of a counter(9).
330 kvm_counter_u64_fetch(kvm_t *kd, u_long base)
335 if (_kvm_pcpu_init(kd) < 0)
339 for (int i = 0; i < mp_ncpus; i++) {
340 if (kvm_read_zpcpu(kd, base, &c, sizeof(c), i) != sizeof(c))