sys/i386/include/counter.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 #ifndef __MACHINE_COUNTER_H__
  32 #define __MACHINE_COUNTER_H__
  33
  34 #include <sys/pcpu.h>
  35 #ifdef INVARIANTS
  36 #include <sys/proc.h>
  37 #endif
  38 #include <machine/md_var.h>
  39 #include <machine/specialreg.h>
  40
  41 #define EARLY_COUNTER   &__pcpu[0].pc_early_dummy_counter
  42
  43 #define counter_enter() do {                            \
  44         if ((cpu_feature & CPUID_CX8) == 0)             \
  45                 critical_enter();                       \
  46 } while (0)
  47
  48 #define counter_exit()  do {                            \
  49         if ((cpu_feature & CPUID_CX8) == 0)             \
  50                 critical_exit();                        \
  51 } while (0)
  52
  53 static inline void
  54 counter_64_inc_8b(uint64_t *p, int64_t inc)
  55 {
  56
  57         __asm __volatile(
  58         "movl   %%fs:(%%esi),%%eax\n\t"
  59         "movl   %%fs:4(%%esi),%%edx\n"
  60 "1:\n\t"
  61         "movl   %%eax,%%ebx\n\t"
  62         "movl   %%edx,%%ecx\n\t"
  63         "addl   (%%edi),%%ebx\n\t"
  64         "adcl   4(%%edi),%%ecx\n\t"
  65         "cmpxchg8b %%fs:(%%esi)\n\t"
  66         "jnz    1b"
  67         :
  68         : "S" ((char *)p - (char *)&__pcpu[0]), "D" (&inc)
  69         : "memory", "cc", "eax", "edx", "ebx", "ecx");
  70 }
  71
  72 #ifdef IN_SUBR_COUNTER_C
  73 struct counter_u64_fetch_cx8_arg {
  74         uint64_t res;
  75         uint64_t *p;
  76 };
  77
  78 static uint64_t
  79 counter_u64_read_one_8b(uint64_t *p)
  80 {
  81         uint32_t res_lo, res_high;
  82
  83         __asm __volatile(
  84         "movl   %%eax,%%ebx\n\t"
  85         "movl   %%edx,%%ecx\n\t"
  86         "cmpxchg8b      (%2)"
  87         : "=a" (res_lo), "=d"(res_high)
  88         : "SD" (p)
  89         : "cc", "ebx", "ecx");
  90         return (res_lo + ((uint64_t)res_high << 32));
  91 }
  92
  93 static void
  94 counter_u64_fetch_cx8_one(void *arg1)
  95 {
  96         struct counter_u64_fetch_cx8_arg *arg;
  97         uint64_t val;
  98
  99         arg = arg1;
 100         val = counter_u64_read_one_8b((uint64_t *)((char *)arg->p +
 101             UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid)));
 102         atomic_add_64(&arg->res, val);
 103 }
 104
 105 static inline uint64_t
 106 counter_u64_fetch_inline(uint64_t *p)
 107 {
 108         struct counter_u64_fetch_cx8_arg arg;
 109         uint64_t res;
 110         int i;
 111
 112         res = 0;
 113         if ((cpu_feature & CPUID_CX8) == 0) {
 114                 /*
 115                  * The machines without cmpxchg8b are not SMP.
 116                  * Disabling the preemption provides atomicity of the
 117                  * counter reading, since update is done in the
 118                  * critical section as well.
 119                  */
 120                 critical_enter();
 121                 CPU_FOREACH(i) {
 122                         res += *(uint64_t *)((char *)p +
 123                             UMA_PCPU_ALLOC_SIZE * i);
 124                 }
 125                 critical_exit();
 126         } else {
 127                 arg.p = p;
 128                 arg.res = 0;
 129                 smp_rendezvous(NULL, counter_u64_fetch_cx8_one, NULL, &arg);
 130                 res = arg.res;
 131         }
 132         return (res);
 133 }
 134
 135 static inline void
 136 counter_u64_zero_one_8b(uint64_t *p)
 137 {
 138
 139         __asm __volatile(
 140         "movl   (%0),%%eax\n\t"
 141         "movl   4(%0),%%edx\n"
 142         "xorl   %%ebx,%%ebx\n\t"
 143         "xorl   %%ecx,%%ecx\n\t"
 144 "1:\n\t"
 145         "cmpxchg8b      (%0)\n\t"
 146         "jnz    1b"
 147         :
 148         : "SD" (p)
 149         : "memory", "cc", "eax", "edx", "ebx", "ecx");
 150 }
 151
 152 static void
 153 counter_u64_zero_one_cpu(void *arg)
 154 {
 155         uint64_t *p;
 156
 157         p = (uint64_t *)((char *)arg + UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid));
 158         counter_u64_zero_one_8b(p);
 159 }
 160
 161 static inline void
 162 counter_u64_zero_inline(counter_u64_t c)
 163 {
 164         int i;
 165
 166         if ((cpu_feature & CPUID_CX8) == 0) {
 167                 critical_enter();
 168                 CPU_FOREACH(i)
 169                         *(uint64_t *)((char *)c + UMA_PCPU_ALLOC_SIZE * i) = 0;
 170                 critical_exit();
 171         } else {
 172                 smp_rendezvous(smp_no_rendezvous_barrier,
 173                     counter_u64_zero_one_cpu, smp_no_rendezvous_barrier, c);
 174         }
 175 }
 176 #endif
 177
 178 #define counter_u64_add_protected(c, inc)       do {    \
 179         if ((cpu_feature & CPUID_CX8) == 0) {           \
 180                 CRITICAL_ASSERT(curthread);             \
 181                 *(uint64_t *)zpcpu_get(c) += (inc);     \
 182         } else                                          \
 183                 counter_64_inc_8b((c), (inc));          \
 184 } while (0)
 185
 186 static inline void
 187 counter_u64_add(counter_u64_t c, int64_t inc)
 188 {
 189
 190         if ((cpu_feature & CPUID_CX8) == 0) {
 191                 critical_enter();
 192                 *(uint64_t *)zpcpu_get(c) += inc;
 193                 critical_exit();
 194         } else {
 195                 counter_64_inc_8b(c, inc);
 196         }
 197 }
 198
 199 #endif  /* ! __MACHINE_COUNTER_H__ */