sys/i386/include/counter.h

   1 /*-
   2  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #ifndef __MACHINE_COUNTER_H__
  30 #define __MACHINE_COUNTER_H__
  31
  32 #include <sys/pcpu.h>
  33 #ifdef INVARIANTS
  34 #include <sys/proc.h>
  35 #endif
  36 #include <machine/md_var.h>
  37 #include <machine/specialreg.h>
  38
  39 #define counter_enter() do {                            \
  40         if ((cpu_feature & CPUID_CX8) == 0)             \
  41                 critical_enter();                       \
  42 } while (0)
  43
  44 #define counter_exit()  do {                            \
  45         if ((cpu_feature & CPUID_CX8) == 0)             \
  46                 critical_exit();                        \
  47 } while (0)
  48
  49 extern struct pcpu __pcpu[MAXCPU];
  50
  51 static inline void
  52 counter_64_inc_8b(uint64_t *p, int64_t inc)
  53 {
  54
  55         __asm __volatile(
  56         "movl   %%fs:(%%esi),%%eax\n\t"
  57         "movl   %%fs:4(%%esi),%%edx\n"
  58 "1:\n\t"
  59         "movl   %%eax,%%ebx\n\t"
  60         "movl   %%edx,%%ecx\n\t"
  61         "addl   (%%edi),%%ebx\n\t"
  62         "adcl   4(%%edi),%%ecx\n\t"
  63         "cmpxchg8b %%fs:(%%esi)\n\t"
  64         "jnz    1b"
  65         :
  66         : "S" ((char *)p - (char *)&__pcpu[0]), "D" (&inc)
  67         : "memory", "cc", "eax", "edx", "ebx", "ecx");
  68 }
  69
  70 #ifdef IN_SUBR_COUNTER_C
  71 struct counter_u64_fetch_cx8_arg {
  72         uint64_t res;
  73         uint64_t *p;
  74 };
  75
  76 static uint64_t
  77 counter_u64_read_one_8b(uint64_t *p)
  78 {
  79         uint32_t res_lo, res_high;
  80
  81         __asm __volatile(
  82         "movl   %%eax,%%ebx\n\t"
  83         "movl   %%edx,%%ecx\n\t"
  84         "cmpxchg8b      (%2)"
  85         : "=a" (res_lo), "=d"(res_high)
  86         : "SD" (p)
  87         : "cc", "ebx", "ecx");
  88         return (res_lo + ((uint64_t)res_high << 32));
  89 }
  90
  91 static void
  92 counter_u64_fetch_cx8_one(void *arg1)
  93 {
  94         struct counter_u64_fetch_cx8_arg *arg;
  95         uint64_t val;
  96
  97         arg = arg1;
  98         val = counter_u64_read_one_8b((uint64_t *)((char *)arg->p +
  99             sizeof(struct pcpu) * PCPU_GET(cpuid)));
 100         atomic_add_64(&arg->res, val);
 101 }
 102
 103 static inline uint64_t
 104 counter_u64_fetch_inline(uint64_t *p)
 105 {
 106         struct counter_u64_fetch_cx8_arg arg;
 107         uint64_t res;
 108         int i;
 109
 110         res = 0;
 111         if ((cpu_feature & CPUID_CX8) == 0) {
 112                 /*
 113                  * The machines without cmpxchg8b are not SMP.
 114                  * Disabling the preemption provides atomicity of the
 115                  * counter reading, since update is done in the
 116                  * critical section as well.
 117                  */
 118                 critical_enter();
 119                 CPU_FOREACH(i) {
 120                         res += *(uint64_t *)((char *)p +
 121                             sizeof(struct pcpu) * i);
 122                 }
 123                 critical_exit();
 124         } else {
 125                 arg.p = p;
 126                 arg.res = 0;
 127                 smp_rendezvous(NULL, counter_u64_fetch_cx8_one, NULL, &arg);
 128                 res = arg.res;
 129         }
 130         return (res);
 131 }
 132
 133 static inline void
 134 counter_u64_zero_one_8b(uint64_t *p)
 135 {
 136
 137         __asm __volatile(
 138         "movl   (%0),%%eax\n\t"
 139         "movl   4(%0),%%edx\n"
 140         "xorl   %%ebx,%%ebx\n\t"
 141         "xorl   %%ecx,%%ecx\n\t"
 142 "1:\n\t"
 143         "cmpxchg8b      (%0)\n\t"
 144         "jnz    1b"
 145         :
 146         : "SD" (p)
 147         : "memory", "cc", "eax", "edx", "ebx", "ecx");
 148 }
 149
 150 static void
 151 counter_u64_zero_one_cpu(void *arg)
 152 {
 153         uint64_t *p;
 154
 155         p = (uint64_t *)((char *)arg + sizeof(struct pcpu) * PCPU_GET(cpuid));
 156         counter_u64_zero_one_8b(p);
 157 }
 158
 159 static inline void
 160 counter_u64_zero_inline(counter_u64_t c)
 161 {
 162         int i;
 163
 164         if ((cpu_feature & CPUID_CX8) == 0) {
 165                 critical_enter();
 166                 CPU_FOREACH(i)
 167                         *(uint64_t *)((char *)c + sizeof(struct pcpu) * i) = 0;
 168                 critical_exit();
 169         } else {
 170                 smp_rendezvous(smp_no_rendezvous_barrier,
 171                     counter_u64_zero_one_cpu, smp_no_rendezvous_barrier, c);
 172         }
 173 }
 174 #endif
 175
 176 #define counter_u64_add_protected(c, inc)       do {    \
 177         if ((cpu_feature & CPUID_CX8) == 0) {           \
 178                 CRITICAL_ASSERT(curthread);             \
 179                 *(uint64_t *)zpcpu_get(c) += (inc);     \
 180         } else                                          \
 181                 counter_64_inc_8b((c), (inc));          \
 182 } while (0)
 183
 184 static inline void
 185 counter_u64_add(counter_u64_t c, int64_t inc)
 186 {
 187
 188         if ((cpu_feature & CPUID_CX8) == 0) {
 189                 critical_enter();
 190                 *(uint64_t *)zpcpu_get(c) += inc;
 191                 critical_exit();
 192         } else {
 193                 counter_64_inc_8b(c, inc);
 194         }
 195 }
 196
 197 #endif  /* ! __MACHINE_COUNTER_H__ */