sys/i386/include/counter.h

   1 /*-
   2  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #ifndef __MACHINE_COUNTER_H__
  30 #define __MACHINE_COUNTER_H__
  31
  32 #include <sys/pcpu.h>
  33 #ifdef INVARIANTS
  34 #include <sys/proc.h>
  35 #endif
  36 #include <machine/md_var.h>
  37 #include <machine/specialreg.h>
  38
  39 #define counter_enter() do {                            \
  40         if ((cpu_feature & CPUID_CX8) == 0)             \
  41                 critical_enter();                       \
  42 } while (0)
  43
  44 #define counter_exit()  do {                            \
  45         if ((cpu_feature & CPUID_CX8) == 0)             \
  46                 critical_exit();                        \
  47 } while (0)
  48
  49 extern struct pcpu __pcpu[MAXCPU];
  50
  51 static inline void
  52 counter_64_inc_8b(uint64_t *p, int64_t inc)
  53 {
  54
  55         __asm __volatile(
  56         "movl   %%fs:(%%esi),%%eax\n\t"
  57         "movl   %%fs:4(%%esi),%%edx\n"
  58 "1:\n\t"
  59         "movl   %%eax,%%ebx\n\t"
  60         "movl   %%edx,%%ecx\n\t"
  61         "addl   (%%edi),%%ebx\n\t"
  62         "adcl   4(%%edi),%%ecx\n\t"
  63         "cmpxchg8b %%fs:(%%esi)\n\t"
  64         "jnz    1b"
  65         :
  66         : "S" ((char *)p - (char *)&__pcpu[0]), "D" (&inc)
  67         : "memory", "cc", "eax", "edx", "ebx", "ecx");
  68 }
  69
  70 #ifdef IN_SUBR_COUNTER_C
  71 static inline uint64_t
  72 counter_u64_read_one_8b(uint64_t *p)
  73 {
  74         uint32_t res_lo, res_high;
  75
  76         __asm __volatile(
  77         "movl   %%eax,%%ebx\n\t"
  78         "movl   %%edx,%%ecx\n\t"
  79         "cmpxchg8b      (%2)"
  80         : "=a" (res_lo), "=d"(res_high)
  81         : "SD" (p)
  82         : "cc", "ebx", "ecx");
  83         return (res_lo + ((uint64_t)res_high << 32));
  84 }
  85
  86 static inline uint64_t
  87 counter_u64_fetch_inline(uint64_t *p)
  88 {
  89         uint64_t res;
  90         int i;
  91
  92         res = 0;
  93         if ((cpu_feature & CPUID_CX8) == 0) {
  94                 /*
  95                  * The machines without cmpxchg8b are not SMP.
  96                  * Disabling the preemption provides atomicity of the
  97                  * counter reading, since update is done in the
  98                  * critical section as well.
  99                  */
 100                 critical_enter();
 101                 for (i = 0; i < mp_ncpus; i++) {
 102                         res += *(uint64_t *)((char *)p +
 103                             sizeof(struct pcpu) * i);
 104                 }
 105                 critical_exit();
 106         } else {
 107                 for (i = 0; i < mp_ncpus; i++)
 108                         res += counter_u64_read_one_8b((uint64_t *)((char *)p +
 109                             sizeof(struct pcpu) * i));
 110         }
 111         return (res);
 112 }
 113
 114 static inline void
 115 counter_u64_zero_one_8b(uint64_t *p)
 116 {
 117
 118         __asm __volatile(
 119         "movl   (%0),%%eax\n\t"
 120         "movl   4(%0),%%edx\n"
 121         "xorl   %%ebx,%%ebx\n\t"
 122         "xorl   %%ecx,%%ecx\n\t"
 123 "1:\n\t"
 124         "cmpxchg8b      (%0)\n\t"
 125         "jnz    1b"
 126         :
 127         : "SD" (p)
 128         : "memory", "cc", "eax", "edx", "ebx", "ecx");
 129 }
 130
 131 static void
 132 counter_u64_zero_one_cpu(void *arg)
 133 {
 134         uint64_t *p;
 135
 136         p = (uint64_t *)((char *)arg + sizeof(struct pcpu) * PCPU_GET(cpuid));
 137         counter_u64_zero_one_8b(p);
 138 }
 139
 140 static inline void
 141 counter_u64_zero_inline(counter_u64_t c)
 142 {
 143         int i;
 144
 145         if ((cpu_feature & CPUID_CX8) == 0) {
 146                 critical_enter();
 147                 for (i = 0; i < mp_ncpus; i++)
 148                         *(uint64_t *)((char *)c + sizeof(struct pcpu) * i) = 0;
 149                 critical_exit();
 150         } else {
 151                 smp_rendezvous(smp_no_rendevous_barrier,
 152                     counter_u64_zero_one_cpu, smp_no_rendevous_barrier, c);
 153         }
 154 }
 155 #endif
 156
 157 #define counter_u64_add_protected(c, inc)       do {    \
 158         if ((cpu_feature & CPUID_CX8) == 0) {           \
 159                 CRITICAL_ASSERT(curthread);             \
 160                 *(uint64_t *)zpcpu_get(c) += (inc);     \
 161         } else                                          \
 162                 counter_64_inc_8b((c), (inc));          \
 163 } while (0)
 164
 165 static inline void
 166 counter_u64_add(counter_u64_t c, int64_t inc)
 167 {
 168
 169         if ((cpu_feature & CPUID_CX8) == 0) {
 170                 critical_enter();
 171                 *(uint64_t *)zpcpu_get(c) += inc;
 172                 critical_exit();
 173         } else {
 174                 counter_64_inc_8b(c, inc);
 175         }
 176 }
 177
 178 #endif  /* ! __MACHINE_COUNTER_H__ */