sys/amd64/amd64/bpf_jit_machdep.h

   1 /*-
   2  * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
   3  * Copyright (C) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org>
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright
  11  * notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  * notice, this list of conditions and the following disclaimer in the
  14  * documentation and/or other materials provided with the distribution.
  15  * 3. Neither the name of the Politecnico di Torino nor the names of its
  16  * contributors may be used to endorse or promote products derived from
  17  * this software without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 #ifndef _BPF_JIT_MACHDEP_H_
  35 #define _BPF_JIT_MACHDEP_H_
  36
  37 /*
  38  * Registers
  39  */
  40 #define RAX     0
  41 #define RCX     1
  42 #define RDX     2
  43 #define RBX     3
  44 #define RSP     4
  45 #define RBP     5
  46 #define RSI     6
  47 #define RDI     7
  48 #define R8      0
  49 #define R9      1
  50 #define R10     2
  51 #define R11     3
  52 #define R12     4
  53 #define R13     5
  54 #define R14     6
  55 #define R15     7
  56
  57 #define EAX     0
  58 #define ECX     1
  59 #define EDX     2
  60 #define EBX     3
  61 #define ESP     4
  62 #define EBP     5
  63 #define ESI     6
  64 #define EDI     7
  65 #define R8D     0
  66 #define R9D     1
  67 #define R10D    2
  68 #define R11D    3
  69 #define R12D    4
  70 #define R13D    5
  71 #define R14D    6
  72 #define R15D    7
  73
  74 #define AX      0
  75 #define CX      1
  76 #define DX      2
  77 #define BX      3
  78 #define SP      4
  79 #define BP      5
  80 #define SI      6
  81 #define DI      7
  82
  83 #define AL      0
  84 #define CL      1
  85 #define DL      2
  86 #define BL      3
  87
  88 /* Optimization flags */
  89 #define BPF_JIT_FRET    0x01
  90 #define BPF_JIT_FPKT    0x02
  91 #define BPF_JIT_FMEM    0x04
  92 #define BPF_JIT_FJMP    0x08
  93 #define BPF_JIT_FLEN    0x10
  94
  95 #define BPF_JIT_FLAG_ALL        \
  96     (BPF_JIT_FPKT | BPF_JIT_FMEM | BPF_JIT_FJMP | BPF_JIT_FLEN)
  97
  98 /* A stream of native binary code */
  99 typedef struct bpf_bin_stream {
 100         /* Current native instruction pointer. */
 101         int             cur_ip;
 102
 103         /*
 104          * Current BPF instruction pointer, i.e. position in
 105          * the BPF program reached by the jitter.
 106          */
 107         int             bpf_pc;
 108
 109         /* Instruction buffer, contains the generated native code. */
 110         char            *ibuf;
 111
 112         /* Jumps reference table. */
 113         u_int           *refs;
 114 } bpf_bin_stream;
 115
 116 /*
 117  * Prototype of the emit functions.
 118  *
 119  * Different emit functions are used to create the reference table and
 120  * to generate the actual filtering code. This allows to have simpler
 121  * instruction macros.
 122  * The first parameter is the stream that will receive the data.
 123  * The second one is a variable containing the data.
 124  * The third one is the length, that can be 1, 2, or 4 since it is possible
 125  * to emit a byte, a short, or a word at a time.
 126  */
 127 typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 128
 129 /*
 130  * Native instruction macros
 131  */
 132
 133 /* movl i32,r32 */
 134 #define MOVid(i32, r32) do {                                            \
 135         emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1);          \
 136         emitm(&stream, i32, 4);                                         \
 137 } while (0)
 138
 139 /* movq i64,r64 */
 140 #define MOViq(i64, r64) do {                                            \
 141         emitm(&stream, 0x48, 1);                                        \
 142         emitm(&stream, (11 << 4) | (1 << 3) | (r64 & 0x7), 1);          \
 143         emitm(&stream, i64, 4);                                         \
 144         emitm(&stream, (i64 >> 32), 4);                                 \
 145 } while (0)
 146
 147 /* movl sr32,dr32 */
 148 #define MOVrd(sr32, dr32) do {                                          \
 149         emitm(&stream, 0x89, 1);                                        \
 150         emitm(&stream,                                                  \
 151             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 152 } while (0)
 153
 154 /* movl sr32,dr32 (dr32 = %r8-15d) */
 155 #define MOVrd2(sr32, dr32) do {                                         \
 156         emitm(&stream, 0x8941, 2);                                      \
 157         emitm(&stream,                                                  \
 158             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 159 } while (0)
 160
 161 /* movl sr32,dr32 (sr32 = %r8-15d) */
 162 #define MOVrd3(sr32, dr32) do {                                         \
 163         emitm(&stream, 0x8944, 2);                                      \
 164         emitm(&stream,                                                  \
 165             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 166 } while (0)
 167
 168 /* movq sr64,dr64 */
 169 #define MOVrq(sr64, dr64) do {                                          \
 170         emitm(&stream, 0x8948, 2);                                      \
 171         emitm(&stream,                                                  \
 172             (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);          \
 173 } while (0)
 174
 175 /* movq sr64,dr64 (dr64 = %r8-15) */
 176 #define MOVrq2(sr64, dr64) do {                                         \
 177         emitm(&stream, 0x8949, 2);                                      \
 178         emitm(&stream,                                                  \
 179             (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);          \
 180 } while (0)
 181
 182 /* movq sr64,dr64 (sr64 = %r8-15) */
 183 #define MOVrq3(sr64, dr64) do {                                         \
 184         emitm(&stream, 0x894c, 2);                                      \
 185         emitm(&stream,                                                  \
 186             (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);          \
 187 } while (0)
 188
 189 /* movl (sr64,or64,1),dr32 */
 190 #define MOVobd(sr64, or64, dr32) do {                                   \
 191         emitm(&stream, 0x8b, 1);                                        \
 192         emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1);                     \
 193         emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);          \
 194 } while (0)
 195
 196 /* movw (sr64,or64,1),dr16 */
 197 #define MOVobw(sr64, or64, dr16) do {                                   \
 198         emitm(&stream, 0x8b66, 2);                                      \
 199         emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1);                     \
 200         emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);          \
 201 } while (0)
 202
 203 /* movb (sr64,or64,1),dr8 */
 204 #define MOVobb(sr64, or64, dr8) do {                                    \
 205         emitm(&stream, 0x8a, 1);                                        \
 206         emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1);                      \
 207         emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);          \
 208 } while (0)
 209
 210 /* movl sr32,(dr64,or64,1) */
 211 #define MOVomd(sr32, dr64, or64) do {                                   \
 212         emitm(&stream, 0x89, 1);                                        \
 213         emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1);                     \
 214         emitm(&stream, ((or64 & 0x7) << 3) | (dr64 & 0x7), 1);          \
 215 } while (0)
 216
 217 /* bswapl dr32 */
 218 #define BSWAP(dr32) do {                                                \
 219         emitm(&stream, 0xf, 1);                                         \
 220         emitm(&stream, (0x19 << 3) | dr32, 1);                          \
 221 } while (0)
 222
 223 /* xchgb %al,%ah */
 224 #define SWAP_AX() do {                                                  \
 225         emitm(&stream, 0xc486, 2);                                      \
 226 } while (0)
 227
 228 /* pushq r64 */
 229 #define PUSH(r64) do {                                                  \
 230         emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1);           \
 231 } while (0)
 232
 233 /* leaveq */
 234 #define LEAVE() do {                                                    \
 235         emitm(&stream, 0xc9, 1);                                        \
 236 } while (0)
 237
 238 /* retq */
 239 #define RET() do {                                                      \
 240         emitm(&stream, 0xc3, 1);                                        \
 241 } while (0)
 242
 243 /* addl sr32,dr32 */
 244 #define ADDrd(sr32, dr32) do {                                          \
 245         emitm(&stream, 0x01, 1);                                        \
 246         emitm(&stream,                                                  \
 247             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 248 } while (0)
 249
 250 /* addl i32,%eax */
 251 #define ADD_EAXi(i32) do {                                              \
 252         emitm(&stream, 0x05, 1);                                        \
 253         emitm(&stream, i32, 4);                                         \
 254 } while (0)
 255
 256 /* addl i8,r32 */
 257 #define ADDib(i8, r32) do {                                             \
 258         emitm(&stream, 0x83, 1);                                        \
 259         emitm(&stream, (24 << 3) | r32, 1);                             \
 260         emitm(&stream, i8, 1);                                          \
 261 } while (0)
 262
 263 /* subl sr32,dr32 */
 264 #define SUBrd(sr32, dr32) do {                                          \
 265         emitm(&stream, 0x29, 1);                                        \
 266         emitm(&stream,                                                  \
 267             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 268 } while (0)
 269
 270 /* subl i32,%eax */
 271 #define SUB_EAXi(i32) do {                                              \
 272         emitm(&stream, 0x2d, 1);                                        \
 273         emitm(&stream, i32, 4);                                         \
 274 } while (0)
 275
 276 /* subq i8,r64 */
 277 #define SUBib(i8, r64) do {                                             \
 278         emitm(&stream, 0x8348, 2);                                      \
 279         emitm(&stream, (29 << 3) | (r64 & 0x7), 1);                     \
 280         emitm(&stream, i8, 1);                                          \
 281 } while (0)
 282
 283 /* mull r32 */
 284 #define MULrd(r32) do {                                                 \
 285         emitm(&stream, 0xf7, 1);                                        \
 286         emitm(&stream, (7 << 5) | (r32 & 0x7), 1);                      \
 287 } while (0)
 288
 289 /* divl r32 */
 290 #define DIVrd(r32) do {                                                 \
 291         emitm(&stream, 0xf7, 1);                                        \
 292         emitm(&stream, (15 << 4) | (r32 & 0x7), 1);                     \
 293 } while (0)
 294
 295 /* andb i8,r8 */
 296 #define ANDib(i8, r8) do {                                              \
 297         if (r8 == AL) {                                                 \
 298                 emitm(&stream, 0x24, 1);                                \
 299         } else {                                                        \
 300                 emitm(&stream, 0x80, 1);                                \
 301                 emitm(&stream, (7 << 5) | r8, 1);                       \
 302         }                                                               \
 303         emitm(&stream, i8, 1);                                          \
 304 } while (0)
 305
 306 /* andl i32,r32 */
 307 #define ANDid(i32, r32) do {                                            \
 308         if (r32 == EAX) {                                               \
 309                 emitm(&stream, 0x25, 1);                                \
 310         } else {                                                        \
 311                 emitm(&stream, 0x81, 1);                                \
 312                 emitm(&stream, (7 << 5) | r32, 1);                      \
 313         }                                                               \
 314         emitm(&stream, i32, 4);                                         \
 315 } while (0)
 316
 317 /* andl sr32,dr32 */
 318 #define ANDrd(sr32, dr32) do {                                          \
 319         emitm(&stream, 0x21, 1);                                        \
 320         emitm(&stream,                                                  \
 321             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 322 } while (0)
 323
 324 /* testl i32,r32 */
 325 #define TESTid(i32, r32) do {                                           \
 326         if (r32 == EAX) {                                               \
 327                 emitm(&stream, 0xa9, 1);                                \
 328         } else {                                                        \
 329                 emitm(&stream, 0xf7, 1);                                \
 330                 emitm(&stream, (3 << 6) | r32, 1);                      \
 331         }                                                               \
 332         emitm(&stream, i32, 4);                                         \
 333 } while (0)
 334
 335 /* testl sr32,dr32 */
 336 #define TESTrd(sr32, dr32) do {                                         \
 337         emitm(&stream, 0x85, 1);                                        \
 338         emitm(&stream,                                                  \
 339             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 340 } while (0)
 341
 342 /* orl sr32,dr32 */
 343 #define ORrd(sr32, dr32) do {                                           \
 344         emitm(&stream, 0x09, 1);                                        \
 345         emitm(&stream,                                                  \
 346             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 347 } while (0)
 348
 349 /* orl i32,r32 */
 350 #define ORid(i32, r32) do {                                             \
 351         if (r32 == EAX) {                                               \
 352                 emitm(&stream, 0x0d, 1);                                \
 353         } else {                                                        \
 354                 emitm(&stream, 0x81, 1);                                \
 355                 emitm(&stream, (25 << 3) | r32, 1);                     \
 356         }                                                               \
 357         emitm(&stream, i32, 4);                                         \
 358 } while (0)
 359
 360 /* shll i8,r32 */
 361 #define SHLib(i8, r32) do {                                             \
 362         emitm(&stream, 0xc1, 1);                                        \
 363         emitm(&stream, (7 << 5) | (r32 & 0x7), 1);                      \
 364         emitm(&stream, i8, 1);                                          \
 365 } while (0)
 366
 367 /* shll %cl,dr32 */
 368 #define SHL_CLrb(dr32) do {                                             \
 369         emitm(&stream, 0xd3, 1);                                        \
 370         emitm(&stream, (7 << 5) | (dr32 & 0x7), 1);                     \
 371 } while (0)
 372
 373 /* shrl i8,r32 */
 374 #define SHRib(i8, r32) do {                                             \
 375         emitm(&stream, 0xc1, 1);                                        \
 376         emitm(&stream, (29 << 3) | (r32 & 0x7), 1);                     \
 377         emitm(&stream, i8, 1);                                          \
 378 } while (0)
 379
 380 /* shrl %cl,dr32 */
 381 #define SHR_CLrb(dr32) do {                                             \
 382         emitm(&stream, 0xd3, 1);                                        \
 383         emitm(&stream, (29 << 3) | (dr32 & 0x7), 1);                    \
 384 } while (0)
 385
 386 /* negl r32 */
 387 #define NEGd(r32) do {                                                  \
 388         emitm(&stream, 0xf7, 1);                                        \
 389         emitm(&stream, (27 << 3) | (r32 & 0x7), 1);                     \
 390 } while (0)
 391
 392 /* cmpl sr32,dr32 */
 393 #define CMPrd(sr32, dr32) do {                                          \
 394         emitm(&stream, 0x39, 1);                                        \
 395         emitm(&stream,                                                  \
 396             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 397 } while (0)
 398
 399 /* cmpl i32,dr32 */
 400 #define CMPid(i32, dr32) do {                                           \
 401         if (dr32 == EAX){                                               \
 402                 emitm(&stream, 0x3d, 1);                                \
 403                 emitm(&stream, i32, 4);                                 \
 404         } else {                                                        \
 405                 emitm(&stream, 0x81, 1);                                \
 406                 emitm(&stream, (0x1f << 3) | (dr32 & 0x7), 1);          \
 407                 emitm(&stream, i32, 4);                                 \
 408         }                                                               \
 409 } while (0)
 410
 411 /* jb off8 */
 412 #define JBb(off8) do {                                                  \
 413         emitm(&stream, 0x72, 1);                                        \
 414         emitm(&stream, off8, 1);                                        \
 415 } while (0)
 416
 417 /* jae off8 */
 418 #define JAEb(off8) do {                                                 \
 419         emitm(&stream, 0x73, 1);                                        \
 420         emitm(&stream, off8, 1);                                        \
 421 } while (0)
 422
 423 /* jne off8 */
 424 #define JNEb(off8) do {                                                 \
 425         emitm(&stream, 0x75, 1);                                        \
 426         emitm(&stream, off8, 1);                                        \
 427 } while (0)
 428
 429 /* ja off8 */
 430 #define JAb(off8) do {                                                  \
 431         emitm(&stream, 0x77, 1);                                        \
 432         emitm(&stream, off8, 1);                                        \
 433 } while (0)
 434
 435 /* jmp off32 */
 436 #define JMP(off32) do {                                                 \
 437         emitm(&stream, 0xe9, 1);                                        \
 438         emitm(&stream, off32, 4);                                       \
 439 } while (0)
 440
 441 /* xorl r32,r32 */
 442 #define ZEROrd(r32) do {                                                \
 443         emitm(&stream, 0x31, 1);                                        \
 444         emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1); \
 445 } while (0)
 446
 447 /*
 448  * Conditional long jumps
 449  */
 450 #define JB      0x82
 451 #define JAE     0x83
 452 #define JE      0x84
 453 #define JNE     0x85
 454 #define JBE     0x86
 455 #define JA      0x87
 456
 457 #define JCC(t, f) do {                                                  \
 458         if (ins->jt != 0 && ins->jf != 0) {                             \
 459                 /* 5 is the size of the following jmp */                \
 460                 emitm(&stream, ((t) << 8) | 0x0f, 2);                   \
 461                 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -   \
 462                     stream.refs[stream.bpf_pc] + 5, 4);                 \
 463                 JMP(stream.refs[stream.bpf_pc + ins->jf] -              \
 464                     stream.refs[stream.bpf_pc]);                        \
 465         } else if (ins->jt != 0) {                                      \
 466                 emitm(&stream, ((t) << 8) | 0x0f, 2);                   \
 467                 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -   \
 468                     stream.refs[stream.bpf_pc], 4);                     \
 469         } else {                                                        \
 470                 emitm(&stream, ((f) << 8) | 0x0f, 2);                   \
 471                 emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] -   \
 472                     stream.refs[stream.bpf_pc], 4);                     \
 473         }                                                               \
 474 } while (0)
 475
 476 #define JUMP(off) do {                                                  \
 477         if ((off) != 0)                                                 \
 478                 JMP(stream.refs[stream.bpf_pc + (off)] -                \
 479                     stream.refs[stream.bpf_pc]);                        \
 480 } while (0)
 481
 482 #endif  /* _BPF_JIT_MACHDEP_H_ */