sys/amd64/amd64/bpf_jit_machdep.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
   5  * Copyright (C) 2005-2016 Jung-uk Kim <jkim@FreeBSD.org>
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  * notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  * notice, this list of conditions and the following disclaimer in the
  16  * documentation and/or other materials provided with the distribution.
  17  * 3. Neither the name of the Politecnico di Torino nor the names of its
  18  * contributors may be used to endorse or promote products derived from
  19  * this software without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * $FreeBSD$
  34  */
  35
  36 #ifndef _BPF_JIT_MACHDEP_H_
  37 #define _BPF_JIT_MACHDEP_H_
  38
  39 /*
  40  * Registers
  41  */
  42 #define RAX     0
  43 #define RCX     1
  44 #define RDX     2
  45 #define RBX     3
  46 #define RSP     4
  47 #define RBP     5
  48 #define RSI     6
  49 #define RDI     7
  50 #define R8      0
  51 #define R9      1
  52 #define R10     2
  53 #define R11     3
  54 #define R12     4
  55 #define R13     5
  56 #define R14     6
  57 #define R15     7
  58
  59 #define EAX     0
  60 #define ECX     1
  61 #define EDX     2
  62 #define EBX     3
  63 #define ESP     4
  64 #define EBP     5
  65 #define ESI     6
  66 #define EDI     7
  67 #define R8D     0
  68 #define R9D     1
  69 #define R10D    2
  70 #define R11D    3
  71 #define R12D    4
  72 #define R13D    5
  73 #define R14D    6
  74 #define R15D    7
  75
  76 #define AX      0
  77 #define CX      1
  78 #define DX      2
  79 #define BX      3
  80 #define SP      4
  81 #define BP      5
  82 #define SI      6
  83 #define DI      7
  84
  85 #define AL      0
  86 #define CL      1
  87 #define DL      2
  88 #define BL      3
  89
  90 /* Optimization flags */
  91 #define BPF_JIT_FRET    0x01
  92 #define BPF_JIT_FPKT    0x02
  93 #define BPF_JIT_FMEM    0x04
  94 #define BPF_JIT_FJMP    0x08
  95 #define BPF_JIT_FLEN    0x10
  96
  97 #define BPF_JIT_FLAG_ALL        \
  98     (BPF_JIT_FPKT | BPF_JIT_FMEM | BPF_JIT_FJMP | BPF_JIT_FLEN)
  99
 100 /* A stream of native binary code */
 101 typedef struct bpf_bin_stream {
 102         /* Current native instruction pointer. */
 103         int             cur_ip;
 104
 105         /*
 106          * Current BPF instruction pointer, i.e. position in
 107          * the BPF program reached by the jitter.
 108          */
 109         int             bpf_pc;
 110
 111         /* Instruction buffer, contains the generated native code. */
 112         char            *ibuf;
 113
 114         /* Jumps reference table. */
 115         u_int           *refs;
 116 } bpf_bin_stream;
 117
 118 /*
 119  * Prototype of the emit functions.
 120  *
 121  * Different emit functions are used to create the reference table and
 122  * to generate the actual filtering code. This allows to have simpler
 123  * instruction macros.
 124  * The first parameter is the stream that will receive the data.
 125  * The second one is a variable containing the data.
 126  * The third one is the length, that can be 1, 2, or 4 since it is possible
 127  * to emit a byte, a short, or a word at a time.
 128  */
 129 typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 130
 131 /*
 132  * Native instruction macros
 133  */
 134
 135 /* movl i32,r32 */
 136 #define MOVid(i32, r32) do {                                            \
 137         emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1);          \
 138         emitm(&stream, i32, 4);                                         \
 139 } while (0)
 140
 141 /* movq i64,r64 */
 142 #define MOViq(i64, r64) do {                                            \
 143         emitm(&stream, 0x48, 1);                                        \
 144         emitm(&stream, (11 << 4) | (1 << 3) | (r64 & 0x7), 1);          \
 145         emitm(&stream, i64, 4);                                         \
 146         emitm(&stream, (i64 >> 32), 4);                                 \
 147 } while (0)
 148
 149 /* movl sr32,dr32 */
 150 #define MOVrd(sr32, dr32) do {                                          \
 151         emitm(&stream, 0x89, 1);                                        \
 152         emitm(&stream,                                                  \
 153             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 154 } while (0)
 155
 156 /* movl sr32,dr32 (dr32 = %r8-15d) */
 157 #define MOVrd2(sr32, dr32) do {                                         \
 158         emitm(&stream, 0x8941, 2);                                      \
 159         emitm(&stream,                                                  \
 160             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 161 } while (0)
 162
 163 /* movl sr32,dr32 (sr32 = %r8-15d) */
 164 #define MOVrd3(sr32, dr32) do {                                         \
 165         emitm(&stream, 0x8944, 2);                                      \
 166         emitm(&stream,                                                  \
 167             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 168 } while (0)
 169
 170 /* movq sr64,dr64 */
 171 #define MOVrq(sr64, dr64) do {                                          \
 172         emitm(&stream, 0x8948, 2);                                      \
 173         emitm(&stream,                                                  \
 174             (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);          \
 175 } while (0)
 176
 177 /* movq sr64,dr64 (dr64 = %r8-15) */
 178 #define MOVrq2(sr64, dr64) do {                                         \
 179         emitm(&stream, 0x8949, 2);                                      \
 180         emitm(&stream,                                                  \
 181             (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);          \
 182 } while (0)
 183
 184 /* movq sr64,dr64 (sr64 = %r8-15) */
 185 #define MOVrq3(sr64, dr64) do {                                         \
 186         emitm(&stream, 0x894c, 2);                                      \
 187         emitm(&stream,                                                  \
 188             (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);          \
 189 } while (0)
 190
 191 /* movl (sr64,or64,1),dr32 */
 192 #define MOVobd(sr64, or64, dr32) do {                                   \
 193         emitm(&stream, 0x8b, 1);                                        \
 194         emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1);                     \
 195         emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);          \
 196 } while (0)
 197
 198 /* movw (sr64,or64,1),dr16 */
 199 #define MOVobw(sr64, or64, dr16) do {                                   \
 200         emitm(&stream, 0x8b66, 2);                                      \
 201         emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1);                     \
 202         emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);          \
 203 } while (0)
 204
 205 /* movb (sr64,or64,1),dr8 */
 206 #define MOVobb(sr64, or64, dr8) do {                                    \
 207         emitm(&stream, 0x8a, 1);                                        \
 208         emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1);                      \
 209         emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);          \
 210 } while (0)
 211
 212 /* movl sr32,(dr64,or64,1) */
 213 #define MOVomd(sr32, dr64, or64) do {                                   \
 214         emitm(&stream, 0x89, 1);                                        \
 215         emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1);                     \
 216         emitm(&stream, ((or64 & 0x7) << 3) | (dr64 & 0x7), 1);          \
 217 } while (0)
 218
 219 /* bswapl dr32 */
 220 #define BSWAP(dr32) do {                                                \
 221         emitm(&stream, 0xf, 1);                                         \
 222         emitm(&stream, (0x19 << 3) | dr32, 1);                          \
 223 } while (0)
 224
 225 /* xchgb %al,%ah */
 226 #define SWAP_AX() do {                                                  \
 227         emitm(&stream, 0xc486, 2);                                      \
 228 } while (0)
 229
 230 /* pushq r64 */
 231 #define PUSH(r64) do {                                                  \
 232         emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1);           \
 233 } while (0)
 234
 235 /* leaveq */
 236 #define LEAVE() do {                                                    \
 237         emitm(&stream, 0xc9, 1);                                        \
 238 } while (0)
 239
 240 /* retq */
 241 #define RET() do {                                                      \
 242         emitm(&stream, 0xc3, 1);                                        \
 243 } while (0)
 244
 245 /* addl sr32,dr32 */
 246 #define ADDrd(sr32, dr32) do {                                          \
 247         emitm(&stream, 0x01, 1);                                        \
 248         emitm(&stream,                                                  \
 249             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 250 } while (0)
 251
 252 /* addl i32,%eax */
 253 #define ADD_EAXi(i32) do {                                              \
 254         emitm(&stream, 0x05, 1);                                        \
 255         emitm(&stream, i32, 4);                                         \
 256 } while (0)
 257
 258 /* addl i8,r32 */
 259 #define ADDib(i8, r32) do {                                             \
 260         emitm(&stream, 0x83, 1);                                        \
 261         emitm(&stream, (24 << 3) | r32, 1);                             \
 262         emitm(&stream, i8, 1);                                          \
 263 } while (0)
 264
 265 /* subl sr32,dr32 */
 266 #define SUBrd(sr32, dr32) do {                                          \
 267         emitm(&stream, 0x29, 1);                                        \
 268         emitm(&stream,                                                  \
 269             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 270 } while (0)
 271
 272 /* subl i32,%eax */
 273 #define SUB_EAXi(i32) do {                                              \
 274         emitm(&stream, 0x2d, 1);                                        \
 275         emitm(&stream, i32, 4);                                         \
 276 } while (0)
 277
 278 /* subq i8,r64 */
 279 #define SUBib(i8, r64) do {                                             \
 280         emitm(&stream, 0x8348, 2);                                      \
 281         emitm(&stream, (29 << 3) | (r64 & 0x7), 1);                     \
 282         emitm(&stream, i8, 1);                                          \
 283 } while (0)
 284
 285 /* mull r32 */
 286 #define MULrd(r32) do {                                                 \
 287         emitm(&stream, 0xf7, 1);                                        \
 288         emitm(&stream, (7 << 5) | (r32 & 0x7), 1);                      \
 289 } while (0)
 290
 291 /* divl r32 */
 292 #define DIVrd(r32) do {                                                 \
 293         emitm(&stream, 0xf7, 1);                                        \
 294         emitm(&stream, (15 << 4) | (r32 & 0x7), 1);                     \
 295 } while (0)
 296
 297 /* andb i8,r8 */
 298 #define ANDib(i8, r8) do {                                              \
 299         if (r8 == AL) {                                                 \
 300                 emitm(&stream, 0x24, 1);                                \
 301         } else {                                                        \
 302                 emitm(&stream, 0x80, 1);                                \
 303                 emitm(&stream, (7 << 5) | r8, 1);                       \
 304         }                                                               \
 305         emitm(&stream, i8, 1);                                          \
 306 } while (0)
 307
 308 /* andl i32,r32 */
 309 #define ANDid(i32, r32) do {                                            \
 310         if (r32 == EAX) {                                               \
 311                 emitm(&stream, 0x25, 1);                                \
 312         } else {                                                        \
 313                 emitm(&stream, 0x81, 1);                                \
 314                 emitm(&stream, (7 << 5) | r32, 1);                      \
 315         }                                                               \
 316         emitm(&stream, i32, 4);                                         \
 317 } while (0)
 318
 319 /* andl sr32,dr32 */
 320 #define ANDrd(sr32, dr32) do {                                          \
 321         emitm(&stream, 0x21, 1);                                        \
 322         emitm(&stream,                                                  \
 323             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 324 } while (0)
 325
 326 /* testl i32,r32 */
 327 #define TESTid(i32, r32) do {                                           \
 328         if (r32 == EAX) {                                               \
 329                 emitm(&stream, 0xa9, 1);                                \
 330         } else {                                                        \
 331                 emitm(&stream, 0xf7, 1);                                \
 332                 emitm(&stream, (3 << 6) | r32, 1);                      \
 333         }                                                               \
 334         emitm(&stream, i32, 4);                                         \
 335 } while (0)
 336
 337 /* testl sr32,dr32 */
 338 #define TESTrd(sr32, dr32) do {                                         \
 339         emitm(&stream, 0x85, 1);                                        \
 340         emitm(&stream,                                                  \
 341             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 342 } while (0)
 343
 344 /* orl sr32,dr32 */
 345 #define ORrd(sr32, dr32) do {                                           \
 346         emitm(&stream, 0x09, 1);                                        \
 347         emitm(&stream,                                                  \
 348             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 349 } while (0)
 350
 351 /* orl i32,r32 */
 352 #define ORid(i32, r32) do {                                             \
 353         if (r32 == EAX) {                                               \
 354                 emitm(&stream, 0x0d, 1);                                \
 355         } else {                                                        \
 356                 emitm(&stream, 0x81, 1);                                \
 357                 emitm(&stream, (25 << 3) | r32, 1);                     \
 358         }                                                               \
 359         emitm(&stream, i32, 4);                                         \
 360 } while (0)
 361
 362 /* xorl sr32,dr32 */
 363 #define XORrd(sr32, dr32) do {                                          \
 364         emitm(&stream, 0x31, 1);                                        \
 365         emitm(&stream,                                                  \
 366             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 367 } while (0)
 368
 369 /* xorl i32,r32 */
 370 #define XORid(i32, r32) do {                                            \
 371         if (r32 == EAX) {                                               \
 372                 emitm(&stream, 0x35, 1);                                \
 373         } else {                                                        \
 374                 emitm(&stream, 0x81, 1);                                \
 375                 emitm(&stream, (25 << 3) | r32, 1);                     \
 376         }                                                               \
 377         emitm(&stream, i32, 4);                                         \
 378 } while (0)
 379
 380 /* shll i8,r32 */
 381 #define SHLib(i8, r32) do {                                             \
 382         emitm(&stream, 0xc1, 1);                                        \
 383         emitm(&stream, (7 << 5) | (r32 & 0x7), 1);                      \
 384         emitm(&stream, i8, 1);                                          \
 385 } while (0)
 386
 387 /* shll %cl,dr32 */
 388 #define SHL_CLrb(dr32) do {                                             \
 389         emitm(&stream, 0xd3, 1);                                        \
 390         emitm(&stream, (7 << 5) | (dr32 & 0x7), 1);                     \
 391 } while (0)
 392
 393 /* shrl i8,r32 */
 394 #define SHRib(i8, r32) do {                                             \
 395         emitm(&stream, 0xc1, 1);                                        \
 396         emitm(&stream, (29 << 3) | (r32 & 0x7), 1);                     \
 397         emitm(&stream, i8, 1);                                          \
 398 } while (0)
 399
 400 /* shrl %cl,dr32 */
 401 #define SHR_CLrb(dr32) do {                                             \
 402         emitm(&stream, 0xd3, 1);                                        \
 403         emitm(&stream, (29 << 3) | (dr32 & 0x7), 1);                    \
 404 } while (0)
 405
 406 /* negl r32 */
 407 #define NEGd(r32) do {                                                  \
 408         emitm(&stream, 0xf7, 1);                                        \
 409         emitm(&stream, (27 << 3) | (r32 & 0x7), 1);                     \
 410 } while (0)
 411
 412 /* cmpl sr32,dr32 */
 413 #define CMPrd(sr32, dr32) do {                                          \
 414         emitm(&stream, 0x39, 1);                                        \
 415         emitm(&stream,                                                  \
 416             (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);          \
 417 } while (0)
 418
 419 /* cmpl i32,dr32 */
 420 #define CMPid(i32, dr32) do {                                           \
 421         if (dr32 == EAX){                                               \
 422                 emitm(&stream, 0x3d, 1);                                \
 423                 emitm(&stream, i32, 4);                                 \
 424         } else {                                                        \
 425                 emitm(&stream, 0x81, 1);                                \
 426                 emitm(&stream, (0x1f << 3) | (dr32 & 0x7), 1);          \
 427                 emitm(&stream, i32, 4);                                 \
 428         }                                                               \
 429 } while (0)
 430
 431 /* jb off8 */
 432 #define JBb(off8) do {                                                  \
 433         emitm(&stream, 0x72, 1);                                        \
 434         emitm(&stream, off8, 1);                                        \
 435 } while (0)
 436
 437 /* jae off8 */
 438 #define JAEb(off8) do {                                                 \
 439         emitm(&stream, 0x73, 1);                                        \
 440         emitm(&stream, off8, 1);                                        \
 441 } while (0)
 442
 443 /* jne off8 */
 444 #define JNEb(off8) do {                                                 \
 445         emitm(&stream, 0x75, 1);                                        \
 446         emitm(&stream, off8, 1);                                        \
 447 } while (0)
 448
 449 /* ja off8 */
 450 #define JAb(off8) do {                                                  \
 451         emitm(&stream, 0x77, 1);                                        \
 452         emitm(&stream, off8, 1);                                        \
 453 } while (0)
 454
 455 /* jmp off32 */
 456 #define JMP(off32) do {                                                 \
 457         emitm(&stream, 0xe9, 1);                                        \
 458         emitm(&stream, off32, 4);                                       \
 459 } while (0)
 460
 461 /* xorl r32,r32 */
 462 #define ZEROrd(r32) do {                                                \
 463         emitm(&stream, 0x31, 1);                                        \
 464         emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1); \
 465 } while (0)
 466
 467 /*
 468  * Conditional long jumps
 469  */
 470 #define JB      0x82
 471 #define JAE     0x83
 472 #define JE      0x84
 473 #define JNE     0x85
 474 #define JBE     0x86
 475 #define JA      0x87
 476
 477 #define JCC(t, f) do {                                                  \
 478         if (ins->jt != 0 && ins->jf != 0) {                             \
 479                 /* 5 is the size of the following jmp */                \
 480                 emitm(&stream, ((t) << 8) | 0x0f, 2);                   \
 481                 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -   \
 482                     stream.refs[stream.bpf_pc] + 5, 4);                 \
 483                 JMP(stream.refs[stream.bpf_pc + ins->jf] -              \
 484                     stream.refs[stream.bpf_pc]);                        \
 485         } else if (ins->jt != 0) {                                      \
 486                 emitm(&stream, ((t) << 8) | 0x0f, 2);                   \
 487                 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -   \
 488                     stream.refs[stream.bpf_pc], 4);                     \
 489         } else {                                                        \
 490                 emitm(&stream, ((f) << 8) | 0x0f, 2);                   \
 491                 emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] -   \
 492                     stream.refs[stream.bpf_pc], 4);                     \
 493         }                                                               \
 494 } while (0)
 495
 496 #define JUMP(off) do {                                                  \
 497         if ((off) != 0)                                                 \
 498                 JMP(stream.refs[stream.bpf_pc + (off)] -                \
 499                     stream.refs[stream.bpf_pc]);                        \
 500 } while (0)
 501
 502 #endif  /* _BPF_JIT_MACHDEP_H_ */