2 * SPDX-License-Identifier: BSD-3-Clause
4 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
5 * Copyright (C) 2005-2017 Jung-uk Kim <jkim@FreeBSD.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the Politecnico di Torino nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/malloc.h>
44 #include <sys/socket.h>
48 #include <vm/vm_extern.h>
49 #include <vm/vm_kern.h>
54 #include <sys/param.h>
57 #include <sys/types.h>
60 #include <net/bpf_jitter.h>
62 #include <amd64/amd64/bpf_jit_machdep.h>
65 * Emit routine to update the jump table.
68 emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len)
71 if (stream->refs != NULL)
72 (stream->refs)[stream->bpf_pc] += len;
73 stream->cur_ip += len;
77 * Emit routine to output the actual binary code.
80 emit_code(bpf_bin_stream *stream, u_int value, u_int len)
85 stream->ibuf[stream->cur_ip] = (u_char)value;
90 *((u_short *)(void *)(stream->ibuf + stream->cur_ip)) =
96 *((u_int *)(void *)(stream->ibuf + stream->cur_ip)) = value;
105 * Scan the filter program and find possible optimization.
108 bpf_jit_optimize(struct bpf_insn *prog, u_int nins)
113 /* Do we return immediately? */
114 if (BPF_CLASS(prog[0].code) == BPF_RET)
115 return (BPF_JIT_FRET);
117 for (flags = 0, i = 0; i < nins; i++) {
118 switch (prog[i].code) {
119 case BPF_LD|BPF_W|BPF_ABS:
120 case BPF_LD|BPF_H|BPF_ABS:
121 case BPF_LD|BPF_B|BPF_ABS:
122 case BPF_LD|BPF_W|BPF_IND:
123 case BPF_LD|BPF_H|BPF_IND:
124 case BPF_LD|BPF_B|BPF_IND:
125 case BPF_LDX|BPF_MSH|BPF_B:
126 flags |= BPF_JIT_FPKT;
129 case BPF_LDX|BPF_MEM:
132 flags |= BPF_JIT_FMEM;
134 case BPF_LD|BPF_W|BPF_LEN:
135 case BPF_LDX|BPF_W|BPF_LEN:
136 flags |= BPF_JIT_FLEN;
139 case BPF_JMP|BPF_JGT|BPF_K:
140 case BPF_JMP|BPF_JGE|BPF_K:
141 case BPF_JMP|BPF_JEQ|BPF_K:
142 case BPF_JMP|BPF_JSET|BPF_K:
143 case BPF_JMP|BPF_JGT|BPF_X:
144 case BPF_JMP|BPF_JGE|BPF_X:
145 case BPF_JMP|BPF_JEQ|BPF_X:
146 case BPF_JMP|BPF_JSET|BPF_X:
147 flags |= BPF_JIT_FJMP;
150 if (flags == BPF_JIT_FLAG_ALL)
158 * Function that does the real stuff.
161 bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
163 bpf_bin_stream stream;
164 struct bpf_insn *ins;
165 int flags, fret, fpkt, fmem, fjmp, flen;
169 * NOTE: Do not modify the name of this variable, as it's used by
170 * the macros to emit code.
174 flags = bpf_jit_optimize(prog, nins);
175 fret = (flags & BPF_JIT_FRET) != 0;
176 fpkt = (flags & BPF_JIT_FPKT) != 0;
177 fmem = (flags & BPF_JIT_FMEM) != 0;
178 fjmp = (flags & BPF_JIT_FJMP) != 0;
179 flen = (flags & BPF_JIT_FLEN) != 0;
184 memset(&stream, 0, sizeof(stream));
186 /* Allocate the reference table for the jumps. */
189 stream.refs = mallocarray(nins + 1, sizeof(u_int), M_BPFJIT,
192 stream.refs = calloc(nins + 1, sizeof(u_int));
194 if (stream.refs == NULL)
199 * The first pass will emit the lengths of the instructions
200 * to create the reference table.
204 for (pass = 0; pass < 2; pass++) {
207 /* Create the procedure header. */
211 SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP);
220 for (i = 0; i < nins; i++) {
244 case BPF_LD|BPF_W|BPF_ABS:
250 CMPid(sizeof(int32_t), ECX);
261 MOVobd(RCX, RSI, EAX);
265 case BPF_LD|BPF_H|BPF_ABS:
272 CMPid(sizeof(int16_t), ECX);
280 MOVobw(RCX, RSI, AX);
284 case BPF_LD|BPF_B|BPF_ABS:
295 MOVobb(RCX, RSI, AL);
298 case BPF_LD|BPF_W|BPF_LEN:
302 case BPF_LDX|BPF_W|BPF_LEN:
306 case BPF_LD|BPF_W|BPF_IND:
317 CMPid(sizeof(int32_t), ECX);
328 MOVobd(RCX, RSI, EAX);
332 case BPF_LD|BPF_H|BPF_IND:
344 CMPid(sizeof(int16_t), ECX);
352 MOVobw(RCX, RSI, AX);
356 case BPF_LD|BPF_B|BPF_IND:
372 MOVobb(RCX, RSI, AL);
375 case BPF_LDX|BPF_MSH|BPF_B:
389 MOVobb(RCX, RSI, DL);
398 case BPF_LDX|BPF_IMM:
403 MOVid(ins->k * sizeof(uint32_t), ESI);
404 MOVobd(RSP, RSI, EAX);
407 case BPF_LDX|BPF_MEM:
408 MOVid(ins->k * sizeof(uint32_t), ESI);
409 MOVobd(RSP, RSI, EDX);
414 * XXX this command and the following could
415 * be optimized if the previous instruction
416 * was already of this type
418 MOVid(ins->k * sizeof(uint32_t), ESI);
419 MOVomd(EAX, RSP, RSI);
423 MOVid(ins->k * sizeof(uint32_t), ESI);
424 MOVomd(EDX, RSP, RSI);
431 case BPF_JMP|BPF_JGT|BPF_K:
432 case BPF_JMP|BPF_JGE|BPF_K:
433 case BPF_JMP|BPF_JEQ|BPF_K:
434 case BPF_JMP|BPF_JSET|BPF_K:
435 case BPF_JMP|BPF_JGT|BPF_X:
436 case BPF_JMP|BPF_JGE|BPF_X:
437 case BPF_JMP|BPF_JEQ|BPF_X:
438 case BPF_JMP|BPF_JSET|BPF_X:
439 if (ins->jt == ins->jf) {
444 case BPF_JMP|BPF_JGT|BPF_K:
449 case BPF_JMP|BPF_JGE|BPF_K:
454 case BPF_JMP|BPF_JEQ|BPF_K:
459 case BPF_JMP|BPF_JSET|BPF_K:
464 case BPF_JMP|BPF_JGT|BPF_X:
469 case BPF_JMP|BPF_JGE|BPF_X:
474 case BPF_JMP|BPF_JEQ|BPF_X:
479 case BPF_JMP|BPF_JSET|BPF_X:
486 case BPF_ALU|BPF_ADD|BPF_X:
490 case BPF_ALU|BPF_SUB|BPF_X:
494 case BPF_ALU|BPF_MUL|BPF_X:
500 case BPF_ALU|BPF_DIV|BPF_X:
501 case BPF_ALU|BPF_MOD|BPF_X:
515 if (BPF_OP(ins->code) == BPF_MOD)
520 case BPF_ALU|BPF_AND|BPF_X:
524 case BPF_ALU|BPF_OR|BPF_X:
528 case BPF_ALU|BPF_XOR|BPF_X:
532 case BPF_ALU|BPF_LSH|BPF_X:
537 case BPF_ALU|BPF_RSH|BPF_X:
542 case BPF_ALU|BPF_ADD|BPF_K:
546 case BPF_ALU|BPF_SUB|BPF_K:
550 case BPF_ALU|BPF_MUL|BPF_K:
557 case BPF_ALU|BPF_DIV|BPF_K:
558 case BPF_ALU|BPF_MOD|BPF_K:
563 if (BPF_OP(ins->code) == BPF_MOD)
568 case BPF_ALU|BPF_AND|BPF_K:
572 case BPF_ALU|BPF_OR|BPF_K:
576 case BPF_ALU|BPF_XOR|BPF_K:
580 case BPF_ALU|BPF_LSH|BPF_K:
581 SHLib((ins->k) & 0xff, EAX);
584 case BPF_ALU|BPF_RSH|BPF_K:
585 SHRib((ins->k) & 0xff, EAX);
588 case BPF_ALU|BPF_NEG:
592 case BPF_MISC|BPF_TAX:
596 case BPF_MISC|BPF_TXA:
606 *size = stream.cur_ip;
609 * We cannot use malloc(9) because DMAP is mapped as NX.
611 stream.ibuf = (void *)kmem_malloc(kernel_arena, *size,
613 if (stream.ibuf == NULL)
616 stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE,
618 if (stream.ibuf == MAP_FAILED) {
625 * Modify the reference table to contain the offsets and
626 * not the lengths of the instructions.
629 for (i = 1; i < nins + 1; i++)
630 stream.refs[i] += stream.refs[i - 1];
632 /* Reset the counters. */
636 /* The second pass creates the actual code. */
641 * The reference table is needed only during compilation,
642 * now we can free it.
646 free(stream.refs, M_BPFJIT);
652 if (stream.ibuf != NULL &&
653 mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) {
654 munmap(stream.ibuf, *size);
659 return ((bpf_filter_func)(void *)stream.ibuf);
663 bpf_jit_free(void *func, size_t size)
667 kmem_free(kernel_arena, (vm_offset_t)func, size);