2 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
3 * Copyright (C) 2005-2017 Jung-uk Kim <jkim@FreeBSD.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the Politecnico di Torino nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
42 #include <sys/socket.h>
46 #include <vm/vm_extern.h>
47 #include <vm/vm_kern.h>
52 #include <sys/param.h>
55 #include <sys/types.h>
58 #include <net/bpf_jitter.h>
60 #include <amd64/amd64/bpf_jit_machdep.h>
63 * Emit routine to update the jump table.
66 emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len)
69 if (stream->refs != NULL)
70 (stream->refs)[stream->bpf_pc] += len;
71 stream->cur_ip += len;
75 * Emit routine to output the actual binary code.
78 emit_code(bpf_bin_stream *stream, u_int value, u_int len)
83 stream->ibuf[stream->cur_ip] = (u_char)value;
88 *((u_short *)(void *)(stream->ibuf + stream->cur_ip)) =
94 *((u_int *)(void *)(stream->ibuf + stream->cur_ip)) = value;
103 * Scan the filter program and find possible optimization.
106 bpf_jit_optimize(struct bpf_insn *prog, u_int nins)
111 /* Do we return immediately? */
112 if (BPF_CLASS(prog[0].code) == BPF_RET)
113 return (BPF_JIT_FRET);
115 for (flags = 0, i = 0; i < nins; i++) {
116 switch (prog[i].code) {
117 case BPF_LD|BPF_W|BPF_ABS:
118 case BPF_LD|BPF_H|BPF_ABS:
119 case BPF_LD|BPF_B|BPF_ABS:
120 case BPF_LD|BPF_W|BPF_IND:
121 case BPF_LD|BPF_H|BPF_IND:
122 case BPF_LD|BPF_B|BPF_IND:
123 case BPF_LDX|BPF_MSH|BPF_B:
124 flags |= BPF_JIT_FPKT;
127 case BPF_LDX|BPF_MEM:
130 flags |= BPF_JIT_FMEM;
132 case BPF_LD|BPF_W|BPF_LEN:
133 case BPF_LDX|BPF_W|BPF_LEN:
134 flags |= BPF_JIT_FLEN;
137 case BPF_JMP|BPF_JGT|BPF_K:
138 case BPF_JMP|BPF_JGE|BPF_K:
139 case BPF_JMP|BPF_JEQ|BPF_K:
140 case BPF_JMP|BPF_JSET|BPF_K:
141 case BPF_JMP|BPF_JGT|BPF_X:
142 case BPF_JMP|BPF_JGE|BPF_X:
143 case BPF_JMP|BPF_JEQ|BPF_X:
144 case BPF_JMP|BPF_JSET|BPF_X:
145 flags |= BPF_JIT_FJMP;
148 if (flags == BPF_JIT_FLAG_ALL)
156 * Function that does the real stuff.
159 bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
161 bpf_bin_stream stream;
162 struct bpf_insn *ins;
163 int flags, fret, fpkt, fmem, fjmp, flen;
167 * NOTE: Do not modify the name of this variable, as it's used by
168 * the macros to emit code.
172 flags = bpf_jit_optimize(prog, nins);
173 fret = (flags & BPF_JIT_FRET) != 0;
174 fpkt = (flags & BPF_JIT_FPKT) != 0;
175 fmem = (flags & BPF_JIT_FMEM) != 0;
176 fjmp = (flags & BPF_JIT_FJMP) != 0;
177 flen = (flags & BPF_JIT_FLEN) != 0;
182 memset(&stream, 0, sizeof(stream));
184 /* Allocate the reference table for the jumps. */
187 stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT,
190 stream.refs = calloc(nins + 1, sizeof(u_int));
192 if (stream.refs == NULL)
197 * The first pass will emit the lengths of the instructions
198 * to create the reference table.
202 for (pass = 0; pass < 2; pass++) {
205 /* Create the procedure header. */
209 SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP);
218 for (i = 0; i < nins; i++) {
242 case BPF_LD|BPF_W|BPF_ABS:
248 CMPid(sizeof(int32_t), ECX);
259 MOVobd(RCX, RSI, EAX);
263 case BPF_LD|BPF_H|BPF_ABS:
270 CMPid(sizeof(int16_t), ECX);
278 MOVobw(RCX, RSI, AX);
282 case BPF_LD|BPF_B|BPF_ABS:
293 MOVobb(RCX, RSI, AL);
296 case BPF_LD|BPF_W|BPF_LEN:
300 case BPF_LDX|BPF_W|BPF_LEN:
304 case BPF_LD|BPF_W|BPF_IND:
315 CMPid(sizeof(int32_t), ECX);
326 MOVobd(RCX, RSI, EAX);
330 case BPF_LD|BPF_H|BPF_IND:
342 CMPid(sizeof(int16_t), ECX);
350 MOVobw(RCX, RSI, AX);
354 case BPF_LD|BPF_B|BPF_IND:
370 MOVobb(RCX, RSI, AL);
373 case BPF_LDX|BPF_MSH|BPF_B:
387 MOVobb(RCX, RSI, DL);
396 case BPF_LDX|BPF_IMM:
401 MOVid(ins->k * sizeof(uint32_t), ESI);
402 MOVobd(RSP, RSI, EAX);
405 case BPF_LDX|BPF_MEM:
406 MOVid(ins->k * sizeof(uint32_t), ESI);
407 MOVobd(RSP, RSI, EDX);
412 * XXX this command and the following could
413 * be optimized if the previous instruction
414 * was already of this type
416 MOVid(ins->k * sizeof(uint32_t), ESI);
417 MOVomd(EAX, RSP, RSI);
421 MOVid(ins->k * sizeof(uint32_t), ESI);
422 MOVomd(EDX, RSP, RSI);
429 case BPF_JMP|BPF_JGT|BPF_K:
430 case BPF_JMP|BPF_JGE|BPF_K:
431 case BPF_JMP|BPF_JEQ|BPF_K:
432 case BPF_JMP|BPF_JSET|BPF_K:
433 case BPF_JMP|BPF_JGT|BPF_X:
434 case BPF_JMP|BPF_JGE|BPF_X:
435 case BPF_JMP|BPF_JEQ|BPF_X:
436 case BPF_JMP|BPF_JSET|BPF_X:
437 if (ins->jt == ins->jf) {
442 case BPF_JMP|BPF_JGT|BPF_K:
447 case BPF_JMP|BPF_JGE|BPF_K:
452 case BPF_JMP|BPF_JEQ|BPF_K:
457 case BPF_JMP|BPF_JSET|BPF_K:
462 case BPF_JMP|BPF_JGT|BPF_X:
467 case BPF_JMP|BPF_JGE|BPF_X:
472 case BPF_JMP|BPF_JEQ|BPF_X:
477 case BPF_JMP|BPF_JSET|BPF_X:
484 case BPF_ALU|BPF_ADD|BPF_X:
488 case BPF_ALU|BPF_SUB|BPF_X:
492 case BPF_ALU|BPF_MUL|BPF_X:
498 case BPF_ALU|BPF_DIV|BPF_X:
499 case BPF_ALU|BPF_MOD|BPF_X:
513 if (BPF_OP(ins->code) == BPF_MOD)
518 case BPF_ALU|BPF_AND|BPF_X:
522 case BPF_ALU|BPF_OR|BPF_X:
526 case BPF_ALU|BPF_XOR|BPF_X:
530 case BPF_ALU|BPF_LSH|BPF_X:
535 case BPF_ALU|BPF_RSH|BPF_X:
540 case BPF_ALU|BPF_ADD|BPF_K:
544 case BPF_ALU|BPF_SUB|BPF_K:
548 case BPF_ALU|BPF_MUL|BPF_K:
555 case BPF_ALU|BPF_DIV|BPF_K:
556 case BPF_ALU|BPF_MOD|BPF_K:
561 if (BPF_OP(ins->code) == BPF_MOD)
566 case BPF_ALU|BPF_AND|BPF_K:
570 case BPF_ALU|BPF_OR|BPF_K:
574 case BPF_ALU|BPF_XOR|BPF_K:
578 case BPF_ALU|BPF_LSH|BPF_K:
579 SHLib((ins->k) & 0xff, EAX);
582 case BPF_ALU|BPF_RSH|BPF_K:
583 SHRib((ins->k) & 0xff, EAX);
586 case BPF_ALU|BPF_NEG:
590 case BPF_MISC|BPF_TAX:
594 case BPF_MISC|BPF_TXA:
604 *size = stream.cur_ip;
607 * We cannot use malloc(9) because DMAP is mapped as NX.
609 stream.ibuf = (void *)kmem_malloc(kernel_arena, *size,
611 if (stream.ibuf == NULL)
614 stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE,
616 if (stream.ibuf == MAP_FAILED) {
623 * Modify the reference table to contain the offsets and
624 * not the lengths of the instructions.
627 for (i = 1; i < nins + 1; i++)
628 stream.refs[i] += stream.refs[i - 1];
630 /* Reset the counters. */
634 /* The second pass creates the actual code. */
639 * The reference table is needed only during compilation,
640 * now we can free it.
644 free(stream.refs, M_BPFJIT);
650 if (stream.ibuf != NULL &&
651 mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) {
652 munmap(stream.ibuf, *size);
657 return ((bpf_filter_func)(void *)stream.ibuf);
661 bpf_jit_free(void *func, size_t size)
665 kmem_free(kernel_arena, (vm_offset_t)func, size);