sys/i386/i386/bpf_jit_machdep.c

   1 /*-
   2  * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
   3  * Copyright (C) 2005-2016 Jung-uk Kim <jkim@FreeBSD.org>
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright
  11  * notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  * notice, this list of conditions and the following disclaimer in the
  14  * documentation and/or other materials provided with the distribution.
  15  * 3. Neither the name of the Politecnico di Torino nor the names of its
  16  * contributors may be used to endorse or promote products derived from
  17  * this software without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #ifdef _KERNEL
  36 #include "opt_bpf.h"
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/mbuf.h>
  41 #include <sys/socket.h>
  42 #include <sys/malloc.h>
  43 #include <net/if.h>
  44 #else
  45 #include <stdlib.h>
  46 #include <string.h>
  47 #include <sys/mman.h>
  48 #include <sys/param.h>
  49 #endif
  50
  51 #include <sys/types.h>
  52
  53 #include <net/bpf.h>
  54 #include <net/bpf_jitter.h>
  55
  56 #include <i386/i386/bpf_jit_machdep.h>
  57
  58 bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
  59
  60 /*
  61  * Emit routine to update the jump table.
  62  */
  63 static void
  64 emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len)
  65 {
  66
  67         if (stream->refs != NULL)
  68                 (stream->refs)[stream->bpf_pc] += len;
  69         stream->cur_ip += len;
  70 }
  71
  72 /*
  73  * Emit routine to output the actual binary code.
  74  */
  75 static void
  76 emit_code(bpf_bin_stream *stream, u_int value, u_int len)
  77 {
  78
  79         switch (len) {
  80         case 1:
  81                 stream->ibuf[stream->cur_ip] = (u_char)value;
  82                 stream->cur_ip++;
  83                 break;
  84
  85         case 2:
  86                 *((u_short *)(void *)(stream->ibuf + stream->cur_ip)) =
  87                     (u_short)value;
  88                 stream->cur_ip += 2;
  89                 break;
  90
  91         case 4:
  92                 *((u_int *)(void *)(stream->ibuf + stream->cur_ip)) = value;
  93                 stream->cur_ip += 4;
  94                 break;
  95         }
  96
  97         return;
  98 }
  99
 100 /*
 101  * Scan the filter program and find possible optimization.
 102  */
 103 static int
 104 bpf_jit_optimize(struct bpf_insn *prog, u_int nins)
 105 {
 106         int flags;
 107         u_int i;
 108
 109         /* Do we return immediately? */
 110         if (BPF_CLASS(prog[0].code) == BPF_RET)
 111                 return (BPF_JIT_FRET);
 112
 113         for (flags = 0, i = 0; i < nins; i++) {
 114                 switch (prog[i].code) {
 115                 case BPF_LD|BPF_W|BPF_ABS:
 116                 case BPF_LD|BPF_H|BPF_ABS:
 117                 case BPF_LD|BPF_B|BPF_ABS:
 118                 case BPF_LD|BPF_W|BPF_IND:
 119                 case BPF_LD|BPF_H|BPF_IND:
 120                 case BPF_LD|BPF_B|BPF_IND:
 121                 case BPF_LDX|BPF_MSH|BPF_B:
 122                         flags |= BPF_JIT_FPKT;
 123                         break;
 124                 case BPF_LD|BPF_MEM:
 125                 case BPF_LDX|BPF_MEM:
 126                 case BPF_ST:
 127                 case BPF_STX:
 128                         flags |= BPF_JIT_FMEM;
 129                         break;
 130                 case BPF_JMP|BPF_JA:
 131                 case BPF_JMP|BPF_JGT|BPF_K:
 132                 case BPF_JMP|BPF_JGE|BPF_K:
 133                 case BPF_JMP|BPF_JEQ|BPF_K:
 134                 case BPF_JMP|BPF_JSET|BPF_K:
 135                 case BPF_JMP|BPF_JGT|BPF_X:
 136                 case BPF_JMP|BPF_JGE|BPF_X:
 137                 case BPF_JMP|BPF_JEQ|BPF_X:
 138                 case BPF_JMP|BPF_JSET|BPF_X:
 139                         flags |= BPF_JIT_FJMP;
 140                         break;
 141                 case BPF_ALU|BPF_DIV|BPF_K:
 142                 case BPF_ALU|BPF_MOD|BPF_K:
 143                         flags |= BPF_JIT_FADK;
 144                         break;
 145                 }
 146                 if (flags == BPF_JIT_FLAG_ALL)
 147                         break;
 148         }
 149
 150         return (flags);
 151 }
 152
 153 /*
 154  * Function that does the real stuff.
 155  */
 156 bpf_filter_func
 157 bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
 158 {
 159         bpf_bin_stream stream;
 160         struct bpf_insn *ins;
 161         int flags, fret, fpkt, fmem, fjmp, fadk;
 162         int save_esp;
 163         u_int i, pass;
 164
 165         /*
 166          * NOTE: Do not modify the name of this variable, as it's used by
 167          * the macros to emit code.
 168          */
 169         emit_func emitm;
 170
 171         flags = bpf_jit_optimize(prog, nins);
 172         fret = (flags & BPF_JIT_FRET) != 0;
 173         fpkt = (flags & BPF_JIT_FPKT) != 0;
 174         fmem = (flags & BPF_JIT_FMEM) != 0;
 175         fjmp = (flags & BPF_JIT_FJMP) != 0;
 176         fadk = (flags & BPF_JIT_FADK) != 0;
 177         save_esp = (fpkt || fmem || fadk);      /* Stack is used. */
 178
 179         if (fret)
 180                 nins = 1;
 181
 182         memset(&stream, 0, sizeof(stream));
 183
 184         /* Allocate the reference table for the jumps. */
 185         if (fjmp) {
 186 #ifdef _KERNEL
 187                 stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT,
 188                     M_NOWAIT | M_ZERO);
 189 #else
 190                 stream.refs = calloc(nins + 1, sizeof(u_int));
 191 #endif
 192                 if (stream.refs == NULL)
 193                         return (NULL);
 194         }
 195
 196         /*
 197          * The first pass will emit the lengths of the instructions
 198          * to create the reference table.
 199          */
 200         emitm = emit_length;
 201
 202         for (pass = 0; pass < 2; pass++) {
 203                 ins = prog;
 204
 205                 /* Create the procedure header. */
 206                 if (save_esp) {
 207                         PUSH(EBP);
 208                         MOVrd(ESP, EBP);
 209                 }
 210                 if (fmem)
 211                         SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP);
 212                 if (save_esp)
 213                         PUSH(ESI);
 214                 if (fpkt) {
 215                         PUSH(EDI);
 216                         PUSH(EBX);
 217                         MOVodd(8, EBP, EBX);
 218                         MOVodd(16, EBP, EDI);
 219                 }
 220
 221                 for (i = 0; i < nins; i++) {
 222                         stream.bpf_pc++;
 223
 224                         switch (ins->code) {
 225                         default:
 226 #ifdef _KERNEL
 227                                 return (NULL);
 228 #else
 229                                 abort();
 230 #endif
 231
 232                         case BPF_RET|BPF_K:
 233                                 MOVid(ins->k, EAX);
 234                                 if (save_esp) {
 235                                         if (fpkt) {
 236                                                 POP(EBX);
 237                                                 POP(EDI);
 238                                         }
 239                                         POP(ESI);
 240                                         LEAVE();
 241                                 }
 242                                 RET();
 243                                 break;
 244
 245                         case BPF_RET|BPF_A:
 246                                 if (save_esp) {
 247                                         if (fpkt) {
 248                                                 POP(EBX);
 249                                                 POP(EDI);
 250                                         }
 251                                         POP(ESI);
 252                                         LEAVE();
 253                                 }
 254                                 RET();
 255                                 break;
 256
 257                         case BPF_LD|BPF_W|BPF_ABS:
 258                                 MOVid(ins->k, ESI);
 259                                 CMPrd(EDI, ESI);
 260                                 JAb(12);
 261                                 MOVrd(EDI, ECX);
 262                                 SUBrd(ESI, ECX);
 263                                 CMPid(sizeof(int32_t), ECX);
 264                                 JAEb(7);
 265                                 ZEROrd(EAX);
 266                                 POP(EBX);
 267                                 POP(EDI);
 268                                 POP(ESI);
 269                                 LEAVE();
 270                                 RET();
 271                                 MOVobd(EBX, ESI, EAX);
 272                                 BSWAP(EAX);
 273                                 break;
 274
 275                         case BPF_LD|BPF_H|BPF_ABS:
 276                                 ZEROrd(EAX);
 277                                 MOVid(ins->k, ESI);
 278                                 CMPrd(EDI, ESI);
 279                                 JAb(12);
 280                                 MOVrd(EDI, ECX);
 281                                 SUBrd(ESI, ECX);
 282                                 CMPid(sizeof(int16_t), ECX);
 283                                 JAEb(5);
 284                                 POP(EBX);
 285                                 POP(EDI);
 286                                 POP(ESI);
 287                                 LEAVE();
 288                                 RET();
 289                                 MOVobw(EBX, ESI, AX);
 290                                 SWAP_AX();
 291                                 break;
 292
 293                         case BPF_LD|BPF_B|BPF_ABS:
 294                                 ZEROrd(EAX);
 295                                 MOVid(ins->k, ESI);
 296                                 CMPrd(EDI, ESI);
 297                                 JBb(5);
 298                                 POP(EBX);
 299                                 POP(EDI);
 300                                 POP(ESI);
 301                                 LEAVE();
 302                                 RET();
 303                                 MOVobb(EBX, ESI, AL);
 304                                 break;
 305
 306                         case BPF_LD|BPF_W|BPF_LEN:
 307                                 if (save_esp)
 308                                         MOVodd(12, EBP, EAX);
 309                                 else {
 310                                         MOVrd(ESP, ECX);
 311                                         MOVodd(12, ECX, EAX);
 312                                 }
 313                                 break;
 314
 315                         case BPF_LDX|BPF_W|BPF_LEN:
 316                                 if (save_esp)
 317                                         MOVodd(12, EBP, EDX);
 318                                 else {
 319                                         MOVrd(ESP, ECX);
 320                                         MOVodd(12, ECX, EDX);
 321                                 }
 322                                 break;
 323
 324                         case BPF_LD|BPF_W|BPF_IND:
 325                                 CMPrd(EDI, EDX);
 326                                 JAb(27);
 327                                 MOVid(ins->k, ESI);
 328                                 MOVrd(EDI, ECX);
 329                                 SUBrd(EDX, ECX);
 330                                 CMPrd(ESI, ECX);
 331                                 JBb(14);
 332                                 ADDrd(EDX, ESI);
 333                                 MOVrd(EDI, ECX);
 334                                 SUBrd(ESI, ECX);
 335                                 CMPid(sizeof(int32_t), ECX);
 336                                 JAEb(7);
 337                                 ZEROrd(EAX);
 338                                 POP(EBX);
 339                                 POP(EDI);
 340                                 POP(ESI);
 341                                 LEAVE();
 342                                 RET();
 343                                 MOVobd(EBX, ESI, EAX);
 344                                 BSWAP(EAX);
 345                                 break;
 346
 347                         case BPF_LD|BPF_H|BPF_IND:
 348                                 ZEROrd(EAX);
 349                                 CMPrd(EDI, EDX);
 350                                 JAb(27);
 351                                 MOVid(ins->k, ESI);
 352                                 MOVrd(EDI, ECX);
 353                                 SUBrd(EDX, ECX);
 354                                 CMPrd(ESI, ECX);
 355                                 JBb(14);
 356                                 ADDrd(EDX, ESI);
 357                                 MOVrd(EDI, ECX);
 358                                 SUBrd(ESI, ECX);
 359                                 CMPid(sizeof(int16_t), ECX);
 360                                 JAEb(5);
 361                                 POP(EBX);
 362                                 POP(EDI);
 363                                 POP(ESI);
 364                                 LEAVE();
 365                                 RET();
 366                                 MOVobw(EBX, ESI, AX);
 367                                 SWAP_AX();
 368                                 break;
 369
 370                         case BPF_LD|BPF_B|BPF_IND:
 371                                 ZEROrd(EAX);
 372                                 CMPrd(EDI, EDX);
 373                                 JAEb(13);
 374                                 MOVid(ins->k, ESI);
 375                                 MOVrd(EDI, ECX);
 376                                 SUBrd(EDX, ECX);
 377                                 CMPrd(ESI, ECX);
 378                                 JAb(5);
 379                                 POP(EBX);
 380                                 POP(EDI);
 381                                 POP(ESI);
 382                                 LEAVE();
 383                                 RET();
 384                                 ADDrd(EDX, ESI);
 385                                 MOVobb(EBX, ESI, AL);
 386                                 break;
 387
 388                         case BPF_LDX|BPF_MSH|BPF_B:
 389                                 MOVid(ins->k, ESI);
 390                                 CMPrd(EDI, ESI);
 391                                 JBb(7);
 392                                 ZEROrd(EAX);
 393                                 POP(EBX);
 394                                 POP(EDI);
 395                                 POP(ESI);
 396                                 LEAVE();
 397                                 RET();
 398                                 ZEROrd(EDX);
 399                                 MOVobb(EBX, ESI, DL);
 400                                 ANDib(0x0f, DL);
 401                                 SHLib(2, EDX);
 402                                 break;
 403
 404                         case BPF_LD|BPF_IMM:
 405                                 MOVid(ins->k, EAX);
 406                                 break;
 407
 408                         case BPF_LDX|BPF_IMM:
 409                                 MOVid(ins->k, EDX);
 410                                 break;
 411
 412                         case BPF_LD|BPF_MEM:
 413                                 MOVrd(EBP, ECX);
 414                                 MOVid(((int)ins->k - BPF_MEMWORDS) *
 415                                     sizeof(uint32_t), ESI);
 416                                 MOVobd(ECX, ESI, EAX);
 417                                 break;
 418
 419                         case BPF_LDX|BPF_MEM:
 420                                 MOVrd(EBP, ECX);
 421                                 MOVid(((int)ins->k - BPF_MEMWORDS) *
 422                                     sizeof(uint32_t), ESI);
 423                                 MOVobd(ECX, ESI, EDX);
 424                                 break;
 425
 426                         case BPF_ST:
 427                                 /*
 428                                  * XXX this command and the following could
 429                                  * be optimized if the previous instruction
 430                                  * was already of this type
 431                                  */
 432                                 MOVrd(EBP, ECX);
 433                                 MOVid(((int)ins->k - BPF_MEMWORDS) *
 434                                     sizeof(uint32_t), ESI);
 435                                 MOVomd(EAX, ECX, ESI);
 436                                 break;
 437
 438                         case BPF_STX:
 439                                 MOVrd(EBP, ECX);
 440                                 MOVid(((int)ins->k - BPF_MEMWORDS) *
 441                                     sizeof(uint32_t), ESI);
 442                                 MOVomd(EDX, ECX, ESI);
 443                                 break;
 444
 445                         case BPF_JMP|BPF_JA:
 446                                 JUMP(ins->k);
 447                                 break;
 448
 449                         case BPF_JMP|BPF_JGT|BPF_K:
 450                         case BPF_JMP|BPF_JGE|BPF_K:
 451                         case BPF_JMP|BPF_JEQ|BPF_K:
 452                         case BPF_JMP|BPF_JSET|BPF_K:
 453                         case BPF_JMP|BPF_JGT|BPF_X:
 454                         case BPF_JMP|BPF_JGE|BPF_X:
 455                         case BPF_JMP|BPF_JEQ|BPF_X:
 456                         case BPF_JMP|BPF_JSET|BPF_X:
 457                                 if (ins->jt == ins->jf) {
 458                                         JUMP(ins->jt);
 459                                         break;
 460                                 }
 461                                 switch (ins->code) {
 462                                 case BPF_JMP|BPF_JGT|BPF_K:
 463                                         CMPid(ins->k, EAX);
 464                                         JCC(JA, JBE);
 465                                         break;
 466
 467                                 case BPF_JMP|BPF_JGE|BPF_K:
 468                                         CMPid(ins->k, EAX);
 469                                         JCC(JAE, JB);
 470                                         break;
 471
 472                                 case BPF_JMP|BPF_JEQ|BPF_K:
 473                                         CMPid(ins->k, EAX);
 474                                         JCC(JE, JNE);
 475                                         break;
 476
 477                                 case BPF_JMP|BPF_JSET|BPF_K:
 478                                         TESTid(ins->k, EAX);
 479                                         JCC(JNE, JE);
 480                                         break;
 481
 482                                 case BPF_JMP|BPF_JGT|BPF_X:
 483                                         CMPrd(EDX, EAX);
 484                                         JCC(JA, JBE);
 485                                         break;
 486
 487                                 case BPF_JMP|BPF_JGE|BPF_X:
 488                                         CMPrd(EDX, EAX);
 489                                         JCC(JAE, JB);
 490                                         break;
 491
 492                                 case BPF_JMP|BPF_JEQ|BPF_X:
 493                                         CMPrd(EDX, EAX);
 494                                         JCC(JE, JNE);
 495                                         break;
 496
 497                                 case BPF_JMP|BPF_JSET|BPF_X:
 498                                         TESTrd(EDX, EAX);
 499                                         JCC(JNE, JE);
 500                                         break;
 501                                 }
 502                                 break;
 503
 504                         case BPF_ALU|BPF_ADD|BPF_X:
 505                                 ADDrd(EDX, EAX);
 506                                 break;
 507
 508                         case BPF_ALU|BPF_SUB|BPF_X:
 509                                 SUBrd(EDX, EAX);
 510                                 break;
 511
 512                         case BPF_ALU|BPF_MUL|BPF_X:
 513                                 MOVrd(EDX, ECX);
 514                                 MULrd(EDX);
 515                                 MOVrd(ECX, EDX);
 516                                 break;
 517
 518                         case BPF_ALU|BPF_DIV|BPF_X:
 519                         case BPF_ALU|BPF_MOD|BPF_X:
 520                                 TESTrd(EDX, EDX);
 521                                 if (save_esp) {
 522                                         if (fpkt) {
 523                                                 JNEb(7);
 524                                                 ZEROrd(EAX);
 525                                                 POP(EBX);
 526                                                 POP(EDI);
 527                                         } else {
 528                                                 JNEb(5);
 529                                                 ZEROrd(EAX);
 530                                         }
 531                                         POP(ESI);
 532                                         LEAVE();
 533                                 } else {
 534                                         JNEb(3);
 535                                         ZEROrd(EAX);
 536                                 }
 537                                 RET();
 538                                 MOVrd(EDX, ECX);
 539                                 ZEROrd(EDX);
 540                                 DIVrd(ECX);
 541                                 if (BPF_OP(ins->code) == BPF_MOD)
 542                                         MOVrd(EDX, EAX);
 543                                 MOVrd(ECX, EDX);
 544                                 break;
 545
 546                         case BPF_ALU|BPF_AND|BPF_X:
 547                                 ANDrd(EDX, EAX);
 548                                 break;
 549
 550                         case BPF_ALU|BPF_OR|BPF_X:
 551                                 ORrd(EDX, EAX);
 552                                 break;
 553
 554                         case BPF_ALU|BPF_XOR|BPF_X:
 555                                 XORrd(EDX, EAX);
 556                                 break;
 557
 558                         case BPF_ALU|BPF_LSH|BPF_X:
 559                                 MOVrd(EDX, ECX);
 560                                 SHL_CLrb(EAX);
 561                                 break;
 562
 563                         case BPF_ALU|BPF_RSH|BPF_X:
 564                                 MOVrd(EDX, ECX);
 565                                 SHR_CLrb(EAX);
 566                                 break;
 567
 568                         case BPF_ALU|BPF_ADD|BPF_K:
 569                                 ADD_EAXi(ins->k);
 570                                 break;
 571
 572                         case BPF_ALU|BPF_SUB|BPF_K:
 573                                 SUB_EAXi(ins->k);
 574                                 break;
 575
 576                         case BPF_ALU|BPF_MUL|BPF_K:
 577                                 MOVrd(EDX, ECX);
 578                                 MOVid(ins->k, EDX);
 579                                 MULrd(EDX);
 580                                 MOVrd(ECX, EDX);
 581                                 break;
 582
 583                         case BPF_ALU|BPF_DIV|BPF_K:
 584                         case BPF_ALU|BPF_MOD|BPF_K:
 585                                 MOVrd(EDX, ECX);
 586                                 ZEROrd(EDX);
 587                                 MOVid(ins->k, ESI);
 588                                 DIVrd(ESI);
 589                                 if (BPF_OP(ins->code) == BPF_MOD)
 590                                         MOVrd(EDX, EAX);
 591                                 MOVrd(ECX, EDX);
 592                                 break;
 593
 594                         case BPF_ALU|BPF_AND|BPF_K:
 595                                 ANDid(ins->k, EAX);
 596                                 break;
 597
 598                         case BPF_ALU|BPF_OR|BPF_K:
 599                                 ORid(ins->k, EAX);
 600                                 break;
 601
 602                         case BPF_ALU|BPF_XOR|BPF_K:
 603                                 XORid(ins->k, EAX);
 604                                 break;
 605
 606                         case BPF_ALU|BPF_LSH|BPF_K:
 607                                 SHLib((ins->k) & 0xff, EAX);
 608                                 break;
 609
 610                         case BPF_ALU|BPF_RSH|BPF_K:
 611                                 SHRib((ins->k) & 0xff, EAX);
 612                                 break;
 613
 614                         case BPF_ALU|BPF_NEG:
 615                                 NEGd(EAX);
 616                                 break;
 617
 618                         case BPF_MISC|BPF_TAX:
 619                                 MOVrd(EAX, EDX);
 620                                 break;
 621
 622                         case BPF_MISC|BPF_TXA:
 623                                 MOVrd(EDX, EAX);
 624                                 break;
 625                         }
 626                         ins++;
 627                 }
 628
 629                 if (pass > 0)
 630                         continue;
 631
 632                 *size = stream.cur_ip;
 633 #ifdef _KERNEL
 634                 stream.ibuf = malloc(*size, M_BPFJIT, M_NOWAIT);
 635                 if (stream.ibuf == NULL)
 636                         break;
 637 #else
 638                 stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE,
 639                     MAP_ANON, -1, 0);
 640                 if (stream.ibuf == MAP_FAILED) {
 641                         stream.ibuf = NULL;
 642                         break;
 643                 }
 644 #endif
 645
 646                 /*
 647                  * Modify the reference table to contain the offsets and
 648                  * not the lengths of the instructions.
 649                  */
 650                 if (fjmp)
 651                         for (i = 1; i < nins + 1; i++)
 652                                 stream.refs[i] += stream.refs[i - 1];
 653
 654                 /* Reset the counters. */
 655                 stream.cur_ip = 0;
 656                 stream.bpf_pc = 0;
 657
 658                 /* The second pass creates the actual code. */
 659                 emitm = emit_code;
 660         }
 661
 662         /*
 663          * The reference table is needed only during compilation,
 664          * now we can free it.
 665          */
 666         if (fjmp)
 667 #ifdef _KERNEL
 668                 free(stream.refs, M_BPFJIT);
 669 #else
 670                 free(stream.refs);
 671 #endif
 672
 673 #ifndef _KERNEL
 674         if (stream.ibuf != NULL &&
 675             mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) {
 676                 munmap(stream.ibuf, *size);
 677                 stream.ibuf = NULL;
 678         }
 679 #endif
 680
 681         return ((bpf_filter_func)(void *)stream.ibuf);
 682 }