sys/netpfil/ipfw/ip_fw_dynamic.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2017-2018 Yandex LLC
   5  * Copyright (c) 2017-2018 Andrey V. Elsukov <ae@FreeBSD.org>
   6  * Copyright (c) 2002 Luigi Rizzo, Universita` di Pisa
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_inet.h"
  34 #include "opt_inet6.h"
  35 #include "opt_ipfw.h"
  36 #ifndef INET
  37 #error IPFIREWALL requires INET.
  38 #endif /* INET */
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #include <sys/hash.h>
  43 #include <sys/mbuf.h>
  44 #include <sys/kernel.h>
  45 #include <sys/lock.h>
  46 #include <sys/pcpu.h>
  47 #include <sys/queue.h>
  48 #include <sys/rmlock.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/syslog.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_var.h>
  56 #include <net/pfil.h>
  57 #include <net/vnet.h>
  58
  59 #include <netinet/in.h>
  60 #include <netinet/ip.h>
  61 #include <netinet/ip_var.h>
  62 #include <netinet/ip_fw.h>
  63 #include <netinet/tcp_var.h>
  64 #include <netinet/udp.h>
  65
  66 #include <netinet/ip6.h>        /* IN6_ARE_ADDR_EQUAL */
  67 #ifdef INET6
  68 #include <netinet6/in6_var.h>
  69 #include <netinet6/ip6_var.h>
  70 #include <netinet6/scope6_var.h>
  71 #endif
  72
  73 #include <netpfil/ipfw/ip_fw_private.h>
  74
  75 #include <machine/in_cksum.h>   /* XXX for in_cksum */
  76
  77 #ifdef MAC
  78 #include <security/mac/mac_framework.h>
  79 #endif
  80
  81 /*
  82  * Description of dynamic states.
  83  *
  84  * Dynamic states are stored in lists accessed through a hash tables
  85  * whose size is curr_dyn_buckets. This value can be modified through
  86  * the sysctl variable dyn_buckets.
  87  *
  88  * Currently there are four tables: dyn_ipv4, dyn_ipv6, dyn_ipv4_parent,
  89  * and dyn_ipv6_parent.
  90  *
  91  * When a packet is received, its address fields hashed, then matched
  92  * against the entries in the corresponding list by addr_type.
  93  * Dynamic states can be used for different purposes:
  94  *  + stateful rules;
  95  *  + enforcing limits on the number of sessions;
  96  *  + in-kernel NAT (not implemented yet)
  97  *
  98  * The lifetime of dynamic states is regulated by dyn_*_lifetime,
  99  * measured in seconds and depending on the flags.
 100  *
 101  * The total number of dynamic states is equal to UMA zone items count.
 102  * The max number of dynamic states is dyn_max. When we reach
 103  * the maximum number of rules we do not create anymore. This is
 104  * done to avoid consuming too much memory, but also too much
 105  * time when searching on each packet (ideally, we should try instead
 106  * to put a limit on the length of the list on each bucket...).
 107  *
 108  * Each state holds a pointer to the parent ipfw rule so we know what
 109  * action to perform. Dynamic rules are removed when the parent rule is
 110  * deleted.
 111  *
 112  * There are some limitations with dynamic rules -- we do not
 113  * obey the 'randomized match', and we do not do multiple
 114  * passes through the firewall. XXX check the latter!!!
 115  */
 116
 117 /* By default use jenkins hash function */
 118 #define IPFIREWALL_JENKINSHASH
 119
 120 #define DYN_COUNTER_INC(d, dir, pktlen) do {    \
 121         (d)->pcnt_ ## dir++;                    \
 122         (d)->bcnt_ ## dir += pktlen;            \
 123         } while (0)
 124
 125 struct dyn_data {
 126         void            *parent;        /* pointer to parent rule */
 127         uint32_t        chain_id;       /* cached ruleset id */
 128         uint32_t        f_pos;          /* cached rule index */
 129
 130         uint32_t        hashval;        /* hash value used for hash resize */
 131         uint16_t        fibnum;         /* fib used to send keepalives */
 132         uint8_t         _pad[3];
 133         uint8_t         set;            /* parent rule set number */
 134         uint16_t        rulenum;        /* parent rule number */
 135         uint32_t        ruleid;         /* parent rule id */
 136
 137         uint32_t        state;          /* TCP session state and flags */
 138         uint32_t        ack_fwd;        /* most recent ACKs in forward */
 139         uint32_t        ack_rev;        /* and reverse direction (used */
 140                                         /* to generate keepalives) */
 141         uint32_t        sync;           /* synchronization time */
 142         uint32_t        expire;         /* expire time */
 143
 144         uint64_t        pcnt_fwd;       /* bytes counter in forward */
 145         uint64_t        bcnt_fwd;       /* packets counter in forward */
 146         uint64_t        pcnt_rev;       /* bytes counter in reverse */
 147         uint64_t        bcnt_rev;       /* packets counter in reverse */
 148 };
 149
 150 #define DPARENT_COUNT_DEC(p)    do {                    \
 151         MPASS(p->count > 0);                            \
 152         ck_pr_dec_32(&(p)->count);                      \
 153 } while (0)
 154 #define DPARENT_COUNT_INC(p)    ck_pr_inc_32(&(p)->count)
 155 #define DPARENT_COUNT(p)        ck_pr_load_32(&(p)->count)
 156 struct dyn_parent {
 157         void            *parent;        /* pointer to parent rule */
 158         uint32_t        count;          /* number of linked states */
 159         uint8_t         _pad;
 160         uint8_t         set;            /* parent rule set number */
 161         uint16_t        rulenum;        /* parent rule number */
 162         uint32_t        ruleid;         /* parent rule id */
 163         uint32_t        hashval;        /* hash value used for hash resize */
 164         uint32_t        expire;         /* expire time */
 165 };
 166
 167 struct dyn_ipv4_state {
 168         uint8_t         type;           /* State type */
 169         uint8_t         proto;          /* UL Protocol */
 170         uint16_t        kidx;           /* named object index */
 171         uint16_t        sport, dport;   /* ULP source and destination ports */
 172         in_addr_t       src, dst;       /* IPv4 source and destination */
 173
 174         union {
 175                 struct dyn_data *data;
 176                 struct dyn_parent *limit;
 177         };
 178         CK_SLIST_ENTRY(dyn_ipv4_state)  entry;
 179         SLIST_ENTRY(dyn_ipv4_state)     expired;
 180 };
 181 CK_SLIST_HEAD(dyn_ipv4ck_slist, dyn_ipv4_state);
 182 static VNET_DEFINE(struct dyn_ipv4ck_slist *, dyn_ipv4);
 183 static VNET_DEFINE(struct dyn_ipv4ck_slist *, dyn_ipv4_parent);
 184
 185 SLIST_HEAD(dyn_ipv4_slist, dyn_ipv4_state);
 186 static VNET_DEFINE(struct dyn_ipv4_slist, dyn_expired_ipv4);
 187 #define V_dyn_ipv4                      VNET(dyn_ipv4)
 188 #define V_dyn_ipv4_parent               VNET(dyn_ipv4_parent)
 189 #define V_dyn_expired_ipv4              VNET(dyn_expired_ipv4)
 190
 191 #ifdef INET6
 192 struct dyn_ipv6_state {
 193         uint8_t         type;           /* State type */
 194         uint8_t         proto;          /* UL Protocol */
 195         uint16_t        kidx;           /* named object index */
 196         uint16_t        sport, dport;   /* ULP source and destination ports */
 197         struct in6_addr src, dst;       /* IPv6 source and destination */
 198         uint32_t        zoneid;         /* IPv6 scope zone id */
 199         union {
 200                 struct dyn_data *data;
 201                 struct dyn_parent *limit;
 202         };
 203         CK_SLIST_ENTRY(dyn_ipv6_state)  entry;
 204         SLIST_ENTRY(dyn_ipv6_state)     expired;
 205 };
 206 CK_SLIST_HEAD(dyn_ipv6ck_slist, dyn_ipv6_state);
 207 static VNET_DEFINE(struct dyn_ipv6ck_slist *, dyn_ipv6);
 208 static VNET_DEFINE(struct dyn_ipv6ck_slist *, dyn_ipv6_parent);
 209
 210 SLIST_HEAD(dyn_ipv6_slist, dyn_ipv6_state);
 211 static VNET_DEFINE(struct dyn_ipv6_slist, dyn_expired_ipv6);
 212 #define V_dyn_ipv6                      VNET(dyn_ipv6)
 213 #define V_dyn_ipv6_parent               VNET(dyn_ipv6_parent)
 214 #define V_dyn_expired_ipv6              VNET(dyn_expired_ipv6)
 215 #endif /* INET6 */
 216
 217 /*
 218  * Per-CPU pointer indicates that specified state is currently in use
 219  * and must not be reclaimed by expiration callout.
 220  */
 221 static void **dyn_hp_cache;
 222 static DPCPU_DEFINE(void *, dyn_hp);
 223 #define DYNSTATE_GET(cpu)       ck_pr_load_ptr(DPCPU_ID_PTR((cpu), dyn_hp))
 224 #define DYNSTATE_PROTECT(v)     ck_pr_store_ptr(DPCPU_PTR(dyn_hp), (v))
 225 #define DYNSTATE_RELEASE()      DYNSTATE_PROTECT(NULL)
 226 #define DYNSTATE_CRITICAL_ENTER()       critical_enter()
 227 #define DYNSTATE_CRITICAL_EXIT()        do {    \
 228         DYNSTATE_RELEASE();                     \
 229         critical_exit();                        \
 230 } while (0);
 231
 232 /*
 233  * We keep two version numbers, one is updated when new entry added to
 234  * the list. Second is updated when an entry deleted from the list.
 235  * Versions are updated under bucket lock.
 236  *
 237  * Bucket "add" version number is used to know, that in the time between
 238  * state lookup (i.e. ipfw_dyn_lookup_state()) and the followed state
 239  * creation (i.e. ipfw_dyn_install_state()) another concurrent thread did
 240  * not install some state in this bucket. Using this info we can avoid
 241  * additional state lookup, because we are sure that we will not install
 242  * the state twice.
 243  *
 244  * Also doing the tracking of bucket "del" version during lookup we can
 245  * be sure, that state entry was not unlinked and freed in time between
 246  * we read the state pointer and protect it with hazard pointer.
 247  *
 248  * An entry unlinked from CK list keeps unchanged until it is freed.
 249  * Unlinked entries are linked into expired lists using "expired" field.
 250  */
 251
 252 /*
 253  * dyn_expire_lock is used to protect access to dyn_expired_xxx lists.
 254  * dyn_bucket_lock is used to get write access to lists in specific bucket.
 255  * Currently one dyn_bucket_lock is used for all ipv4, ipv4_parent, ipv6,
 256  * and ipv6_parent lists.
 257  */
 258 static VNET_DEFINE(struct mtx, dyn_expire_lock);
 259 static VNET_DEFINE(struct mtx *, dyn_bucket_lock);
 260 #define V_dyn_expire_lock               VNET(dyn_expire_lock)
 261 #define V_dyn_bucket_lock               VNET(dyn_bucket_lock)
 262
 263 /*
 264  * Bucket's add/delete generation versions.
 265  */
 266 static VNET_DEFINE(uint32_t *, dyn_ipv4_add);
 267 static VNET_DEFINE(uint32_t *, dyn_ipv4_del);
 268 static VNET_DEFINE(uint32_t *, dyn_ipv4_parent_add);
 269 static VNET_DEFINE(uint32_t *, dyn_ipv4_parent_del);
 270 #define V_dyn_ipv4_add                  VNET(dyn_ipv4_add)
 271 #define V_dyn_ipv4_del                  VNET(dyn_ipv4_del)
 272 #define V_dyn_ipv4_parent_add           VNET(dyn_ipv4_parent_add)
 273 #define V_dyn_ipv4_parent_del           VNET(dyn_ipv4_parent_del)
 274
 275 #ifdef INET6
 276 static VNET_DEFINE(uint32_t *, dyn_ipv6_add);
 277 static VNET_DEFINE(uint32_t *, dyn_ipv6_del);
 278 static VNET_DEFINE(uint32_t *, dyn_ipv6_parent_add);
 279 static VNET_DEFINE(uint32_t *, dyn_ipv6_parent_del);
 280 #define V_dyn_ipv6_add                  VNET(dyn_ipv6_add)
 281 #define V_dyn_ipv6_del                  VNET(dyn_ipv6_del)
 282 #define V_dyn_ipv6_parent_add           VNET(dyn_ipv6_parent_add)
 283 #define V_dyn_ipv6_parent_del           VNET(dyn_ipv6_parent_del)
 284 #endif /* INET6 */
 285
 286 #define DYN_BUCKET(h, b)                ((h) & (b - 1))
 287 #define DYN_BUCKET_VERSION(b, v)        ck_pr_load_32(&V_dyn_ ## v[(b)])
 288 #define DYN_BUCKET_VERSION_BUMP(b, v)   ck_pr_inc_32(&V_dyn_ ## v[(b)])
 289
 290 #define DYN_BUCKET_LOCK_INIT(lock, b)           \
 291     mtx_init(&lock[(b)], "IPFW dynamic bucket", NULL, MTX_DEF)
 292 #define DYN_BUCKET_LOCK_DESTROY(lock, b)        mtx_destroy(&lock[(b)])
 293 #define DYN_BUCKET_LOCK(b)      mtx_lock(&V_dyn_bucket_lock[(b)])
 294 #define DYN_BUCKET_UNLOCK(b)    mtx_unlock(&V_dyn_bucket_lock[(b)])
 295 #define DYN_BUCKET_ASSERT(b)    mtx_assert(&V_dyn_bucket_lock[(b)], MA_OWNED)
 296
 297 #define DYN_EXPIRED_LOCK_INIT()         \
 298     mtx_init(&V_dyn_expire_lock, "IPFW expired states list", NULL, MTX_DEF)
 299 #define DYN_EXPIRED_LOCK_DESTROY()      mtx_destroy(&V_dyn_expire_lock)
 300 #define DYN_EXPIRED_LOCK()              mtx_lock(&V_dyn_expire_lock)
 301 #define DYN_EXPIRED_UNLOCK()            mtx_unlock(&V_dyn_expire_lock)
 302
 303 static VNET_DEFINE(uint32_t, dyn_buckets_max);
 304 static VNET_DEFINE(uint32_t, curr_dyn_buckets);
 305 static VNET_DEFINE(struct callout, dyn_timeout);
 306 #define V_dyn_buckets_max               VNET(dyn_buckets_max)
 307 #define V_curr_dyn_buckets              VNET(curr_dyn_buckets)
 308 #define V_dyn_timeout                   VNET(dyn_timeout)
 309
 310 /* Maximum length of states chain in a bucket */
 311 static VNET_DEFINE(uint32_t, curr_max_length);
 312 #define V_curr_max_length               VNET(curr_max_length)
 313
 314 static VNET_DEFINE(uma_zone_t, dyn_data_zone);
 315 static VNET_DEFINE(uma_zone_t, dyn_parent_zone);
 316 static VNET_DEFINE(uma_zone_t, dyn_ipv4_zone);
 317 #ifdef INET6
 318 static VNET_DEFINE(uma_zone_t, dyn_ipv6_zone);
 319 #define V_dyn_ipv6_zone                 VNET(dyn_ipv6_zone)
 320 #endif /* INET6 */
 321 #define V_dyn_data_zone                 VNET(dyn_data_zone)
 322 #define V_dyn_parent_zone               VNET(dyn_parent_zone)
 323 #define V_dyn_ipv4_zone                 VNET(dyn_ipv4_zone)
 324
 325 /*
 326  * Timeouts for various events in handing dynamic rules.
 327  */
 328 static VNET_DEFINE(uint32_t, dyn_ack_lifetime);
 329 static VNET_DEFINE(uint32_t, dyn_syn_lifetime);
 330 static VNET_DEFINE(uint32_t, dyn_fin_lifetime);
 331 static VNET_DEFINE(uint32_t, dyn_rst_lifetime);
 332 static VNET_DEFINE(uint32_t, dyn_udp_lifetime);
 333 static VNET_DEFINE(uint32_t, dyn_short_lifetime);
 334
 335 #define V_dyn_ack_lifetime              VNET(dyn_ack_lifetime)
 336 #define V_dyn_syn_lifetime              VNET(dyn_syn_lifetime)
 337 #define V_dyn_fin_lifetime              VNET(dyn_fin_lifetime)
 338 #define V_dyn_rst_lifetime              VNET(dyn_rst_lifetime)
 339 #define V_dyn_udp_lifetime              VNET(dyn_udp_lifetime)
 340 #define V_dyn_short_lifetime            VNET(dyn_short_lifetime)
 341
 342 /*
 343  * Keepalives are sent if dyn_keepalive is set. They are sent every
 344  * dyn_keepalive_period seconds, in the last dyn_keepalive_interval
 345  * seconds of lifetime of a rule.
 346  * dyn_rst_lifetime and dyn_fin_lifetime should be strictly lower
 347  * than dyn_keepalive_period.
 348  */
 349 #define DYN_KEEPALIVE_MAXQ              512
 350 static VNET_DEFINE(uint32_t, dyn_keepalive_interval);
 351 static VNET_DEFINE(uint32_t, dyn_keepalive_period);
 352 static VNET_DEFINE(uint32_t, dyn_keepalive);
 353 static VNET_DEFINE(time_t, dyn_keepalive_last);
 354
 355 #define V_dyn_keepalive_interval        VNET(dyn_keepalive_interval)
 356 #define V_dyn_keepalive_period          VNET(dyn_keepalive_period)
 357 #define V_dyn_keepalive                 VNET(dyn_keepalive)
 358 #define V_dyn_keepalive_last            VNET(dyn_keepalive_last)
 359
 360 static VNET_DEFINE(uint32_t, dyn_max);          /* max # of dynamic states */
 361 static VNET_DEFINE(uint32_t, dyn_count);        /* number of states */
 362 static VNET_DEFINE(uint32_t, dyn_parent_max);   /* max # of parent states */
 363 static VNET_DEFINE(uint32_t, dyn_parent_count); /* number of parent states */
 364 #define V_dyn_max                       VNET(dyn_max)
 365 #define V_dyn_count                     VNET(dyn_count)
 366 #define V_dyn_parent_max                VNET(dyn_parent_max)
 367 #define V_dyn_parent_count              VNET(dyn_parent_count)
 368
 369 #define DYN_COUNT_DEC(name)     do {                    \
 370         MPASS((V_ ## name) > 0);                        \
 371         ck_pr_dec_32(&(V_ ## name));                    \
 372 } while (0)
 373 #define DYN_COUNT_INC(name)     ck_pr_inc_32(&(V_ ## name))
 374 #define DYN_COUNT(name)         ck_pr_load_32(&(V_ ## name))
 375
 376 static time_t last_log; /* Log ratelimiting */
 377
 378 /*
 379  * Get/set maximum number of dynamic states in given VNET instance.
 380  */
 381 static int
 382 sysctl_dyn_max(SYSCTL_HANDLER_ARGS)
 383 {
 384         uint32_t nstates;
 385         int error;
 386
 387         nstates = V_dyn_max;
 388         error = sysctl_handle_32(oidp, &nstates, 0, req);
 389         /* Read operation or some error */
 390         if ((error != 0) || (req->newptr == NULL))
 391                 return (error);
 392
 393         V_dyn_max = nstates;
 394         uma_zone_set_max(V_dyn_data_zone, V_dyn_max);
 395         return (0);
 396 }
 397
 398 static int
 399 sysctl_dyn_parent_max(SYSCTL_HANDLER_ARGS)
 400 {
 401         uint32_t nstates;
 402         int error;
 403
 404         nstates = V_dyn_parent_max;
 405         error = sysctl_handle_32(oidp, &nstates, 0, req);
 406         /* Read operation or some error */
 407         if ((error != 0) || (req->newptr == NULL))
 408                 return (error);
 409
 410         V_dyn_parent_max = nstates;
 411         uma_zone_set_max(V_dyn_parent_zone, V_dyn_parent_max);
 412         return (0);
 413 }
 414
 415 static int
 416 sysctl_dyn_buckets(SYSCTL_HANDLER_ARGS)
 417 {
 418         uint32_t nbuckets;
 419         int error;
 420
 421         nbuckets = V_dyn_buckets_max;
 422         error = sysctl_handle_32(oidp, &nbuckets, 0, req);
 423         /* Read operation or some error */
 424         if ((error != 0) || (req->newptr == NULL))
 425                 return (error);
 426
 427         if (nbuckets > 256)
 428                 V_dyn_buckets_max = 1 << fls(nbuckets - 1);
 429         else
 430                 return (EINVAL);
 431         return (0);
 432 }
 433
 434 SYSCTL_DECL(_net_inet_ip_fw);
 435
 436 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_count,
 437     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(dyn_count), 0,
 438     "Current number of dynamic states.");
 439 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_parent_count,
 440     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(dyn_parent_count), 0,
 441     "Current number of parent states. ");
 442 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, curr_dyn_buckets,
 443     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(curr_dyn_buckets), 0,
 444     "Current number of buckets for states hash table.");
 445 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, curr_max_length,
 446     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(curr_max_length), 0,
 447     "Current maximum length of states chains in hash buckets.");
 448 SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_buckets,
 449     CTLFLAG_VNET | CTLTYPE_U32 | CTLFLAG_RW, 0, 0, sysctl_dyn_buckets,
 450     "IU", "Max number of buckets for dynamic states hash table.");
 451 SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_max,
 452     CTLFLAG_VNET | CTLTYPE_U32 | CTLFLAG_RW, 0, 0, sysctl_dyn_max,
 453     "IU", "Max number of dynamic states.");
 454 SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_parent_max,
 455     CTLFLAG_VNET | CTLTYPE_U32 | CTLFLAG_RW, 0, 0, sysctl_dyn_parent_max,
 456     "IU", "Max number of parent dynamic states.");
 457 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_ack_lifetime,
 458     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_ack_lifetime), 0,
 459     "Lifetime of dynamic states for TCP ACK.");
 460 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_syn_lifetime,
 461     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_syn_lifetime), 0,
 462     "Lifetime of dynamic states for TCP SYN.");
 463 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_fin_lifetime,
 464     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_fin_lifetime), 0,
 465     "Lifetime of dynamic states for TCP FIN.");
 466 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_rst_lifetime,
 467     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_rst_lifetime), 0,
 468     "Lifetime of dynamic states for TCP RST.");
 469 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_udp_lifetime,
 470     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_udp_lifetime), 0,
 471     "Lifetime of dynamic states for UDP.");
 472 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_short_lifetime,
 473     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_short_lifetime), 0,
 474     "Lifetime of dynamic states for other situations.");
 475 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_keepalive,
 476     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_keepalive), 0,
 477     "Enable keepalives for dynamic states.");
 478
 479 #ifdef IPFIREWALL_DYNDEBUG
 480 #define DYN_DEBUG(fmt, ...)     do {                    \
 481         printf("%s: " fmt "\n", __func__, __VA_ARGS__); \
 482 } while (0)
 483 #else
 484 #define DYN_DEBUG(fmt, ...)
 485 #endif /* !IPFIREWALL_DYNDEBUG */
 486
 487 #ifdef INET6
 488 /* Functions to work with IPv6 states */
 489 static struct dyn_ipv6_state *dyn_lookup_ipv6_state(
 490     const struct ipfw_flow_id *, uint32_t, const void *,
 491     struct ipfw_dyn_info *, int);
 492 static int dyn_lookup_ipv6_state_locked(const struct ipfw_flow_id *,
 493     uint32_t, const void *, int, const void *, uint32_t, uint16_t, uint32_t,
 494     uint16_t);
 495 static struct dyn_ipv6_state *dyn_alloc_ipv6_state(
 496     const struct ipfw_flow_id *, uint32_t, uint16_t, uint8_t);
 497 static int dyn_add_ipv6_state(void *, uint32_t, uint16_t, uint8_t,
 498     const struct ipfw_flow_id *, uint32_t, const void *, int, uint32_t,
 499     struct ipfw_dyn_info *, uint16_t, uint16_t, uint8_t);
 500 static void dyn_export_ipv6_state(const struct dyn_ipv6_state *,
 501     ipfw_dyn_rule *);
 502
 503 static uint32_t dyn_getscopeid(const struct ip_fw_args *);
 504 static void dyn_make_keepalive_ipv6(struct mbuf *, const struct in6_addr *,
 505     const struct in6_addr *, uint32_t, uint32_t, uint32_t, uint16_t,
 506     uint16_t);
 507 static void dyn_enqueue_keepalive_ipv6(struct mbufq *,
 508     const struct dyn_ipv6_state *);
 509 static void dyn_send_keepalive_ipv6(struct ip_fw_chain *);
 510
 511 static struct dyn_ipv6_state *dyn_lookup_ipv6_parent(
 512     const struct ipfw_flow_id *, uint32_t, const void *, uint32_t, uint16_t,
 513     uint32_t);
 514 static struct dyn_ipv6_state *dyn_lookup_ipv6_parent_locked(
 515     const struct ipfw_flow_id *, uint32_t, const void *, uint32_t, uint16_t,
 516     uint32_t);
 517 static struct dyn_ipv6_state *dyn_add_ipv6_parent(void *, uint32_t, uint16_t,
 518     uint8_t, const struct ipfw_flow_id *, uint32_t, uint32_t, uint32_t,
 519     uint16_t);
 520 #endif /* INET6 */
 521
 522 /* Functions to work with limit states */
 523 static void *dyn_get_parent_state(const struct ipfw_flow_id *, uint32_t,
 524     struct ip_fw *, uint32_t, uint32_t, uint16_t);
 525 static struct dyn_ipv4_state *dyn_lookup_ipv4_parent(
 526     const struct ipfw_flow_id *, const void *, uint32_t, uint16_t, uint32_t);
 527 static struct dyn_ipv4_state *dyn_lookup_ipv4_parent_locked(
 528     const struct ipfw_flow_id *, const void *, uint32_t, uint16_t, uint32_t);
 529 static struct dyn_parent *dyn_alloc_parent(void *, uint32_t, uint16_t,
 530     uint8_t, uint32_t);
 531 static struct dyn_ipv4_state *dyn_add_ipv4_parent(void *, uint32_t, uint16_t,
 532     uint8_t, const struct ipfw_flow_id *, uint32_t, uint32_t, uint16_t);
 533
 534 static void dyn_tick(void *);
 535 static void dyn_expire_states(struct ip_fw_chain *, ipfw_range_tlv *);
 536 static void dyn_free_states(struct ip_fw_chain *);
 537 static void dyn_export_parent(const struct dyn_parent *, uint16_t,
 538     ipfw_dyn_rule *);
 539 static void dyn_export_data(const struct dyn_data *, uint16_t, uint8_t,
 540     ipfw_dyn_rule *);
 541 static uint32_t dyn_update_tcp_state(struct dyn_data *,
 542     const struct ipfw_flow_id *, const struct tcphdr *, int);
 543 static void dyn_update_proto_state(struct dyn_data *,
 544     const struct ipfw_flow_id *, const void *, int, int);
 545
 546 /* Functions to work with IPv4 states */
 547 struct dyn_ipv4_state *dyn_lookup_ipv4_state(const struct ipfw_flow_id *,
 548     const void *, struct ipfw_dyn_info *, int);
 549 static int dyn_lookup_ipv4_state_locked(const struct ipfw_flow_id *,
 550     const void *, int, const void *, uint32_t, uint16_t, uint32_t, uint16_t);
 551 static struct dyn_ipv4_state *dyn_alloc_ipv4_state(
 552     const struct ipfw_flow_id *, uint16_t, uint8_t);
 553 static int dyn_add_ipv4_state(void *, uint32_t, uint16_t, uint8_t,
 554     const struct ipfw_flow_id *, const void *, int, uint32_t,
 555     struct ipfw_dyn_info *, uint16_t, uint16_t, uint8_t);
 556 static void dyn_export_ipv4_state(const struct dyn_ipv4_state *,
 557     ipfw_dyn_rule *);
 558
 559 /*
 560  * Named states support.
 561  */
 562 static char *default_state_name = "default";
 563 struct dyn_state_obj {
 564         struct named_object     no;
 565         char                    name[64];
 566 };
 567
 568 #define DYN_STATE_OBJ(ch, cmd)  \
 569     ((struct dyn_state_obj *)SRV_OBJECT(ch, (cmd)->arg1))
 570 /*
 571  * Classifier callback.
 572  * Return 0 if opcode contains object that should be referenced
 573  * or rewritten.
 574  */
 575 static int
 576 dyn_classify(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype)
 577 {
 578
 579         DYN_DEBUG("opcode %d, arg1 %d", cmd->opcode, cmd->arg1);
 580         /* Don't rewrite "check-state any" */
 581         if (cmd->arg1 == 0 &&
 582             cmd->opcode == O_CHECK_STATE)
 583                 return (1);
 584
 585         *puidx = cmd->arg1;
 586         *ptype = 0;
 587         return (0);
 588 }
 589
 590 static void
 591 dyn_update(ipfw_insn *cmd, uint16_t idx)
 592 {
 593
 594         cmd->arg1 = idx;
 595         DYN_DEBUG("opcode %d, arg1 %d", cmd->opcode, cmd->arg1);
 596 }
 597
 598 static int
 599 dyn_findbyname(struct ip_fw_chain *ch, struct tid_info *ti,
 600     struct named_object **pno)
 601 {
 602         ipfw_obj_ntlv *ntlv;
 603         const char *name;
 604
 605         DYN_DEBUG("uidx %d", ti->uidx);
 606         if (ti->uidx != 0) {
 607                 if (ti->tlvs == NULL)
 608                         return (EINVAL);
 609                 /* Search ntlv in the buffer provided by user */
 610                 ntlv = ipfw_find_name_tlv_type(ti->tlvs, ti->tlen, ti->uidx,
 611                     IPFW_TLV_STATE_NAME);
 612                 if (ntlv == NULL)
 613                         return (EINVAL);
 614                 name = ntlv->name;
 615         } else
 616                 name = default_state_name;
 617         /*
 618          * Search named object with corresponding name.
 619          * Since states objects are global - ignore the set value
 620          * and use zero instead.
 621          */
 622         *pno = ipfw_objhash_lookup_name_type(CHAIN_TO_SRV(ch), 0,
 623             IPFW_TLV_STATE_NAME, name);
 624         /*
 625          * We always return success here.
 626          * The caller will check *pno and mark object as unresolved,
 627          * then it will automatically create "default" object.
 628          */
 629         return (0);
 630 }
 631
 632 static struct named_object *
 633 dyn_findbykidx(struct ip_fw_chain *ch, uint16_t idx)
 634 {
 635
 636         DYN_DEBUG("kidx %d", idx);
 637         return (ipfw_objhash_lookup_kidx(CHAIN_TO_SRV(ch), idx));
 638 }
 639
 640 static int
 641 dyn_create(struct ip_fw_chain *ch, struct tid_info *ti,
 642     uint16_t *pkidx)
 643 {
 644         struct namedobj_instance *ni;
 645         struct dyn_state_obj *obj;
 646         struct named_object *no;
 647         ipfw_obj_ntlv *ntlv;
 648         char *name;
 649
 650         DYN_DEBUG("uidx %d", ti->uidx);
 651         if (ti->uidx != 0) {
 652                 if (ti->tlvs == NULL)
 653                         return (EINVAL);
 654                 ntlv = ipfw_find_name_tlv_type(ti->tlvs, ti->tlen, ti->uidx,
 655                     IPFW_TLV_STATE_NAME);
 656                 if (ntlv == NULL)
 657                         return (EINVAL);
 658                 name = ntlv->name;
 659         } else
 660                 name = default_state_name;
 661
 662         ni = CHAIN_TO_SRV(ch);
 663         obj = malloc(sizeof(*obj), M_IPFW, M_WAITOK | M_ZERO);
 664         obj->no.name = obj->name;
 665         obj->no.etlv = IPFW_TLV_STATE_NAME;
 666         strlcpy(obj->name, name, sizeof(obj->name));
 667
 668         IPFW_UH_WLOCK(ch);
 669         no = ipfw_objhash_lookup_name_type(ni, 0,
 670             IPFW_TLV_STATE_NAME, name);
 671         if (no != NULL) {
 672                 /*
 673                  * Object is already created.
 674                  * Just return its kidx and bump refcount.
 675                  */
 676                 *pkidx = no->kidx;
 677                 no->refcnt++;
 678                 IPFW_UH_WUNLOCK(ch);
 679                 free(obj, M_IPFW);
 680                 DYN_DEBUG("\tfound kidx %d", *pkidx);
 681                 return (0);
 682         }
 683         if (ipfw_objhash_alloc_idx(ni, &obj->no.kidx) != 0) {
 684                 DYN_DEBUG("\talloc_idx failed for %s", name);
 685                 IPFW_UH_WUNLOCK(ch);
 686                 free(obj, M_IPFW);
 687                 return (ENOSPC);
 688         }
 689         ipfw_objhash_add(ni, &obj->no);
 690         SRV_OBJECT(ch, obj->no.kidx) = obj;
 691         obj->no.refcnt++;
 692         *pkidx = obj->no.kidx;
 693         IPFW_UH_WUNLOCK(ch);
 694         DYN_DEBUG("\tcreated kidx %d", *pkidx);
 695         return (0);
 696 }
 697
 698 static void
 699 dyn_destroy(struct ip_fw_chain *ch, struct named_object *no)
 700 {
 701         struct dyn_state_obj *obj;
 702
 703         IPFW_UH_WLOCK_ASSERT(ch);
 704
 705         KASSERT(no->refcnt == 1,
 706             ("Destroying object '%s' (type %u, idx %u) with refcnt %u",
 707             no->name, no->etlv, no->kidx, no->refcnt));
 708         DYN_DEBUG("kidx %d", no->kidx);
 709         obj = SRV_OBJECT(ch, no->kidx);
 710         SRV_OBJECT(ch, no->kidx) = NULL;
 711         ipfw_objhash_del(CHAIN_TO_SRV(ch), no);
 712         ipfw_objhash_free_idx(CHAIN_TO_SRV(ch), no->kidx);
 713
 714         free(obj, M_IPFW);
 715 }
 716
 717 static struct opcode_obj_rewrite dyn_opcodes[] = {
 718         {
 719                 O_KEEP_STATE, IPFW_TLV_STATE_NAME,
 720                 dyn_classify, dyn_update,
 721                 dyn_findbyname, dyn_findbykidx,
 722                 dyn_create, dyn_destroy
 723         },
 724         {
 725                 O_CHECK_STATE, IPFW_TLV_STATE_NAME,
 726                 dyn_classify, dyn_update,
 727                 dyn_findbyname, dyn_findbykidx,
 728                 dyn_create, dyn_destroy
 729         },
 730         {
 731                 O_PROBE_STATE, IPFW_TLV_STATE_NAME,
 732                 dyn_classify, dyn_update,
 733                 dyn_findbyname, dyn_findbykidx,
 734                 dyn_create, dyn_destroy
 735         },
 736         {
 737                 O_LIMIT, IPFW_TLV_STATE_NAME,
 738                 dyn_classify, dyn_update,
 739                 dyn_findbyname, dyn_findbykidx,
 740                 dyn_create, dyn_destroy
 741         },
 742 };
 743
 744 /*
 745  * IMPORTANT: the hash function for dynamic rules must be commutative
 746  * in source and destination (ip,port), because rules are bidirectional
 747  * and we want to find both in the same bucket.
 748  */
 749 #ifndef IPFIREWALL_JENKINSHASH
 750 static __inline uint32_t
 751 hash_packet(const struct ipfw_flow_id *id)
 752 {
 753         uint32_t i;
 754
 755 #ifdef INET6
 756         if (IS_IP6_FLOW_ID(id))
 757                 i = ntohl((id->dst_ip6.__u6_addr.__u6_addr32[2]) ^
 758                     (id->dst_ip6.__u6_addr.__u6_addr32[3]) ^
 759                     (id->src_ip6.__u6_addr.__u6_addr32[2]) ^
 760                     (id->src_ip6.__u6_addr.__u6_addr32[3]));
 761         else
 762 #endif /* INET6 */
 763         i = (id->dst_ip) ^ (id->src_ip);
 764         i ^= (id->dst_port) ^ (id->src_port);
 765         return (i);
 766 }
 767
 768 static __inline uint32_t
 769 hash_parent(const struct ipfw_flow_id *id, const void *rule)
 770 {
 771
 772         return (hash_packet(id) ^ ((uintptr_t)rule));
 773 }
 774
 775 #else /* IPFIREWALL_JENKINSHASH */
 776
 777 static VNET_DEFINE(uint32_t, dyn_hashseed);
 778 #define V_dyn_hashseed          VNET(dyn_hashseed)
 779
 780 static __inline int
 781 addrcmp4(const struct ipfw_flow_id *id)
 782 {
 783
 784         if (id->src_ip < id->dst_ip)
 785                 return (0);
 786         if (id->src_ip > id->dst_ip)
 787                 return (1);
 788         if (id->src_port <= id->dst_port)
 789                 return (0);
 790         return (1);
 791 }
 792
 793 #ifdef INET6
 794 static __inline int
 795 addrcmp6(const struct ipfw_flow_id *id)
 796 {
 797         int ret;
 798
 799         ret = memcmp(&id->src_ip6, &id->dst_ip6, sizeof(struct in6_addr));
 800         if (ret < 0)
 801                 return (0);
 802         if (ret > 0)
 803                 return (1);
 804         if (id->src_port <= id->dst_port)
 805                 return (0);
 806         return (1);
 807 }
 808
 809 static __inline uint32_t
 810 hash_packet6(const struct ipfw_flow_id *id)
 811 {
 812         struct tuple6 {
 813                 struct in6_addr addr[2];
 814                 uint16_t        port[2];
 815         } t6;
 816
 817         if (addrcmp6(id) == 0) {
 818                 t6.addr[0] = id->src_ip6;
 819                 t6.addr[1] = id->dst_ip6;
 820                 t6.port[0] = id->src_port;
 821                 t6.port[1] = id->dst_port;
 822         } else {
 823                 t6.addr[0] = id->dst_ip6;
 824                 t6.addr[1] = id->src_ip6;
 825                 t6.port[0] = id->dst_port;
 826                 t6.port[1] = id->src_port;
 827         }
 828         return (jenkins_hash32((const uint32_t *)&t6,
 829             sizeof(t6) / sizeof(uint32_t), V_dyn_hashseed));
 830 }
 831 #endif
 832
 833 static __inline uint32_t
 834 hash_packet(const struct ipfw_flow_id *id)
 835 {
 836         struct tuple4 {
 837                 in_addr_t       addr[2];
 838                 uint16_t        port[2];
 839         } t4;
 840
 841         if (IS_IP4_FLOW_ID(id)) {
 842                 /* All fields are in host byte order */
 843                 if (addrcmp4(id) == 0) {
 844                         t4.addr[0] = id->src_ip;
 845                         t4.addr[1] = id->dst_ip;
 846                         t4.port[0] = id->src_port;
 847                         t4.port[1] = id->dst_port;
 848                 } else {
 849                         t4.addr[0] = id->dst_ip;
 850                         t4.addr[1] = id->src_ip;
 851                         t4.port[0] = id->dst_port;
 852                         t4.port[1] = id->src_port;
 853                 }
 854                 return (jenkins_hash32((const uint32_t *)&t4,
 855                     sizeof(t4) / sizeof(uint32_t), V_dyn_hashseed));
 856         } else
 857 #ifdef INET6
 858         if (IS_IP6_FLOW_ID(id))
 859                 return (hash_packet6(id));
 860 #endif
 861         return (0);
 862 }
 863
 864 static __inline uint32_t
 865 hash_parent(const struct ipfw_flow_id *id, const void *rule)
 866 {
 867
 868         return (jenkins_hash32((const uint32_t *)&rule,
 869             sizeof(rule) / sizeof(uint32_t), hash_packet(id)));
 870 }
 871 #endif /* IPFIREWALL_JENKINSHASH */
 872
 873 /*
 874  * Print customizable flow id description via log(9) facility.
 875  */
 876 static void
 877 print_dyn_rule_flags(const struct ipfw_flow_id *id, int dyn_type,
 878     int log_flags, char *prefix, char *postfix)
 879 {
 880         struct in_addr da;
 881 #ifdef INET6
 882         char src[INET6_ADDRSTRLEN], dst[INET6_ADDRSTRLEN];
 883 #else
 884         char src[INET_ADDRSTRLEN], dst[INET_ADDRSTRLEN];
 885 #endif
 886
 887 #ifdef INET6
 888         if (IS_IP6_FLOW_ID(id)) {
 889                 ip6_sprintf(src, &id->src_ip6);
 890                 ip6_sprintf(dst, &id->dst_ip6);
 891         } else
 892 #endif
 893         {
 894                 da.s_addr = htonl(id->src_ip);
 895                 inet_ntop(AF_INET, &da, src, sizeof(src));
 896                 da.s_addr = htonl(id->dst_ip);
 897                 inet_ntop(AF_INET, &da, dst, sizeof(dst));
 898         }
 899         log(log_flags, "ipfw: %s type %d %s %d -> %s %d, %d %s\n",
 900             prefix, dyn_type, src, id->src_port, dst,
 901             id->dst_port, V_dyn_count, postfix);
 902 }
 903
 904 #define print_dyn_rule(id, dtype, prefix, postfix)      \
 905         print_dyn_rule_flags(id, dtype, LOG_DEBUG, prefix, postfix)
 906
 907 #define TIME_LEQ(a,b)   ((int)((a)-(b)) <= 0)
 908 #define TIME_LE(a,b)    ((int)((a)-(b)) < 0)
 909 #define _SEQ_GE(a,b)    ((int)((a)-(b)) >= 0)
 910 #define BOTH_SYN        (TH_SYN | (TH_SYN << 8))
 911 #define BOTH_FIN        (TH_FIN | (TH_FIN << 8))
 912 #define TCP_FLAGS       (TH_FLAGS | (TH_FLAGS << 8))
 913 #define ACK_FWD         0x00010000      /* fwd ack seen */
 914 #define ACK_REV         0x00020000      /* rev ack seen */
 915 #define ACK_BOTH        (ACK_FWD | ACK_REV)
 916
 917 static uint32_t
 918 dyn_update_tcp_state(struct dyn_data *data, const struct ipfw_flow_id *pkt,
 919     const struct tcphdr *tcp, int dir)
 920 {
 921         uint32_t ack, expire;
 922         uint32_t state, old;
 923         uint8_t th_flags;
 924
 925         expire = data->expire;
 926         old = state = data->state;
 927         th_flags = pkt->_flags & (TH_FIN | TH_SYN | TH_RST);
 928         state |= (dir == MATCH_FORWARD) ? th_flags: (th_flags << 8);
 929         switch (state & TCP_FLAGS) {
 930         case TH_SYN:                    /* opening */
 931                 expire = time_uptime + V_dyn_syn_lifetime;
 932                 break;
 933
 934         case BOTH_SYN:                  /* move to established */
 935         case BOTH_SYN | TH_FIN:         /* one side tries to close */
 936         case BOTH_SYN | (TH_FIN << 8):
 937                 if (tcp == NULL)
 938                         break;
 939                 ack = ntohl(tcp->th_ack);
 940                 if (dir == MATCH_FORWARD) {
 941                         if (data->ack_fwd == 0 ||
 942                             _SEQ_GE(ack, data->ack_fwd)) {
 943                                 state |= ACK_FWD;
 944                                 if (data->ack_fwd != ack)
 945                                         ck_pr_store_32(&data->ack_fwd, ack);
 946                         }
 947                 } else {
 948                         if (data->ack_rev == 0 ||
 949                             _SEQ_GE(ack, data->ack_rev)) {
 950                                 state |= ACK_REV;
 951                                 if (data->ack_rev != ack)
 952                                         ck_pr_store_32(&data->ack_rev, ack);
 953                         }
 954                 }
 955                 if ((state & ACK_BOTH) == ACK_BOTH) {
 956                         /*
 957                          * Set expire time to V_dyn_ack_lifetime only if
 958                          * we got ACKs for both directions.
 959                          * We use XOR here to avoid possible state
 960                          * overwriting in concurrent thread.
 961                          */
 962                         expire = time_uptime + V_dyn_ack_lifetime;
 963                         ck_pr_xor_32(&data->state, ACK_BOTH);
 964                 } else if ((data->state & ACK_BOTH) != (state & ACK_BOTH))
 965                         ck_pr_or_32(&data->state, state & ACK_BOTH);
 966                 break;
 967
 968         case BOTH_SYN | BOTH_FIN:       /* both sides closed */
 969                 if (V_dyn_fin_lifetime >= V_dyn_keepalive_period)
 970                         V_dyn_fin_lifetime = V_dyn_keepalive_period - 1;
 971                 expire = time_uptime + V_dyn_fin_lifetime;
 972                 break;
 973
 974         default:
 975                 if (V_dyn_rst_lifetime >= V_dyn_keepalive_period)
 976                         V_dyn_rst_lifetime = V_dyn_keepalive_period - 1;
 977                 expire = time_uptime + V_dyn_rst_lifetime;
 978         }
 979         /* Save TCP state if it was changed */
 980         if ((state & TCP_FLAGS) != (old & TCP_FLAGS))
 981                 ck_pr_or_32(&data->state, state & TCP_FLAGS);
 982         return (expire);
 983 }
 984
 985 /*
 986  * Update ULP specific state.
 987  * For TCP we keep sequence numbers and flags. For other protocols
 988  * currently we update only expire time. Packets and bytes counters
 989  * are also updated here.
 990  */
 991 static void
 992 dyn_update_proto_state(struct dyn_data *data, const struct ipfw_flow_id *pkt,
 993     const void *ulp, int pktlen, int dir)
 994 {
 995         uint32_t expire;
 996
 997         /* NOTE: we are in critical section here. */
 998         switch (pkt->proto) {
 999         case IPPROTO_UDP:
1000         case IPPROTO_UDPLITE:
1001                 expire = time_uptime + V_dyn_udp_lifetime;
1002                 break;
1003         case IPPROTO_TCP:
1004                 expire = dyn_update_tcp_state(data, pkt, ulp, dir);
1005                 break;
1006         default:
1007                 expire = time_uptime + V_dyn_short_lifetime;
1008         }
1009         /*
1010          * Expiration timer has the per-second granularity, no need to update
1011          * it every time when state is matched.
1012          */
1013         if (data->expire != expire)
1014                 ck_pr_store_32(&data->expire, expire);
1015
1016         if (dir == MATCH_FORWARD)
1017                 DYN_COUNTER_INC(data, fwd, pktlen);
1018         else
1019                 DYN_COUNTER_INC(data, rev, pktlen);
1020 }
1021
1022 /*
1023  * Lookup IPv4 state.
1024  * Must be called in critical section.
1025  */
1026 struct dyn_ipv4_state *
1027 dyn_lookup_ipv4_state(const struct ipfw_flow_id *pkt, const void *ulp,
1028     struct ipfw_dyn_info *info, int pktlen)
1029 {
1030         struct dyn_ipv4_state *s;
1031         uint32_t version, bucket;
1032
1033         bucket = DYN_BUCKET(info->hashval, V_curr_dyn_buckets);
1034         info->version = DYN_BUCKET_VERSION(bucket, ipv4_add);
1035 restart:
1036         version = DYN_BUCKET_VERSION(bucket, ipv4_del);
1037         CK_SLIST_FOREACH(s, &V_dyn_ipv4[bucket], entry) {
1038                 DYNSTATE_PROTECT(s);
1039                 if (version != DYN_BUCKET_VERSION(bucket, ipv4_del))
1040                         goto restart;
1041                 if (s->proto != pkt->proto)
1042                         continue;
1043                 if (info->kidx != 0 && s->kidx != info->kidx)
1044                         continue;
1045                 if (s->sport == pkt->src_port && s->dport == pkt->dst_port &&
1046                     s->src == pkt->src_ip && s->dst == pkt->dst_ip) {
1047                         info->direction = MATCH_FORWARD;
1048                         break;
1049                 }
1050                 if (s->sport == pkt->dst_port && s->dport == pkt->src_port &&
1051                     s->src == pkt->dst_ip && s->dst == pkt->src_ip) {
1052                         info->direction = MATCH_REVERSE;
1053                         break;
1054                 }
1055         }
1056
1057         if (s != NULL)
1058                 dyn_update_proto_state(s->data, pkt, ulp, pktlen,
1059                     info->direction);
1060         return (s);
1061 }
1062
1063 /*
1064  * Lookup IPv4 state.
1065  * Simplifed version is used to check that matching state doesn't exist.
1066  */
1067 static int
1068 dyn_lookup_ipv4_state_locked(const struct ipfw_flow_id *pkt,
1069     const void *ulp, int pktlen, const void *parent, uint32_t ruleid,
1070     uint16_t rulenum, uint32_t bucket, uint16_t kidx)
1071 {
1072         struct dyn_ipv4_state *s;
1073         int dir;
1074
1075         dir = MATCH_NONE;
1076         DYN_BUCKET_ASSERT(bucket);
1077         CK_SLIST_FOREACH(s, &V_dyn_ipv4[bucket], entry) {
1078                 if (s->proto != pkt->proto ||
1079                     s->kidx != kidx)
1080                         continue;
1081                 /*
1082                  * XXXAE: Install synchronized state only when there are
1083                  *        no matching states.
1084                  */
1085                 if (pktlen != 0 && (
1086                     s->data->parent != parent ||
1087                     s->data->ruleid != ruleid ||
1088                     s->data->rulenum != rulenum))
1089                         continue;
1090                 if (s->sport == pkt->src_port &&
1091                     s->dport == pkt->dst_port &&
1092                     s->src == pkt->src_ip && s->dst == pkt->dst_ip) {
1093                         dir = MATCH_FORWARD;
1094                         break;
1095                 }
1096                 if (s->sport == pkt->dst_port && s->dport == pkt->src_port &&
1097                     s->src == pkt->dst_ip && s->dst == pkt->src_ip) {
1098                         dir = MATCH_REVERSE;
1099                         break;
1100                 }
1101         }
1102         if (s != NULL)
1103                 dyn_update_proto_state(s->data, pkt, ulp, pktlen, dir);
1104         return (s != NULL);
1105 }
1106
1107 struct dyn_ipv4_state *
1108 dyn_lookup_ipv4_parent(const struct ipfw_flow_id *pkt, const void *rule,
1109     uint32_t ruleid, uint16_t rulenum, uint32_t hashval)
1110 {
1111         struct dyn_ipv4_state *s;
1112         uint32_t version, bucket;
1113
1114         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1115 restart:
1116         version = DYN_BUCKET_VERSION(bucket, ipv4_parent_del);
1117         CK_SLIST_FOREACH(s, &V_dyn_ipv4_parent[bucket], entry) {
1118                 DYNSTATE_PROTECT(s);
1119                 if (version != DYN_BUCKET_VERSION(bucket, ipv4_parent_del))
1120                         goto restart;
1121                 /*
1122                  * NOTE: we do not need to check kidx, because parent rule
1123                  * can not create states with different kidx.
1124                  * And parent rule always created for forward direction.
1125                  */
1126                 if (s->limit->parent == rule &&
1127                     s->limit->ruleid == ruleid &&
1128                     s->limit->rulenum == rulenum &&
1129                     s->proto == pkt->proto &&
1130                     s->sport == pkt->src_port &&
1131                     s->dport == pkt->dst_port &&
1132                     s->src == pkt->src_ip && s->dst == pkt->dst_ip) {
1133                         if (s->limit->expire != time_uptime +
1134                             V_dyn_short_lifetime)
1135                                 ck_pr_store_32(&s->limit->expire,
1136                                     time_uptime + V_dyn_short_lifetime);
1137                         break;
1138                 }
1139         }
1140         return (s);
1141 }
1142
1143 static struct dyn_ipv4_state *
1144 dyn_lookup_ipv4_parent_locked(const struct ipfw_flow_id *pkt,
1145     const void *rule, uint32_t ruleid, uint16_t rulenum, uint32_t bucket)
1146 {
1147         struct dyn_ipv4_state *s;
1148
1149         DYN_BUCKET_ASSERT(bucket);
1150         CK_SLIST_FOREACH(s, &V_dyn_ipv4_parent[bucket], entry) {
1151                 if (s->limit->parent == rule &&
1152                     s->limit->ruleid == ruleid &&
1153                     s->limit->rulenum == rulenum &&
1154                     s->proto == pkt->proto &&
1155                     s->sport == pkt->src_port &&
1156                     s->dport == pkt->dst_port &&
1157                     s->src == pkt->src_ip && s->dst == pkt->dst_ip)
1158                         break;
1159         }
1160         return (s);
1161 }
1162
1163
1164 #ifdef INET6
1165 static uint32_t
1166 dyn_getscopeid(const struct ip_fw_args *args)
1167 {
1168
1169         /*
1170          * If source or destination address is an scopeid address, we need
1171          * determine the scope zone id to resolve address scope ambiguity.
1172          */
1173         if (IN6_IS_ADDR_LINKLOCAL(&args->f_id.src_ip6) ||
1174             IN6_IS_ADDR_LINKLOCAL(&args->f_id.dst_ip6)) {
1175                 MPASS(args->oif != NULL ||
1176                     args->m->m_pkthdr.rcvif != NULL);
1177                 return (in6_getscopezone(args->oif != NULL ? args->oif:
1178                     args->m->m_pkthdr.rcvif, IPV6_ADDR_SCOPE_LINKLOCAL));
1179         }
1180         return (0);
1181 }
1182
1183 /*
1184  * Lookup IPv6 state.
1185  * Must be called in critical section.
1186  */
1187 static struct dyn_ipv6_state *
1188 dyn_lookup_ipv6_state(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1189     const void *ulp, struct ipfw_dyn_info *info, int pktlen)
1190 {
1191         struct dyn_ipv6_state *s;
1192         uint32_t version, bucket;
1193
1194         bucket = DYN_BUCKET(info->hashval, V_curr_dyn_buckets);
1195         info->version = DYN_BUCKET_VERSION(bucket, ipv6_add);
1196 restart:
1197         version = DYN_BUCKET_VERSION(bucket, ipv6_del);
1198         CK_SLIST_FOREACH(s, &V_dyn_ipv6[bucket], entry) {
1199                 DYNSTATE_PROTECT(s);
1200                 if (version != DYN_BUCKET_VERSION(bucket, ipv6_del))
1201                         goto restart;
1202                 if (s->proto != pkt->proto || s->zoneid != zoneid)
1203                         continue;
1204                 if (info->kidx != 0 && s->kidx != info->kidx)
1205                         continue;
1206                 if (s->sport == pkt->src_port && s->dport == pkt->dst_port &&
1207                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->src_ip6) &&
1208                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->dst_ip6)) {
1209                         info->direction = MATCH_FORWARD;
1210                         break;
1211                 }
1212                 if (s->sport == pkt->dst_port && s->dport == pkt->src_port &&
1213                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->dst_ip6) &&
1214                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->src_ip6)) {
1215                         info->direction = MATCH_REVERSE;
1216                         break;
1217                 }
1218         }
1219         if (s != NULL)
1220                 dyn_update_proto_state(s->data, pkt, ulp, pktlen,
1221                     info->direction);
1222         return (s);
1223 }
1224
1225 /*
1226  * Lookup IPv6 state.
1227  * Simplifed version is used to check that matching state doesn't exist.
1228  */
1229 static int
1230 dyn_lookup_ipv6_state_locked(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1231     const void *ulp, int pktlen, const void *parent, uint32_t ruleid,
1232     uint16_t rulenum, uint32_t bucket, uint16_t kidx)
1233 {
1234         struct dyn_ipv6_state *s;
1235         int dir;
1236
1237         dir = MATCH_NONE;
1238         DYN_BUCKET_ASSERT(bucket);
1239         CK_SLIST_FOREACH(s, &V_dyn_ipv6[bucket], entry) {
1240                 if (s->proto != pkt->proto || s->kidx != kidx ||
1241                     s->zoneid != zoneid)
1242                         continue;
1243                 /*
1244                  * XXXAE: Install synchronized state only when there are
1245                  *        no matching states.
1246                  */
1247                 if (pktlen != 0 && (
1248                     s->data->parent != parent ||
1249                     s->data->ruleid != ruleid ||
1250                     s->data->rulenum != rulenum))
1251                         continue;
1252                 if (s->sport == pkt->src_port && s->dport == pkt->dst_port &&
1253                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->src_ip6) &&
1254                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->dst_ip6)) {
1255                         dir = MATCH_FORWARD;
1256                         break;
1257                 }
1258                 if (s->sport == pkt->dst_port && s->dport == pkt->src_port &&
1259                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->dst_ip6) &&
1260                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->src_ip6)) {
1261                         dir = MATCH_REVERSE;
1262                         break;
1263                 }
1264         }
1265         if (s != NULL)
1266                 dyn_update_proto_state(s->data, pkt, ulp, pktlen, dir);
1267         return (s != NULL);
1268 }
1269
1270 static struct dyn_ipv6_state *
1271 dyn_lookup_ipv6_parent(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1272     const void *rule, uint32_t ruleid, uint16_t rulenum, uint32_t hashval)
1273 {
1274         struct dyn_ipv6_state *s;
1275         uint32_t version, bucket;
1276
1277         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1278 restart:
1279         version = DYN_BUCKET_VERSION(bucket, ipv6_parent_del);
1280         CK_SLIST_FOREACH(s, &V_dyn_ipv6_parent[bucket], entry) {
1281                 DYNSTATE_PROTECT(s);
1282                 if (version != DYN_BUCKET_VERSION(bucket, ipv6_parent_del))
1283                         goto restart;
1284                 /*
1285                  * NOTE: we do not need to check kidx, because parent rule
1286                  * can not create states with different kidx.
1287                  * Also parent rule always created for forward direction.
1288                  */
1289                 if (s->limit->parent == rule &&
1290                     s->limit->ruleid == ruleid &&
1291                     s->limit->rulenum == rulenum &&
1292                     s->proto == pkt->proto &&
1293                     s->sport == pkt->src_port &&
1294                     s->dport == pkt->dst_port && s->zoneid == zoneid &&
1295                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->src_ip6) &&
1296                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->dst_ip6)) {
1297                         if (s->limit->expire != time_uptime +
1298                             V_dyn_short_lifetime)
1299                                 ck_pr_store_32(&s->limit->expire,
1300                                     time_uptime + V_dyn_short_lifetime);
1301                         break;
1302                 }
1303         }
1304         return (s);
1305 }
1306
1307 static struct dyn_ipv6_state *
1308 dyn_lookup_ipv6_parent_locked(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1309     const void *rule, uint32_t ruleid, uint16_t rulenum, uint32_t bucket)
1310 {
1311         struct dyn_ipv6_state *s;
1312
1313         DYN_BUCKET_ASSERT(bucket);
1314         CK_SLIST_FOREACH(s, &V_dyn_ipv6_parent[bucket], entry) {
1315                 if (s->limit->parent == rule &&
1316                     s->limit->ruleid == ruleid &&
1317                     s->limit->rulenum == rulenum &&
1318                     s->proto == pkt->proto &&
1319                     s->sport == pkt->src_port &&
1320                     s->dport == pkt->dst_port && s->zoneid == zoneid &&
1321                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->src_ip6) &&
1322                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->dst_ip6))
1323                         break;
1324         }
1325         return (s);
1326 }
1327
1328 #endif /* INET6 */
1329
1330 /*
1331  * Lookup dynamic state.
1332  *  pkt - filled by ipfw_chk() ipfw_flow_id;
1333  *  ulp - determined by ipfw_chk() upper level protocol header;
1334  *  dyn_info - info about matched state to return back;
1335  * Returns pointer to state's parent rule and dyn_info. If there is
1336  * no state, NULL is returned.
1337  * On match ipfw_dyn_lookup() updates state's counters.
1338  */
1339 struct ip_fw *
1340 ipfw_dyn_lookup_state(const struct ip_fw_args *args, const void *ulp,
1341     int pktlen, const ipfw_insn *cmd, struct ipfw_dyn_info *info)
1342 {
1343         struct dyn_data *data;
1344         struct ip_fw *rule;
1345
1346         IPFW_RLOCK_ASSERT(&V_layer3_chain);
1347
1348         data = NULL;
1349         rule = NULL;
1350         info->kidx = cmd->arg1;
1351         info->direction = MATCH_NONE;
1352         info->hashval = hash_packet(&args->f_id);
1353
1354         DYNSTATE_CRITICAL_ENTER();
1355         if (IS_IP4_FLOW_ID(&args->f_id)) {
1356                 struct dyn_ipv4_state *s;
1357
1358                 s = dyn_lookup_ipv4_state(&args->f_id, ulp, info, pktlen);
1359                 if (s != NULL) {
1360                         /*
1361                          * Dynamic states are created using the same 5-tuple,
1362                          * so it is assumed, that parent rule for O_LIMIT
1363                          * state has the same address family.
1364                          */
1365                         data = s->data;
1366                         if (s->type == O_LIMIT) {
1367                                 s = data->parent;
1368                                 rule = s->limit->parent;
1369                         } else
1370                                 rule = data->parent;
1371                 }
1372         }
1373 #ifdef INET6
1374         else if (IS_IP6_FLOW_ID(&args->f_id)) {
1375                 struct dyn_ipv6_state *s;
1376
1377                 s = dyn_lookup_ipv6_state(&args->f_id, dyn_getscopeid(args),
1378                     ulp, info, pktlen);
1379                 if (s != NULL) {
1380                         data = s->data;
1381                         if (s->type == O_LIMIT) {
1382                                 s = data->parent;
1383                                 rule = s->limit->parent;
1384                         } else
1385                                 rule = data->parent;
1386                 }
1387         }
1388 #endif
1389         if (data != NULL) {
1390                 /*
1391                  * If cached chain id is the same, we can avoid rule index
1392                  * lookup. Otherwise do lookup and update chain_id and f_pos.
1393                  * It is safe even if there is concurrent thread that want
1394                  * update the same state, because chain->id can be changed
1395                  * only under IPFW_WLOCK().
1396                  */
1397                 if (data->chain_id != V_layer3_chain.id) {
1398                         data->f_pos = ipfw_find_rule(&V_layer3_chain,
1399                             data->rulenum, data->ruleid);
1400                         /*
1401                          * Check that found state has not orphaned.
1402                          * When chain->id being changed the parent
1403                          * rule can be deleted. If found rule doesn't
1404                          * match the parent pointer, consider this
1405                          * result as MATCH_NONE and return NULL.
1406                          *
1407                          * This will lead to creation of new similar state
1408                          * that will be added into head of this bucket.
1409                          * And the state that we currently have matched
1410                          * should be deleted by dyn_expire_states().
1411                          */
1412                         if (V_layer3_chain.map[data->f_pos] == rule)
1413                                 data->chain_id = V_layer3_chain.id;
1414                         else {
1415                                 rule = NULL;
1416                                 info->direction = MATCH_NONE;
1417                                 DYN_DEBUG("rule %p  [%u, %u] is considered "
1418                                     "invalid in data %p", rule, data->ruleid,
1419                                     data->rulenum, data);
1420                         }
1421                 }
1422                 info->f_pos = data->f_pos;
1423         }
1424         DYNSTATE_CRITICAL_EXIT();
1425 #if 0
1426         /*
1427          * Return MATCH_NONE if parent rule is in disabled set.
1428          * This will lead to creation of new similar state that
1429          * will be added into head of this bucket.
1430          *
1431          * XXXAE: we need to be able update state's set when parent
1432          *        rule set is changed.
1433          */
1434         if (rule != NULL && (V_set_disable & (1 << rule->set))) {
1435                 rule = NULL;
1436                 info->direction = MATCH_NONE;
1437         }
1438 #endif
1439         return (rule);
1440 }
1441
1442 static struct dyn_parent *
1443 dyn_alloc_parent(void *parent, uint32_t ruleid, uint16_t rulenum,
1444     uint8_t set, uint32_t hashval)
1445 {
1446         struct dyn_parent *limit;
1447
1448         limit = uma_zalloc(V_dyn_parent_zone, M_NOWAIT | M_ZERO);
1449         if (limit == NULL) {
1450                 if (last_log != time_uptime) {
1451                         last_log = time_uptime;
1452                         log(LOG_DEBUG,
1453                             "ipfw: Cannot allocate parent dynamic state, "
1454                             "consider increasing "
1455                             "net.inet.ip.fw.dyn_parent_max\n");
1456                 }
1457                 return (NULL);
1458         }
1459
1460         limit->parent = parent;
1461         limit->ruleid = ruleid;
1462         limit->rulenum = rulenum;
1463         limit->set = set;
1464         limit->hashval = hashval;
1465         limit->expire = time_uptime + V_dyn_short_lifetime;
1466         return (limit);
1467 }
1468
1469 static struct dyn_data *
1470 dyn_alloc_dyndata(void *parent, uint32_t ruleid, uint16_t rulenum,
1471     uint8_t set, const struct ipfw_flow_id *pkt, const void *ulp, int pktlen,
1472     uint32_t hashval, uint16_t fibnum)
1473 {
1474         struct dyn_data *data;
1475
1476         data = uma_zalloc(V_dyn_data_zone, M_NOWAIT | M_ZERO);
1477         if (data == NULL) {
1478                 if (last_log != time_uptime) {
1479                         last_log = time_uptime;
1480                         log(LOG_DEBUG,
1481                             "ipfw: Cannot allocate dynamic state, "
1482                             "consider increasing net.inet.ip.fw.dyn_max\n");
1483                 }
1484                 return (NULL);
1485         }
1486
1487         data->parent = parent;
1488         data->ruleid = ruleid;
1489         data->rulenum = rulenum;
1490         data->set = set;
1491         data->fibnum = fibnum;
1492         data->hashval = hashval;
1493         data->expire = time_uptime + V_dyn_syn_lifetime;
1494         dyn_update_proto_state(data, pkt, ulp, pktlen, MATCH_FORWARD);
1495         return (data);
1496 }
1497
1498 static struct dyn_ipv4_state *
1499 dyn_alloc_ipv4_state(const struct ipfw_flow_id *pkt, uint16_t kidx,
1500     uint8_t type)
1501 {
1502         struct dyn_ipv4_state *s;
1503
1504         s = uma_zalloc(V_dyn_ipv4_zone, M_NOWAIT | M_ZERO);
1505         if (s == NULL)
1506                 return (NULL);
1507
1508         s->type = type;
1509         s->kidx = kidx;
1510         s->proto = pkt->proto;
1511         s->sport = pkt->src_port;
1512         s->dport = pkt->dst_port;
1513         s->src = pkt->src_ip;
1514         s->dst = pkt->dst_ip;
1515         return (s);
1516 }
1517
1518 /*
1519  * Add IPv4 parent state.
1520  * Returns pointer to parent state. When it is not NULL we are in
1521  * critical section and pointer protected by hazard pointer.
1522  * When some error occurs, it returns NULL and exit from critical section
1523  * is not needed.
1524  */
1525 static struct dyn_ipv4_state *
1526 dyn_add_ipv4_parent(void *rule, uint32_t ruleid, uint16_t rulenum,
1527     uint8_t set, const struct ipfw_flow_id *pkt, uint32_t hashval,
1528     uint32_t version, uint16_t kidx)
1529 {
1530         struct dyn_ipv4_state *s;
1531         struct dyn_parent *limit;
1532         uint32_t bucket;
1533
1534         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1535         DYN_BUCKET_LOCK(bucket);
1536         if (version != DYN_BUCKET_VERSION(bucket, ipv4_parent_add)) {
1537                 /*
1538                  * Bucket version has been changed since last lookup,
1539                  * do lookup again to be sure that state does not exist.
1540                  */
1541                 s = dyn_lookup_ipv4_parent_locked(pkt, rule, ruleid,
1542                     rulenum, bucket);
1543                 if (s != NULL) {
1544                         /*
1545                          * Simultaneous thread has already created this
1546                          * state. Just return it.
1547                          */
1548                         DYNSTATE_CRITICAL_ENTER();
1549                         DYNSTATE_PROTECT(s);
1550                         DYN_BUCKET_UNLOCK(bucket);
1551                         return (s);
1552                 }
1553         }
1554
1555         limit = dyn_alloc_parent(rule, ruleid, rulenum, set, hashval);
1556         if (limit == NULL) {
1557                 DYN_BUCKET_UNLOCK(bucket);
1558                 return (NULL);
1559         }
1560
1561         s = dyn_alloc_ipv4_state(pkt, kidx, O_LIMIT_PARENT);
1562         if (s == NULL) {
1563                 DYN_BUCKET_UNLOCK(bucket);
1564                 uma_zfree(V_dyn_parent_zone, limit);
1565                 return (NULL);
1566         }
1567
1568         s->limit = limit;
1569         CK_SLIST_INSERT_HEAD(&V_dyn_ipv4_parent[bucket], s, entry);
1570         DYN_COUNT_INC(dyn_parent_count);
1571         DYN_BUCKET_VERSION_BUMP(bucket, ipv4_parent_add);
1572         DYNSTATE_CRITICAL_ENTER();
1573         DYNSTATE_PROTECT(s);
1574         DYN_BUCKET_UNLOCK(bucket);
1575         return (s);
1576 }
1577
1578 static int
1579 dyn_add_ipv4_state(void *parent, uint32_t ruleid, uint16_t rulenum,
1580     uint8_t set, const struct ipfw_flow_id *pkt, const void *ulp, int pktlen,
1581     uint32_t hashval, struct ipfw_dyn_info *info, uint16_t fibnum,
1582     uint16_t kidx, uint8_t type)
1583 {
1584         struct dyn_ipv4_state *s;
1585         void *data;
1586         uint32_t bucket;
1587
1588         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1589         DYN_BUCKET_LOCK(bucket);
1590         if (info->direction == MATCH_UNKNOWN ||
1591             info->kidx != kidx ||
1592             info->hashval != hashval ||
1593             info->version != DYN_BUCKET_VERSION(bucket, ipv4_add)) {
1594                 /*
1595                  * Bucket version has been changed since last lookup,
1596                  * do lookup again to be sure that state does not exist.
1597                  */
1598                 if (dyn_lookup_ipv4_state_locked(pkt, ulp, pktlen, parent,
1599                     ruleid, rulenum, bucket, kidx) != 0) {
1600                         DYN_BUCKET_UNLOCK(bucket);
1601                         return (EEXIST);
1602                 }
1603         }
1604
1605         data = dyn_alloc_dyndata(parent, ruleid, rulenum, set, pkt, ulp,
1606             pktlen, hashval, fibnum);
1607         if (data == NULL) {
1608                 DYN_BUCKET_UNLOCK(bucket);
1609                 return (ENOMEM);
1610         }
1611
1612         s = dyn_alloc_ipv4_state(pkt, kidx, type);
1613         if (s == NULL) {
1614                 DYN_BUCKET_UNLOCK(bucket);
1615                 uma_zfree(V_dyn_data_zone, data);
1616                 return (ENOMEM);
1617         }
1618
1619         s->data = data;
1620         CK_SLIST_INSERT_HEAD(&V_dyn_ipv4[bucket], s, entry);
1621         DYN_COUNT_INC(dyn_count);
1622         DYN_BUCKET_VERSION_BUMP(bucket, ipv4_add);
1623         DYN_BUCKET_UNLOCK(bucket);
1624         return (0);
1625 }
1626
1627 #ifdef INET6
1628 static struct dyn_ipv6_state *
1629 dyn_alloc_ipv6_state(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1630     uint16_t kidx, uint8_t type)
1631 {
1632         struct dyn_ipv6_state *s;
1633
1634         s = uma_zalloc(V_dyn_ipv6_zone, M_NOWAIT | M_ZERO);
1635         if (s == NULL)
1636                 return (NULL);
1637
1638         s->type = type;
1639         s->kidx = kidx;
1640         s->zoneid = zoneid;
1641         s->proto = pkt->proto;
1642         s->sport = pkt->src_port;
1643         s->dport = pkt->dst_port;
1644         s->src = pkt->src_ip6;
1645         s->dst = pkt->dst_ip6;
1646         return (s);
1647 }
1648
1649 /*
1650  * Add IPv6 parent state.
1651  * Returns pointer to parent state. When it is not NULL we are in
1652  * critical section and pointer protected by hazard pointer.
1653  * When some error occurs, it return NULL and exit from critical section
1654  * is not needed.
1655  */
1656 static struct dyn_ipv6_state *
1657 dyn_add_ipv6_parent(void *rule, uint32_t ruleid, uint16_t rulenum,
1658     uint8_t set, const struct ipfw_flow_id *pkt, uint32_t zoneid,
1659     uint32_t hashval, uint32_t version, uint16_t kidx)
1660 {
1661         struct dyn_ipv6_state *s;
1662         struct dyn_parent *limit;
1663         uint32_t bucket;
1664
1665         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1666         DYN_BUCKET_LOCK(bucket);
1667         if (version != DYN_BUCKET_VERSION(bucket, ipv6_parent_add)) {
1668                 /*
1669                  * Bucket version has been changed since last lookup,
1670                  * do lookup again to be sure that state does not exist.
1671                  */
1672                 s = dyn_lookup_ipv6_parent_locked(pkt, zoneid, rule, ruleid,
1673                     rulenum, bucket);
1674                 if (s != NULL) {
1675                         /*
1676                          * Simultaneous thread has already created this
1677                          * state. Just return it.
1678                          */
1679                         DYNSTATE_CRITICAL_ENTER();
1680                         DYNSTATE_PROTECT(s);
1681                         DYN_BUCKET_UNLOCK(bucket);
1682                         return (s);
1683                 }
1684         }
1685
1686         limit = dyn_alloc_parent(rule, ruleid, rulenum, set, hashval);
1687         if (limit == NULL) {
1688                 DYN_BUCKET_UNLOCK(bucket);
1689                 return (NULL);
1690         }
1691
1692         s = dyn_alloc_ipv6_state(pkt, zoneid, kidx, O_LIMIT_PARENT);
1693         if (s == NULL) {
1694                 DYN_BUCKET_UNLOCK(bucket);
1695                 uma_zfree(V_dyn_parent_zone, limit);
1696                 return (NULL);
1697         }
1698
1699         s->limit = limit;
1700         CK_SLIST_INSERT_HEAD(&V_dyn_ipv6_parent[bucket], s, entry);
1701         DYN_COUNT_INC(dyn_parent_count);
1702         DYN_BUCKET_VERSION_BUMP(bucket, ipv6_parent_add);
1703         DYNSTATE_CRITICAL_ENTER();
1704         DYNSTATE_PROTECT(s);
1705         DYN_BUCKET_UNLOCK(bucket);
1706         return (s);
1707 }
1708
1709 static int
1710 dyn_add_ipv6_state(void *parent, uint32_t ruleid, uint16_t rulenum,
1711     uint8_t set, const struct ipfw_flow_id *pkt, uint32_t zoneid,
1712     const void *ulp, int pktlen, uint32_t hashval, struct ipfw_dyn_info *info,
1713     uint16_t fibnum, uint16_t kidx, uint8_t type)
1714 {
1715         struct dyn_ipv6_state *s;
1716         struct dyn_data *data;
1717         uint32_t bucket;
1718
1719         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1720         DYN_BUCKET_LOCK(bucket);
1721         if (info->direction == MATCH_UNKNOWN ||
1722             info->kidx != kidx ||
1723             info->hashval != hashval ||
1724             info->version != DYN_BUCKET_VERSION(bucket, ipv6_add)) {
1725                 /*
1726                  * Bucket version has been changed since last lookup,
1727                  * do lookup again to be sure that state does not exist.
1728                  */
1729                 if (dyn_lookup_ipv6_state_locked(pkt, zoneid, ulp, pktlen,
1730                     parent, ruleid, rulenum, bucket, kidx) != 0) {
1731                         DYN_BUCKET_UNLOCK(bucket);
1732                         return (EEXIST);
1733                 }
1734         }
1735
1736         data = dyn_alloc_dyndata(parent, ruleid, rulenum, set, pkt, ulp,
1737             pktlen, hashval, fibnum);
1738         if (data == NULL) {
1739                 DYN_BUCKET_UNLOCK(bucket);
1740                 return (ENOMEM);
1741         }
1742
1743         s = dyn_alloc_ipv6_state(pkt, zoneid, kidx, type);
1744         if (s == NULL) {
1745                 DYN_BUCKET_UNLOCK(bucket);
1746                 uma_zfree(V_dyn_data_zone, data);
1747                 return (ENOMEM);
1748         }
1749
1750         s->data = data;
1751         CK_SLIST_INSERT_HEAD(&V_dyn_ipv6[bucket], s, entry);
1752         DYN_COUNT_INC(dyn_count);
1753         DYN_BUCKET_VERSION_BUMP(bucket, ipv6_add);
1754         DYN_BUCKET_UNLOCK(bucket);
1755         return (0);
1756 }
1757 #endif /* INET6 */
1758
1759 static void *
1760 dyn_get_parent_state(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1761     struct ip_fw *rule, uint32_t hashval, uint32_t limit, uint16_t kidx)
1762 {
1763         char sbuf[24];
1764         struct dyn_parent *p;
1765         void *ret;
1766         uint32_t bucket, version;
1767
1768         p = NULL;
1769         ret = NULL;
1770         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1771         DYNSTATE_CRITICAL_ENTER();
1772         if (IS_IP4_FLOW_ID(pkt)) {
1773                 struct dyn_ipv4_state *s;
1774
1775                 version = DYN_BUCKET_VERSION(bucket, ipv4_parent_add);
1776                 s = dyn_lookup_ipv4_parent(pkt, rule, rule->id,
1777                     rule->rulenum, bucket);
1778                 if (s == NULL) {
1779                         /*
1780                          * Exit from critical section because dyn_add_parent()
1781                          * will acquire bucket lock.
1782                          */
1783                         DYNSTATE_CRITICAL_EXIT();
1784
1785                         s = dyn_add_ipv4_parent(rule, rule->id,
1786                             rule->rulenum, rule->set, pkt, hashval,
1787                             version, kidx);
1788                         if (s == NULL)
1789                                 return (NULL);
1790                         /* Now we are in critical section again. */
1791                 }
1792                 ret = s;
1793                 p = s->limit;
1794         }
1795 #ifdef INET6
1796         else if (IS_IP6_FLOW_ID(pkt)) {
1797                 struct dyn_ipv6_state *s;
1798
1799                 version = DYN_BUCKET_VERSION(bucket, ipv6_parent_add);
1800                 s = dyn_lookup_ipv6_parent(pkt, zoneid, rule, rule->id,
1801                     rule->rulenum, bucket);
1802                 if (s == NULL) {
1803                         /*
1804                          * Exit from critical section because dyn_add_parent()
1805                          * can acquire bucket mutex.
1806                          */
1807                         DYNSTATE_CRITICAL_EXIT();
1808
1809                         s = dyn_add_ipv6_parent(rule, rule->id,
1810                             rule->rulenum, rule->set, pkt, zoneid, hashval,
1811                             version, kidx);
1812                         if (s == NULL)
1813                                 return (NULL);
1814                         /* Now we are in critical section again. */
1815                 }
1816                 ret = s;
1817                 p = s->limit;
1818         }
1819 #endif
1820         else {
1821                 DYNSTATE_CRITICAL_EXIT();
1822                 return (NULL);
1823         }
1824
1825         /* Check the limit */
1826         if (DPARENT_COUNT(p) >= limit) {
1827                 DYNSTATE_CRITICAL_EXIT();
1828                 if (V_fw_verbose && last_log != time_uptime) {
1829                         last_log = time_uptime;
1830                         snprintf(sbuf, sizeof(sbuf), "%u drop session",
1831                             rule->rulenum);
1832                         print_dyn_rule_flags(pkt, O_LIMIT,
1833                             LOG_SECURITY | LOG_DEBUG, sbuf,
1834                             "too many entries");
1835                 }
1836                 return (NULL);
1837         }
1838
1839         /* Take new session into account. */
1840         DPARENT_COUNT_INC(p);
1841         /*
1842          * We must exit from critical section because the following code
1843          * can acquire bucket mutex.
1844          * We rely on the the 'count' field. The state will not expire
1845          * until it has some child states, i.e. 'count' field is not zero.
1846          * Return state pointer, it will be used by child states as parent.
1847          */
1848         DYNSTATE_CRITICAL_EXIT();
1849         return (ret);
1850 }
1851
1852 static int
1853 dyn_install_state(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1854     uint16_t fibnum, const void *ulp, int pktlen, void *rule,
1855     uint32_t ruleid, uint16_t rulenum, uint8_t set,
1856     struct ipfw_dyn_info *info, uint32_t limit, uint16_t limit_mask,
1857     uint16_t kidx, uint8_t type)
1858 {
1859         struct ipfw_flow_id id;
1860         uint32_t hashval, parent_hashval;
1861         int ret;
1862
1863         MPASS(type == O_LIMIT || type == O_KEEP_STATE);
1864
1865         if (type == O_LIMIT) {
1866                 /* Create masked flow id and calculate bucket */
1867                 id.addr_type = pkt->addr_type;
1868                 id.proto = pkt->proto;
1869                 id.fib = fibnum; /* unused */
1870                 id.src_port = (limit_mask & DYN_SRC_PORT) ?
1871                     pkt->src_port: 0;
1872                 id.dst_port = (limit_mask & DYN_DST_PORT) ?
1873                     pkt->dst_port: 0;
1874                 if (IS_IP4_FLOW_ID(pkt)) {
1875                         id.src_ip = (limit_mask & DYN_SRC_ADDR) ?
1876                             pkt->src_ip: 0;
1877                         id.dst_ip = (limit_mask & DYN_DST_ADDR) ?
1878                             pkt->dst_ip: 0;
1879                 }
1880 #ifdef INET6
1881                 else if (IS_IP6_FLOW_ID(pkt)) {
1882                         if (limit_mask & DYN_SRC_ADDR)
1883                                 id.src_ip6 = pkt->src_ip6;
1884                         else
1885                                 memset(&id.src_ip6, 0, sizeof(id.src_ip6));
1886                         if (limit_mask & DYN_DST_ADDR)
1887                                 id.dst_ip6 = pkt->dst_ip6;
1888                         else
1889                                 memset(&id.dst_ip6, 0, sizeof(id.dst_ip6));
1890                 }
1891 #endif
1892                 else
1893                         return (EAFNOSUPPORT);
1894
1895                 parent_hashval = hash_parent(&id, rule);
1896                 rule = dyn_get_parent_state(&id, zoneid, rule, parent_hashval,
1897                     limit, kidx);
1898                 if (rule == NULL) {
1899 #if 0
1900                         if (V_fw_verbose && last_log != time_uptime) {
1901                                 last_log = time_uptime;
1902                                 snprintf(sbuf, sizeof(sbuf),
1903                                     "%u drop session", rule->rulenum);
1904                         print_dyn_rule_flags(pkt, O_LIMIT,
1905                             LOG_SECURITY | LOG_DEBUG, sbuf,
1906                             "too many entries");
1907                         }
1908 #endif
1909                         return (EACCES);
1910                 }
1911                 /*
1912                  * Limit is not reached, create new state.
1913                  * Now rule points to parent state.
1914                  */
1915         }
1916
1917         hashval = hash_packet(pkt);
1918         if (IS_IP4_FLOW_ID(pkt))
1919                 ret = dyn_add_ipv4_state(rule, ruleid, rulenum, set, pkt,
1920                     ulp, pktlen, hashval, info, fibnum, kidx, type);
1921 #ifdef INET6
1922         else if (IS_IP6_FLOW_ID(pkt))
1923                 ret = dyn_add_ipv6_state(rule, ruleid, rulenum, set, pkt,
1924                     zoneid, ulp, pktlen, hashval, info, fibnum, kidx, type);
1925 #endif /* INET6 */
1926         else
1927                 ret = EAFNOSUPPORT;
1928
1929         if (type == O_LIMIT) {
1930                 if (ret != 0) {
1931                         /*
1932                          * We failed to create child state for O_LIMIT
1933                          * opcode. Since we already counted it in the parent,
1934                          * we must revert counter back. The 'rule' points to
1935                          * parent state, use it to get dyn_parent.
1936                          *
1937                          * XXXAE: it should be safe to use 'rule' pointer
1938                          * without extra lookup, parent state is referenced
1939                          * and should not be freed.
1940                          */
1941                         if (IS_IP4_FLOW_ID(&id))
1942                                 DPARENT_COUNT_DEC(
1943                                     ((struct dyn_ipv4_state *)rule)->limit);
1944 #ifdef INET6
1945                         else if (IS_IP6_FLOW_ID(&id))
1946                                 DPARENT_COUNT_DEC(
1947                                     ((struct dyn_ipv6_state *)rule)->limit);
1948 #endif
1949                 }
1950         }
1951         /*
1952          * EEXIST means that simultaneous thread has created this
1953          * state. Consider this as success.
1954          *
1955          * XXXAE: should we invalidate 'info' content here?
1956          */
1957         if (ret == EEXIST)
1958                 return (0);
1959         return (ret);
1960 }
1961
1962 /*
1963  * Install dynamic state.
1964  *  chain - ipfw's instance;
1965  *  rule - the parent rule that installs the state;
1966  *  cmd - opcode that installs the state;
1967  *  args - ipfw arguments;
1968  *  ulp - upper level protocol header;
1969  *  pktlen - packet length;
1970  *  info - dynamic state lookup info;
1971  *  tablearg - tablearg id.
1972  *
1973  * Returns non-zero value (failure) if state is not installed because
1974  * of errors or because session limitations are enforced.
1975  */
1976 int
1977 ipfw_dyn_install_state(struct ip_fw_chain *chain, struct ip_fw *rule,
1978     const ipfw_insn_limit *cmd, const struct ip_fw_args *args,
1979     const void *ulp, int pktlen, struct ipfw_dyn_info *info,
1980     uint32_t tablearg)
1981 {
1982         uint32_t limit;
1983         uint16_t limit_mask;
1984
1985         if (cmd->o.opcode == O_LIMIT) {
1986                 limit = IP_FW_ARG_TABLEARG(chain, cmd->conn_limit, limit);
1987                 limit_mask = cmd->limit_mask;
1988         } else {
1989                 limit = 0;
1990                 limit_mask = 0;
1991         }
1992         return (dyn_install_state(&args->f_id,
1993 #ifdef INET6
1994             IS_IP6_FLOW_ID(&args->f_id) ? dyn_getscopeid(args):
1995 #endif
1996             0, M_GETFIB(args->m), ulp, pktlen, rule, rule->id, rule->rulenum,
1997             rule->set, info, limit, limit_mask, cmd->o.arg1, cmd->o.opcode));
1998 }
1999
2000 /*
2001  * Free safe to remove state entries from expired lists.
2002  */
2003 static void
2004 dyn_free_states(struct ip_fw_chain *chain)
2005 {
2006         struct dyn_ipv4_state *s4, *s4n;
2007 #ifdef INET6
2008         struct dyn_ipv6_state *s6, *s6n;
2009 #endif
2010         int cached_count, i;
2011
2012         /*
2013          * We keep pointers to objects that are in use on each CPU
2014          * in the per-cpu dyn_hp pointer. When object is going to be
2015          * removed, first of it is unlinked from the corresponding
2016          * list. This leads to changing of dyn_bucket_xxx_delver version.
2017          * Unlinked objects is placed into corresponding dyn_expired_xxx
2018          * list. Reader that is going to dereference object pointer checks
2019          * dyn_bucket_xxx_delver version before and after storing pointer
2020          * into dyn_hp. If version is the same, the object is protected
2021          * from freeing and it is safe to dereference. Othervise reader
2022          * tries to iterate list again from the beginning, but this object
2023          * now unlinked and thus will not be accessible.
2024          *
2025          * Copy dyn_hp pointers for each CPU into dyn_hp_cache array.
2026          * It does not matter that some pointer can be changed in
2027          * time while we are copying. We need to check, that objects
2028          * removed in the previous pass are not in use. And if dyn_hp
2029          * pointer does not contain it in the time when we are copying,
2030          * it will not appear there, because it is already unlinked.
2031          * And for new pointers we will not free objects that will be
2032          * unlinked in this pass.
2033          */
2034         cached_count = 0;
2035         CPU_FOREACH(i) {
2036                 dyn_hp_cache[cached_count] = DYNSTATE_GET(i);
2037                 if (dyn_hp_cache[cached_count] != NULL)
2038                         cached_count++;
2039         }
2040
2041         /*
2042          * Free expired states that are safe to free.
2043          * Check each entry from previous pass in the dyn_expired_xxx
2044          * list, if pointer to the object is in the dyn_hp_cache array,
2045          * keep it until next pass. Otherwise it is safe to free the
2046          * object.
2047          *
2048          * XXXAE: optimize this to use SLIST_REMOVE_AFTER.
2049          */
2050 #define DYN_FREE_STATES(s, next, name)          do {                    \
2051         s = SLIST_FIRST(&V_dyn_expired_ ## name);                       \
2052         while (s != NULL) {                                             \
2053                 next = SLIST_NEXT(s, expired);                          \
2054                 for (i = 0; i < cached_count; i++)                      \
2055                         if (dyn_hp_cache[i] == s)                       \
2056                                 break;                                  \
2057                 if (i == cached_count) {                                \
2058                         if (s->type == O_LIMIT_PARENT &&                \
2059                             s->limit->count != 0) {                     \
2060                                 s = next;                               \
2061                                 continue;                               \
2062                         }                                               \
2063                         SLIST_REMOVE(&V_dyn_expired_ ## name,           \
2064                             s, dyn_ ## name ## _state, expired);        \
2065                         if (s->type == O_LIMIT_PARENT)                  \
2066                                 uma_zfree(V_dyn_parent_zone, s->limit); \
2067                         else                                            \
2068                                 uma_zfree(V_dyn_data_zone, s->data);    \
2069                         uma_zfree(V_dyn_ ## name ## _zone, s);          \
2070                 }                                                       \
2071                 s = next;                                               \
2072         }                                                               \
2073 } while (0)
2074
2075         /*
2076          * Protect access to expired lists with DYN_EXPIRED_LOCK.
2077          * Userland can invoke ipfw_expire_dyn_states() to delete
2078          * specific states, this will lead to modification of expired
2079          * lists.
2080          *
2081          * XXXAE: do we need DYN_EXPIRED_LOCK? We can just use
2082          *        IPFW_UH_WLOCK to protect access to these lists.
2083          */
2084         DYN_EXPIRED_LOCK();
2085         DYN_FREE_STATES(s4, s4n, ipv4);
2086 #ifdef INET6
2087         DYN_FREE_STATES(s6, s6n, ipv6);
2088 #endif
2089         DYN_EXPIRED_UNLOCK();
2090 #undef DYN_FREE_STATES
2091 }
2092
2093 /*
2094  * Returns 1 when state is matched by specified range, otherwise returns 0.
2095  */
2096 static int
2097 dyn_match_range(uint16_t rulenum, uint8_t set, const ipfw_range_tlv *rt)
2098 {
2099
2100         MPASS(rt != NULL);
2101         /* flush all states */
2102         if (rt->flags & IPFW_RCFLAG_ALL)
2103                 return (1);
2104         if ((rt->flags & IPFW_RCFLAG_SET) != 0 && set != rt->set)
2105                 return (0);
2106         if ((rt->flags & IPFW_RCFLAG_RANGE) != 0 &&
2107             (rulenum < rt->start_rule || rulenum > rt->end_rule))
2108                 return (0);
2109         return (1);
2110 }
2111
2112 static int
2113 dyn_match_ipv4_state(struct dyn_ipv4_state *s, const ipfw_range_tlv *rt)
2114 {
2115
2116         if (s->type == O_LIMIT_PARENT)
2117                 return (dyn_match_range(s->limit->rulenum,
2118                     s->limit->set, rt));
2119
2120         if (s->type == O_LIMIT)
2121                 return (dyn_match_range(s->data->rulenum, s->data->set, rt));
2122
2123         if (dyn_match_range(s->data->rulenum, s->data->set, rt))
2124                 return (1);
2125
2126         return (0);
2127 }
2128
2129 #ifdef INET6
2130 static int
2131 dyn_match_ipv6_state(struct dyn_ipv6_state *s, const ipfw_range_tlv *rt)
2132 {
2133
2134         if (s->type == O_LIMIT_PARENT)
2135                 return (dyn_match_range(s->limit->rulenum,
2136                     s->limit->set, rt));
2137
2138         if (s->type == O_LIMIT)
2139                 return (dyn_match_range(s->data->rulenum, s->data->set, rt));
2140
2141         if (dyn_match_range(s->data->rulenum, s->data->set, rt))
2142                 return (1);
2143
2144         return (0);
2145 }
2146 #endif
2147
2148 /*
2149  * Unlink expired entries from states lists.
2150  * @rt can be used to specify the range of states for deletion.
2151  */
2152 static void
2153 dyn_expire_states(struct ip_fw_chain *chain, ipfw_range_tlv *rt)
2154 {
2155         struct dyn_ipv4_slist expired_ipv4;
2156 #ifdef INET6
2157         struct dyn_ipv6_slist expired_ipv6;
2158         struct dyn_ipv6_state *s6, *s6n, *s6p;
2159 #endif
2160         struct dyn_ipv4_state *s4, *s4n, *s4p;
2161         int bucket, removed, length, max_length;
2162
2163         /*
2164          * Unlink expired states from each bucket.
2165          * With acquired bucket lock iterate entries of each lists:
2166          * ipv4, ipv4_parent, ipv6, and ipv6_parent. Check expired time
2167          * and unlink entry from the list, link entry into temporary
2168          * expired_xxx lists then bump "del" bucket version.
2169          *
2170          * When an entry is removed, corresponding states counter is
2171          * decremented. If entry has O_LIMIT type, parent's reference
2172          * counter is decremented.
2173          *
2174          * NOTE: this function can be called from userspace context
2175          * when user deletes rules. In this case all matched states
2176          * will be forcedly unlinked. O_LIMIT_PARENT states will be kept
2177          * in the expired lists until reference counter become zero.
2178          */
2179 #define DYN_UNLINK_STATES(s, prev, next, exp, af, name, extra)  do {    \
2180         length = 0;                                                     \
2181         removed = 0;                                                    \
2182         prev = NULL;                                                    \
2183         s = CK_SLIST_FIRST(&V_dyn_ ## name [bucket]);                   \
2184         while (s != NULL) {                                             \
2185                 next = CK_SLIST_NEXT(s, entry);                         \
2186                 if ((TIME_LEQ((s)->exp, time_uptime) && extra) ||       \
2187                     (rt != NULL && dyn_match_ ## af ## _state(s, rt))) {\
2188                         if (prev != NULL)                               \
2189                                 CK_SLIST_REMOVE_AFTER(prev, entry);     \
2190                         else                                            \
2191                                 CK_SLIST_REMOVE_HEAD(                   \
2192                                     &V_dyn_ ## name [bucket], entry);   \
2193                         removed++;                                      \
2194                         SLIST_INSERT_HEAD(&expired_ ## af, s, expired); \
2195                         if (s->type == O_LIMIT_PARENT)                  \
2196                                 DYN_COUNT_DEC(dyn_parent_count);        \
2197                         else {                                          \
2198                                 DYN_COUNT_DEC(dyn_count);               \
2199                                 if (s->type == O_LIMIT) {               \
2200                                         s = s->data->parent;            \
2201                                         DPARENT_COUNT_DEC(s->limit);    \
2202                                 }                                       \
2203                         }                                               \
2204                 } else {                                                \
2205                         prev = s;                                       \
2206                         length++;                                       \
2207                 }                                                       \
2208                 s = next;                                               \
2209         }                                                               \
2210         if (removed != 0)                                               \
2211                 DYN_BUCKET_VERSION_BUMP(bucket, name ## _del);          \
2212         if (length > max_length)                                \
2213                 max_length = length;                            \
2214 } while (0)
2215
2216         SLIST_INIT(&expired_ipv4);
2217 #ifdef INET6
2218         SLIST_INIT(&expired_ipv6);
2219 #endif
2220         max_length = 0;
2221         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2222                 DYN_BUCKET_LOCK(bucket);
2223                 DYN_UNLINK_STATES(s4, s4p, s4n, data->expire, ipv4, ipv4, 1);
2224                 DYN_UNLINK_STATES(s4, s4p, s4n, limit->expire, ipv4,
2225                     ipv4_parent, (s4->limit->count == 0));
2226 #ifdef INET6
2227                 DYN_UNLINK_STATES(s6, s6p, s6n, data->expire, ipv6, ipv6, 1);
2228                 DYN_UNLINK_STATES(s6, s6p, s6n, limit->expire, ipv6,
2229                     ipv6_parent, (s6->limit->count == 0));
2230 #endif
2231                 DYN_BUCKET_UNLOCK(bucket);
2232         }
2233         /* Update curr_max_length for statistics. */
2234         V_curr_max_length = max_length;
2235         /*
2236          * Concatenate temporary lists with global expired lists.
2237          */
2238         DYN_EXPIRED_LOCK();
2239         SLIST_CONCAT(&V_dyn_expired_ipv4, &expired_ipv4,
2240             dyn_ipv4_state, expired);
2241 #ifdef INET6
2242         SLIST_CONCAT(&V_dyn_expired_ipv6, &expired_ipv6,
2243             dyn_ipv6_state, expired);
2244 #endif
2245         DYN_EXPIRED_UNLOCK();
2246 #undef DYN_UNLINK_STATES
2247 #undef DYN_UNREF_STATES
2248 }
2249
2250 static struct mbuf *
2251 dyn_mgethdr(int len, uint16_t fibnum)
2252 {
2253         struct mbuf *m;
2254
2255         m = m_gethdr(M_NOWAIT, MT_DATA);
2256         if (m == NULL)
2257                 return (NULL);
2258 #ifdef MAC
2259         mac_netinet_firewall_send(m);
2260 #endif
2261         M_SETFIB(m, fibnum);
2262         m->m_data += max_linkhdr;
2263         m->m_flags |= M_SKIP_FIREWALL;
2264         m->m_len = m->m_pkthdr.len = len;
2265         bzero(m->m_data, len);
2266         return (m);
2267 }
2268
2269 static void
2270 dyn_make_keepalive_ipv4(struct mbuf *m, in_addr_t src, in_addr_t dst,
2271     uint32_t seq, uint32_t ack, uint16_t sport, uint16_t dport)
2272 {
2273         struct tcphdr *tcp;
2274         struct ip *ip;
2275
2276         ip = mtod(m, struct ip *);
2277         ip->ip_v = 4;
2278         ip->ip_hl = sizeof(*ip) >> 2;
2279         ip->ip_tos = IPTOS_LOWDELAY;
2280         ip->ip_len = htons(m->m_len);
2281         ip->ip_off |= htons(IP_DF);
2282         ip->ip_ttl = V_ip_defttl;
2283         ip->ip_p = IPPROTO_TCP;
2284         ip->ip_src.s_addr = htonl(src);
2285         ip->ip_dst.s_addr = htonl(dst);
2286
2287         tcp = mtodo(m, sizeof(struct ip));
2288         tcp->th_sport = htons(sport);
2289         tcp->th_dport = htons(dport);
2290         tcp->th_off = sizeof(struct tcphdr) >> 2;
2291         tcp->th_seq = htonl(seq);
2292         tcp->th_ack = htonl(ack);
2293         tcp->th_flags = TH_ACK;
2294         tcp->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2295             htons(sizeof(struct tcphdr) + IPPROTO_TCP));
2296
2297         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2298         m->m_pkthdr.csum_flags = CSUM_TCP;
2299 }
2300
2301 static void
2302 dyn_enqueue_keepalive_ipv4(struct mbufq *q, const struct dyn_ipv4_state *s)
2303 {
2304         struct mbuf *m;
2305
2306         if ((s->data->state & ACK_FWD) == 0 && s->data->ack_fwd > 0) {
2307                 m = dyn_mgethdr(sizeof(struct ip) + sizeof(struct tcphdr),
2308                     s->data->fibnum);
2309                 if (m != NULL) {
2310                         dyn_make_keepalive_ipv4(m, s->dst, s->src,
2311                             s->data->ack_fwd - 1, s->data->ack_rev,
2312                             s->dport, s->sport);
2313                         if (mbufq_enqueue(q, m)) {
2314                                 m_freem(m);
2315                                 log(LOG_DEBUG, "ipfw: limit for IPv4 "
2316                                     "keepalive queue is reached.\n");
2317                                 return;
2318                         }
2319                 }
2320         }
2321
2322         if ((s->data->state & ACK_REV) == 0 && s->data->ack_rev > 0) {
2323                 m = dyn_mgethdr(sizeof(struct ip) + sizeof(struct tcphdr),
2324                     s->data->fibnum);
2325                 if (m != NULL) {
2326                         dyn_make_keepalive_ipv4(m, s->src, s->dst,
2327                             s->data->ack_rev - 1, s->data->ack_fwd,
2328                             s->sport, s->dport);
2329                         if (mbufq_enqueue(q, m)) {
2330                                 m_freem(m);
2331                                 log(LOG_DEBUG, "ipfw: limit for IPv4 "
2332                                     "keepalive queue is reached.\n");
2333                                 return;
2334                         }
2335                 }
2336         }
2337 }
2338
2339 /*
2340  * Prepare and send keep-alive packets.
2341  */
2342 static void
2343 dyn_send_keepalive_ipv4(struct ip_fw_chain *chain)
2344 {
2345         struct mbufq q;
2346         struct mbuf *m;
2347         struct dyn_ipv4_state *s;
2348         uint32_t bucket;
2349
2350         mbufq_init(&q, DYN_KEEPALIVE_MAXQ);
2351         IPFW_UH_RLOCK(chain);
2352         /*
2353          * It is safe to not use hazard pointer and just do lockless
2354          * access to the lists, because states entries can not be deleted
2355          * while we hold IPFW_UH_RLOCK.
2356          */
2357         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2358                 CK_SLIST_FOREACH(s, &V_dyn_ipv4[bucket], entry) {
2359                         /*
2360                          * Only established TCP connections that will
2361                          * become expired withing dyn_keepalive_interval.
2362                          */
2363                         if (s->proto != IPPROTO_TCP ||
2364                             (s->data->state & BOTH_SYN) != BOTH_SYN ||
2365                             TIME_LEQ(time_uptime + V_dyn_keepalive_interval,
2366                                 s->data->expire))
2367                                 continue;
2368                         dyn_enqueue_keepalive_ipv4(&q, s);
2369                 }
2370         }
2371         IPFW_UH_RUNLOCK(chain);
2372         while ((m = mbufq_dequeue(&q)) != NULL)
2373                 ip_output(m, NULL, NULL, 0, NULL, NULL);
2374 }
2375
2376 #ifdef INET6
2377 static void
2378 dyn_make_keepalive_ipv6(struct mbuf *m, const struct in6_addr *src,
2379     const struct in6_addr *dst, uint32_t zoneid, uint32_t seq, uint32_t ack,
2380     uint16_t sport, uint16_t dport)
2381 {
2382         struct tcphdr *tcp;
2383         struct ip6_hdr *ip6;
2384
2385         ip6 = mtod(m, struct ip6_hdr *);
2386         ip6->ip6_vfc |= IPV6_VERSION;
2387         ip6->ip6_plen = htons(sizeof(struct tcphdr));
2388         ip6->ip6_nxt = IPPROTO_TCP;
2389         ip6->ip6_hlim = IPV6_DEFHLIM;
2390         ip6->ip6_src = *src;
2391         if (IN6_IS_ADDR_LINKLOCAL(src))
2392                 ip6->ip6_src.s6_addr16[1] = htons(zoneid & 0xffff);
2393         ip6->ip6_dst = *dst;
2394         if (IN6_IS_ADDR_LINKLOCAL(dst))
2395                 ip6->ip6_dst.s6_addr16[1] = htons(zoneid & 0xffff);
2396
2397         tcp = mtodo(m, sizeof(struct ip6_hdr));
2398         tcp->th_sport = htons(sport);
2399         tcp->th_dport = htons(dport);
2400         tcp->th_off = sizeof(struct tcphdr) >> 2;
2401         tcp->th_seq = htonl(seq);
2402         tcp->th_ack = htonl(ack);
2403         tcp->th_flags = TH_ACK;
2404         tcp->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr),
2405             IPPROTO_TCP, 0);
2406
2407         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2408         m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
2409 }
2410
2411 static void
2412 dyn_enqueue_keepalive_ipv6(struct mbufq *q, const struct dyn_ipv6_state *s)
2413 {
2414         struct mbuf *m;
2415
2416         if ((s->data->state & ACK_FWD) == 0 && s->data->ack_fwd > 0) {
2417                 m = dyn_mgethdr(sizeof(struct ip6_hdr) +
2418                     sizeof(struct tcphdr), s->data->fibnum);
2419                 if (m != NULL) {
2420                         dyn_make_keepalive_ipv6(m, &s->dst, &s->src,
2421                             s->zoneid, s->data->ack_fwd - 1, s->data->ack_rev,
2422                             s->dport, s->sport);
2423                         if (mbufq_enqueue(q, m)) {
2424                                 m_freem(m);
2425                                 log(LOG_DEBUG, "ipfw: limit for IPv6 "
2426                                     "keepalive queue is reached.\n");
2427                                 return;
2428                         }
2429                 }
2430         }
2431
2432         if ((s->data->state & ACK_REV) == 0 && s->data->ack_rev > 0) {
2433                 m = dyn_mgethdr(sizeof(struct ip6_hdr) +
2434                     sizeof(struct tcphdr), s->data->fibnum);
2435                 if (m != NULL) {
2436                         dyn_make_keepalive_ipv6(m, &s->src, &s->dst,
2437                             s->zoneid, s->data->ack_rev - 1, s->data->ack_fwd,
2438                             s->sport, s->dport);
2439                         if (mbufq_enqueue(q, m)) {
2440                                 m_freem(m);
2441                                 log(LOG_DEBUG, "ipfw: limit for IPv6 "
2442                                     "keepalive queue is reached.\n");
2443                                 return;
2444                         }
2445                 }
2446         }
2447 }
2448
2449 static void
2450 dyn_send_keepalive_ipv6(struct ip_fw_chain *chain)
2451 {
2452         struct mbufq q;
2453         struct mbuf *m;
2454         struct dyn_ipv6_state *s;
2455         uint32_t bucket;
2456
2457         mbufq_init(&q, DYN_KEEPALIVE_MAXQ);
2458         IPFW_UH_RLOCK(chain);
2459         /*
2460          * It is safe to not use hazard pointer and just do lockless
2461          * access to the lists, because states entries can not be deleted
2462          * while we hold IPFW_UH_RLOCK.
2463          */
2464         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2465                 CK_SLIST_FOREACH(s, &V_dyn_ipv6[bucket], entry) {
2466                         /*
2467                          * Only established TCP connections that will
2468                          * become expired withing dyn_keepalive_interval.
2469                          */
2470                         if (s->proto != IPPROTO_TCP ||
2471                             (s->data->state & BOTH_SYN) != BOTH_SYN ||
2472                             TIME_LEQ(time_uptime + V_dyn_keepalive_interval,
2473                                 s->data->expire))
2474                                 continue;
2475                         dyn_enqueue_keepalive_ipv6(&q, s);
2476                 }
2477         }
2478         IPFW_UH_RUNLOCK(chain);
2479         while ((m = mbufq_dequeue(&q)) != NULL)
2480                 ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL);
2481 }
2482 #endif /* INET6 */
2483
2484 static void
2485 dyn_grow_hashtable(struct ip_fw_chain *chain, uint32_t new)
2486 {
2487 #ifdef INET6
2488         struct dyn_ipv6ck_slist *ipv6, *ipv6_parent;
2489         uint32_t *ipv6_add, *ipv6_del, *ipv6_parent_add, *ipv6_parent_del;
2490         struct dyn_ipv6_state *s6;
2491 #endif
2492         struct dyn_ipv4ck_slist *ipv4, *ipv4_parent;
2493         uint32_t *ipv4_add, *ipv4_del, *ipv4_parent_add, *ipv4_parent_del;
2494         struct dyn_ipv4_state *s4;
2495         struct mtx *bucket_lock;
2496         void *tmp;
2497         uint32_t bucket;
2498
2499         MPASS(powerof2(new));
2500         DYN_DEBUG("grow hash size %u -> %u", V_curr_dyn_buckets, new);
2501         /*
2502          * Allocate and initialize new lists.
2503          * XXXAE: on memory pressure this can disable callout timer.
2504          */
2505         bucket_lock = malloc(new * sizeof(struct mtx), M_IPFW,
2506             M_WAITOK | M_ZERO);
2507         ipv4 = malloc(new * sizeof(struct dyn_ipv4ck_slist), M_IPFW,
2508             M_WAITOK | M_ZERO);
2509         ipv4_parent = malloc(new * sizeof(struct dyn_ipv4ck_slist), M_IPFW,
2510             M_WAITOK | M_ZERO);
2511         ipv4_add = malloc(new * sizeof(uint32_t), M_IPFW, M_WAITOK | M_ZERO);
2512         ipv4_del = malloc(new * sizeof(uint32_t), M_IPFW, M_WAITOK | M_ZERO);
2513         ipv4_parent_add = malloc(new * sizeof(uint32_t), M_IPFW,
2514             M_WAITOK | M_ZERO);
2515         ipv4_parent_del = malloc(new * sizeof(uint32_t), M_IPFW,
2516             M_WAITOK | M_ZERO);
2517 #ifdef INET6
2518         ipv6 = malloc(new * sizeof(struct dyn_ipv6ck_slist), M_IPFW,
2519             M_WAITOK | M_ZERO);
2520         ipv6_parent = malloc(new * sizeof(struct dyn_ipv6ck_slist), M_IPFW,
2521             M_WAITOK | M_ZERO);
2522         ipv6_add = malloc(new * sizeof(uint32_t), M_IPFW, M_WAITOK | M_ZERO);
2523         ipv6_del = malloc(new * sizeof(uint32_t), M_IPFW, M_WAITOK | M_ZERO);
2524         ipv6_parent_add = malloc(new * sizeof(uint32_t), M_IPFW,
2525             M_WAITOK | M_ZERO);
2526         ipv6_parent_del = malloc(new * sizeof(uint32_t), M_IPFW,
2527             M_WAITOK | M_ZERO);
2528 #endif
2529         for (bucket = 0; bucket < new; bucket++) {
2530                 DYN_BUCKET_LOCK_INIT(bucket_lock, bucket);
2531                 CK_SLIST_INIT(&ipv4[bucket]);
2532                 CK_SLIST_INIT(&ipv4_parent[bucket]);
2533 #ifdef INET6
2534                 CK_SLIST_INIT(&ipv6[bucket]);
2535                 CK_SLIST_INIT(&ipv6_parent[bucket]);
2536 #endif
2537         }
2538
2539 #define DYN_RELINK_STATES(s, hval, i, head, ohead)      do {            \
2540         while ((s = CK_SLIST_FIRST(&V_dyn_ ## ohead[i])) != NULL) {     \
2541                 CK_SLIST_REMOVE_HEAD(&V_dyn_ ## ohead[i], entry);       \
2542                 CK_SLIST_INSERT_HEAD(&head[DYN_BUCKET(s->hval, new)],   \
2543                     s, entry);                                          \
2544         }                                                               \
2545 } while (0)
2546         /*
2547          * Prevent rules changing from userland.
2548          */
2549         IPFW_UH_WLOCK(chain);
2550         /*
2551          * Hold traffic processing until we finish resize to
2552          * prevent access to states lists.
2553          */
2554         IPFW_WLOCK(chain);
2555         /* Re-link all dynamic states */
2556         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2557                 DYN_RELINK_STATES(s4, data->hashval, bucket, ipv4, ipv4);
2558                 DYN_RELINK_STATES(s4, limit->hashval, bucket, ipv4_parent,
2559                     ipv4_parent);
2560 #ifdef INET6
2561                 DYN_RELINK_STATES(s6, data->hashval, bucket, ipv6, ipv6);
2562                 DYN_RELINK_STATES(s6, limit->hashval, bucket, ipv6_parent,
2563                     ipv6_parent);
2564 #endif
2565         }
2566
2567 #define DYN_SWAP_PTR(old, new, tmp)     do {            \
2568         tmp = old;                                      \
2569         old = new;                                      \
2570         new = tmp;                                      \
2571 } while (0)
2572         /* Swap pointers */
2573         DYN_SWAP_PTR(V_dyn_bucket_lock, bucket_lock, tmp);
2574         DYN_SWAP_PTR(V_dyn_ipv4, ipv4, tmp);
2575         DYN_SWAP_PTR(V_dyn_ipv4_parent, ipv4_parent, tmp);
2576         DYN_SWAP_PTR(V_dyn_ipv4_add, ipv4_add, tmp);
2577         DYN_SWAP_PTR(V_dyn_ipv4_parent_add, ipv4_parent_add, tmp);
2578         DYN_SWAP_PTR(V_dyn_ipv4_del, ipv4_del, tmp);
2579         DYN_SWAP_PTR(V_dyn_ipv4_parent_del, ipv4_parent_del, tmp);
2580
2581 #ifdef INET6
2582         DYN_SWAP_PTR(V_dyn_ipv6, ipv6, tmp);
2583         DYN_SWAP_PTR(V_dyn_ipv6_parent, ipv6_parent, tmp);
2584         DYN_SWAP_PTR(V_dyn_ipv6_add, ipv6_add, tmp);
2585         DYN_SWAP_PTR(V_dyn_ipv6_parent_add, ipv6_parent_add, tmp);
2586         DYN_SWAP_PTR(V_dyn_ipv6_del, ipv6_del, tmp);
2587         DYN_SWAP_PTR(V_dyn_ipv6_parent_del, ipv6_parent_del, tmp);
2588 #endif
2589         bucket = V_curr_dyn_buckets;
2590         V_curr_dyn_buckets = new;
2591
2592         IPFW_WUNLOCK(chain);
2593         IPFW_UH_WUNLOCK(chain);
2594
2595         /* Release old resources */
2596         while (bucket-- != 0)
2597                 DYN_BUCKET_LOCK_DESTROY(bucket_lock, bucket);
2598         free(bucket_lock, M_IPFW);
2599         free(ipv4, M_IPFW);
2600         free(ipv4_parent, M_IPFW);
2601         free(ipv4_add, M_IPFW);
2602         free(ipv4_parent_add, M_IPFW);
2603         free(ipv4_del, M_IPFW);
2604         free(ipv4_parent_del, M_IPFW);
2605 #ifdef INET6
2606         free(ipv6, M_IPFW);
2607         free(ipv6_parent, M_IPFW);
2608         free(ipv6_add, M_IPFW);
2609         free(ipv6_parent_add, M_IPFW);
2610         free(ipv6_del, M_IPFW);
2611         free(ipv6_parent_del, M_IPFW);
2612 #endif
2613 }
2614
2615 /*
2616  * This function is used to perform various maintenance
2617  * on dynamic hash lists. Currently it is called every second.
2618  */
2619 static void
2620 dyn_tick(void *vnetx)
2621 {
2622         uint32_t buckets;
2623
2624         CURVNET_SET((struct vnet *)vnetx);
2625         /*
2626          * First free states unlinked in previous passes.
2627          */
2628         dyn_free_states(&V_layer3_chain);
2629         /*
2630          * Now unlink others expired states.
2631          * We use IPFW_UH_WLOCK to avoid concurrent call of
2632          * dyn_expire_states(). It is the only function that does
2633          * deletion of state entries from states lists.
2634          */
2635         IPFW_UH_WLOCK(&V_layer3_chain);
2636         dyn_expire_states(&V_layer3_chain, NULL);
2637         IPFW_UH_WUNLOCK(&V_layer3_chain);
2638         /*
2639          * Send keepalives if they are enabled and the time has come.
2640          */
2641         if (V_dyn_keepalive != 0 &&
2642             V_dyn_keepalive_last + V_dyn_keepalive_period <= time_uptime) {
2643                 V_dyn_keepalive_last = time_uptime;
2644                 dyn_send_keepalive_ipv4(&V_layer3_chain);
2645 #ifdef INET6
2646                 dyn_send_keepalive_ipv6(&V_layer3_chain);
2647 #endif
2648         }
2649         /*
2650          * Check if we need to resize the hash:
2651          * if current number of states exceeds number of buckets in hash,
2652          * and dyn_buckets_max permits to grow the number of buckets, then
2653          * do it. Grow hash size to the minimum power of 2 which is bigger
2654          * than current states count.
2655          */
2656         if (V_curr_dyn_buckets < V_dyn_buckets_max &&
2657             (V_curr_dyn_buckets < V_dyn_count / 2 || (
2658             V_curr_dyn_buckets < V_dyn_count && V_curr_max_length > 8))) {
2659                 buckets = 1 << fls(V_dyn_count);
2660                 if (buckets > V_dyn_buckets_max)
2661                         buckets = V_dyn_buckets_max;
2662                 dyn_grow_hashtable(&V_layer3_chain, buckets);
2663         }
2664
2665         callout_reset_on(&V_dyn_timeout, hz, dyn_tick, vnetx, 0);
2666         CURVNET_RESTORE();
2667 }
2668
2669 void
2670 ipfw_expire_dyn_states(struct ip_fw_chain *chain, ipfw_range_tlv *rt)
2671 {
2672         /*
2673          * Do not perform any checks if we currently have no dynamic states
2674          */
2675         if (V_dyn_count == 0)
2676                 return;
2677
2678         IPFW_UH_WLOCK_ASSERT(chain);
2679         dyn_expire_states(chain, rt);
2680 }
2681
2682 /*
2683  * Returns size of dynamic states in legacy format
2684  */
2685 int
2686 ipfw_dyn_len(void)
2687 {
2688
2689         return ((V_dyn_count + V_dyn_parent_count) * sizeof(ipfw_dyn_rule));
2690 }
2691
2692 /*
2693  * Returns number of dynamic states.
2694  * Used by dump format v1 (current).
2695  */
2696 uint32_t
2697 ipfw_dyn_get_count(void)
2698 {
2699
2700         return (V_dyn_count + V_dyn_parent_count);
2701 }
2702
2703 /*
2704  * Check if rule contains at least one dynamic opcode.
2705  *
2706  * Returns 1 if such opcode is found, 0 otherwise.
2707  */
2708 int
2709 ipfw_is_dyn_rule(struct ip_fw *rule)
2710 {
2711         int cmdlen, l;
2712         ipfw_insn *cmd;
2713
2714         l = rule->cmd_len;
2715         cmd = rule->cmd;
2716         cmdlen = 0;
2717         for ( ; l > 0 ; l -= cmdlen, cmd += cmdlen) {
2718                 cmdlen = F_LEN(cmd);
2719
2720                 switch (cmd->opcode) {
2721                 case O_LIMIT:
2722                 case O_KEEP_STATE:
2723                 case O_PROBE_STATE:
2724                 case O_CHECK_STATE:
2725                         return (1);
2726                 }
2727         }
2728
2729         return (0);
2730 }
2731
2732 static void
2733 dyn_export_parent(const struct dyn_parent *p, uint16_t kidx,
2734     ipfw_dyn_rule *dst)
2735 {
2736
2737         dst->dyn_type = O_LIMIT_PARENT;
2738         dst->kidx = kidx;
2739         dst->count = (uint16_t)DPARENT_COUNT(p);
2740         dst->expire = TIME_LEQ(p->expire, time_uptime) ?  0:
2741             p->expire - time_uptime;
2742
2743         /* 'rule' is used to pass up the rule number and set */
2744         memcpy(&dst->rule, &p->rulenum, sizeof(p->rulenum));
2745         /* store set number into high word of dst->rule pointer. */
2746         memcpy((char *)&dst->rule + sizeof(p->rulenum), &p->set,
2747             sizeof(p->set));
2748
2749         /* unused fields */
2750         dst->pcnt = 0;
2751         dst->bcnt = 0;
2752         dst->parent = NULL;
2753         dst->state = 0;
2754         dst->ack_fwd = 0;
2755         dst->ack_rev = 0;
2756         dst->bucket = p->hashval;
2757         /*
2758          * The legacy userland code will interpret a NULL here as a marker
2759          * for the last dynamic rule.
2760          */
2761         dst->next = (ipfw_dyn_rule *)1;
2762 }
2763
2764 static void
2765 dyn_export_data(const struct dyn_data *data, uint16_t kidx, uint8_t type,
2766     ipfw_dyn_rule *dst)
2767 {
2768
2769         dst->dyn_type = type;
2770         dst->kidx = kidx;
2771         dst->pcnt = data->pcnt_fwd + data->pcnt_rev;
2772         dst->bcnt = data->bcnt_fwd + data->bcnt_rev;
2773         dst->expire = TIME_LEQ(data->expire, time_uptime) ?  0:
2774             data->expire - time_uptime;
2775
2776         /* 'rule' is used to pass up the rule number and set */
2777         memcpy(&dst->rule, &data->rulenum, sizeof(data->rulenum));
2778         /* store set number into high word of dst->rule pointer. */
2779         memcpy((char *)&dst->rule + sizeof(data->rulenum), &data->set,
2780             sizeof(data->set));
2781
2782         /* unused fields */
2783         dst->parent = NULL;
2784         dst->state = data->state;
2785         dst->ack_fwd = data->ack_fwd;
2786         dst->ack_rev = data->ack_rev;
2787         dst->count = 0;
2788         dst->bucket = data->hashval;
2789         /*
2790          * The legacy userland code will interpret a NULL here as a marker
2791          * for the last dynamic rule.
2792          */
2793         dst->next = (ipfw_dyn_rule *)1;
2794 }
2795
2796 static void
2797 dyn_export_ipv4_state(const struct dyn_ipv4_state *s, ipfw_dyn_rule *dst)
2798 {
2799
2800         switch (s->type) {
2801         case O_LIMIT_PARENT:
2802                 dyn_export_parent(s->limit, s->kidx, dst);
2803                 break;
2804         default:
2805                 dyn_export_data(s->data, s->kidx, s->type, dst);
2806         }
2807
2808         dst->id.dst_ip = s->dst;
2809         dst->id.src_ip = s->src;
2810         dst->id.dst_port = s->dport;
2811         dst->id.src_port = s->sport;
2812         dst->id.fib = s->data->fibnum;
2813         dst->id.proto = s->proto;
2814         dst->id._flags = 0;
2815         dst->id.addr_type = 4;
2816
2817         memset(&dst->id.dst_ip6, 0, sizeof(dst->id.dst_ip6));
2818         memset(&dst->id.src_ip6, 0, sizeof(dst->id.src_ip6));
2819         dst->id.flow_id6 = dst->id.extra = 0;
2820 }
2821
2822 #ifdef INET6
2823 static void
2824 dyn_export_ipv6_state(const struct dyn_ipv6_state *s, ipfw_dyn_rule *dst)
2825 {
2826
2827         switch (s->type) {
2828         case O_LIMIT_PARENT:
2829                 dyn_export_parent(s->limit, s->kidx, dst);
2830                 break;
2831         default:
2832                 dyn_export_data(s->data, s->kidx, s->type, dst);
2833         }
2834
2835         dst->id.src_ip6 = s->src;
2836         dst->id.dst_ip6 = s->dst;
2837         dst->id.dst_port = s->dport;
2838         dst->id.src_port = s->sport;
2839         dst->id.fib = s->data->fibnum;
2840         dst->id.proto = s->proto;
2841         dst->id._flags = 0;
2842         dst->id.addr_type = 6;
2843
2844         dst->id.dst_ip = dst->id.src_ip = 0;
2845         dst->id.flow_id6 = dst->id.extra = 0;
2846 }
2847 #endif /* INET6 */
2848
2849 /*
2850  * Fills the buffer given by @sd with dynamic states.
2851  * Used by dump format v1 (current).
2852  *
2853  * Returns 0 on success.
2854  */
2855 int
2856 ipfw_dump_states(struct ip_fw_chain *chain, struct sockopt_data *sd)
2857 {
2858 #ifdef INET6
2859         struct dyn_ipv6_state *s6;
2860 #endif
2861         struct dyn_ipv4_state *s4;
2862         ipfw_obj_dyntlv *dst, *last;
2863         ipfw_obj_ctlv *ctlv;
2864         uint32_t bucket;
2865
2866         if (V_dyn_count == 0)
2867                 return (0);
2868
2869         /*
2870          * IPFW_UH_RLOCK garantees that another userland request
2871          * and callout thread will not delete entries from states
2872          * lists.
2873          */
2874         IPFW_UH_RLOCK_ASSERT(chain);
2875
2876         ctlv = (ipfw_obj_ctlv *)ipfw_get_sopt_space(sd, sizeof(*ctlv));
2877         if (ctlv == NULL)
2878                 return (ENOMEM);
2879         ctlv->head.type = IPFW_TLV_DYNSTATE_LIST;
2880         ctlv->objsize = sizeof(ipfw_obj_dyntlv);
2881         last = NULL;
2882
2883 #define DYN_EXPORT_STATES(s, af, h, b)                          \
2884         CK_SLIST_FOREACH(s, &V_dyn_ ## h[b], entry) {                   \
2885                 dst = (ipfw_obj_dyntlv *)ipfw_get_sopt_space(sd,        \
2886                     sizeof(ipfw_obj_dyntlv));                           \
2887                 if (dst == NULL)                                        \
2888                         return (ENOMEM);                                \
2889                 dyn_export_ ## af ## _state(s, &dst->state);            \
2890                 dst->head.length = sizeof(ipfw_obj_dyntlv);             \
2891                 dst->head.type = IPFW_TLV_DYN_ENT;                      \
2892                 last = dst;                                             \
2893         }
2894
2895         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2896                 DYN_EXPORT_STATES(s4, ipv4, ipv4_parent, bucket);
2897                 DYN_EXPORT_STATES(s4, ipv4, ipv4, bucket);
2898 #ifdef INET6
2899                 DYN_EXPORT_STATES(s6, ipv6, ipv6_parent, bucket);
2900                 DYN_EXPORT_STATES(s6, ipv6, ipv6, bucket);
2901 #endif /* INET6 */
2902         }
2903
2904         /* mark last dynamic rule */
2905         if (last != NULL)
2906                 last->head.flags = IPFW_DF_LAST; /* XXX: unused */
2907         return (0);
2908 #undef DYN_EXPORT_STATES
2909 }
2910
2911 /*
2912  * Fill given buffer with dynamic states (legacy format).
2913  * IPFW_UH_RLOCK has to be held while calling.
2914  */
2915 void
2916 ipfw_get_dynamic(struct ip_fw_chain *chain, char **pbp, const char *ep)
2917 {
2918 #ifdef INET6
2919         struct dyn_ipv6_state *s6;
2920 #endif
2921         struct dyn_ipv4_state *s4;
2922         ipfw_dyn_rule *p, *last = NULL;
2923         char *bp;
2924         uint32_t bucket;
2925
2926         if (V_dyn_count == 0)
2927                 return;
2928         bp = *pbp;
2929
2930         IPFW_UH_RLOCK_ASSERT(chain);
2931
2932 #define DYN_EXPORT_STATES(s, af, head, b)                               \
2933         CK_SLIST_FOREACH(s, &V_dyn_ ## head[b], entry) {                \
2934                 if (bp + sizeof(*p) > ep)                               \
2935                         break;                                          \
2936                 p = (ipfw_dyn_rule *)bp;                                \
2937                 dyn_export_ ## af ## _state(s, p);                      \
2938                 last = p;                                               \
2939                 bp += sizeof(*p);                                       \
2940         }
2941
2942         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2943                 DYN_EXPORT_STATES(s4, ipv4, ipv4_parent, bucket);
2944                 DYN_EXPORT_STATES(s4, ipv4, ipv4, bucket);
2945 #ifdef INET6
2946                 DYN_EXPORT_STATES(s6, ipv6, ipv6_parent, bucket);
2947                 DYN_EXPORT_STATES(s6, ipv6, ipv6, bucket);
2948 #endif /* INET6 */
2949         }
2950
2951         if (last != NULL) /* mark last dynamic rule */
2952                 last->next = NULL;
2953         *pbp = bp;
2954 #undef DYN_EXPORT_STATES
2955 }
2956
2957 void
2958 ipfw_dyn_init(struct ip_fw_chain *chain)
2959 {
2960
2961 #ifdef IPFIREWALL_JENKINSHASH
2962         V_dyn_hashseed = arc4random();
2963 #endif
2964         V_dyn_max = 16384;              /* max # of states */
2965         V_dyn_parent_max = 4096;        /* max # of parent states */
2966         V_dyn_buckets_max = 8192;       /* must be power of 2 */
2967
2968         V_dyn_ack_lifetime = 300;
2969         V_dyn_syn_lifetime = 20;
2970         V_dyn_fin_lifetime = 1;
2971         V_dyn_rst_lifetime = 1;
2972         V_dyn_udp_lifetime = 10;
2973         V_dyn_short_lifetime = 5;
2974
2975         V_dyn_keepalive_interval = 20;
2976         V_dyn_keepalive_period = 5;
2977         V_dyn_keepalive = 1;            /* send keepalives */
2978         V_dyn_keepalive_last = time_uptime;
2979
2980         V_dyn_data_zone = uma_zcreate("IPFW dynamic states data",
2981             sizeof(struct dyn_data), NULL, NULL, NULL, NULL,
2982             UMA_ALIGN_PTR, 0);
2983         uma_zone_set_max(V_dyn_data_zone, V_dyn_max);
2984
2985         V_dyn_parent_zone = uma_zcreate("IPFW parent dynamic states",
2986             sizeof(struct dyn_parent), NULL, NULL, NULL, NULL,
2987             UMA_ALIGN_PTR, 0);
2988         uma_zone_set_max(V_dyn_parent_zone, V_dyn_parent_max);
2989
2990         SLIST_INIT(&V_dyn_expired_ipv4);
2991         V_dyn_ipv4 = NULL;
2992         V_dyn_ipv4_parent = NULL;
2993         V_dyn_ipv4_zone = uma_zcreate("IPFW IPv4 dynamic states",
2994             sizeof(struct dyn_ipv4_state), NULL, NULL, NULL, NULL,
2995             UMA_ALIGN_PTR, 0);
2996
2997 #ifdef INET6
2998         SLIST_INIT(&V_dyn_expired_ipv6);
2999         V_dyn_ipv6 = NULL;
3000         V_dyn_ipv6_parent = NULL;
3001         V_dyn_ipv6_zone = uma_zcreate("IPFW IPv6 dynamic states",
3002             sizeof(struct dyn_ipv6_state), NULL, NULL, NULL, NULL,
3003             UMA_ALIGN_PTR, 0);
3004 #endif
3005
3006         /* Initialize buckets. */
3007         V_curr_dyn_buckets = 0;
3008         V_dyn_bucket_lock = NULL;
3009         dyn_grow_hashtable(chain, 256);
3010
3011         if (IS_DEFAULT_VNET(curvnet))
3012                 dyn_hp_cache = malloc(mp_ncpus * sizeof(void *), M_IPFW,
3013                     M_WAITOK | M_ZERO);
3014
3015         DYN_EXPIRED_LOCK_INIT();
3016         callout_init(&V_dyn_timeout, 1);
3017         callout_reset(&V_dyn_timeout, hz, dyn_tick, curvnet);
3018         IPFW_ADD_OBJ_REWRITER(IS_DEFAULT_VNET(curvnet), dyn_opcodes);
3019 }
3020
3021 void
3022 ipfw_dyn_uninit(int pass)
3023 {
3024 #ifdef INET6
3025         struct dyn_ipv6_state *s6;
3026 #endif
3027         struct dyn_ipv4_state *s4;
3028         int bucket;
3029
3030         if (pass == 0) {
3031                 callout_drain(&V_dyn_timeout);
3032                 return;
3033         }
3034         IPFW_DEL_OBJ_REWRITER(IS_DEFAULT_VNET(curvnet), dyn_opcodes);
3035         DYN_EXPIRED_LOCK_DESTROY();
3036
3037 #define DYN_FREE_STATES_FORCED(CK, s, af, name, en)     do {            \
3038         while ((s = CK ## SLIST_FIRST(&V_dyn_ ## name)) != NULL) {      \
3039                 CK ## SLIST_REMOVE_HEAD(&V_dyn_ ## name, en);   \
3040                 if (s->type == O_LIMIT_PARENT)                          \
3041                         uma_zfree(V_dyn_parent_zone, s->limit);         \
3042                 else                                                    \
3043                         uma_zfree(V_dyn_data_zone, s->data);            \
3044                 uma_zfree(V_dyn_ ## af ## _zone, s);                    \
3045         }                                                               \
3046 } while (0)
3047         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
3048                 DYN_BUCKET_LOCK_DESTROY(V_dyn_bucket_lock, bucket);
3049
3050                 DYN_FREE_STATES_FORCED(CK_, s4, ipv4, ipv4[bucket], entry);
3051                 DYN_FREE_STATES_FORCED(CK_, s4, ipv4, ipv4_parent[bucket],
3052                     entry);
3053 #ifdef INET6
3054                 DYN_FREE_STATES_FORCED(CK_, s6, ipv6, ipv6[bucket], entry);
3055                 DYN_FREE_STATES_FORCED(CK_, s6, ipv6, ipv6_parent[bucket],
3056                     entry);
3057 #endif /* INET6 */
3058         }
3059         DYN_FREE_STATES_FORCED(, s4, ipv4, expired_ipv4, expired);
3060 #ifdef INET6
3061         DYN_FREE_STATES_FORCED(, s6, ipv6, expired_ipv6, expired);
3062 #endif
3063 #undef DYN_FREE_STATES_FORCED
3064
3065         uma_zdestroy(V_dyn_ipv4_zone);
3066         uma_zdestroy(V_dyn_data_zone);
3067         uma_zdestroy(V_dyn_parent_zone);
3068 #ifdef INET6
3069         uma_zdestroy(V_dyn_ipv6_zone);
3070         free(V_dyn_ipv6, M_IPFW);
3071         free(V_dyn_ipv6_parent, M_IPFW);
3072         free(V_dyn_ipv6_add, M_IPFW);
3073         free(V_dyn_ipv6_parent_add, M_IPFW);
3074         free(V_dyn_ipv6_del, M_IPFW);
3075         free(V_dyn_ipv6_parent_del, M_IPFW);
3076 #endif
3077         free(V_dyn_bucket_lock, M_IPFW);
3078         free(V_dyn_ipv4, M_IPFW);
3079         free(V_dyn_ipv4_parent, M_IPFW);
3080         free(V_dyn_ipv4_add, M_IPFW);
3081         free(V_dyn_ipv4_parent_add, M_IPFW);
3082         free(V_dyn_ipv4_del, M_IPFW);
3083         free(V_dyn_ipv4_parent_del, M_IPFW);
3084         if (IS_DEFAULT_VNET(curvnet))
3085                 free(dyn_hp_cache, M_IPFW);
3086 }
3087
3088