sys/netpfil/ipfw/ip_fw_dynamic.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2017-2018 Yandex LLC
   5  * Copyright (c) 2017-2018 Andrey V. Elsukov <ae@FreeBSD.org>
   6  * Copyright (c) 2002 Luigi Rizzo, Universita` di Pisa
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_inet.h"
  34 #include "opt_inet6.h"
  35 #include "opt_ipfw.h"
  36 #ifndef INET
  37 #error IPFIREWALL requires INET.
  38 #endif /* INET */
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #include <sys/hash.h>
  43 #include <sys/mbuf.h>
  44 #include <sys/kernel.h>
  45 #include <sys/lock.h>
  46 #include <sys/pcpu.h>
  47 #include <sys/queue.h>
  48 #include <sys/rmlock.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/syslog.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_var.h>
  56 #include <net/pfil.h>
  57 #include <net/vnet.h>
  58
  59 #include <netinet/in.h>
  60 #include <netinet/ip.h>
  61 #include <netinet/ip_var.h>
  62 #include <netinet/ip_fw.h>
  63 #include <netinet/tcp_var.h>
  64 #include <netinet/udp.h>
  65
  66 #include <netinet/ip6.h>        /* IN6_ARE_ADDR_EQUAL */
  67 #ifdef INET6
  68 #include <netinet6/in6_var.h>
  69 #include <netinet6/ip6_var.h>
  70 #include <netinet6/scope6_var.h>
  71 #endif
  72
  73 #include <netpfil/ipfw/ip_fw_private.h>
  74
  75 #include <machine/in_cksum.h>   /* XXX for in_cksum */
  76
  77 #ifdef MAC
  78 #include <security/mac/mac_framework.h>
  79 #endif
  80
  81 /*
  82  * Description of dynamic states.
  83  *
  84  * Dynamic states are stored in lists accessed through a hash tables
  85  * whose size is curr_dyn_buckets. This value can be modified through
  86  * the sysctl variable dyn_buckets.
  87  *
  88  * Currently there are four tables: dyn_ipv4, dyn_ipv6, dyn_ipv4_parent,
  89  * and dyn_ipv6_parent.
  90  *
  91  * When a packet is received, its address fields hashed, then matched
  92  * against the entries in the corresponding list by addr_type.
  93  * Dynamic states can be used for different purposes:
  94  *  + stateful rules;
  95  *  + enforcing limits on the number of sessions;
  96  *  + in-kernel NAT (not implemented yet)
  97  *
  98  * The lifetime of dynamic states is regulated by dyn_*_lifetime,
  99  * measured in seconds and depending on the flags.
 100  *
 101  * The total number of dynamic states is equal to UMA zone items count.
 102  * The max number of dynamic states is dyn_max. When we reach
 103  * the maximum number of rules we do not create anymore. This is
 104  * done to avoid consuming too much memory, but also too much
 105  * time when searching on each packet (ideally, we should try instead
 106  * to put a limit on the length of the list on each bucket...).
 107  *
 108  * Each state holds a pointer to the parent ipfw rule so we know what
 109  * action to perform. Dynamic rules are removed when the parent rule is
 110  * deleted.
 111  *
 112  * There are some limitations with dynamic rules -- we do not
 113  * obey the 'randomized match', and we do not do multiple
 114  * passes through the firewall. XXX check the latter!!!
 115  */
 116
 117 /* By default use jenkins hash function */
 118 #define IPFIREWALL_JENKINSHASH
 119
 120 #define DYN_COUNTER_INC(d, dir, pktlen) do {    \
 121         (d)->pcnt_ ## dir++;                    \
 122         (d)->bcnt_ ## dir += pktlen;            \
 123         } while (0)
 124
 125 struct dyn_data {
 126         void            *parent;        /* pointer to parent rule */
 127         uint32_t        chain_id;       /* cached ruleset id */
 128         uint32_t        f_pos;          /* cached rule index */
 129
 130         uint32_t        hashval;        /* hash value used for hash resize */
 131         uint16_t        fibnum;         /* fib used to send keepalives */
 132         uint8_t         _pad[3];
 133         uint8_t         set;            /* parent rule set number */
 134         uint16_t        rulenum;        /* parent rule number */
 135         uint32_t        ruleid;         /* parent rule id */
 136
 137         uint32_t        state;          /* TCP session state and flags */
 138         uint32_t        ack_fwd;        /* most recent ACKs in forward */
 139         uint32_t        ack_rev;        /* and reverse direction (used */
 140                                         /* to generate keepalives) */
 141         uint32_t        sync;           /* synchronization time */
 142         uint32_t        expire;         /* expire time */
 143
 144         uint64_t        pcnt_fwd;       /* bytes counter in forward */
 145         uint64_t        bcnt_fwd;       /* packets counter in forward */
 146         uint64_t        pcnt_rev;       /* bytes counter in reverse */
 147         uint64_t        bcnt_rev;       /* packets counter in reverse */
 148 };
 149
 150 #define DPARENT_COUNT_DEC(p)    do {                    \
 151         MPASS(p->count > 0);                            \
 152         ck_pr_dec_32(&(p)->count);                      \
 153 } while (0)
 154 #define DPARENT_COUNT_INC(p)    ck_pr_inc_32(&(p)->count)
 155 #define DPARENT_COUNT(p)        ck_pr_load_32(&(p)->count)
 156 struct dyn_parent {
 157         void            *parent;        /* pointer to parent rule */
 158         uint32_t        count;          /* number of linked states */
 159         uint8_t         _pad;
 160         uint8_t         set;            /* parent rule set number */
 161         uint16_t        rulenum;        /* parent rule number */
 162         uint32_t        ruleid;         /* parent rule id */
 163         uint32_t        hashval;        /* hash value used for hash resize */
 164         uint32_t        expire;         /* expire time */
 165 };
 166
 167 struct dyn_ipv4_state {
 168         uint8_t         type;           /* State type */
 169         uint8_t         proto;          /* UL Protocol */
 170         uint16_t        kidx;           /* named object index */
 171         uint16_t        sport, dport;   /* ULP source and destination ports */
 172         in_addr_t       src, dst;       /* IPv4 source and destination */
 173
 174         union {
 175                 struct dyn_data *data;
 176                 struct dyn_parent *limit;
 177         };
 178         CK_SLIST_ENTRY(dyn_ipv4_state)  entry;
 179         SLIST_ENTRY(dyn_ipv4_state)     expired;
 180 };
 181 CK_SLIST_HEAD(dyn_ipv4ck_slist, dyn_ipv4_state);
 182 VNET_DEFINE_STATIC(struct dyn_ipv4ck_slist *, dyn_ipv4);
 183 VNET_DEFINE_STATIC(struct dyn_ipv4ck_slist *, dyn_ipv4_parent);
 184
 185 SLIST_HEAD(dyn_ipv4_slist, dyn_ipv4_state);
 186 VNET_DEFINE_STATIC(struct dyn_ipv4_slist, dyn_expired_ipv4);
 187 #define V_dyn_ipv4                      VNET(dyn_ipv4)
 188 #define V_dyn_ipv4_parent               VNET(dyn_ipv4_parent)
 189 #define V_dyn_expired_ipv4              VNET(dyn_expired_ipv4)
 190
 191 #ifdef INET6
 192 struct dyn_ipv6_state {
 193         uint8_t         type;           /* State type */
 194         uint8_t         proto;          /* UL Protocol */
 195         uint16_t        kidx;           /* named object index */
 196         uint16_t        sport, dport;   /* ULP source and destination ports */
 197         struct in6_addr src, dst;       /* IPv6 source and destination */
 198         uint32_t        zoneid;         /* IPv6 scope zone id */
 199         union {
 200                 struct dyn_data *data;
 201                 struct dyn_parent *limit;
 202         };
 203         CK_SLIST_ENTRY(dyn_ipv6_state)  entry;
 204         SLIST_ENTRY(dyn_ipv6_state)     expired;
 205 };
 206 CK_SLIST_HEAD(dyn_ipv6ck_slist, dyn_ipv6_state);
 207 VNET_DEFINE_STATIC(struct dyn_ipv6ck_slist *, dyn_ipv6);
 208 VNET_DEFINE_STATIC(struct dyn_ipv6ck_slist *, dyn_ipv6_parent);
 209
 210 SLIST_HEAD(dyn_ipv6_slist, dyn_ipv6_state);
 211 VNET_DEFINE_STATIC(struct dyn_ipv6_slist, dyn_expired_ipv6);
 212 #define V_dyn_ipv6                      VNET(dyn_ipv6)
 213 #define V_dyn_ipv6_parent               VNET(dyn_ipv6_parent)
 214 #define V_dyn_expired_ipv6              VNET(dyn_expired_ipv6)
 215 #endif /* INET6 */
 216
 217 /*
 218  * Per-CPU pointer indicates that specified state is currently in use
 219  * and must not be reclaimed by expiration callout.
 220  */
 221 static void **dyn_hp_cache;
 222 DPCPU_DEFINE_STATIC(void *, dyn_hp);
 223 #define DYNSTATE_GET(cpu)       ck_pr_load_ptr(DPCPU_ID_PTR((cpu), dyn_hp))
 224 #define DYNSTATE_PROTECT(v)     ck_pr_store_ptr(DPCPU_PTR(dyn_hp), (v))
 225 #define DYNSTATE_RELEASE()      DYNSTATE_PROTECT(NULL)
 226 #define DYNSTATE_CRITICAL_ENTER()       critical_enter()
 227 #define DYNSTATE_CRITICAL_EXIT()        do {    \
 228         DYNSTATE_RELEASE();                     \
 229         critical_exit();                        \
 230 } while (0);
 231
 232 /*
 233  * We keep two version numbers, one is updated when new entry added to
 234  * the list. Second is updated when an entry deleted from the list.
 235  * Versions are updated under bucket lock.
 236  *
 237  * Bucket "add" version number is used to know, that in the time between
 238  * state lookup (i.e. ipfw_dyn_lookup_state()) and the followed state
 239  * creation (i.e. ipfw_dyn_install_state()) another concurrent thread did
 240  * not install some state in this bucket. Using this info we can avoid
 241  * additional state lookup, because we are sure that we will not install
 242  * the state twice.
 243  *
 244  * Also doing the tracking of bucket "del" version during lookup we can
 245  * be sure, that state entry was not unlinked and freed in time between
 246  * we read the state pointer and protect it with hazard pointer.
 247  *
 248  * An entry unlinked from CK list keeps unchanged until it is freed.
 249  * Unlinked entries are linked into expired lists using "expired" field.
 250  */
 251
 252 /*
 253  * dyn_expire_lock is used to protect access to dyn_expired_xxx lists.
 254  * dyn_bucket_lock is used to get write access to lists in specific bucket.
 255  * Currently one dyn_bucket_lock is used for all ipv4, ipv4_parent, ipv6,
 256  * and ipv6_parent lists.
 257  */
 258 VNET_DEFINE_STATIC(struct mtx, dyn_expire_lock);
 259 VNET_DEFINE_STATIC(struct mtx *, dyn_bucket_lock);
 260 #define V_dyn_expire_lock               VNET(dyn_expire_lock)
 261 #define V_dyn_bucket_lock               VNET(dyn_bucket_lock)
 262
 263 /*
 264  * Bucket's add/delete generation versions.
 265  */
 266 VNET_DEFINE_STATIC(uint32_t *, dyn_ipv4_add);
 267 VNET_DEFINE_STATIC(uint32_t *, dyn_ipv4_del);
 268 VNET_DEFINE_STATIC(uint32_t *, dyn_ipv4_parent_add);
 269 VNET_DEFINE_STATIC(uint32_t *, dyn_ipv4_parent_del);
 270 #define V_dyn_ipv4_add                  VNET(dyn_ipv4_add)
 271 #define V_dyn_ipv4_del                  VNET(dyn_ipv4_del)
 272 #define V_dyn_ipv4_parent_add           VNET(dyn_ipv4_parent_add)
 273 #define V_dyn_ipv4_parent_del           VNET(dyn_ipv4_parent_del)
 274
 275 #ifdef INET6
 276 VNET_DEFINE_STATIC(uint32_t *, dyn_ipv6_add);
 277 VNET_DEFINE_STATIC(uint32_t *, dyn_ipv6_del);
 278 VNET_DEFINE_STATIC(uint32_t *, dyn_ipv6_parent_add);
 279 VNET_DEFINE_STATIC(uint32_t *, dyn_ipv6_parent_del);
 280 #define V_dyn_ipv6_add                  VNET(dyn_ipv6_add)
 281 #define V_dyn_ipv6_del                  VNET(dyn_ipv6_del)
 282 #define V_dyn_ipv6_parent_add           VNET(dyn_ipv6_parent_add)
 283 #define V_dyn_ipv6_parent_del           VNET(dyn_ipv6_parent_del)
 284 #endif /* INET6 */
 285
 286 #define DYN_BUCKET(h, b)                ((h) & (b - 1))
 287 #define DYN_BUCKET_VERSION(b, v)        ck_pr_load_32(&V_dyn_ ## v[(b)])
 288 #define DYN_BUCKET_VERSION_BUMP(b, v)   ck_pr_inc_32(&V_dyn_ ## v[(b)])
 289
 290 #define DYN_BUCKET_LOCK_INIT(lock, b)           \
 291     mtx_init(&lock[(b)], "IPFW dynamic bucket", NULL, MTX_DEF)
 292 #define DYN_BUCKET_LOCK_DESTROY(lock, b)        mtx_destroy(&lock[(b)])
 293 #define DYN_BUCKET_LOCK(b)      mtx_lock(&V_dyn_bucket_lock[(b)])
 294 #define DYN_BUCKET_UNLOCK(b)    mtx_unlock(&V_dyn_bucket_lock[(b)])
 295 #define DYN_BUCKET_ASSERT(b)    mtx_assert(&V_dyn_bucket_lock[(b)], MA_OWNED)
 296
 297 #define DYN_EXPIRED_LOCK_INIT()         \
 298     mtx_init(&V_dyn_expire_lock, "IPFW expired states list", NULL, MTX_DEF)
 299 #define DYN_EXPIRED_LOCK_DESTROY()      mtx_destroy(&V_dyn_expire_lock)
 300 #define DYN_EXPIRED_LOCK()              mtx_lock(&V_dyn_expire_lock)
 301 #define DYN_EXPIRED_UNLOCK()            mtx_unlock(&V_dyn_expire_lock)
 302
 303 VNET_DEFINE_STATIC(uint32_t, dyn_buckets_max);
 304 VNET_DEFINE_STATIC(uint32_t, curr_dyn_buckets);
 305 VNET_DEFINE_STATIC(struct callout, dyn_timeout);
 306 #define V_dyn_buckets_max               VNET(dyn_buckets_max)
 307 #define V_curr_dyn_buckets              VNET(curr_dyn_buckets)
 308 #define V_dyn_timeout                   VNET(dyn_timeout)
 309
 310 /* Maximum length of states chain in a bucket */
 311 VNET_DEFINE_STATIC(uint32_t, curr_max_length);
 312 #define V_curr_max_length               VNET(curr_max_length)
 313
 314 VNET_DEFINE_STATIC(uint32_t, dyn_keep_states);
 315 #define V_dyn_keep_states               VNET(dyn_keep_states)
 316
 317 VNET_DEFINE_STATIC(uma_zone_t, dyn_data_zone);
 318 VNET_DEFINE_STATIC(uma_zone_t, dyn_parent_zone);
 319 VNET_DEFINE_STATIC(uma_zone_t, dyn_ipv4_zone);
 320 #ifdef INET6
 321 VNET_DEFINE_STATIC(uma_zone_t, dyn_ipv6_zone);
 322 #define V_dyn_ipv6_zone                 VNET(dyn_ipv6_zone)
 323 #endif /* INET6 */
 324 #define V_dyn_data_zone                 VNET(dyn_data_zone)
 325 #define V_dyn_parent_zone               VNET(dyn_parent_zone)
 326 #define V_dyn_ipv4_zone                 VNET(dyn_ipv4_zone)
 327
 328 /*
 329  * Timeouts for various events in handing dynamic rules.
 330  */
 331 VNET_DEFINE_STATIC(uint32_t, dyn_ack_lifetime);
 332 VNET_DEFINE_STATIC(uint32_t, dyn_syn_lifetime);
 333 VNET_DEFINE_STATIC(uint32_t, dyn_fin_lifetime);
 334 VNET_DEFINE_STATIC(uint32_t, dyn_rst_lifetime);
 335 VNET_DEFINE_STATIC(uint32_t, dyn_udp_lifetime);
 336 VNET_DEFINE_STATIC(uint32_t, dyn_short_lifetime);
 337
 338 #define V_dyn_ack_lifetime              VNET(dyn_ack_lifetime)
 339 #define V_dyn_syn_lifetime              VNET(dyn_syn_lifetime)
 340 #define V_dyn_fin_lifetime              VNET(dyn_fin_lifetime)
 341 #define V_dyn_rst_lifetime              VNET(dyn_rst_lifetime)
 342 #define V_dyn_udp_lifetime              VNET(dyn_udp_lifetime)
 343 #define V_dyn_short_lifetime            VNET(dyn_short_lifetime)
 344
 345 /*
 346  * Keepalives are sent if dyn_keepalive is set. They are sent every
 347  * dyn_keepalive_period seconds, in the last dyn_keepalive_interval
 348  * seconds of lifetime of a rule.
 349  * dyn_rst_lifetime and dyn_fin_lifetime should be strictly lower
 350  * than dyn_keepalive_period.
 351  */
 352 VNET_DEFINE_STATIC(uint32_t, dyn_keepalive_interval);
 353 VNET_DEFINE_STATIC(uint32_t, dyn_keepalive_period);
 354 VNET_DEFINE_STATIC(uint32_t, dyn_keepalive);
 355 VNET_DEFINE_STATIC(time_t, dyn_keepalive_last);
 356
 357 #define V_dyn_keepalive_interval        VNET(dyn_keepalive_interval)
 358 #define V_dyn_keepalive_period          VNET(dyn_keepalive_period)
 359 #define V_dyn_keepalive                 VNET(dyn_keepalive)
 360 #define V_dyn_keepalive_last            VNET(dyn_keepalive_last)
 361
 362 VNET_DEFINE_STATIC(uint32_t, dyn_max);          /* max # of dynamic states */
 363 VNET_DEFINE_STATIC(uint32_t, dyn_count);        /* number of states */
 364 VNET_DEFINE_STATIC(uint32_t, dyn_parent_max);   /* max # of parent states */
 365 VNET_DEFINE_STATIC(uint32_t, dyn_parent_count); /* number of parent states */
 366
 367 #define V_dyn_max                       VNET(dyn_max)
 368 #define V_dyn_count                     VNET(dyn_count)
 369 #define V_dyn_parent_max                VNET(dyn_parent_max)
 370 #define V_dyn_parent_count              VNET(dyn_parent_count)
 371
 372 #define DYN_COUNT_DEC(name)     do {                    \
 373         MPASS((V_ ## name) > 0);                        \
 374         ck_pr_dec_32(&(V_ ## name));                    \
 375 } while (0)
 376 #define DYN_COUNT_INC(name)     ck_pr_inc_32(&(V_ ## name))
 377 #define DYN_COUNT(name)         ck_pr_load_32(&(V_ ## name))
 378
 379 static time_t last_log; /* Log ratelimiting */
 380
 381 /*
 382  * Get/set maximum number of dynamic states in given VNET instance.
 383  */
 384 static int
 385 sysctl_dyn_max(SYSCTL_HANDLER_ARGS)
 386 {
 387         uint32_t nstates;
 388         int error;
 389
 390         nstates = V_dyn_max;
 391         error = sysctl_handle_32(oidp, &nstates, 0, req);
 392         /* Read operation or some error */
 393         if ((error != 0) || (req->newptr == NULL))
 394                 return (error);
 395
 396         V_dyn_max = nstates;
 397         uma_zone_set_max(V_dyn_data_zone, V_dyn_max);
 398         return (0);
 399 }
 400
 401 static int
 402 sysctl_dyn_parent_max(SYSCTL_HANDLER_ARGS)
 403 {
 404         uint32_t nstates;
 405         int error;
 406
 407         nstates = V_dyn_parent_max;
 408         error = sysctl_handle_32(oidp, &nstates, 0, req);
 409         /* Read operation or some error */
 410         if ((error != 0) || (req->newptr == NULL))
 411                 return (error);
 412
 413         V_dyn_parent_max = nstates;
 414         uma_zone_set_max(V_dyn_parent_zone, V_dyn_parent_max);
 415         return (0);
 416 }
 417
 418 static int
 419 sysctl_dyn_buckets(SYSCTL_HANDLER_ARGS)
 420 {
 421         uint32_t nbuckets;
 422         int error;
 423
 424         nbuckets = V_dyn_buckets_max;
 425         error = sysctl_handle_32(oidp, &nbuckets, 0, req);
 426         /* Read operation or some error */
 427         if ((error != 0) || (req->newptr == NULL))
 428                 return (error);
 429
 430         if (nbuckets > 256)
 431                 V_dyn_buckets_max = 1 << fls(nbuckets - 1);
 432         else
 433                 return (EINVAL);
 434         return (0);
 435 }
 436
 437 SYSCTL_DECL(_net_inet_ip_fw);
 438
 439 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_count,
 440     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(dyn_count), 0,
 441     "Current number of dynamic states.");
 442 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_parent_count,
 443     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(dyn_parent_count), 0,
 444     "Current number of parent states. ");
 445 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, curr_dyn_buckets,
 446     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(curr_dyn_buckets), 0,
 447     "Current number of buckets for states hash table.");
 448 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, curr_max_length,
 449     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(curr_max_length), 0,
 450     "Current maximum length of states chains in hash buckets.");
 451 SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_buckets,
 452     CTLFLAG_VNET | CTLTYPE_U32 | CTLFLAG_RW, 0, 0, sysctl_dyn_buckets,
 453     "IU", "Max number of buckets for dynamic states hash table.");
 454 SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_max,
 455     CTLFLAG_VNET | CTLTYPE_U32 | CTLFLAG_RW, 0, 0, sysctl_dyn_max,
 456     "IU", "Max number of dynamic states.");
 457 SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_parent_max,
 458     CTLFLAG_VNET | CTLTYPE_U32 | CTLFLAG_RW, 0, 0, sysctl_dyn_parent_max,
 459     "IU", "Max number of parent dynamic states.");
 460 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_ack_lifetime,
 461     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_ack_lifetime), 0,
 462     "Lifetime of dynamic states for TCP ACK.");
 463 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_syn_lifetime,
 464     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_syn_lifetime), 0,
 465     "Lifetime of dynamic states for TCP SYN.");
 466 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_fin_lifetime,
 467     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_fin_lifetime), 0,
 468     "Lifetime of dynamic states for TCP FIN.");
 469 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_rst_lifetime,
 470     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_rst_lifetime), 0,
 471     "Lifetime of dynamic states for TCP RST.");
 472 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_udp_lifetime,
 473     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_udp_lifetime), 0,
 474     "Lifetime of dynamic states for UDP.");
 475 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_short_lifetime,
 476     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_short_lifetime), 0,
 477     "Lifetime of dynamic states for other situations.");
 478 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_keepalive,
 479     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_keepalive), 0,
 480     "Enable keepalives for dynamic states.");
 481 SYSCTL_U32(_net_inet_ip_fw, OID_AUTO, dyn_keep_states,
 482     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_keep_states), 0,
 483     "Do not flush dynamic states on rule deletion");
 484
 485
 486 #ifdef IPFIREWALL_DYNDEBUG
 487 #define DYN_DEBUG(fmt, ...)     do {                    \
 488         printf("%s: " fmt "\n", __func__, __VA_ARGS__); \
 489 } while (0)
 490 #else
 491 #define DYN_DEBUG(fmt, ...)
 492 #endif /* !IPFIREWALL_DYNDEBUG */
 493
 494 #ifdef INET6
 495 /* Functions to work with IPv6 states */
 496 static struct dyn_ipv6_state *dyn_lookup_ipv6_state(
 497     const struct ipfw_flow_id *, uint32_t, const void *,
 498     struct ipfw_dyn_info *, int);
 499 static int dyn_lookup_ipv6_state_locked(const struct ipfw_flow_id *,
 500     uint32_t, const void *, int, uint32_t, uint16_t);
 501 static struct dyn_ipv6_state *dyn_alloc_ipv6_state(
 502     const struct ipfw_flow_id *, uint32_t, uint16_t, uint8_t);
 503 static int dyn_add_ipv6_state(void *, uint32_t, uint16_t, uint8_t,
 504     const struct ipfw_flow_id *, uint32_t, const void *, int, uint32_t,
 505     struct ipfw_dyn_info *, uint16_t, uint16_t, uint8_t);
 506 static void dyn_export_ipv6_state(const struct dyn_ipv6_state *,
 507     ipfw_dyn_rule *);
 508
 509 static uint32_t dyn_getscopeid(const struct ip_fw_args *);
 510 static void dyn_make_keepalive_ipv6(struct mbuf *, const struct in6_addr *,
 511     const struct in6_addr *, uint32_t, uint32_t, uint32_t, uint16_t,
 512     uint16_t);
 513 static void dyn_enqueue_keepalive_ipv6(struct mbufq *,
 514     const struct dyn_ipv6_state *);
 515 static void dyn_send_keepalive_ipv6(struct ip_fw_chain *);
 516
 517 static struct dyn_ipv6_state *dyn_lookup_ipv6_parent(
 518     const struct ipfw_flow_id *, uint32_t, const void *, uint32_t, uint16_t,
 519     uint32_t);
 520 static struct dyn_ipv6_state *dyn_lookup_ipv6_parent_locked(
 521     const struct ipfw_flow_id *, uint32_t, const void *, uint32_t, uint16_t,
 522     uint32_t);
 523 static struct dyn_ipv6_state *dyn_add_ipv6_parent(void *, uint32_t, uint16_t,
 524     uint8_t, const struct ipfw_flow_id *, uint32_t, uint32_t, uint32_t,
 525     uint16_t);
 526 #endif /* INET6 */
 527
 528 /* Functions to work with limit states */
 529 static void *dyn_get_parent_state(const struct ipfw_flow_id *, uint32_t,
 530     struct ip_fw *, uint32_t, uint32_t, uint16_t);
 531 static struct dyn_ipv4_state *dyn_lookup_ipv4_parent(
 532     const struct ipfw_flow_id *, const void *, uint32_t, uint16_t, uint32_t);
 533 static struct dyn_ipv4_state *dyn_lookup_ipv4_parent_locked(
 534     const struct ipfw_flow_id *, const void *, uint32_t, uint16_t, uint32_t);
 535 static struct dyn_parent *dyn_alloc_parent(void *, uint32_t, uint16_t,
 536     uint8_t, uint32_t);
 537 static struct dyn_ipv4_state *dyn_add_ipv4_parent(void *, uint32_t, uint16_t,
 538     uint8_t, const struct ipfw_flow_id *, uint32_t, uint32_t, uint16_t);
 539
 540 static void dyn_tick(void *);
 541 static void dyn_expire_states(struct ip_fw_chain *, ipfw_range_tlv *);
 542 static void dyn_free_states(struct ip_fw_chain *);
 543 static void dyn_export_parent(const struct dyn_parent *, uint16_t,
 544     ipfw_dyn_rule *);
 545 static void dyn_export_data(const struct dyn_data *, uint16_t, uint8_t,
 546     ipfw_dyn_rule *);
 547 static uint32_t dyn_update_tcp_state(struct dyn_data *,
 548     const struct ipfw_flow_id *, const struct tcphdr *, int);
 549 static void dyn_update_proto_state(struct dyn_data *,
 550     const struct ipfw_flow_id *, const void *, int, int);
 551
 552 /* Functions to work with IPv4 states */
 553 struct dyn_ipv4_state *dyn_lookup_ipv4_state(const struct ipfw_flow_id *,
 554     const void *, struct ipfw_dyn_info *, int);
 555 static int dyn_lookup_ipv4_state_locked(const struct ipfw_flow_id *,
 556     const void *, int, uint32_t, uint16_t);
 557 static struct dyn_ipv4_state *dyn_alloc_ipv4_state(
 558     const struct ipfw_flow_id *, uint16_t, uint8_t);
 559 static int dyn_add_ipv4_state(void *, uint32_t, uint16_t, uint8_t,
 560     const struct ipfw_flow_id *, const void *, int, uint32_t,
 561     struct ipfw_dyn_info *, uint16_t, uint16_t, uint8_t);
 562 static void dyn_export_ipv4_state(const struct dyn_ipv4_state *,
 563     ipfw_dyn_rule *);
 564
 565 /*
 566  * Named states support.
 567  */
 568 static char *default_state_name = "default";
 569 struct dyn_state_obj {
 570         struct named_object     no;
 571         char                    name[64];
 572 };
 573
 574 #define DYN_STATE_OBJ(ch, cmd)  \
 575     ((struct dyn_state_obj *)SRV_OBJECT(ch, (cmd)->arg1))
 576 /*
 577  * Classifier callback.
 578  * Return 0 if opcode contains object that should be referenced
 579  * or rewritten.
 580  */
 581 static int
 582 dyn_classify(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype)
 583 {
 584
 585         DYN_DEBUG("opcode %d, arg1 %d", cmd->opcode, cmd->arg1);
 586         /* Don't rewrite "check-state any" */
 587         if (cmd->arg1 == 0 &&
 588             cmd->opcode == O_CHECK_STATE)
 589                 return (1);
 590
 591         *puidx = cmd->arg1;
 592         *ptype = 0;
 593         return (0);
 594 }
 595
 596 static void
 597 dyn_update(ipfw_insn *cmd, uint16_t idx)
 598 {
 599
 600         cmd->arg1 = idx;
 601         DYN_DEBUG("opcode %d, arg1 %d", cmd->opcode, cmd->arg1);
 602 }
 603
 604 static int
 605 dyn_findbyname(struct ip_fw_chain *ch, struct tid_info *ti,
 606     struct named_object **pno)
 607 {
 608         ipfw_obj_ntlv *ntlv;
 609         const char *name;
 610
 611         DYN_DEBUG("uidx %d", ti->uidx);
 612         if (ti->uidx != 0) {
 613                 if (ti->tlvs == NULL)
 614                         return (EINVAL);
 615                 /* Search ntlv in the buffer provided by user */
 616                 ntlv = ipfw_find_name_tlv_type(ti->tlvs, ti->tlen, ti->uidx,
 617                     IPFW_TLV_STATE_NAME);
 618                 if (ntlv == NULL)
 619                         return (EINVAL);
 620                 name = ntlv->name;
 621         } else
 622                 name = default_state_name;
 623         /*
 624          * Search named object with corresponding name.
 625          * Since states objects are global - ignore the set value
 626          * and use zero instead.
 627          */
 628         *pno = ipfw_objhash_lookup_name_type(CHAIN_TO_SRV(ch), 0,
 629             IPFW_TLV_STATE_NAME, name);
 630         /*
 631          * We always return success here.
 632          * The caller will check *pno and mark object as unresolved,
 633          * then it will automatically create "default" object.
 634          */
 635         return (0);
 636 }
 637
 638 static struct named_object *
 639 dyn_findbykidx(struct ip_fw_chain *ch, uint16_t idx)
 640 {
 641
 642         DYN_DEBUG("kidx %d", idx);
 643         return (ipfw_objhash_lookup_kidx(CHAIN_TO_SRV(ch), idx));
 644 }
 645
 646 static int
 647 dyn_create(struct ip_fw_chain *ch, struct tid_info *ti,
 648     uint16_t *pkidx)
 649 {
 650         struct namedobj_instance *ni;
 651         struct dyn_state_obj *obj;
 652         struct named_object *no;
 653         ipfw_obj_ntlv *ntlv;
 654         char *name;
 655
 656         DYN_DEBUG("uidx %d", ti->uidx);
 657         if (ti->uidx != 0) {
 658                 if (ti->tlvs == NULL)
 659                         return (EINVAL);
 660                 ntlv = ipfw_find_name_tlv_type(ti->tlvs, ti->tlen, ti->uidx,
 661                     IPFW_TLV_STATE_NAME);
 662                 if (ntlv == NULL)
 663                         return (EINVAL);
 664                 name = ntlv->name;
 665         } else
 666                 name = default_state_name;
 667
 668         ni = CHAIN_TO_SRV(ch);
 669         obj = malloc(sizeof(*obj), M_IPFW, M_WAITOK | M_ZERO);
 670         obj->no.name = obj->name;
 671         obj->no.etlv = IPFW_TLV_STATE_NAME;
 672         strlcpy(obj->name, name, sizeof(obj->name));
 673
 674         IPFW_UH_WLOCK(ch);
 675         no = ipfw_objhash_lookup_name_type(ni, 0,
 676             IPFW_TLV_STATE_NAME, name);
 677         if (no != NULL) {
 678                 /*
 679                  * Object is already created.
 680                  * Just return its kidx and bump refcount.
 681                  */
 682                 *pkidx = no->kidx;
 683                 no->refcnt++;
 684                 IPFW_UH_WUNLOCK(ch);
 685                 free(obj, M_IPFW);
 686                 DYN_DEBUG("\tfound kidx %d", *pkidx);
 687                 return (0);
 688         }
 689         if (ipfw_objhash_alloc_idx(ni, &obj->no.kidx) != 0) {
 690                 DYN_DEBUG("\talloc_idx failed for %s", name);
 691                 IPFW_UH_WUNLOCK(ch);
 692                 free(obj, M_IPFW);
 693                 return (ENOSPC);
 694         }
 695         ipfw_objhash_add(ni, &obj->no);
 696         SRV_OBJECT(ch, obj->no.kidx) = obj;
 697         obj->no.refcnt++;
 698         *pkidx = obj->no.kidx;
 699         IPFW_UH_WUNLOCK(ch);
 700         DYN_DEBUG("\tcreated kidx %d", *pkidx);
 701         return (0);
 702 }
 703
 704 static void
 705 dyn_destroy(struct ip_fw_chain *ch, struct named_object *no)
 706 {
 707         struct dyn_state_obj *obj;
 708
 709         IPFW_UH_WLOCK_ASSERT(ch);
 710
 711         KASSERT(no->etlv == IPFW_TLV_STATE_NAME,
 712             ("%s: wrong object type %u", __func__, no->etlv));
 713         KASSERT(no->refcnt == 1,
 714             ("Destroying object '%s' (type %u, idx %u) with refcnt %u",
 715             no->name, no->etlv, no->kidx, no->refcnt));
 716         DYN_DEBUG("kidx %d", no->kidx);
 717         obj = SRV_OBJECT(ch, no->kidx);
 718         SRV_OBJECT(ch, no->kidx) = NULL;
 719         ipfw_objhash_del(CHAIN_TO_SRV(ch), no);
 720         ipfw_objhash_free_idx(CHAIN_TO_SRV(ch), no->kidx);
 721
 722         free(obj, M_IPFW);
 723 }
 724
 725 static struct opcode_obj_rewrite dyn_opcodes[] = {
 726         {
 727                 O_KEEP_STATE, IPFW_TLV_STATE_NAME,
 728                 dyn_classify, dyn_update,
 729                 dyn_findbyname, dyn_findbykidx,
 730                 dyn_create, dyn_destroy
 731         },
 732         {
 733                 O_CHECK_STATE, IPFW_TLV_STATE_NAME,
 734                 dyn_classify, dyn_update,
 735                 dyn_findbyname, dyn_findbykidx,
 736                 dyn_create, dyn_destroy
 737         },
 738         {
 739                 O_PROBE_STATE, IPFW_TLV_STATE_NAME,
 740                 dyn_classify, dyn_update,
 741                 dyn_findbyname, dyn_findbykidx,
 742                 dyn_create, dyn_destroy
 743         },
 744         {
 745                 O_LIMIT, IPFW_TLV_STATE_NAME,
 746                 dyn_classify, dyn_update,
 747                 dyn_findbyname, dyn_findbykidx,
 748                 dyn_create, dyn_destroy
 749         },
 750 };
 751
 752 /*
 753  * IMPORTANT: the hash function for dynamic rules must be commutative
 754  * in source and destination (ip,port), because rules are bidirectional
 755  * and we want to find both in the same bucket.
 756  */
 757 #ifndef IPFIREWALL_JENKINSHASH
 758 static __inline uint32_t
 759 hash_packet(const struct ipfw_flow_id *id)
 760 {
 761         uint32_t i;
 762
 763 #ifdef INET6
 764         if (IS_IP6_FLOW_ID(id))
 765                 i = ntohl((id->dst_ip6.__u6_addr.__u6_addr32[2]) ^
 766                     (id->dst_ip6.__u6_addr.__u6_addr32[3]) ^
 767                     (id->src_ip6.__u6_addr.__u6_addr32[2]) ^
 768                     (id->src_ip6.__u6_addr.__u6_addr32[3]));
 769         else
 770 #endif /* INET6 */
 771         i = (id->dst_ip) ^ (id->src_ip);
 772         i ^= (id->dst_port) ^ (id->src_port);
 773         return (i);
 774 }
 775
 776 static __inline uint32_t
 777 hash_parent(const struct ipfw_flow_id *id, const void *rule)
 778 {
 779
 780         return (hash_packet(id) ^ ((uintptr_t)rule));
 781 }
 782
 783 #else /* IPFIREWALL_JENKINSHASH */
 784
 785 VNET_DEFINE_STATIC(uint32_t, dyn_hashseed);
 786 #define V_dyn_hashseed          VNET(dyn_hashseed)
 787
 788 static __inline int
 789 addrcmp4(const struct ipfw_flow_id *id)
 790 {
 791
 792         if (id->src_ip < id->dst_ip)
 793                 return (0);
 794         if (id->src_ip > id->dst_ip)
 795                 return (1);
 796         if (id->src_port <= id->dst_port)
 797                 return (0);
 798         return (1);
 799 }
 800
 801 #ifdef INET6
 802 static __inline int
 803 addrcmp6(const struct ipfw_flow_id *id)
 804 {
 805         int ret;
 806
 807         ret = memcmp(&id->src_ip6, &id->dst_ip6, sizeof(struct in6_addr));
 808         if (ret < 0)
 809                 return (0);
 810         if (ret > 0)
 811                 return (1);
 812         if (id->src_port <= id->dst_port)
 813                 return (0);
 814         return (1);
 815 }
 816
 817 static __inline uint32_t
 818 hash_packet6(const struct ipfw_flow_id *id)
 819 {
 820         struct tuple6 {
 821                 struct in6_addr addr[2];
 822                 uint16_t        port[2];
 823         } t6;
 824
 825         if (addrcmp6(id) == 0) {
 826                 t6.addr[0] = id->src_ip6;
 827                 t6.addr[1] = id->dst_ip6;
 828                 t6.port[0] = id->src_port;
 829                 t6.port[1] = id->dst_port;
 830         } else {
 831                 t6.addr[0] = id->dst_ip6;
 832                 t6.addr[1] = id->src_ip6;
 833                 t6.port[0] = id->dst_port;
 834                 t6.port[1] = id->src_port;
 835         }
 836         return (jenkins_hash32((const uint32_t *)&t6,
 837             sizeof(t6) / sizeof(uint32_t), V_dyn_hashseed));
 838 }
 839 #endif
 840
 841 static __inline uint32_t
 842 hash_packet(const struct ipfw_flow_id *id)
 843 {
 844         struct tuple4 {
 845                 in_addr_t       addr[2];
 846                 uint16_t        port[2];
 847         } t4;
 848
 849         if (IS_IP4_FLOW_ID(id)) {
 850                 /* All fields are in host byte order */
 851                 if (addrcmp4(id) == 0) {
 852                         t4.addr[0] = id->src_ip;
 853                         t4.addr[1] = id->dst_ip;
 854                         t4.port[0] = id->src_port;
 855                         t4.port[1] = id->dst_port;
 856                 } else {
 857                         t4.addr[0] = id->dst_ip;
 858                         t4.addr[1] = id->src_ip;
 859                         t4.port[0] = id->dst_port;
 860                         t4.port[1] = id->src_port;
 861                 }
 862                 return (jenkins_hash32((const uint32_t *)&t4,
 863                     sizeof(t4) / sizeof(uint32_t), V_dyn_hashseed));
 864         } else
 865 #ifdef INET6
 866         if (IS_IP6_FLOW_ID(id))
 867                 return (hash_packet6(id));
 868 #endif
 869         return (0);
 870 }
 871
 872 static __inline uint32_t
 873 hash_parent(const struct ipfw_flow_id *id, const void *rule)
 874 {
 875
 876         return (jenkins_hash32((const uint32_t *)&rule,
 877             sizeof(rule) / sizeof(uint32_t), hash_packet(id)));
 878 }
 879 #endif /* IPFIREWALL_JENKINSHASH */
 880
 881 /*
 882  * Print customizable flow id description via log(9) facility.
 883  */
 884 static void
 885 print_dyn_rule_flags(const struct ipfw_flow_id *id, int dyn_type,
 886     int log_flags, char *prefix, char *postfix)
 887 {
 888         struct in_addr da;
 889 #ifdef INET6
 890         char src[INET6_ADDRSTRLEN], dst[INET6_ADDRSTRLEN];
 891 #else
 892         char src[INET_ADDRSTRLEN], dst[INET_ADDRSTRLEN];
 893 #endif
 894
 895 #ifdef INET6
 896         if (IS_IP6_FLOW_ID(id)) {
 897                 ip6_sprintf(src, &id->src_ip6);
 898                 ip6_sprintf(dst, &id->dst_ip6);
 899         } else
 900 #endif
 901         {
 902                 da.s_addr = htonl(id->src_ip);
 903                 inet_ntop(AF_INET, &da, src, sizeof(src));
 904                 da.s_addr = htonl(id->dst_ip);
 905                 inet_ntop(AF_INET, &da, dst, sizeof(dst));
 906         }
 907         log(log_flags, "ipfw: %s type %d %s %d -> %s %d, %d %s\n",
 908             prefix, dyn_type, src, id->src_port, dst,
 909             id->dst_port, V_dyn_count, postfix);
 910 }
 911
 912 #define print_dyn_rule(id, dtype, prefix, postfix)      \
 913         print_dyn_rule_flags(id, dtype, LOG_DEBUG, prefix, postfix)
 914
 915 #define TIME_LEQ(a,b)   ((int)((a)-(b)) <= 0)
 916 #define TIME_LE(a,b)    ((int)((a)-(b)) < 0)
 917 #define _SEQ_GE(a,b)    ((int)((a)-(b)) >= 0)
 918 #define BOTH_SYN        (TH_SYN | (TH_SYN << 8))
 919 #define BOTH_FIN        (TH_FIN | (TH_FIN << 8))
 920 #define TCP_FLAGS       (TH_FLAGS | (TH_FLAGS << 8))
 921 #define ACK_FWD         0x00010000      /* fwd ack seen */
 922 #define ACK_REV         0x00020000      /* rev ack seen */
 923 #define ACK_BOTH        (ACK_FWD | ACK_REV)
 924
 925 static uint32_t
 926 dyn_update_tcp_state(struct dyn_data *data, const struct ipfw_flow_id *pkt,
 927     const struct tcphdr *tcp, int dir)
 928 {
 929         uint32_t ack, expire;
 930         uint32_t state, old;
 931         uint8_t th_flags;
 932
 933         expire = data->expire;
 934         old = state = data->state;
 935         th_flags = pkt->_flags & (TH_FIN | TH_SYN | TH_RST);
 936         state |= (dir == MATCH_FORWARD) ? th_flags: (th_flags << 8);
 937         switch (state & TCP_FLAGS) {
 938         case TH_SYN:                    /* opening */
 939                 expire = time_uptime + V_dyn_syn_lifetime;
 940                 break;
 941
 942         case BOTH_SYN:                  /* move to established */
 943         case BOTH_SYN | TH_FIN:         /* one side tries to close */
 944         case BOTH_SYN | (TH_FIN << 8):
 945                 if (tcp == NULL)
 946                         break;
 947                 ack = ntohl(tcp->th_ack);
 948                 if (dir == MATCH_FORWARD) {
 949                         if (data->ack_fwd == 0 ||
 950                             _SEQ_GE(ack, data->ack_fwd)) {
 951                                 state |= ACK_FWD;
 952                                 if (data->ack_fwd != ack)
 953                                         ck_pr_store_32(&data->ack_fwd, ack);
 954                         }
 955                 } else {
 956                         if (data->ack_rev == 0 ||
 957                             _SEQ_GE(ack, data->ack_rev)) {
 958                                 state |= ACK_REV;
 959                                 if (data->ack_rev != ack)
 960                                         ck_pr_store_32(&data->ack_rev, ack);
 961                         }
 962                 }
 963                 if ((state & ACK_BOTH) == ACK_BOTH) {
 964                         /*
 965                          * Set expire time to V_dyn_ack_lifetime only if
 966                          * we got ACKs for both directions.
 967                          * We use XOR here to avoid possible state
 968                          * overwriting in concurrent thread.
 969                          */
 970                         expire = time_uptime + V_dyn_ack_lifetime;
 971                         ck_pr_xor_32(&data->state, ACK_BOTH);
 972                 } else if ((data->state & ACK_BOTH) != (state & ACK_BOTH))
 973                         ck_pr_or_32(&data->state, state & ACK_BOTH);
 974                 break;
 975
 976         case BOTH_SYN | BOTH_FIN:       /* both sides closed */
 977                 if (V_dyn_fin_lifetime >= V_dyn_keepalive_period)
 978                         V_dyn_fin_lifetime = V_dyn_keepalive_period - 1;
 979                 expire = time_uptime + V_dyn_fin_lifetime;
 980                 break;
 981
 982         default:
 983                 if (V_dyn_keepalive != 0 &&
 984                     V_dyn_rst_lifetime >= V_dyn_keepalive_period)
 985                         V_dyn_rst_lifetime = V_dyn_keepalive_period - 1;
 986                 expire = time_uptime + V_dyn_rst_lifetime;
 987         }
 988         /* Save TCP state if it was changed */
 989         if ((state & TCP_FLAGS) != (old & TCP_FLAGS))
 990                 ck_pr_or_32(&data->state, state & TCP_FLAGS);
 991         return (expire);
 992 }
 993
 994 /*
 995  * Update ULP specific state.
 996  * For TCP we keep sequence numbers and flags. For other protocols
 997  * currently we update only expire time. Packets and bytes counters
 998  * are also updated here.
 999  */
1000 static void
1001 dyn_update_proto_state(struct dyn_data *data, const struct ipfw_flow_id *pkt,
1002     const void *ulp, int pktlen, int dir)
1003 {
1004         uint32_t expire;
1005
1006         /* NOTE: we are in critical section here. */
1007         switch (pkt->proto) {
1008         case IPPROTO_UDP:
1009         case IPPROTO_UDPLITE:
1010                 expire = time_uptime + V_dyn_udp_lifetime;
1011                 break;
1012         case IPPROTO_TCP:
1013                 expire = dyn_update_tcp_state(data, pkt, ulp, dir);
1014                 break;
1015         default:
1016                 expire = time_uptime + V_dyn_short_lifetime;
1017         }
1018         /*
1019          * Expiration timer has the per-second granularity, no need to update
1020          * it every time when state is matched.
1021          */
1022         if (data->expire != expire)
1023                 ck_pr_store_32(&data->expire, expire);
1024
1025         if (dir == MATCH_FORWARD)
1026                 DYN_COUNTER_INC(data, fwd, pktlen);
1027         else
1028                 DYN_COUNTER_INC(data, rev, pktlen);
1029 }
1030
1031 /*
1032  * Lookup IPv4 state.
1033  * Must be called in critical section.
1034  */
1035 struct dyn_ipv4_state *
1036 dyn_lookup_ipv4_state(const struct ipfw_flow_id *pkt, const void *ulp,
1037     struct ipfw_dyn_info *info, int pktlen)
1038 {
1039         struct dyn_ipv4_state *s;
1040         uint32_t version, bucket;
1041
1042         bucket = DYN_BUCKET(info->hashval, V_curr_dyn_buckets);
1043         info->version = DYN_BUCKET_VERSION(bucket, ipv4_add);
1044 restart:
1045         version = DYN_BUCKET_VERSION(bucket, ipv4_del);
1046         CK_SLIST_FOREACH(s, &V_dyn_ipv4[bucket], entry) {
1047                 DYNSTATE_PROTECT(s);
1048                 if (version != DYN_BUCKET_VERSION(bucket, ipv4_del))
1049                         goto restart;
1050                 if (s->proto != pkt->proto)
1051                         continue;
1052                 if (info->kidx != 0 && s->kidx != info->kidx)
1053                         continue;
1054                 if (s->sport == pkt->src_port && s->dport == pkt->dst_port &&
1055                     s->src == pkt->src_ip && s->dst == pkt->dst_ip) {
1056                         info->direction = MATCH_FORWARD;
1057                         break;
1058                 }
1059                 if (s->sport == pkt->dst_port && s->dport == pkt->src_port &&
1060                     s->src == pkt->dst_ip && s->dst == pkt->src_ip) {
1061                         info->direction = MATCH_REVERSE;
1062                         break;
1063                 }
1064         }
1065
1066         if (s != NULL)
1067                 dyn_update_proto_state(s->data, pkt, ulp, pktlen,
1068                     info->direction);
1069         return (s);
1070 }
1071
1072 /*
1073  * Lookup IPv4 state.
1074  * Simplifed version is used to check that matching state doesn't exist.
1075  */
1076 static int
1077 dyn_lookup_ipv4_state_locked(const struct ipfw_flow_id *pkt,
1078     const void *ulp, int pktlen, uint32_t bucket, uint16_t kidx)
1079 {
1080         struct dyn_ipv4_state *s;
1081         int dir;
1082
1083         dir = MATCH_NONE;
1084         DYN_BUCKET_ASSERT(bucket);
1085         CK_SLIST_FOREACH(s, &V_dyn_ipv4[bucket], entry) {
1086                 if (s->proto != pkt->proto ||
1087                     s->kidx != kidx)
1088                         continue;
1089                 if (s->sport == pkt->src_port &&
1090                     s->dport == pkt->dst_port &&
1091                     s->src == pkt->src_ip && s->dst == pkt->dst_ip) {
1092                         dir = MATCH_FORWARD;
1093                         break;
1094                 }
1095                 if (s->sport == pkt->dst_port && s->dport == pkt->src_port &&
1096                     s->src == pkt->dst_ip && s->dst == pkt->src_ip) {
1097                         dir = MATCH_REVERSE;
1098                         break;
1099                 }
1100         }
1101         if (s != NULL)
1102                 dyn_update_proto_state(s->data, pkt, ulp, pktlen, dir);
1103         return (s != NULL);
1104 }
1105
1106 struct dyn_ipv4_state *
1107 dyn_lookup_ipv4_parent(const struct ipfw_flow_id *pkt, const void *rule,
1108     uint32_t ruleid, uint16_t rulenum, uint32_t hashval)
1109 {
1110         struct dyn_ipv4_state *s;
1111         uint32_t version, bucket;
1112
1113         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1114 restart:
1115         version = DYN_BUCKET_VERSION(bucket, ipv4_parent_del);
1116         CK_SLIST_FOREACH(s, &V_dyn_ipv4_parent[bucket], entry) {
1117                 DYNSTATE_PROTECT(s);
1118                 if (version != DYN_BUCKET_VERSION(bucket, ipv4_parent_del))
1119                         goto restart;
1120                 /*
1121                  * NOTE: we do not need to check kidx, because parent rule
1122                  * can not create states with different kidx.
1123                  * And parent rule always created for forward direction.
1124                  */
1125                 if (s->limit->parent == rule &&
1126                     s->limit->ruleid == ruleid &&
1127                     s->limit->rulenum == rulenum &&
1128                     s->proto == pkt->proto &&
1129                     s->sport == pkt->src_port &&
1130                     s->dport == pkt->dst_port &&
1131                     s->src == pkt->src_ip && s->dst == pkt->dst_ip) {
1132                         if (s->limit->expire != time_uptime +
1133                             V_dyn_short_lifetime)
1134                                 ck_pr_store_32(&s->limit->expire,
1135                                     time_uptime + V_dyn_short_lifetime);
1136                         break;
1137                 }
1138         }
1139         return (s);
1140 }
1141
1142 static struct dyn_ipv4_state *
1143 dyn_lookup_ipv4_parent_locked(const struct ipfw_flow_id *pkt,
1144     const void *rule, uint32_t ruleid, uint16_t rulenum, uint32_t bucket)
1145 {
1146         struct dyn_ipv4_state *s;
1147
1148         DYN_BUCKET_ASSERT(bucket);
1149         CK_SLIST_FOREACH(s, &V_dyn_ipv4_parent[bucket], entry) {
1150                 if (s->limit->parent == rule &&
1151                     s->limit->ruleid == ruleid &&
1152                     s->limit->rulenum == rulenum &&
1153                     s->proto == pkt->proto &&
1154                     s->sport == pkt->src_port &&
1155                     s->dport == pkt->dst_port &&
1156                     s->src == pkt->src_ip && s->dst == pkt->dst_ip)
1157                         break;
1158         }
1159         return (s);
1160 }
1161
1162
1163 #ifdef INET6
1164 static uint32_t
1165 dyn_getscopeid(const struct ip_fw_args *args)
1166 {
1167
1168         /*
1169          * If source or destination address is an scopeid address, we need
1170          * determine the scope zone id to resolve address scope ambiguity.
1171          */
1172         if (IN6_IS_ADDR_LINKLOCAL(&args->f_id.src_ip6) ||
1173             IN6_IS_ADDR_LINKLOCAL(&args->f_id.dst_ip6)) {
1174                 MPASS(args->oif != NULL ||
1175                     args->m->m_pkthdr.rcvif != NULL);
1176                 return (in6_getscopezone(args->oif != NULL ? args->oif:
1177                     args->m->m_pkthdr.rcvif, IPV6_ADDR_SCOPE_LINKLOCAL));
1178         }
1179         return (0);
1180 }
1181
1182 /*
1183  * Lookup IPv6 state.
1184  * Must be called in critical section.
1185  */
1186 static struct dyn_ipv6_state *
1187 dyn_lookup_ipv6_state(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1188     const void *ulp, struct ipfw_dyn_info *info, int pktlen)
1189 {
1190         struct dyn_ipv6_state *s;
1191         uint32_t version, bucket;
1192
1193         bucket = DYN_BUCKET(info->hashval, V_curr_dyn_buckets);
1194         info->version = DYN_BUCKET_VERSION(bucket, ipv6_add);
1195 restart:
1196         version = DYN_BUCKET_VERSION(bucket, ipv6_del);
1197         CK_SLIST_FOREACH(s, &V_dyn_ipv6[bucket], entry) {
1198                 DYNSTATE_PROTECT(s);
1199                 if (version != DYN_BUCKET_VERSION(bucket, ipv6_del))
1200                         goto restart;
1201                 if (s->proto != pkt->proto || s->zoneid != zoneid)
1202                         continue;
1203                 if (info->kidx != 0 && s->kidx != info->kidx)
1204                         continue;
1205                 if (s->sport == pkt->src_port && s->dport == pkt->dst_port &&
1206                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->src_ip6) &&
1207                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->dst_ip6)) {
1208                         info->direction = MATCH_FORWARD;
1209                         break;
1210                 }
1211                 if (s->sport == pkt->dst_port && s->dport == pkt->src_port &&
1212                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->dst_ip6) &&
1213                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->src_ip6)) {
1214                         info->direction = MATCH_REVERSE;
1215                         break;
1216                 }
1217         }
1218         if (s != NULL)
1219                 dyn_update_proto_state(s->data, pkt, ulp, pktlen,
1220                     info->direction);
1221         return (s);
1222 }
1223
1224 /*
1225  * Lookup IPv6 state.
1226  * Simplifed version is used to check that matching state doesn't exist.
1227  */
1228 static int
1229 dyn_lookup_ipv6_state_locked(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1230     const void *ulp, int pktlen, uint32_t bucket, uint16_t kidx)
1231 {
1232         struct dyn_ipv6_state *s;
1233         int dir;
1234
1235         dir = MATCH_NONE;
1236         DYN_BUCKET_ASSERT(bucket);
1237         CK_SLIST_FOREACH(s, &V_dyn_ipv6[bucket], entry) {
1238                 if (s->proto != pkt->proto || s->kidx != kidx ||
1239                     s->zoneid != zoneid)
1240                         continue;
1241                 if (s->sport == pkt->src_port && s->dport == pkt->dst_port &&
1242                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->src_ip6) &&
1243                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->dst_ip6)) {
1244                         dir = MATCH_FORWARD;
1245                         break;
1246                 }
1247                 if (s->sport == pkt->dst_port && s->dport == pkt->src_port &&
1248                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->dst_ip6) &&
1249                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->src_ip6)) {
1250                         dir = MATCH_REVERSE;
1251                         break;
1252                 }
1253         }
1254         if (s != NULL)
1255                 dyn_update_proto_state(s->data, pkt, ulp, pktlen, dir);
1256         return (s != NULL);
1257 }
1258
1259 static struct dyn_ipv6_state *
1260 dyn_lookup_ipv6_parent(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1261     const void *rule, uint32_t ruleid, uint16_t rulenum, uint32_t hashval)
1262 {
1263         struct dyn_ipv6_state *s;
1264         uint32_t version, bucket;
1265
1266         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1267 restart:
1268         version = DYN_BUCKET_VERSION(bucket, ipv6_parent_del);
1269         CK_SLIST_FOREACH(s, &V_dyn_ipv6_parent[bucket], entry) {
1270                 DYNSTATE_PROTECT(s);
1271                 if (version != DYN_BUCKET_VERSION(bucket, ipv6_parent_del))
1272                         goto restart;
1273                 /*
1274                  * NOTE: we do not need to check kidx, because parent rule
1275                  * can not create states with different kidx.
1276                  * Also parent rule always created for forward direction.
1277                  */
1278                 if (s->limit->parent == rule &&
1279                     s->limit->ruleid == ruleid &&
1280                     s->limit->rulenum == rulenum &&
1281                     s->proto == pkt->proto &&
1282                     s->sport == pkt->src_port &&
1283                     s->dport == pkt->dst_port && s->zoneid == zoneid &&
1284                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->src_ip6) &&
1285                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->dst_ip6)) {
1286                         if (s->limit->expire != time_uptime +
1287                             V_dyn_short_lifetime)
1288                                 ck_pr_store_32(&s->limit->expire,
1289                                     time_uptime + V_dyn_short_lifetime);
1290                         break;
1291                 }
1292         }
1293         return (s);
1294 }
1295
1296 static struct dyn_ipv6_state *
1297 dyn_lookup_ipv6_parent_locked(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1298     const void *rule, uint32_t ruleid, uint16_t rulenum, uint32_t bucket)
1299 {
1300         struct dyn_ipv6_state *s;
1301
1302         DYN_BUCKET_ASSERT(bucket);
1303         CK_SLIST_FOREACH(s, &V_dyn_ipv6_parent[bucket], entry) {
1304                 if (s->limit->parent == rule &&
1305                     s->limit->ruleid == ruleid &&
1306                     s->limit->rulenum == rulenum &&
1307                     s->proto == pkt->proto &&
1308                     s->sport == pkt->src_port &&
1309                     s->dport == pkt->dst_port && s->zoneid == zoneid &&
1310                     IN6_ARE_ADDR_EQUAL(&s->src, &pkt->src_ip6) &&
1311                     IN6_ARE_ADDR_EQUAL(&s->dst, &pkt->dst_ip6))
1312                         break;
1313         }
1314         return (s);
1315 }
1316
1317 #endif /* INET6 */
1318
1319 /*
1320  * Lookup dynamic state.
1321  *  pkt - filled by ipfw_chk() ipfw_flow_id;
1322  *  ulp - determined by ipfw_chk() upper level protocol header;
1323  *  dyn_info - info about matched state to return back;
1324  * Returns pointer to state's parent rule and dyn_info. If there is
1325  * no state, NULL is returned.
1326  * On match ipfw_dyn_lookup() updates state's counters.
1327  */
1328 struct ip_fw *
1329 ipfw_dyn_lookup_state(const struct ip_fw_args *args, const void *ulp,
1330     int pktlen, const ipfw_insn *cmd, struct ipfw_dyn_info *info)
1331 {
1332         struct dyn_data *data;
1333         struct ip_fw *rule;
1334
1335         IPFW_RLOCK_ASSERT(&V_layer3_chain);
1336
1337         data = NULL;
1338         rule = NULL;
1339         info->kidx = cmd->arg1;
1340         info->direction = MATCH_NONE;
1341         info->hashval = hash_packet(&args->f_id);
1342
1343         DYNSTATE_CRITICAL_ENTER();
1344         if (IS_IP4_FLOW_ID(&args->f_id)) {
1345                 struct dyn_ipv4_state *s;
1346
1347                 s = dyn_lookup_ipv4_state(&args->f_id, ulp, info, pktlen);
1348                 if (s != NULL) {
1349                         /*
1350                          * Dynamic states are created using the same 5-tuple,
1351                          * so it is assumed, that parent rule for O_LIMIT
1352                          * state has the same address family.
1353                          */
1354                         data = s->data;
1355                         if (s->type == O_LIMIT) {
1356                                 s = data->parent;
1357                                 rule = s->limit->parent;
1358                         } else
1359                                 rule = data->parent;
1360                 }
1361         }
1362 #ifdef INET6
1363         else if (IS_IP6_FLOW_ID(&args->f_id)) {
1364                 struct dyn_ipv6_state *s;
1365
1366                 s = dyn_lookup_ipv6_state(&args->f_id, dyn_getscopeid(args),
1367                     ulp, info, pktlen);
1368                 if (s != NULL) {
1369                         data = s->data;
1370                         if (s->type == O_LIMIT) {
1371                                 s = data->parent;
1372                                 rule = s->limit->parent;
1373                         } else
1374                                 rule = data->parent;
1375                 }
1376         }
1377 #endif
1378         if (data != NULL) {
1379                 /*
1380                  * If cached chain id is the same, we can avoid rule index
1381                  * lookup. Otherwise do lookup and update chain_id and f_pos.
1382                  * It is safe even if there is concurrent thread that want
1383                  * update the same state, because chain->id can be changed
1384                  * only under IPFW_WLOCK().
1385                  */
1386                 if (data->chain_id != V_layer3_chain.id) {
1387                         data->f_pos = ipfw_find_rule(&V_layer3_chain,
1388                             data->rulenum, data->ruleid);
1389                         /*
1390                          * Check that found state has not orphaned.
1391                          * When chain->id being changed the parent
1392                          * rule can be deleted. If found rule doesn't
1393                          * match the parent pointer, consider this
1394                          * result as MATCH_NONE and return NULL.
1395                          *
1396                          * This will lead to creation of new similar state
1397                          * that will be added into head of this bucket.
1398                          * And the state that we currently have matched
1399                          * should be deleted by dyn_expire_states().
1400                          *
1401                          * In case when dyn_keep_states is enabled, return
1402                          * pointer to default rule and corresponding f_pos
1403                          * value.
1404                          * XXX: In this case we lose the cache efficiency,
1405                          *      since f_pos is not cached, because it seems
1406                          *      there is no easy way to atomically switch
1407                          *      all fields related to parent rule of given
1408                          *      state.
1409                          */
1410                         if (V_layer3_chain.map[data->f_pos] == rule) {
1411                                 data->chain_id = V_layer3_chain.id;
1412                                 info->f_pos = data->f_pos;
1413                         } else if (V_dyn_keep_states != 0) {
1414                                 rule = V_layer3_chain.default_rule;
1415                                 info->f_pos = V_layer3_chain.n_rules - 1;
1416                         } else {
1417                                 rule = NULL;
1418                                 info->direction = MATCH_NONE;
1419                                 DYN_DEBUG("rule %p  [%u, %u] is considered "
1420                                     "invalid in data %p", rule, data->ruleid,
1421                                     data->rulenum, data);
1422                                 /* info->f_pos doesn't matter here. */
1423                         }
1424                 } else
1425                         info->f_pos = data->f_pos;
1426         }
1427         DYNSTATE_CRITICAL_EXIT();
1428 #if 0
1429         /*
1430          * Return MATCH_NONE if parent rule is in disabled set.
1431          * This will lead to creation of new similar state that
1432          * will be added into head of this bucket.
1433          *
1434          * XXXAE: we need to be able update state's set when parent
1435          *        rule set is changed.
1436          */
1437         if (rule != NULL && (V_set_disable & (1 << rule->set))) {
1438                 rule = NULL;
1439                 info->direction = MATCH_NONE;
1440         }
1441 #endif
1442         return (rule);
1443 }
1444
1445 static struct dyn_parent *
1446 dyn_alloc_parent(void *parent, uint32_t ruleid, uint16_t rulenum,
1447     uint8_t set, uint32_t hashval)
1448 {
1449         struct dyn_parent *limit;
1450
1451         limit = uma_zalloc(V_dyn_parent_zone, M_NOWAIT | M_ZERO);
1452         if (limit == NULL) {
1453                 if (last_log != time_uptime) {
1454                         last_log = time_uptime;
1455                         log(LOG_DEBUG,
1456                             "ipfw: Cannot allocate parent dynamic state, "
1457                             "consider increasing "
1458                             "net.inet.ip.fw.dyn_parent_max\n");
1459                 }
1460                 return (NULL);
1461         }
1462
1463         limit->parent = parent;
1464         limit->ruleid = ruleid;
1465         limit->rulenum = rulenum;
1466         limit->set = set;
1467         limit->hashval = hashval;
1468         limit->expire = time_uptime + V_dyn_short_lifetime;
1469         return (limit);
1470 }
1471
1472 static struct dyn_data *
1473 dyn_alloc_dyndata(void *parent, uint32_t ruleid, uint16_t rulenum,
1474     uint8_t set, const struct ipfw_flow_id *pkt, const void *ulp, int pktlen,
1475     uint32_t hashval, uint16_t fibnum)
1476 {
1477         struct dyn_data *data;
1478
1479         data = uma_zalloc(V_dyn_data_zone, M_NOWAIT | M_ZERO);
1480         if (data == NULL) {
1481                 if (last_log != time_uptime) {
1482                         last_log = time_uptime;
1483                         log(LOG_DEBUG,
1484                             "ipfw: Cannot allocate dynamic state, "
1485                             "consider increasing net.inet.ip.fw.dyn_max\n");
1486                 }
1487                 return (NULL);
1488         }
1489
1490         data->parent = parent;
1491         data->ruleid = ruleid;
1492         data->rulenum = rulenum;
1493         data->set = set;
1494         data->fibnum = fibnum;
1495         data->hashval = hashval;
1496         data->expire = time_uptime + V_dyn_syn_lifetime;
1497         dyn_update_proto_state(data, pkt, ulp, pktlen, MATCH_FORWARD);
1498         return (data);
1499 }
1500
1501 static struct dyn_ipv4_state *
1502 dyn_alloc_ipv4_state(const struct ipfw_flow_id *pkt, uint16_t kidx,
1503     uint8_t type)
1504 {
1505         struct dyn_ipv4_state *s;
1506
1507         s = uma_zalloc(V_dyn_ipv4_zone, M_NOWAIT | M_ZERO);
1508         if (s == NULL)
1509                 return (NULL);
1510
1511         s->type = type;
1512         s->kidx = kidx;
1513         s->proto = pkt->proto;
1514         s->sport = pkt->src_port;
1515         s->dport = pkt->dst_port;
1516         s->src = pkt->src_ip;
1517         s->dst = pkt->dst_ip;
1518         return (s);
1519 }
1520
1521 /*
1522  * Add IPv4 parent state.
1523  * Returns pointer to parent state. When it is not NULL we are in
1524  * critical section and pointer protected by hazard pointer.
1525  * When some error occurs, it returns NULL and exit from critical section
1526  * is not needed.
1527  */
1528 static struct dyn_ipv4_state *
1529 dyn_add_ipv4_parent(void *rule, uint32_t ruleid, uint16_t rulenum,
1530     uint8_t set, const struct ipfw_flow_id *pkt, uint32_t hashval,
1531     uint32_t version, uint16_t kidx)
1532 {
1533         struct dyn_ipv4_state *s;
1534         struct dyn_parent *limit;
1535         uint32_t bucket;
1536
1537         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1538         DYN_BUCKET_LOCK(bucket);
1539         if (version != DYN_BUCKET_VERSION(bucket, ipv4_parent_add)) {
1540                 /*
1541                  * Bucket version has been changed since last lookup,
1542                  * do lookup again to be sure that state does not exist.
1543                  */
1544                 s = dyn_lookup_ipv4_parent_locked(pkt, rule, ruleid,
1545                     rulenum, bucket);
1546                 if (s != NULL) {
1547                         /*
1548                          * Simultaneous thread has already created this
1549                          * state. Just return it.
1550                          */
1551                         DYNSTATE_CRITICAL_ENTER();
1552                         DYNSTATE_PROTECT(s);
1553                         DYN_BUCKET_UNLOCK(bucket);
1554                         return (s);
1555                 }
1556         }
1557
1558         limit = dyn_alloc_parent(rule, ruleid, rulenum, set, hashval);
1559         if (limit == NULL) {
1560                 DYN_BUCKET_UNLOCK(bucket);
1561                 return (NULL);
1562         }
1563
1564         s = dyn_alloc_ipv4_state(pkt, kidx, O_LIMIT_PARENT);
1565         if (s == NULL) {
1566                 DYN_BUCKET_UNLOCK(bucket);
1567                 uma_zfree(V_dyn_parent_zone, limit);
1568                 return (NULL);
1569         }
1570
1571         s->limit = limit;
1572         CK_SLIST_INSERT_HEAD(&V_dyn_ipv4_parent[bucket], s, entry);
1573         DYN_COUNT_INC(dyn_parent_count);
1574         DYN_BUCKET_VERSION_BUMP(bucket, ipv4_parent_add);
1575         DYNSTATE_CRITICAL_ENTER();
1576         DYNSTATE_PROTECT(s);
1577         DYN_BUCKET_UNLOCK(bucket);
1578         return (s);
1579 }
1580
1581 static int
1582 dyn_add_ipv4_state(void *parent, uint32_t ruleid, uint16_t rulenum,
1583     uint8_t set, const struct ipfw_flow_id *pkt, const void *ulp, int pktlen,
1584     uint32_t hashval, struct ipfw_dyn_info *info, uint16_t fibnum,
1585     uint16_t kidx, uint8_t type)
1586 {
1587         struct dyn_ipv4_state *s;
1588         void *data;
1589         uint32_t bucket;
1590
1591         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1592         DYN_BUCKET_LOCK(bucket);
1593         if (info->direction == MATCH_UNKNOWN ||
1594             info->kidx != kidx ||
1595             info->hashval != hashval ||
1596             info->version != DYN_BUCKET_VERSION(bucket, ipv4_add)) {
1597                 /*
1598                  * Bucket version has been changed since last lookup,
1599                  * do lookup again to be sure that state does not exist.
1600                  */
1601                 if (dyn_lookup_ipv4_state_locked(pkt, ulp, pktlen,
1602                     bucket, kidx) != 0) {
1603                         DYN_BUCKET_UNLOCK(bucket);
1604                         return (EEXIST);
1605                 }
1606         }
1607
1608         data = dyn_alloc_dyndata(parent, ruleid, rulenum, set, pkt, ulp,
1609             pktlen, hashval, fibnum);
1610         if (data == NULL) {
1611                 DYN_BUCKET_UNLOCK(bucket);
1612                 return (ENOMEM);
1613         }
1614
1615         s = dyn_alloc_ipv4_state(pkt, kidx, type);
1616         if (s == NULL) {
1617                 DYN_BUCKET_UNLOCK(bucket);
1618                 uma_zfree(V_dyn_data_zone, data);
1619                 return (ENOMEM);
1620         }
1621
1622         s->data = data;
1623         CK_SLIST_INSERT_HEAD(&V_dyn_ipv4[bucket], s, entry);
1624         DYN_COUNT_INC(dyn_count);
1625         DYN_BUCKET_VERSION_BUMP(bucket, ipv4_add);
1626         DYN_BUCKET_UNLOCK(bucket);
1627         return (0);
1628 }
1629
1630 #ifdef INET6
1631 static struct dyn_ipv6_state *
1632 dyn_alloc_ipv6_state(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1633     uint16_t kidx, uint8_t type)
1634 {
1635         struct dyn_ipv6_state *s;
1636
1637         s = uma_zalloc(V_dyn_ipv6_zone, M_NOWAIT | M_ZERO);
1638         if (s == NULL)
1639                 return (NULL);
1640
1641         s->type = type;
1642         s->kidx = kidx;
1643         s->zoneid = zoneid;
1644         s->proto = pkt->proto;
1645         s->sport = pkt->src_port;
1646         s->dport = pkt->dst_port;
1647         s->src = pkt->src_ip6;
1648         s->dst = pkt->dst_ip6;
1649         return (s);
1650 }
1651
1652 /*
1653  * Add IPv6 parent state.
1654  * Returns pointer to parent state. When it is not NULL we are in
1655  * critical section and pointer protected by hazard pointer.
1656  * When some error occurs, it return NULL and exit from critical section
1657  * is not needed.
1658  */
1659 static struct dyn_ipv6_state *
1660 dyn_add_ipv6_parent(void *rule, uint32_t ruleid, uint16_t rulenum,
1661     uint8_t set, const struct ipfw_flow_id *pkt, uint32_t zoneid,
1662     uint32_t hashval, uint32_t version, uint16_t kidx)
1663 {
1664         struct dyn_ipv6_state *s;
1665         struct dyn_parent *limit;
1666         uint32_t bucket;
1667
1668         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1669         DYN_BUCKET_LOCK(bucket);
1670         if (version != DYN_BUCKET_VERSION(bucket, ipv6_parent_add)) {
1671                 /*
1672                  * Bucket version has been changed since last lookup,
1673                  * do lookup again to be sure that state does not exist.
1674                  */
1675                 s = dyn_lookup_ipv6_parent_locked(pkt, zoneid, rule, ruleid,
1676                     rulenum, bucket);
1677                 if (s != NULL) {
1678                         /*
1679                          * Simultaneous thread has already created this
1680                          * state. Just return it.
1681                          */
1682                         DYNSTATE_CRITICAL_ENTER();
1683                         DYNSTATE_PROTECT(s);
1684                         DYN_BUCKET_UNLOCK(bucket);
1685                         return (s);
1686                 }
1687         }
1688
1689         limit = dyn_alloc_parent(rule, ruleid, rulenum, set, hashval);
1690         if (limit == NULL) {
1691                 DYN_BUCKET_UNLOCK(bucket);
1692                 return (NULL);
1693         }
1694
1695         s = dyn_alloc_ipv6_state(pkt, zoneid, kidx, O_LIMIT_PARENT);
1696         if (s == NULL) {
1697                 DYN_BUCKET_UNLOCK(bucket);
1698                 uma_zfree(V_dyn_parent_zone, limit);
1699                 return (NULL);
1700         }
1701
1702         s->limit = limit;
1703         CK_SLIST_INSERT_HEAD(&V_dyn_ipv6_parent[bucket], s, entry);
1704         DYN_COUNT_INC(dyn_parent_count);
1705         DYN_BUCKET_VERSION_BUMP(bucket, ipv6_parent_add);
1706         DYNSTATE_CRITICAL_ENTER();
1707         DYNSTATE_PROTECT(s);
1708         DYN_BUCKET_UNLOCK(bucket);
1709         return (s);
1710 }
1711
1712 static int
1713 dyn_add_ipv6_state(void *parent, uint32_t ruleid, uint16_t rulenum,
1714     uint8_t set, const struct ipfw_flow_id *pkt, uint32_t zoneid,
1715     const void *ulp, int pktlen, uint32_t hashval, struct ipfw_dyn_info *info,
1716     uint16_t fibnum, uint16_t kidx, uint8_t type)
1717 {
1718         struct dyn_ipv6_state *s;
1719         struct dyn_data *data;
1720         uint32_t bucket;
1721
1722         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1723         DYN_BUCKET_LOCK(bucket);
1724         if (info->direction == MATCH_UNKNOWN ||
1725             info->kidx != kidx ||
1726             info->hashval != hashval ||
1727             info->version != DYN_BUCKET_VERSION(bucket, ipv6_add)) {
1728                 /*
1729                  * Bucket version has been changed since last lookup,
1730                  * do lookup again to be sure that state does not exist.
1731                  */
1732                 if (dyn_lookup_ipv6_state_locked(pkt, zoneid, ulp, pktlen,
1733                     bucket, kidx) != 0) {
1734                         DYN_BUCKET_UNLOCK(bucket);
1735                         return (EEXIST);
1736                 }
1737         }
1738
1739         data = dyn_alloc_dyndata(parent, ruleid, rulenum, set, pkt, ulp,
1740             pktlen, hashval, fibnum);
1741         if (data == NULL) {
1742                 DYN_BUCKET_UNLOCK(bucket);
1743                 return (ENOMEM);
1744         }
1745
1746         s = dyn_alloc_ipv6_state(pkt, zoneid, kidx, type);
1747         if (s == NULL) {
1748                 DYN_BUCKET_UNLOCK(bucket);
1749                 uma_zfree(V_dyn_data_zone, data);
1750                 return (ENOMEM);
1751         }
1752
1753         s->data = data;
1754         CK_SLIST_INSERT_HEAD(&V_dyn_ipv6[bucket], s, entry);
1755         DYN_COUNT_INC(dyn_count);
1756         DYN_BUCKET_VERSION_BUMP(bucket, ipv6_add);
1757         DYN_BUCKET_UNLOCK(bucket);
1758         return (0);
1759 }
1760 #endif /* INET6 */
1761
1762 static void *
1763 dyn_get_parent_state(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1764     struct ip_fw *rule, uint32_t hashval, uint32_t limit, uint16_t kidx)
1765 {
1766         char sbuf[24];
1767         struct dyn_parent *p;
1768         void *ret;
1769         uint32_t bucket, version;
1770
1771         p = NULL;
1772         ret = NULL;
1773         bucket = DYN_BUCKET(hashval, V_curr_dyn_buckets);
1774         DYNSTATE_CRITICAL_ENTER();
1775         if (IS_IP4_FLOW_ID(pkt)) {
1776                 struct dyn_ipv4_state *s;
1777
1778                 version = DYN_BUCKET_VERSION(bucket, ipv4_parent_add);
1779                 s = dyn_lookup_ipv4_parent(pkt, rule, rule->id,
1780                     rule->rulenum, bucket);
1781                 if (s == NULL) {
1782                         /*
1783                          * Exit from critical section because dyn_add_parent()
1784                          * will acquire bucket lock.
1785                          */
1786                         DYNSTATE_CRITICAL_EXIT();
1787
1788                         s = dyn_add_ipv4_parent(rule, rule->id,
1789                             rule->rulenum, rule->set, pkt, hashval,
1790                             version, kidx);
1791                         if (s == NULL)
1792                                 return (NULL);
1793                         /* Now we are in critical section again. */
1794                 }
1795                 ret = s;
1796                 p = s->limit;
1797         }
1798 #ifdef INET6
1799         else if (IS_IP6_FLOW_ID(pkt)) {
1800                 struct dyn_ipv6_state *s;
1801
1802                 version = DYN_BUCKET_VERSION(bucket, ipv6_parent_add);
1803                 s = dyn_lookup_ipv6_parent(pkt, zoneid, rule, rule->id,
1804                     rule->rulenum, bucket);
1805                 if (s == NULL) {
1806                         /*
1807                          * Exit from critical section because dyn_add_parent()
1808                          * can acquire bucket mutex.
1809                          */
1810                         DYNSTATE_CRITICAL_EXIT();
1811
1812                         s = dyn_add_ipv6_parent(rule, rule->id,
1813                             rule->rulenum, rule->set, pkt, zoneid, hashval,
1814                             version, kidx);
1815                         if (s == NULL)
1816                                 return (NULL);
1817                         /* Now we are in critical section again. */
1818                 }
1819                 ret = s;
1820                 p = s->limit;
1821         }
1822 #endif
1823         else {
1824                 DYNSTATE_CRITICAL_EXIT();
1825                 return (NULL);
1826         }
1827
1828         /* Check the limit */
1829         if (DPARENT_COUNT(p) >= limit) {
1830                 DYNSTATE_CRITICAL_EXIT();
1831                 if (V_fw_verbose && last_log != time_uptime) {
1832                         last_log = time_uptime;
1833                         snprintf(sbuf, sizeof(sbuf), "%u drop session",
1834                             rule->rulenum);
1835                         print_dyn_rule_flags(pkt, O_LIMIT,
1836                             LOG_SECURITY | LOG_DEBUG, sbuf,
1837                             "too many entries");
1838                 }
1839                 return (NULL);
1840         }
1841
1842         /* Take new session into account. */
1843         DPARENT_COUNT_INC(p);
1844         /*
1845          * We must exit from critical section because the following code
1846          * can acquire bucket mutex.
1847          * We rely on the the 'count' field. The state will not expire
1848          * until it has some child states, i.e. 'count' field is not zero.
1849          * Return state pointer, it will be used by child states as parent.
1850          */
1851         DYNSTATE_CRITICAL_EXIT();
1852         return (ret);
1853 }
1854
1855 static int
1856 dyn_install_state(const struct ipfw_flow_id *pkt, uint32_t zoneid,
1857     uint16_t fibnum, const void *ulp, int pktlen, void *rule,
1858     uint32_t ruleid, uint16_t rulenum, uint8_t set,
1859     struct ipfw_dyn_info *info, uint32_t limit, uint16_t limit_mask,
1860     uint16_t kidx, uint8_t type)
1861 {
1862         struct ipfw_flow_id id;
1863         uint32_t hashval, parent_hashval;
1864         int ret;
1865
1866         MPASS(type == O_LIMIT || type == O_KEEP_STATE);
1867
1868         if (type == O_LIMIT) {
1869                 /* Create masked flow id and calculate bucket */
1870                 id.addr_type = pkt->addr_type;
1871                 id.proto = pkt->proto;
1872                 id.fib = fibnum; /* unused */
1873                 id.src_port = (limit_mask & DYN_SRC_PORT) ?
1874                     pkt->src_port: 0;
1875                 id.dst_port = (limit_mask & DYN_DST_PORT) ?
1876                     pkt->dst_port: 0;
1877                 if (IS_IP4_FLOW_ID(pkt)) {
1878                         id.src_ip = (limit_mask & DYN_SRC_ADDR) ?
1879                             pkt->src_ip: 0;
1880                         id.dst_ip = (limit_mask & DYN_DST_ADDR) ?
1881                             pkt->dst_ip: 0;
1882                 }
1883 #ifdef INET6
1884                 else if (IS_IP6_FLOW_ID(pkt)) {
1885                         if (limit_mask & DYN_SRC_ADDR)
1886                                 id.src_ip6 = pkt->src_ip6;
1887                         else
1888                                 memset(&id.src_ip6, 0, sizeof(id.src_ip6));
1889                         if (limit_mask & DYN_DST_ADDR)
1890                                 id.dst_ip6 = pkt->dst_ip6;
1891                         else
1892                                 memset(&id.dst_ip6, 0, sizeof(id.dst_ip6));
1893                 }
1894 #endif
1895                 else
1896                         return (EAFNOSUPPORT);
1897
1898                 parent_hashval = hash_parent(&id, rule);
1899                 rule = dyn_get_parent_state(&id, zoneid, rule, parent_hashval,
1900                     limit, kidx);
1901                 if (rule == NULL) {
1902 #if 0
1903                         if (V_fw_verbose && last_log != time_uptime) {
1904                                 last_log = time_uptime;
1905                                 snprintf(sbuf, sizeof(sbuf),
1906                                     "%u drop session", rule->rulenum);
1907                         print_dyn_rule_flags(pkt, O_LIMIT,
1908                             LOG_SECURITY | LOG_DEBUG, sbuf,
1909                             "too many entries");
1910                         }
1911 #endif
1912                         return (EACCES);
1913                 }
1914                 /*
1915                  * Limit is not reached, create new state.
1916                  * Now rule points to parent state.
1917                  */
1918         }
1919
1920         hashval = hash_packet(pkt);
1921         if (IS_IP4_FLOW_ID(pkt))
1922                 ret = dyn_add_ipv4_state(rule, ruleid, rulenum, set, pkt,
1923                     ulp, pktlen, hashval, info, fibnum, kidx, type);
1924 #ifdef INET6
1925         else if (IS_IP6_FLOW_ID(pkt))
1926                 ret = dyn_add_ipv6_state(rule, ruleid, rulenum, set, pkt,
1927                     zoneid, ulp, pktlen, hashval, info, fibnum, kidx, type);
1928 #endif /* INET6 */
1929         else
1930                 ret = EAFNOSUPPORT;
1931
1932         if (type == O_LIMIT) {
1933                 if (ret != 0) {
1934                         /*
1935                          * We failed to create child state for O_LIMIT
1936                          * opcode. Since we already counted it in the parent,
1937                          * we must revert counter back. The 'rule' points to
1938                          * parent state, use it to get dyn_parent.
1939                          *
1940                          * XXXAE: it should be safe to use 'rule' pointer
1941                          * without extra lookup, parent state is referenced
1942                          * and should not be freed.
1943                          */
1944                         if (IS_IP4_FLOW_ID(&id))
1945                                 DPARENT_COUNT_DEC(
1946                                     ((struct dyn_ipv4_state *)rule)->limit);
1947 #ifdef INET6
1948                         else if (IS_IP6_FLOW_ID(&id))
1949                                 DPARENT_COUNT_DEC(
1950                                     ((struct dyn_ipv6_state *)rule)->limit);
1951 #endif
1952                 }
1953         }
1954         /*
1955          * EEXIST means that simultaneous thread has created this
1956          * state. Consider this as success.
1957          *
1958          * XXXAE: should we invalidate 'info' content here?
1959          */
1960         if (ret == EEXIST)
1961                 return (0);
1962         return (ret);
1963 }
1964
1965 /*
1966  * Install dynamic state.
1967  *  chain - ipfw's instance;
1968  *  rule - the parent rule that installs the state;
1969  *  cmd - opcode that installs the state;
1970  *  args - ipfw arguments;
1971  *  ulp - upper level protocol header;
1972  *  pktlen - packet length;
1973  *  info - dynamic state lookup info;
1974  *  tablearg - tablearg id.
1975  *
1976  * Returns non-zero value (failure) if state is not installed because
1977  * of errors or because session limitations are enforced.
1978  */
1979 int
1980 ipfw_dyn_install_state(struct ip_fw_chain *chain, struct ip_fw *rule,
1981     const ipfw_insn_limit *cmd, const struct ip_fw_args *args,
1982     const void *ulp, int pktlen, struct ipfw_dyn_info *info,
1983     uint32_t tablearg)
1984 {
1985         uint32_t limit;
1986         uint16_t limit_mask;
1987
1988         if (cmd->o.opcode == O_LIMIT) {
1989                 limit = IP_FW_ARG_TABLEARG(chain, cmd->conn_limit, limit);
1990                 limit_mask = cmd->limit_mask;
1991         } else {
1992                 limit = 0;
1993                 limit_mask = 0;
1994         }
1995         return (dyn_install_state(&args->f_id,
1996 #ifdef INET6
1997             IS_IP6_FLOW_ID(&args->f_id) ? dyn_getscopeid(args):
1998 #endif
1999             0, M_GETFIB(args->m), ulp, pktlen, rule, rule->id, rule->rulenum,
2000             rule->set, info, limit, limit_mask, cmd->o.arg1, cmd->o.opcode));
2001 }
2002
2003 /*
2004  * Free safe to remove state entries from expired lists.
2005  */
2006 static void
2007 dyn_free_states(struct ip_fw_chain *chain)
2008 {
2009         struct dyn_ipv4_state *s4, *s4n;
2010 #ifdef INET6
2011         struct dyn_ipv6_state *s6, *s6n;
2012 #endif
2013         int cached_count, i;
2014
2015         /*
2016          * We keep pointers to objects that are in use on each CPU
2017          * in the per-cpu dyn_hp pointer. When object is going to be
2018          * removed, first of it is unlinked from the corresponding
2019          * list. This leads to changing of dyn_bucket_xxx_delver version.
2020          * Unlinked objects is placed into corresponding dyn_expired_xxx
2021          * list. Reader that is going to dereference object pointer checks
2022          * dyn_bucket_xxx_delver version before and after storing pointer
2023          * into dyn_hp. If version is the same, the object is protected
2024          * from freeing and it is safe to dereference. Othervise reader
2025          * tries to iterate list again from the beginning, but this object
2026          * now unlinked and thus will not be accessible.
2027          *
2028          * Copy dyn_hp pointers for each CPU into dyn_hp_cache array.
2029          * It does not matter that some pointer can be changed in
2030          * time while we are copying. We need to check, that objects
2031          * removed in the previous pass are not in use. And if dyn_hp
2032          * pointer does not contain it in the time when we are copying,
2033          * it will not appear there, because it is already unlinked.
2034          * And for new pointers we will not free objects that will be
2035          * unlinked in this pass.
2036          */
2037         cached_count = 0;
2038         CPU_FOREACH(i) {
2039                 dyn_hp_cache[cached_count] = DYNSTATE_GET(i);
2040                 if (dyn_hp_cache[cached_count] != NULL)
2041                         cached_count++;
2042         }
2043
2044         /*
2045          * Free expired states that are safe to free.
2046          * Check each entry from previous pass in the dyn_expired_xxx
2047          * list, if pointer to the object is in the dyn_hp_cache array,
2048          * keep it until next pass. Otherwise it is safe to free the
2049          * object.
2050          *
2051          * XXXAE: optimize this to use SLIST_REMOVE_AFTER.
2052          */
2053 #define DYN_FREE_STATES(s, next, name)          do {                    \
2054         s = SLIST_FIRST(&V_dyn_expired_ ## name);                       \
2055         while (s != NULL) {                                             \
2056                 next = SLIST_NEXT(s, expired);                          \
2057                 for (i = 0; i < cached_count; i++)                      \
2058                         if (dyn_hp_cache[i] == s)                       \
2059                                 break;                                  \
2060                 if (i == cached_count) {                                \
2061                         if (s->type == O_LIMIT_PARENT &&                \
2062                             s->limit->count != 0) {                     \
2063                                 s = next;                               \
2064                                 continue;                               \
2065                         }                                               \
2066                         SLIST_REMOVE(&V_dyn_expired_ ## name,           \
2067                             s, dyn_ ## name ## _state, expired);        \
2068                         if (s->type == O_LIMIT_PARENT)                  \
2069                                 uma_zfree(V_dyn_parent_zone, s->limit); \
2070                         else                                            \
2071                                 uma_zfree(V_dyn_data_zone, s->data);    \
2072                         uma_zfree(V_dyn_ ## name ## _zone, s);          \
2073                 }                                                       \
2074                 s = next;                                               \
2075         }                                                               \
2076 } while (0)
2077
2078         /*
2079          * Protect access to expired lists with DYN_EXPIRED_LOCK.
2080          * Userland can invoke ipfw_expire_dyn_states() to delete
2081          * specific states, this will lead to modification of expired
2082          * lists.
2083          *
2084          * XXXAE: do we need DYN_EXPIRED_LOCK? We can just use
2085          *        IPFW_UH_WLOCK to protect access to these lists.
2086          */
2087         DYN_EXPIRED_LOCK();
2088         DYN_FREE_STATES(s4, s4n, ipv4);
2089 #ifdef INET6
2090         DYN_FREE_STATES(s6, s6n, ipv6);
2091 #endif
2092         DYN_EXPIRED_UNLOCK();
2093 #undef DYN_FREE_STATES
2094 }
2095
2096 /*
2097  * Returns 1 when state is matched by specified range, otherwise returns 0.
2098  */
2099 static int
2100 dyn_match_range(uint16_t rulenum, uint8_t set, const ipfw_range_tlv *rt)
2101 {
2102
2103         MPASS(rt != NULL);
2104         /* flush all states */
2105         if (rt->flags & IPFW_RCFLAG_ALL)
2106                 return (1);
2107         if ((rt->flags & IPFW_RCFLAG_SET) != 0 && set != rt->set)
2108                 return (0);
2109         if ((rt->flags & IPFW_RCFLAG_RANGE) != 0 &&
2110             (rulenum < rt->start_rule || rulenum > rt->end_rule))
2111                 return (0);
2112         return (1);
2113 }
2114
2115 static int
2116 dyn_match_ipv4_state(struct dyn_ipv4_state *s, const ipfw_range_tlv *rt)
2117 {
2118
2119         if (s->type == O_LIMIT_PARENT)
2120                 return (dyn_match_range(s->limit->rulenum,
2121                     s->limit->set, rt));
2122
2123         if (s->type == O_LIMIT)
2124                 return (dyn_match_range(s->data->rulenum, s->data->set, rt));
2125
2126         if (V_dyn_keep_states == 0 &&
2127             dyn_match_range(s->data->rulenum, s->data->set, rt))
2128                 return (1);
2129
2130         return (0);
2131 }
2132
2133 #ifdef INET6
2134 static int
2135 dyn_match_ipv6_state(struct dyn_ipv6_state *s, const ipfw_range_tlv *rt)
2136 {
2137
2138         if (s->type == O_LIMIT_PARENT)
2139                 return (dyn_match_range(s->limit->rulenum,
2140                     s->limit->set, rt));
2141
2142         if (s->type == O_LIMIT)
2143                 return (dyn_match_range(s->data->rulenum, s->data->set, rt));
2144
2145         if (V_dyn_keep_states == 0 &&
2146             dyn_match_range(s->data->rulenum, s->data->set, rt))
2147                 return (1);
2148
2149         return (0);
2150 }
2151 #endif
2152
2153 /*
2154  * Unlink expired entries from states lists.
2155  * @rt can be used to specify the range of states for deletion.
2156  */
2157 static void
2158 dyn_expire_states(struct ip_fw_chain *chain, ipfw_range_tlv *rt)
2159 {
2160         struct dyn_ipv4_slist expired_ipv4;
2161 #ifdef INET6
2162         struct dyn_ipv6_slist expired_ipv6;
2163         struct dyn_ipv6_state *s6, *s6n, *s6p;
2164 #endif
2165         struct dyn_ipv4_state *s4, *s4n, *s4p;
2166         int bucket, removed, length, max_length;
2167
2168         /*
2169          * Unlink expired states from each bucket.
2170          * With acquired bucket lock iterate entries of each lists:
2171          * ipv4, ipv4_parent, ipv6, and ipv6_parent. Check expired time
2172          * and unlink entry from the list, link entry into temporary
2173          * expired_xxx lists then bump "del" bucket version.
2174          *
2175          * When an entry is removed, corresponding states counter is
2176          * decremented. If entry has O_LIMIT type, parent's reference
2177          * counter is decremented.
2178          *
2179          * NOTE: this function can be called from userspace context
2180          * when user deletes rules. In this case all matched states
2181          * will be forcedly unlinked. O_LIMIT_PARENT states will be kept
2182          * in the expired lists until reference counter become zero.
2183          */
2184 #define DYN_UNLINK_STATES(s, prev, next, exp, af, name, extra)  do {    \
2185         length = 0;                                                     \
2186         removed = 0;                                                    \
2187         prev = NULL;                                                    \
2188         s = CK_SLIST_FIRST(&V_dyn_ ## name [bucket]);                   \
2189         while (s != NULL) {                                             \
2190                 next = CK_SLIST_NEXT(s, entry);                         \
2191                 if ((TIME_LEQ((s)->exp, time_uptime) && extra) ||       \
2192                     (rt != NULL && dyn_match_ ## af ## _state(s, rt))) {\
2193                         if (prev != NULL)                               \
2194                                 CK_SLIST_REMOVE_AFTER(prev, entry);     \
2195                         else                                            \
2196                                 CK_SLIST_REMOVE_HEAD(                   \
2197                                     &V_dyn_ ## name [bucket], entry);   \
2198                         removed++;                                      \
2199                         SLIST_INSERT_HEAD(&expired_ ## af, s, expired); \
2200                         if (s->type == O_LIMIT_PARENT)                  \
2201                                 DYN_COUNT_DEC(dyn_parent_count);        \
2202                         else {                                          \
2203                                 DYN_COUNT_DEC(dyn_count);               \
2204                                 if (s->type == O_LIMIT) {               \
2205                                         s = s->data->parent;            \
2206                                         DPARENT_COUNT_DEC(s->limit);    \
2207                                 }                                       \
2208                         }                                               \
2209                 } else {                                                \
2210                         prev = s;                                       \
2211                         length++;                                       \
2212                 }                                                       \
2213                 s = next;                                               \
2214         }                                                               \
2215         if (removed != 0)                                               \
2216                 DYN_BUCKET_VERSION_BUMP(bucket, name ## _del);          \
2217         if (length > max_length)                                \
2218                 max_length = length;                            \
2219 } while (0)
2220
2221         SLIST_INIT(&expired_ipv4);
2222 #ifdef INET6
2223         SLIST_INIT(&expired_ipv6);
2224 #endif
2225         max_length = 0;
2226         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2227                 DYN_BUCKET_LOCK(bucket);
2228                 DYN_UNLINK_STATES(s4, s4p, s4n, data->expire, ipv4, ipv4, 1);
2229                 DYN_UNLINK_STATES(s4, s4p, s4n, limit->expire, ipv4,
2230                     ipv4_parent, (s4->limit->count == 0));
2231 #ifdef INET6
2232                 DYN_UNLINK_STATES(s6, s6p, s6n, data->expire, ipv6, ipv6, 1);
2233                 DYN_UNLINK_STATES(s6, s6p, s6n, limit->expire, ipv6,
2234                     ipv6_parent, (s6->limit->count == 0));
2235 #endif
2236                 DYN_BUCKET_UNLOCK(bucket);
2237         }
2238         /* Update curr_max_length for statistics. */
2239         V_curr_max_length = max_length;
2240         /*
2241          * Concatenate temporary lists with global expired lists.
2242          */
2243         DYN_EXPIRED_LOCK();
2244         SLIST_CONCAT(&V_dyn_expired_ipv4, &expired_ipv4,
2245             dyn_ipv4_state, expired);
2246 #ifdef INET6
2247         SLIST_CONCAT(&V_dyn_expired_ipv6, &expired_ipv6,
2248             dyn_ipv6_state, expired);
2249 #endif
2250         DYN_EXPIRED_UNLOCK();
2251 #undef DYN_UNLINK_STATES
2252 #undef DYN_UNREF_STATES
2253 }
2254
2255 static struct mbuf *
2256 dyn_mgethdr(int len, uint16_t fibnum)
2257 {
2258         struct mbuf *m;
2259
2260         m = m_gethdr(M_NOWAIT, MT_DATA);
2261         if (m == NULL)
2262                 return (NULL);
2263 #ifdef MAC
2264         mac_netinet_firewall_send(m);
2265 #endif
2266         M_SETFIB(m, fibnum);
2267         m->m_data += max_linkhdr;
2268         m->m_flags |= M_SKIP_FIREWALL;
2269         m->m_len = m->m_pkthdr.len = len;
2270         bzero(m->m_data, len);
2271         return (m);
2272 }
2273
2274 static void
2275 dyn_make_keepalive_ipv4(struct mbuf *m, in_addr_t src, in_addr_t dst,
2276     uint32_t seq, uint32_t ack, uint16_t sport, uint16_t dport)
2277 {
2278         struct tcphdr *tcp;
2279         struct ip *ip;
2280
2281         ip = mtod(m, struct ip *);
2282         ip->ip_v = 4;
2283         ip->ip_hl = sizeof(*ip) >> 2;
2284         ip->ip_tos = IPTOS_LOWDELAY;
2285         ip->ip_len = htons(m->m_len);
2286         ip->ip_off |= htons(IP_DF);
2287         ip->ip_ttl = V_ip_defttl;
2288         ip->ip_p = IPPROTO_TCP;
2289         ip->ip_src.s_addr = htonl(src);
2290         ip->ip_dst.s_addr = htonl(dst);
2291
2292         tcp = mtodo(m, sizeof(struct ip));
2293         tcp->th_sport = htons(sport);
2294         tcp->th_dport = htons(dport);
2295         tcp->th_off = sizeof(struct tcphdr) >> 2;
2296         tcp->th_seq = htonl(seq);
2297         tcp->th_ack = htonl(ack);
2298         tcp->th_flags = TH_ACK;
2299         tcp->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2300             htons(sizeof(struct tcphdr) + IPPROTO_TCP));
2301
2302         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2303         m->m_pkthdr.csum_flags = CSUM_TCP;
2304 }
2305
2306 static void
2307 dyn_enqueue_keepalive_ipv4(struct mbufq *q, const struct dyn_ipv4_state *s)
2308 {
2309         struct mbuf *m;
2310
2311         if ((s->data->state & ACK_FWD) == 0 && s->data->ack_fwd > 0) {
2312                 m = dyn_mgethdr(sizeof(struct ip) + sizeof(struct tcphdr),
2313                     s->data->fibnum);
2314                 if (m != NULL) {
2315                         dyn_make_keepalive_ipv4(m, s->dst, s->src,
2316                             s->data->ack_fwd - 1, s->data->ack_rev,
2317                             s->dport, s->sport);
2318                         if (mbufq_enqueue(q, m)) {
2319                                 m_freem(m);
2320                                 log(LOG_DEBUG, "ipfw: limit for IPv4 "
2321                                     "keepalive queue is reached.\n");
2322                                 return;
2323                         }
2324                 }
2325         }
2326
2327         if ((s->data->state & ACK_REV) == 0 && s->data->ack_rev > 0) {
2328                 m = dyn_mgethdr(sizeof(struct ip) + sizeof(struct tcphdr),
2329                     s->data->fibnum);
2330                 if (m != NULL) {
2331                         dyn_make_keepalive_ipv4(m, s->src, s->dst,
2332                             s->data->ack_rev - 1, s->data->ack_fwd,
2333                             s->sport, s->dport);
2334                         if (mbufq_enqueue(q, m)) {
2335                                 m_freem(m);
2336                                 log(LOG_DEBUG, "ipfw: limit for IPv4 "
2337                                     "keepalive queue is reached.\n");
2338                                 return;
2339                         }
2340                 }
2341         }
2342 }
2343
2344 /*
2345  * Prepare and send keep-alive packets.
2346  */
2347 static void
2348 dyn_send_keepalive_ipv4(struct ip_fw_chain *chain)
2349 {
2350         struct mbufq q;
2351         struct mbuf *m;
2352         struct dyn_ipv4_state *s;
2353         uint32_t bucket;
2354
2355         mbufq_init(&q, INT_MAX);
2356         IPFW_UH_RLOCK(chain);
2357         /*
2358          * It is safe to not use hazard pointer and just do lockless
2359          * access to the lists, because states entries can not be deleted
2360          * while we hold IPFW_UH_RLOCK.
2361          */
2362         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2363                 CK_SLIST_FOREACH(s, &V_dyn_ipv4[bucket], entry) {
2364                         /*
2365                          * Only established TCP connections that will
2366                          * become expired withing dyn_keepalive_interval.
2367                          */
2368                         if (s->proto != IPPROTO_TCP ||
2369                             (s->data->state & BOTH_SYN) != BOTH_SYN ||
2370                             TIME_LEQ(time_uptime + V_dyn_keepalive_interval,
2371                                 s->data->expire))
2372                                 continue;
2373                         dyn_enqueue_keepalive_ipv4(&q, s);
2374                 }
2375         }
2376         IPFW_UH_RUNLOCK(chain);
2377         while ((m = mbufq_dequeue(&q)) != NULL)
2378                 ip_output(m, NULL, NULL, 0, NULL, NULL);
2379 }
2380
2381 #ifdef INET6
2382 static void
2383 dyn_make_keepalive_ipv6(struct mbuf *m, const struct in6_addr *src,
2384     const struct in6_addr *dst, uint32_t zoneid, uint32_t seq, uint32_t ack,
2385     uint16_t sport, uint16_t dport)
2386 {
2387         struct tcphdr *tcp;
2388         struct ip6_hdr *ip6;
2389
2390         ip6 = mtod(m, struct ip6_hdr *);
2391         ip6->ip6_vfc |= IPV6_VERSION;
2392         ip6->ip6_plen = htons(sizeof(struct tcphdr));
2393         ip6->ip6_nxt = IPPROTO_TCP;
2394         ip6->ip6_hlim = IPV6_DEFHLIM;
2395         ip6->ip6_src = *src;
2396         if (IN6_IS_ADDR_LINKLOCAL(src))
2397                 ip6->ip6_src.s6_addr16[1] = htons(zoneid & 0xffff);
2398         ip6->ip6_dst = *dst;
2399         if (IN6_IS_ADDR_LINKLOCAL(dst))
2400                 ip6->ip6_dst.s6_addr16[1] = htons(zoneid & 0xffff);
2401
2402         tcp = mtodo(m, sizeof(struct ip6_hdr));
2403         tcp->th_sport = htons(sport);
2404         tcp->th_dport = htons(dport);
2405         tcp->th_off = sizeof(struct tcphdr) >> 2;
2406         tcp->th_seq = htonl(seq);
2407         tcp->th_ack = htonl(ack);
2408         tcp->th_flags = TH_ACK;
2409         tcp->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr),
2410             IPPROTO_TCP, 0);
2411
2412         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2413         m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
2414 }
2415
2416 static void
2417 dyn_enqueue_keepalive_ipv6(struct mbufq *q, const struct dyn_ipv6_state *s)
2418 {
2419         struct mbuf *m;
2420
2421         if ((s->data->state & ACK_FWD) == 0 && s->data->ack_fwd > 0) {
2422                 m = dyn_mgethdr(sizeof(struct ip6_hdr) +
2423                     sizeof(struct tcphdr), s->data->fibnum);
2424                 if (m != NULL) {
2425                         dyn_make_keepalive_ipv6(m, &s->dst, &s->src,
2426                             s->zoneid, s->data->ack_fwd - 1, s->data->ack_rev,
2427                             s->dport, s->sport);
2428                         if (mbufq_enqueue(q, m)) {
2429                                 m_freem(m);
2430                                 log(LOG_DEBUG, "ipfw: limit for IPv6 "
2431                                     "keepalive queue is reached.\n");
2432                                 return;
2433                         }
2434                 }
2435         }
2436
2437         if ((s->data->state & ACK_REV) == 0 && s->data->ack_rev > 0) {
2438                 m = dyn_mgethdr(sizeof(struct ip6_hdr) +
2439                     sizeof(struct tcphdr), s->data->fibnum);
2440                 if (m != NULL) {
2441                         dyn_make_keepalive_ipv6(m, &s->src, &s->dst,
2442                             s->zoneid, s->data->ack_rev - 1, s->data->ack_fwd,
2443                             s->sport, s->dport);
2444                         if (mbufq_enqueue(q, m)) {
2445                                 m_freem(m);
2446                                 log(LOG_DEBUG, "ipfw: limit for IPv6 "
2447                                     "keepalive queue is reached.\n");
2448                                 return;
2449                         }
2450                 }
2451         }
2452 }
2453
2454 static void
2455 dyn_send_keepalive_ipv6(struct ip_fw_chain *chain)
2456 {
2457         struct mbufq q;
2458         struct mbuf *m;
2459         struct dyn_ipv6_state *s;
2460         uint32_t bucket;
2461
2462         mbufq_init(&q, INT_MAX);
2463         IPFW_UH_RLOCK(chain);
2464         /*
2465          * It is safe to not use hazard pointer and just do lockless
2466          * access to the lists, because states entries can not be deleted
2467          * while we hold IPFW_UH_RLOCK.
2468          */
2469         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2470                 CK_SLIST_FOREACH(s, &V_dyn_ipv6[bucket], entry) {
2471                         /*
2472                          * Only established TCP connections that will
2473                          * become expired withing dyn_keepalive_interval.
2474                          */
2475                         if (s->proto != IPPROTO_TCP ||
2476                             (s->data->state & BOTH_SYN) != BOTH_SYN ||
2477                             TIME_LEQ(time_uptime + V_dyn_keepalive_interval,
2478                                 s->data->expire))
2479                                 continue;
2480                         dyn_enqueue_keepalive_ipv6(&q, s);
2481                 }
2482         }
2483         IPFW_UH_RUNLOCK(chain);
2484         while ((m = mbufq_dequeue(&q)) != NULL)
2485                 ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL);
2486 }
2487 #endif /* INET6 */
2488
2489 static void
2490 dyn_grow_hashtable(struct ip_fw_chain *chain, uint32_t new)
2491 {
2492 #ifdef INET6
2493         struct dyn_ipv6ck_slist *ipv6, *ipv6_parent;
2494         uint32_t *ipv6_add, *ipv6_del, *ipv6_parent_add, *ipv6_parent_del;
2495         struct dyn_ipv6_state *s6;
2496 #endif
2497         struct dyn_ipv4ck_slist *ipv4, *ipv4_parent;
2498         uint32_t *ipv4_add, *ipv4_del, *ipv4_parent_add, *ipv4_parent_del;
2499         struct dyn_ipv4_state *s4;
2500         struct mtx *bucket_lock;
2501         void *tmp;
2502         uint32_t bucket;
2503
2504         MPASS(powerof2(new));
2505         DYN_DEBUG("grow hash size %u -> %u", V_curr_dyn_buckets, new);
2506         /*
2507          * Allocate and initialize new lists.
2508          * XXXAE: on memory pressure this can disable callout timer.
2509          */
2510         bucket_lock = malloc(new * sizeof(struct mtx), M_IPFW,
2511             M_WAITOK | M_ZERO);
2512         ipv4 = malloc(new * sizeof(struct dyn_ipv4ck_slist), M_IPFW,
2513             M_WAITOK | M_ZERO);
2514         ipv4_parent = malloc(new * sizeof(struct dyn_ipv4ck_slist), M_IPFW,
2515             M_WAITOK | M_ZERO);
2516         ipv4_add = malloc(new * sizeof(uint32_t), M_IPFW, M_WAITOK | M_ZERO);
2517         ipv4_del = malloc(new * sizeof(uint32_t), M_IPFW, M_WAITOK | M_ZERO);
2518         ipv4_parent_add = malloc(new * sizeof(uint32_t), M_IPFW,
2519             M_WAITOK | M_ZERO);
2520         ipv4_parent_del = malloc(new * sizeof(uint32_t), M_IPFW,
2521             M_WAITOK | M_ZERO);
2522 #ifdef INET6
2523         ipv6 = malloc(new * sizeof(struct dyn_ipv6ck_slist), M_IPFW,
2524             M_WAITOK | M_ZERO);
2525         ipv6_parent = malloc(new * sizeof(struct dyn_ipv6ck_slist), M_IPFW,
2526             M_WAITOK | M_ZERO);
2527         ipv6_add = malloc(new * sizeof(uint32_t), M_IPFW, M_WAITOK | M_ZERO);
2528         ipv6_del = malloc(new * sizeof(uint32_t), M_IPFW, M_WAITOK | M_ZERO);
2529         ipv6_parent_add = malloc(new * sizeof(uint32_t), M_IPFW,
2530             M_WAITOK | M_ZERO);
2531         ipv6_parent_del = malloc(new * sizeof(uint32_t), M_IPFW,
2532             M_WAITOK | M_ZERO);
2533 #endif
2534         for (bucket = 0; bucket < new; bucket++) {
2535                 DYN_BUCKET_LOCK_INIT(bucket_lock, bucket);
2536                 CK_SLIST_INIT(&ipv4[bucket]);
2537                 CK_SLIST_INIT(&ipv4_parent[bucket]);
2538 #ifdef INET6
2539                 CK_SLIST_INIT(&ipv6[bucket]);
2540                 CK_SLIST_INIT(&ipv6_parent[bucket]);
2541 #endif
2542         }
2543
2544 #define DYN_RELINK_STATES(s, hval, i, head, ohead)      do {            \
2545         while ((s = CK_SLIST_FIRST(&V_dyn_ ## ohead[i])) != NULL) {     \
2546                 CK_SLIST_REMOVE_HEAD(&V_dyn_ ## ohead[i], entry);       \
2547                 CK_SLIST_INSERT_HEAD(&head[DYN_BUCKET(s->hval, new)],   \
2548                     s, entry);                                          \
2549         }                                                               \
2550 } while (0)
2551         /*
2552          * Prevent rules changing from userland.
2553          */
2554         IPFW_UH_WLOCK(chain);
2555         /*
2556          * Hold traffic processing until we finish resize to
2557          * prevent access to states lists.
2558          */
2559         IPFW_WLOCK(chain);
2560         /* Re-link all dynamic states */
2561         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2562                 DYN_RELINK_STATES(s4, data->hashval, bucket, ipv4, ipv4);
2563                 DYN_RELINK_STATES(s4, limit->hashval, bucket, ipv4_parent,
2564                     ipv4_parent);
2565 #ifdef INET6
2566                 DYN_RELINK_STATES(s6, data->hashval, bucket, ipv6, ipv6);
2567                 DYN_RELINK_STATES(s6, limit->hashval, bucket, ipv6_parent,
2568                     ipv6_parent);
2569 #endif
2570         }
2571
2572 #define DYN_SWAP_PTR(old, new, tmp)     do {            \
2573         tmp = old;                                      \
2574         old = new;                                      \
2575         new = tmp;                                      \
2576 } while (0)
2577         /* Swap pointers */
2578         DYN_SWAP_PTR(V_dyn_bucket_lock, bucket_lock, tmp);
2579         DYN_SWAP_PTR(V_dyn_ipv4, ipv4, tmp);
2580         DYN_SWAP_PTR(V_dyn_ipv4_parent, ipv4_parent, tmp);
2581         DYN_SWAP_PTR(V_dyn_ipv4_add, ipv4_add, tmp);
2582         DYN_SWAP_PTR(V_dyn_ipv4_parent_add, ipv4_parent_add, tmp);
2583         DYN_SWAP_PTR(V_dyn_ipv4_del, ipv4_del, tmp);
2584         DYN_SWAP_PTR(V_dyn_ipv4_parent_del, ipv4_parent_del, tmp);
2585
2586 #ifdef INET6
2587         DYN_SWAP_PTR(V_dyn_ipv6, ipv6, tmp);
2588         DYN_SWAP_PTR(V_dyn_ipv6_parent, ipv6_parent, tmp);
2589         DYN_SWAP_PTR(V_dyn_ipv6_add, ipv6_add, tmp);
2590         DYN_SWAP_PTR(V_dyn_ipv6_parent_add, ipv6_parent_add, tmp);
2591         DYN_SWAP_PTR(V_dyn_ipv6_del, ipv6_del, tmp);
2592         DYN_SWAP_PTR(V_dyn_ipv6_parent_del, ipv6_parent_del, tmp);
2593 #endif
2594         bucket = V_curr_dyn_buckets;
2595         V_curr_dyn_buckets = new;
2596
2597         IPFW_WUNLOCK(chain);
2598         IPFW_UH_WUNLOCK(chain);
2599
2600         /* Release old resources */
2601         while (bucket-- != 0)
2602                 DYN_BUCKET_LOCK_DESTROY(bucket_lock, bucket);
2603         free(bucket_lock, M_IPFW);
2604         free(ipv4, M_IPFW);
2605         free(ipv4_parent, M_IPFW);
2606         free(ipv4_add, M_IPFW);
2607         free(ipv4_parent_add, M_IPFW);
2608         free(ipv4_del, M_IPFW);
2609         free(ipv4_parent_del, M_IPFW);
2610 #ifdef INET6
2611         free(ipv6, M_IPFW);
2612         free(ipv6_parent, M_IPFW);
2613         free(ipv6_add, M_IPFW);
2614         free(ipv6_parent_add, M_IPFW);
2615         free(ipv6_del, M_IPFW);
2616         free(ipv6_parent_del, M_IPFW);
2617 #endif
2618 }
2619
2620 /*
2621  * This function is used to perform various maintenance
2622  * on dynamic hash lists. Currently it is called every second.
2623  */
2624 static void
2625 dyn_tick(void *vnetx)
2626 {
2627         uint32_t buckets;
2628
2629         CURVNET_SET((struct vnet *)vnetx);
2630         /*
2631          * First free states unlinked in previous passes.
2632          */
2633         dyn_free_states(&V_layer3_chain);
2634         /*
2635          * Now unlink others expired states.
2636          * We use IPFW_UH_WLOCK to avoid concurrent call of
2637          * dyn_expire_states(). It is the only function that does
2638          * deletion of state entries from states lists.
2639          */
2640         IPFW_UH_WLOCK(&V_layer3_chain);
2641         dyn_expire_states(&V_layer3_chain, NULL);
2642         IPFW_UH_WUNLOCK(&V_layer3_chain);
2643         /*
2644          * Send keepalives if they are enabled and the time has come.
2645          */
2646         if (V_dyn_keepalive != 0 &&
2647             V_dyn_keepalive_last + V_dyn_keepalive_period <= time_uptime) {
2648                 V_dyn_keepalive_last = time_uptime;
2649                 dyn_send_keepalive_ipv4(&V_layer3_chain);
2650 #ifdef INET6
2651                 dyn_send_keepalive_ipv6(&V_layer3_chain);
2652 #endif
2653         }
2654         /*
2655          * Check if we need to resize the hash:
2656          * if current number of states exceeds number of buckets in hash,
2657          * and dyn_buckets_max permits to grow the number of buckets, then
2658          * do it. Grow hash size to the minimum power of 2 which is bigger
2659          * than current states count.
2660          */
2661         if (V_curr_dyn_buckets < V_dyn_buckets_max &&
2662             (V_curr_dyn_buckets < V_dyn_count / 2 || (
2663             V_curr_dyn_buckets < V_dyn_count && V_curr_max_length > 8))) {
2664                 buckets = 1 << fls(V_dyn_count);
2665                 if (buckets > V_dyn_buckets_max)
2666                         buckets = V_dyn_buckets_max;
2667                 dyn_grow_hashtable(&V_layer3_chain, buckets);
2668         }
2669
2670         callout_reset_on(&V_dyn_timeout, hz, dyn_tick, vnetx, 0);
2671         CURVNET_RESTORE();
2672 }
2673
2674 void
2675 ipfw_expire_dyn_states(struct ip_fw_chain *chain, ipfw_range_tlv *rt)
2676 {
2677         /*
2678          * Do not perform any checks if we currently have no dynamic states
2679          */
2680         if (V_dyn_count == 0)
2681                 return;
2682
2683         IPFW_UH_WLOCK_ASSERT(chain);
2684         dyn_expire_states(chain, rt);
2685 }
2686
2687 /*
2688  * Returns size of dynamic states in legacy format
2689  */
2690 int
2691 ipfw_dyn_len(void)
2692 {
2693
2694         return ((V_dyn_count + V_dyn_parent_count) * sizeof(ipfw_dyn_rule));
2695 }
2696
2697 /*
2698  * Returns number of dynamic states.
2699  * Used by dump format v1 (current).
2700  */
2701 uint32_t
2702 ipfw_dyn_get_count(void)
2703 {
2704
2705         return (V_dyn_count + V_dyn_parent_count);
2706 }
2707
2708 /*
2709  * Check if rule contains at least one dynamic opcode.
2710  *
2711  * Returns 1 if such opcode is found, 0 otherwise.
2712  */
2713 int
2714 ipfw_is_dyn_rule(struct ip_fw *rule)
2715 {
2716         int cmdlen, l;
2717         ipfw_insn *cmd;
2718
2719         l = rule->cmd_len;
2720         cmd = rule->cmd;
2721         cmdlen = 0;
2722         for ( ; l > 0 ; l -= cmdlen, cmd += cmdlen) {
2723                 cmdlen = F_LEN(cmd);
2724
2725                 switch (cmd->opcode) {
2726                 case O_LIMIT:
2727                 case O_KEEP_STATE:
2728                 case O_PROBE_STATE:
2729                 case O_CHECK_STATE:
2730                         return (1);
2731                 }
2732         }
2733
2734         return (0);
2735 }
2736
2737 static void
2738 dyn_export_parent(const struct dyn_parent *p, uint16_t kidx,
2739     ipfw_dyn_rule *dst)
2740 {
2741
2742         dst->dyn_type = O_LIMIT_PARENT;
2743         dst->kidx = kidx;
2744         dst->count = (uint16_t)DPARENT_COUNT(p);
2745         dst->expire = TIME_LEQ(p->expire, time_uptime) ?  0:
2746             p->expire - time_uptime;
2747
2748         /* 'rule' is used to pass up the rule number and set */
2749         memcpy(&dst->rule, &p->rulenum, sizeof(p->rulenum));
2750         /* store set number into high word of dst->rule pointer. */
2751         memcpy((char *)&dst->rule + sizeof(p->rulenum), &p->set,
2752             sizeof(p->set));
2753
2754         /* unused fields */
2755         dst->pcnt = 0;
2756         dst->bcnt = 0;
2757         dst->parent = NULL;
2758         dst->state = 0;
2759         dst->ack_fwd = 0;
2760         dst->ack_rev = 0;
2761         dst->bucket = p->hashval;
2762         /*
2763          * The legacy userland code will interpret a NULL here as a marker
2764          * for the last dynamic rule.
2765          */
2766         dst->next = (ipfw_dyn_rule *)1;
2767 }
2768
2769 static void
2770 dyn_export_data(const struct dyn_data *data, uint16_t kidx, uint8_t type,
2771     ipfw_dyn_rule *dst)
2772 {
2773
2774         dst->dyn_type = type;
2775         dst->kidx = kidx;
2776         dst->pcnt = data->pcnt_fwd + data->pcnt_rev;
2777         dst->bcnt = data->bcnt_fwd + data->bcnt_rev;
2778         dst->expire = TIME_LEQ(data->expire, time_uptime) ?  0:
2779             data->expire - time_uptime;
2780
2781         /* 'rule' is used to pass up the rule number and set */
2782         memcpy(&dst->rule, &data->rulenum, sizeof(data->rulenum));
2783         /* store set number into high word of dst->rule pointer. */
2784         memcpy((char *)&dst->rule + sizeof(data->rulenum), &data->set,
2785             sizeof(data->set));
2786
2787         /* unused fields */
2788         dst->parent = NULL;
2789         dst->state = data->state;
2790         dst->ack_fwd = data->ack_fwd;
2791         dst->ack_rev = data->ack_rev;
2792         dst->count = 0;
2793         dst->bucket = data->hashval;
2794         /*
2795          * The legacy userland code will interpret a NULL here as a marker
2796          * for the last dynamic rule.
2797          */
2798         dst->next = (ipfw_dyn_rule *)1;
2799 }
2800
2801 static void
2802 dyn_export_ipv4_state(const struct dyn_ipv4_state *s, ipfw_dyn_rule *dst)
2803 {
2804
2805         switch (s->type) {
2806         case O_LIMIT_PARENT:
2807                 dyn_export_parent(s->limit, s->kidx, dst);
2808                 break;
2809         default:
2810                 dyn_export_data(s->data, s->kidx, s->type, dst);
2811         }
2812
2813         dst->id.dst_ip = s->dst;
2814         dst->id.src_ip = s->src;
2815         dst->id.dst_port = s->dport;
2816         dst->id.src_port = s->sport;
2817         dst->id.fib = s->data->fibnum;
2818         dst->id.proto = s->proto;
2819         dst->id._flags = 0;
2820         dst->id.addr_type = 4;
2821
2822         memset(&dst->id.dst_ip6, 0, sizeof(dst->id.dst_ip6));
2823         memset(&dst->id.src_ip6, 0, sizeof(dst->id.src_ip6));
2824         dst->id.flow_id6 = dst->id.extra = 0;
2825 }
2826
2827 #ifdef INET6
2828 static void
2829 dyn_export_ipv6_state(const struct dyn_ipv6_state *s, ipfw_dyn_rule *dst)
2830 {
2831
2832         switch (s->type) {
2833         case O_LIMIT_PARENT:
2834                 dyn_export_parent(s->limit, s->kidx, dst);
2835                 break;
2836         default:
2837                 dyn_export_data(s->data, s->kidx, s->type, dst);
2838         }
2839
2840         dst->id.src_ip6 = s->src;
2841         dst->id.dst_ip6 = s->dst;
2842         dst->id.dst_port = s->dport;
2843         dst->id.src_port = s->sport;
2844         dst->id.fib = s->data->fibnum;
2845         dst->id.proto = s->proto;
2846         dst->id._flags = 0;
2847         dst->id.addr_type = 6;
2848
2849         dst->id.dst_ip = dst->id.src_ip = 0;
2850         dst->id.flow_id6 = dst->id.extra = 0;
2851 }
2852 #endif /* INET6 */
2853
2854 /*
2855  * Fills the buffer given by @sd with dynamic states.
2856  * Used by dump format v1 (current).
2857  *
2858  * Returns 0 on success.
2859  */
2860 int
2861 ipfw_dump_states(struct ip_fw_chain *chain, struct sockopt_data *sd)
2862 {
2863 #ifdef INET6
2864         struct dyn_ipv6_state *s6;
2865 #endif
2866         struct dyn_ipv4_state *s4;
2867         ipfw_obj_dyntlv *dst, *last;
2868         ipfw_obj_ctlv *ctlv;
2869         uint32_t bucket;
2870
2871         if (V_dyn_count == 0)
2872                 return (0);
2873
2874         /*
2875          * IPFW_UH_RLOCK garantees that another userland request
2876          * and callout thread will not delete entries from states
2877          * lists.
2878          */
2879         IPFW_UH_RLOCK_ASSERT(chain);
2880
2881         ctlv = (ipfw_obj_ctlv *)ipfw_get_sopt_space(sd, sizeof(*ctlv));
2882         if (ctlv == NULL)
2883                 return (ENOMEM);
2884         ctlv->head.type = IPFW_TLV_DYNSTATE_LIST;
2885         ctlv->objsize = sizeof(ipfw_obj_dyntlv);
2886         last = NULL;
2887
2888 #define DYN_EXPORT_STATES(s, af, h, b)                          \
2889         CK_SLIST_FOREACH(s, &V_dyn_ ## h[b], entry) {                   \
2890                 dst = (ipfw_obj_dyntlv *)ipfw_get_sopt_space(sd,        \
2891                     sizeof(ipfw_obj_dyntlv));                           \
2892                 if (dst == NULL)                                        \
2893                         return (ENOMEM);                                \
2894                 dyn_export_ ## af ## _state(s, &dst->state);            \
2895                 dst->head.length = sizeof(ipfw_obj_dyntlv);             \
2896                 dst->head.type = IPFW_TLV_DYN_ENT;                      \
2897                 last = dst;                                             \
2898         }
2899
2900         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2901                 DYN_EXPORT_STATES(s4, ipv4, ipv4_parent, bucket);
2902                 DYN_EXPORT_STATES(s4, ipv4, ipv4, bucket);
2903 #ifdef INET6
2904                 DYN_EXPORT_STATES(s6, ipv6, ipv6_parent, bucket);
2905                 DYN_EXPORT_STATES(s6, ipv6, ipv6, bucket);
2906 #endif /* INET6 */
2907         }
2908
2909         /* mark last dynamic rule */
2910         if (last != NULL)
2911                 last->head.flags = IPFW_DF_LAST; /* XXX: unused */
2912         return (0);
2913 #undef DYN_EXPORT_STATES
2914 }
2915
2916 /*
2917  * Fill given buffer with dynamic states (legacy format).
2918  * IPFW_UH_RLOCK has to be held while calling.
2919  */
2920 void
2921 ipfw_get_dynamic(struct ip_fw_chain *chain, char **pbp, const char *ep)
2922 {
2923 #ifdef INET6
2924         struct dyn_ipv6_state *s6;
2925 #endif
2926         struct dyn_ipv4_state *s4;
2927         ipfw_dyn_rule *p, *last = NULL;
2928         char *bp;
2929         uint32_t bucket;
2930
2931         if (V_dyn_count == 0)
2932                 return;
2933         bp = *pbp;
2934
2935         IPFW_UH_RLOCK_ASSERT(chain);
2936
2937 #define DYN_EXPORT_STATES(s, af, head, b)                               \
2938         CK_SLIST_FOREACH(s, &V_dyn_ ## head[b], entry) {                \
2939                 if (bp + sizeof(*p) > ep)                               \
2940                         break;                                          \
2941                 p = (ipfw_dyn_rule *)bp;                                \
2942                 dyn_export_ ## af ## _state(s, p);                      \
2943                 last = p;                                               \
2944                 bp += sizeof(*p);                                       \
2945         }
2946
2947         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
2948                 DYN_EXPORT_STATES(s4, ipv4, ipv4_parent, bucket);
2949                 DYN_EXPORT_STATES(s4, ipv4, ipv4, bucket);
2950 #ifdef INET6
2951                 DYN_EXPORT_STATES(s6, ipv6, ipv6_parent, bucket);
2952                 DYN_EXPORT_STATES(s6, ipv6, ipv6, bucket);
2953 #endif /* INET6 */
2954         }
2955
2956         if (last != NULL) /* mark last dynamic rule */
2957                 last->next = NULL;
2958         *pbp = bp;
2959 #undef DYN_EXPORT_STATES
2960 }
2961
2962 void
2963 ipfw_dyn_init(struct ip_fw_chain *chain)
2964 {
2965
2966 #ifdef IPFIREWALL_JENKINSHASH
2967         V_dyn_hashseed = arc4random();
2968 #endif
2969         V_dyn_max = 16384;              /* max # of states */
2970         V_dyn_parent_max = 4096;        /* max # of parent states */
2971         V_dyn_buckets_max = 8192;       /* must be power of 2 */
2972
2973         V_dyn_ack_lifetime = 300;
2974         V_dyn_syn_lifetime = 20;
2975         V_dyn_fin_lifetime = 1;
2976         V_dyn_rst_lifetime = 1;
2977         V_dyn_udp_lifetime = 10;
2978         V_dyn_short_lifetime = 5;
2979
2980         V_dyn_keepalive_interval = 20;
2981         V_dyn_keepalive_period = 5;
2982         V_dyn_keepalive = 1;            /* send keepalives */
2983         V_dyn_keepalive_last = time_uptime;
2984
2985         V_dyn_data_zone = uma_zcreate("IPFW dynamic states data",
2986             sizeof(struct dyn_data), NULL, NULL, NULL, NULL,
2987             UMA_ALIGN_PTR, 0);
2988         uma_zone_set_max(V_dyn_data_zone, V_dyn_max);
2989
2990         V_dyn_parent_zone = uma_zcreate("IPFW parent dynamic states",
2991             sizeof(struct dyn_parent), NULL, NULL, NULL, NULL,
2992             UMA_ALIGN_PTR, 0);
2993         uma_zone_set_max(V_dyn_parent_zone, V_dyn_parent_max);
2994
2995         SLIST_INIT(&V_dyn_expired_ipv4);
2996         V_dyn_ipv4 = NULL;
2997         V_dyn_ipv4_parent = NULL;
2998         V_dyn_ipv4_zone = uma_zcreate("IPFW IPv4 dynamic states",
2999             sizeof(struct dyn_ipv4_state), NULL, NULL, NULL, NULL,
3000             UMA_ALIGN_PTR, 0);
3001
3002 #ifdef INET6
3003         SLIST_INIT(&V_dyn_expired_ipv6);
3004         V_dyn_ipv6 = NULL;
3005         V_dyn_ipv6_parent = NULL;
3006         V_dyn_ipv6_zone = uma_zcreate("IPFW IPv6 dynamic states",
3007             sizeof(struct dyn_ipv6_state), NULL, NULL, NULL, NULL,
3008             UMA_ALIGN_PTR, 0);
3009 #endif
3010
3011         /* Initialize buckets. */
3012         V_curr_dyn_buckets = 0;
3013         V_dyn_bucket_lock = NULL;
3014         dyn_grow_hashtable(chain, 256);
3015
3016         if (IS_DEFAULT_VNET(curvnet))
3017                 dyn_hp_cache = malloc(mp_ncpus * sizeof(void *), M_IPFW,
3018                     M_WAITOK | M_ZERO);
3019
3020         DYN_EXPIRED_LOCK_INIT();
3021         callout_init(&V_dyn_timeout, 1);
3022         callout_reset(&V_dyn_timeout, hz, dyn_tick, curvnet);
3023         IPFW_ADD_OBJ_REWRITER(IS_DEFAULT_VNET(curvnet), dyn_opcodes);
3024 }
3025
3026 void
3027 ipfw_dyn_uninit(int pass)
3028 {
3029 #ifdef INET6
3030         struct dyn_ipv6_state *s6;
3031 #endif
3032         struct dyn_ipv4_state *s4;
3033         int bucket;
3034
3035         if (pass == 0) {
3036                 callout_drain(&V_dyn_timeout);
3037                 return;
3038         }
3039         IPFW_DEL_OBJ_REWRITER(IS_DEFAULT_VNET(curvnet), dyn_opcodes);
3040         DYN_EXPIRED_LOCK_DESTROY();
3041
3042 #define DYN_FREE_STATES_FORCED(CK, s, af, name, en)     do {            \
3043         while ((s = CK ## SLIST_FIRST(&V_dyn_ ## name)) != NULL) {      \
3044                 CK ## SLIST_REMOVE_HEAD(&V_dyn_ ## name, en);   \
3045                 if (s->type == O_LIMIT_PARENT)                          \
3046                         uma_zfree(V_dyn_parent_zone, s->limit);         \
3047                 else                                                    \
3048                         uma_zfree(V_dyn_data_zone, s->data);            \
3049                 uma_zfree(V_dyn_ ## af ## _zone, s);                    \
3050         }                                                               \
3051 } while (0)
3052         for (bucket = 0; bucket < V_curr_dyn_buckets; bucket++) {
3053                 DYN_BUCKET_LOCK_DESTROY(V_dyn_bucket_lock, bucket);
3054
3055                 DYN_FREE_STATES_FORCED(CK_, s4, ipv4, ipv4[bucket], entry);
3056                 DYN_FREE_STATES_FORCED(CK_, s4, ipv4, ipv4_parent[bucket],
3057                     entry);
3058 #ifdef INET6
3059                 DYN_FREE_STATES_FORCED(CK_, s6, ipv6, ipv6[bucket], entry);
3060                 DYN_FREE_STATES_FORCED(CK_, s6, ipv6, ipv6_parent[bucket],
3061                     entry);
3062 #endif /* INET6 */
3063         }
3064         DYN_FREE_STATES_FORCED(, s4, ipv4, expired_ipv4, expired);
3065 #ifdef INET6
3066         DYN_FREE_STATES_FORCED(, s6, ipv6, expired_ipv6, expired);
3067 #endif
3068 #undef DYN_FREE_STATES_FORCED
3069
3070         uma_zdestroy(V_dyn_ipv4_zone);
3071         uma_zdestroy(V_dyn_data_zone);
3072         uma_zdestroy(V_dyn_parent_zone);
3073 #ifdef INET6
3074         uma_zdestroy(V_dyn_ipv6_zone);
3075         free(V_dyn_ipv6, M_IPFW);
3076         free(V_dyn_ipv6_parent, M_IPFW);
3077         free(V_dyn_ipv6_add, M_IPFW);
3078         free(V_dyn_ipv6_parent_add, M_IPFW);
3079         free(V_dyn_ipv6_del, M_IPFW);
3080         free(V_dyn_ipv6_parent_del, M_IPFW);
3081 #endif
3082         free(V_dyn_bucket_lock, M_IPFW);
3083         free(V_dyn_ipv4, M_IPFW);
3084         free(V_dyn_ipv4_parent, M_IPFW);
3085         free(V_dyn_ipv4_add, M_IPFW);
3086         free(V_dyn_ipv4_parent_add, M_IPFW);
3087         free(V_dyn_ipv4_del, M_IPFW);
3088         free(V_dyn_ipv4_parent_del, M_IPFW);
3089         if (IS_DEFAULT_VNET(curvnet))
3090                 free(dyn_hp_cache, M_IPFW);
3091 }
3092
3093