sys/netinet/tcp_fastopen.c

   1 /*-
   2  * Copyright (c) 2015-2017 Patrick Kelsey
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 /*
  28  * This is an implementation of TCP Fast Open (TFO) [RFC7413]. To include
  29  * this code, add the following line to your kernel config:
  30  *
  31  * options TCP_RFC7413
  32  *
  33  *
  34  * The generated TFO cookies are the 64-bit output of
  35  * SipHash24(key=<16-byte-key>, msg=<client-ip>).  Multiple concurrent valid
  36  * keys are supported so that time-based rolling cookie invalidation
  37  * policies can be implemented in the system.  The default number of
  38  * concurrent keys is 2.  This can be adjusted in the kernel config as
  39  * follows:
  40  *
  41  * options TCP_RFC7413_MAX_KEYS=<num-keys>
  42  *
  43  *
  44  * In addition to the facilities defined in RFC7413, this implementation
  45  * supports a pre-shared key (PSK) mode of operation in which the TFO server
  46  * requires the client to be in posession of a shared secret in order for
  47  * the client to be able to successfully open TFO connections with the
  48  * server.  This is useful, for example, in environments where TFO servers
  49  * are exposed to both internal and external clients and only wish to allow
  50  * TFO connections from internal clients.
  51  *
  52  * In the PSK mode of operation, the server generates and sends TFO cookies
  53  * to requesting clients as usual.  However, when validating cookies
  54  * received in TFO SYNs from clients, the server requires the
  55  * client-supplied cookie to equal SipHash24(key=<16-byte-psk>,
  56  * msg=<cookie-sent-to-client>).
  57  *
  58  * Multiple concurrent valid pre-shared keys are supported so that
  59  * time-based rolling PSK invalidation policies can be implemented in the
  60  * system.  The default number of concurrent pre-shared keys is 2.  This can
  61  * be adjusted in the kernel config as follows:
  62  *
  63  * options TCP_RFC7413_MAX_PSKS=<num-psks>
  64  *
  65  *
  66  * The following TFO-specific sysctls are defined:
  67  *
  68  * net.inet.tcp.fastopen.acceptany (RW, default 0)
  69  *     When non-zero, all client-supplied TFO cookies will be considered to
  70  *     be valid.
  71  *
  72  * net.inet.tcp.fastopen.autokey (RW, default 120)
  73  *     When this and net.inet.tcp.fastopen.server_enable are non-zero, a new
  74  *     key will be automatically generated after this many seconds.
  75  *
  76  * net.inet.tcp.fastopen.ccache_bucket_limit
  77  *                     (RWTUN, default TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT)
  78  *     The maximum number of entries in a client cookie cache bucket.
  79  *
  80  * net.inet.tcp.fastopen.ccache_buckets
  81  *                          (RDTUN, default TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT)
  82  *     The number of client cookie cache buckets.
  83  *
  84  * net.inet.tcp.fastopen.client_enable (RW, default 0)
  85  *     When zero, no new active (i.e., client) TFO connections can be
  86  *     created.  On the transition from enabled to disabled, the client
  87  *     cookie cache is cleared and disabled.  The transition from enabled to
  88  *     disabled does not affect any active TFO connections in progress; it
  89  *     only prevents new ones from being made.
  90  *
  91  * net.inet.tcp.fastopen.keylen (RD)
  92  *     The key length in bytes.
  93  *
  94  * net.inet.tcp.fastopen.maxkeys (RD)
  95  *     The maximum number of keys supported.
  96  *
  97  * net.inet.tcp.fastopen.maxpsks (RD)
  98  *     The maximum number of pre-shared keys supported.
  99  *
 100  * net.inet.tcp.fastopen.numkeys (RD)
 101  *     The current number of keys installed.
 102  *
 103  * net.inet.tcp.fastopen.numpsks (RD)
 104  *     The current number of pre-shared keys installed.
 105  *
 106  * net.inet.tcp.fastopen.path_disable_time
 107  *                          (RW, default TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT)
 108  *     When a failure occurs while trying to create a new active (i.e.,
 109  *     client) TFO connection, new active connections on the same path, as
 110  *     determined by the tuple {client_ip, server_ip, server_port}, will be
 111  *     forced to be non-TFO for this many seconds.  Note that the path
 112  *     disable mechanism relies on state stored in client cookie cache
 113  *     entries, so it is possible for the disable time for a given path to
 114  *     be reduced if the corresponding client cookie cache entry is reused
 115  *     due to resource pressure before the disable period has elapsed.
 116  *
 117  * net.inet.tcp.fastopen.psk_enable (RW, default 0)
 118  *     When non-zero, pre-shared key (PSK) mode is enabled for all TFO
 119  *     servers.  On the transition from enabled to disabled, all installed
 120  *     pre-shared keys are removed.
 121  *
 122  * net.inet.tcp.fastopen.server_enable (RW, default 0)
 123  *     When zero, no new passive (i.e., server) TFO connections can be
 124  *     created.  On the transition from enabled to disabled, all installed
 125  *     keys and pre-shared keys are removed.  On the transition from
 126  *     disabled to enabled, if net.inet.tcp.fastopen.autokey is non-zero and
 127  *     there are no keys installed, a new key will be generated immediately.
 128  *     The transition from enabled to disabled does not affect any passive
 129  *     TFO connections in progress; it only prevents new ones from being
 130  *     made.
 131  *
 132  * net.inet.tcp.fastopen.setkey (WR)
 133  *     Install a new key by writing net.inet.tcp.fastopen.keylen bytes to
 134  *     this sysctl.
 135  *
 136  * net.inet.tcp.fastopen.setpsk (WR)
 137  *     Install a new pre-shared key by writing net.inet.tcp.fastopen.keylen
 138  *     bytes to this sysctl.
 139  *
 140  * In order for TFO connections to be created via a listen socket, that
 141  * socket must have the TCP_FASTOPEN socket option set on it.  This option
 142  * can be set on the socket either before or after the listen() is invoked.
 143  * Clearing this option on a listen socket after it has been set has no
 144  * effect on existing TFO connections or TFO connections in progress; it
 145  * only prevents new TFO connections from being made.
 146  *
 147  * For passively-created sockets, the TCP_FASTOPEN socket option can be
 148  * queried to determine whether the connection was established using TFO.
 149  * Note that connections that are established via a TFO SYN, but that fall
 150  * back to using a non-TFO SYN|ACK will have the TCP_FASTOPEN socket option
 151  * set.
 152  *
 153  * Per the RFC, this implementation limits the number of TFO connections
 154  * that can be in the SYN_RECEIVED state on a per listen-socket basis.
 155  * Whenever this limit is exceeded, requests for new TFO connections are
 156  * serviced as non-TFO requests.  Without such a limit, given a valid TFO
 157  * cookie, an attacker could keep the listen queue in an overflow condition
 158  * using a TFO SYN flood.  This implementation sets the limit at half the
 159  * configured listen backlog.
 160  *
 161  */
 162
 163 #include <sys/cdefs.h>
 164 __FBSDID("$FreeBSD$");
 165
 166 #include "opt_inet.h"
 167
 168 #include <sys/param.h>
 169 #include <sys/kernel.h>
 170 #include <sys/hash.h>
 171 #include <sys/limits.h>
 172 #include <sys/lock.h>
 173 #include <sys/rmlock.h>
 174 #include <sys/socket.h>
 175 #include <sys/socketvar.h>
 176 #include <sys/sysctl.h>
 177 #include <sys/systm.h>
 178
 179 #include <crypto/siphash/siphash.h>
 180
 181 #include <net/vnet.h>
 182
 183 #include <netinet/in.h>
 184 #include <netinet/in_pcb.h>
 185 #include <netinet/tcp_var.h>
 186 #include <netinet/tcp_fastopen.h>
 187
 188
 189 #define TCP_FASTOPEN_KEY_LEN    SIPHASH_KEY_LENGTH
 190
 191 #if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN
 192 #error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN
 193 #endif
 194
 195 /*
 196  * Because a PSK-mode setsockopt() uses tcpcb.t_tfo_cookie.client to hold
 197  * the PSK until the connect occurs.
 198  */
 199 #if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN
 200 #error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN
 201 #endif
 202
 203 #define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT        16
 204 #define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT             2048 /* must be power of 2 */
 205
 206 #define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT          900 /* seconds */
 207
 208 #if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1)
 209 #define TCP_FASTOPEN_MAX_KEYS   2
 210 #else
 211 #define TCP_FASTOPEN_MAX_KEYS   TCP_RFC7413_MAX_KEYS
 212 #endif
 213
 214 #if TCP_FASTOPEN_MAX_KEYS > 10
 215 #undef TCP_FASTOPEN_MAX_KEYS
 216 #define TCP_FASTOPEN_MAX_KEYS   10
 217 #endif
 218
 219 #if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1)
 220 #define TCP_FASTOPEN_MAX_PSKS   2
 221 #else
 222 #define TCP_FASTOPEN_MAX_PSKS   TCP_RFC7413_MAX_PSKS
 223 #endif
 224
 225 #if TCP_FASTOPEN_MAX_PSKS > 10
 226 #undef TCP_FASTOPEN_MAX_PSKS
 227 #define TCP_FASTOPEN_MAX_PSKS   10
 228 #endif
 229
 230 struct tcp_fastopen_keylist {
 231         unsigned int newest;
 232         unsigned int newest_psk;
 233         uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN];
 234         uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN];
 235 };
 236
 237 struct tcp_fastopen_callout {
 238         struct callout c;
 239         struct vnet *v;
 240 };
 241
 242 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup(
 243     struct in_conninfo *, struct tcp_fastopen_ccache_bucket **);
 244 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create(
 245     struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t,
 246     uint8_t *);
 247 static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *,
 248     unsigned int);
 249 static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *,
 250     struct tcp_fastopen_ccache_bucket *);
 251
 252 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW, 0, "TCP Fast Open");
 253
 254 static VNET_DEFINE(int, tcp_fastopen_acceptany) = 0;
 255 #define V_tcp_fastopen_acceptany        VNET(tcp_fastopen_acceptany)
 256 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany,
 257     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0,
 258     "Accept any non-empty cookie");
 259
 260 static VNET_DEFINE(unsigned int, tcp_fastopen_autokey) = 120;
 261 #define V_tcp_fastopen_autokey  VNET(tcp_fastopen_autokey)
 262 static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS);
 263 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey,
 264     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, NULL, 0,
 265     &sysctl_net_inet_tcp_fastopen_autokey, "IU",
 266     "Number of seconds between auto-generation of a new key; zero disables");
 267
 268 static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS);
 269 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_bucket_limit,
 270     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN, NULL, 0,
 271     &sysctl_net_inet_tcp_fastopen_ccache_bucket_limit, "IU",
 272     "Max entries per bucket in client cookie cache");
 273
 274 static VNET_DEFINE(unsigned int, tcp_fastopen_ccache_buckets) =
 275     TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
 276 #define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets)
 277 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets,
 278     CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0,
 279     "Client cookie cache number of buckets (power of 2)");
 280
 281 VNET_DEFINE(unsigned int, tcp_fastopen_client_enable) = 0;
 282 static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS);
 283 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, client_enable,
 284     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, NULL, 0,
 285     &sysctl_net_inet_tcp_fastopen_client_enable, "IU",
 286     "Enable/disable TCP Fast Open client functionality");
 287
 288 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, keylen,
 289     CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_KEY_LEN,
 290     "Key length in bytes");
 291
 292 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxkeys,
 293     CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_KEYS,
 294     "Maximum number of keys supported");
 295
 296 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxpsks,
 297     CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_PSKS,
 298     "Maximum number of pre-shared keys supported");
 299
 300 static VNET_DEFINE(unsigned int, tcp_fastopen_numkeys) = 0;
 301 #define V_tcp_fastopen_numkeys  VNET(tcp_fastopen_numkeys)
 302 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numkeys,
 303     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0,
 304     "Number of keys installed");
 305
 306 static VNET_DEFINE(unsigned int, tcp_fastopen_numpsks) = 0;
 307 #define V_tcp_fastopen_numpsks  VNET(tcp_fastopen_numpsks)
 308 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numpsks,
 309     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0,
 310     "Number of pre-shared keys installed");
 311
 312 static VNET_DEFINE(unsigned int, tcp_fastopen_path_disable_time) =
 313     TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT;
 314 #define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time)
 315 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, path_disable_time,
 316     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0,
 317     "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path");
 318
 319 static VNET_DEFINE(unsigned int, tcp_fastopen_psk_enable) = 0;
 320 #define V_tcp_fastopen_psk_enable       VNET(tcp_fastopen_psk_enable)
 321 static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS);
 322 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, psk_enable,
 323     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, NULL, 0,
 324     &sysctl_net_inet_tcp_fastopen_psk_enable, "IU",
 325     "Enable/disable TCP Fast Open server pre-shared key mode");
 326
 327 VNET_DEFINE(unsigned int, tcp_fastopen_server_enable) = 0;
 328 static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS);
 329 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, server_enable,
 330     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, NULL, 0,
 331     &sysctl_net_inet_tcp_fastopen_server_enable, "IU",
 332     "Enable/disable TCP Fast Open server functionality");
 333
 334 static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS);
 335 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setkey,
 336     CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR, NULL, 0,
 337     &sysctl_net_inet_tcp_fastopen_setkey, "",
 338     "Install a new key");
 339
 340 static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS);
 341 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setpsk,
 342     CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR, NULL, 0,
 343     &sysctl_net_inet_tcp_fastopen_setpsk, "",
 344     "Install a new pre-shared key");
 345
 346 static VNET_DEFINE(struct rmlock, tcp_fastopen_keylock);
 347 #define V_tcp_fastopen_keylock  VNET(tcp_fastopen_keylock)
 348
 349 #define TCP_FASTOPEN_KEYS_RLOCK(t)      rm_rlock(&V_tcp_fastopen_keylock, (t))
 350 #define TCP_FASTOPEN_KEYS_RUNLOCK(t)    rm_runlock(&V_tcp_fastopen_keylock, (t))
 351 #define TCP_FASTOPEN_KEYS_WLOCK()       rm_wlock(&V_tcp_fastopen_keylock)
 352 #define TCP_FASTOPEN_KEYS_WUNLOCK()     rm_wunlock(&V_tcp_fastopen_keylock)
 353
 354 static VNET_DEFINE(struct tcp_fastopen_keylist, tcp_fastopen_keys);
 355 #define V_tcp_fastopen_keys     VNET(tcp_fastopen_keys)
 356
 357 static VNET_DEFINE(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx);
 358 #define V_tcp_fastopen_autokey_ctx      VNET(tcp_fastopen_autokey_ctx)
 359
 360 static VNET_DEFINE(uma_zone_t, counter_zone);
 361 #define V_counter_zone                  VNET(counter_zone)
 362
 363 static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets");
 364
 365 static VNET_DEFINE(struct tcp_fastopen_ccache, tcp_fastopen_ccache);
 366 #define V_tcp_fastopen_ccache   VNET(tcp_fastopen_ccache)
 367
 368 #define CCB_LOCK(ccb)           mtx_lock(&(ccb)->ccb_mtx)
 369 #define CCB_UNLOCK(ccb)         mtx_unlock(&(ccb)->ccb_mtx)
 370 #define CCB_LOCK_ASSERT(ccb)    mtx_assert(&(ccb)->ccb_mtx, MA_OWNED)
 371
 372
 373 void
 374 tcp_fastopen_init(void)
 375 {
 376         unsigned int i;
 377
 378         V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int),
 379             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 380         rm_init(&V_tcp_fastopen_keylock, "tfo_keylock");
 381         callout_init_rm(&V_tcp_fastopen_autokey_ctx.c,
 382             &V_tcp_fastopen_keylock, 0);
 383         V_tcp_fastopen_autokey_ctx.v = curvnet;
 384         V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
 385         V_tcp_fastopen_keys.newest_psk = TCP_FASTOPEN_MAX_PSKS - 1;
 386
 387         /* May already be non-zero if kernel tunable was set */
 388         if (V_tcp_fastopen_ccache.bucket_limit == 0)
 389                 V_tcp_fastopen_ccache.bucket_limit =
 390                     TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT;
 391
 392         /* May already be non-zero if kernel tunable was set */
 393         if ((V_tcp_fastopen_ccache_buckets == 0) ||
 394             !powerof2(V_tcp_fastopen_ccache_buckets))
 395                 V_tcp_fastopen_ccache.buckets =
 396                         TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
 397         else
 398                 V_tcp_fastopen_ccache.buckets = V_tcp_fastopen_ccache_buckets;
 399
 400         V_tcp_fastopen_ccache.mask = V_tcp_fastopen_ccache.buckets - 1;
 401         V_tcp_fastopen_ccache.secret = arc4random();
 402
 403         V_tcp_fastopen_ccache.base = malloc(V_tcp_fastopen_ccache.buckets *
 404             sizeof(struct tcp_fastopen_ccache_bucket), M_TCP_FASTOPEN_CCACHE,
 405             M_WAITOK | M_ZERO);
 406
 407         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
 408                 TAILQ_INIT(&V_tcp_fastopen_ccache.base[i].ccb_entries);
 409                 mtx_init(&V_tcp_fastopen_ccache.base[i].ccb_mtx, "tfo_ccache_bucket",
 410                          NULL, MTX_DEF);
 411                 V_tcp_fastopen_ccache.base[i].ccb_num_entries = -1; /* bucket disabled */
 412                 V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache;
 413         }
 414
 415         /*
 416          * Note that while the total number of entries in the cookie cache
 417          * is limited by the table management logic to
 418          * V_tcp_fastopen_ccache.buckets *
 419          * V_tcp_fastopen_ccache.bucket_limit, the total number of items in
 420          * this zone can exceed that amount by the number of CPUs in the
 421          * system times the maximum number of unallocated items that can be
 422          * present in each UMA per-CPU cache for this zone.
 423          */
 424         V_tcp_fastopen_ccache.zone = uma_zcreate("tfo_ccache_entries",
 425             sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL,
 426             UMA_ALIGN_CACHE, 0);
 427 }
 428
 429 void
 430 tcp_fastopen_destroy(void)
 431 {
 432         struct tcp_fastopen_ccache_bucket *ccb;
 433         unsigned int i;
 434
 435         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
 436                 ccb = &V_tcp_fastopen_ccache.base[i];
 437                 tcp_fastopen_ccache_bucket_trim(ccb, 0);
 438                 mtx_destroy(&ccb->ccb_mtx);
 439         }
 440
 441         KASSERT(uma_zone_get_cur(V_tcp_fastopen_ccache.zone) == 0,
 442             ("%s: TFO ccache zone allocation count not 0", __func__));
 443         uma_zdestroy(V_tcp_fastopen_ccache.zone);
 444         free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE);
 445
 446         callout_drain(&V_tcp_fastopen_autokey_ctx.c);
 447         rm_destroy(&V_tcp_fastopen_keylock);
 448         uma_zdestroy(V_counter_zone);
 449 }
 450
 451 unsigned int *
 452 tcp_fastopen_alloc_counter(void)
 453 {
 454         unsigned int *counter;
 455         counter = uma_zalloc(V_counter_zone, M_NOWAIT);
 456         if (counter)
 457                 *counter = 1;
 458         return (counter);
 459 }
 460
 461 void
 462 tcp_fastopen_decrement_counter(unsigned int *counter)
 463 {
 464         if (*counter == 1)
 465                 uma_zfree(V_counter_zone, counter);
 466         else
 467                 atomic_subtract_int(counter, 1);
 468 }
 469
 470 static void
 471 tcp_fastopen_addkey_locked(uint8_t *key)
 472 {
 473
 474         V_tcp_fastopen_keys.newest++;
 475         if (V_tcp_fastopen_keys.newest == TCP_FASTOPEN_MAX_KEYS)
 476                 V_tcp_fastopen_keys.newest = 0;
 477         memcpy(V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], key,
 478             TCP_FASTOPEN_KEY_LEN);
 479         if (V_tcp_fastopen_numkeys < TCP_FASTOPEN_MAX_KEYS)
 480                 V_tcp_fastopen_numkeys++;
 481 }
 482
 483 static void
 484 tcp_fastopen_addpsk_locked(uint8_t *psk)
 485 {
 486
 487         V_tcp_fastopen_keys.newest_psk++;
 488         if (V_tcp_fastopen_keys.newest_psk == TCP_FASTOPEN_MAX_PSKS)
 489                 V_tcp_fastopen_keys.newest_psk = 0;
 490         memcpy(V_tcp_fastopen_keys.psk[V_tcp_fastopen_keys.newest_psk], psk,
 491             TCP_FASTOPEN_KEY_LEN);
 492         if (V_tcp_fastopen_numpsks < TCP_FASTOPEN_MAX_PSKS)
 493                 V_tcp_fastopen_numpsks++;
 494 }
 495
 496 static void
 497 tcp_fastopen_autokey_locked(void)
 498 {
 499         uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
 500
 501         arc4rand(newkey, TCP_FASTOPEN_KEY_LEN, 0);
 502         tcp_fastopen_addkey_locked(newkey);
 503 }
 504
 505 static void
 506 tcp_fastopen_autokey_callout(void *arg)
 507 {
 508         struct tcp_fastopen_callout *ctx = arg;
 509
 510         CURVNET_SET(ctx->v);
 511         tcp_fastopen_autokey_locked();
 512         callout_reset(&ctx->c, V_tcp_fastopen_autokey * hz,
 513                       tcp_fastopen_autokey_callout, ctx);
 514         CURVNET_RESTORE();
 515 }
 516
 517
 518 static uint64_t
 519 tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc)
 520 {
 521         SIPHASH_CTX ctx;
 522         uint64_t siphash;
 523
 524         SipHash24_Init(&ctx);
 525         SipHash_SetKey(&ctx, key);
 526         switch (inc->inc_flags & INC_ISIPV6) {
 527 #ifdef INET
 528         case 0:
 529                 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(inc->inc_faddr));
 530                 break;
 531 #endif
 532 #ifdef INET6
 533         case INC_ISIPV6:
 534                 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(inc->inc6_faddr));
 535                 break;
 536 #endif
 537         }
 538         SipHash_Final((u_int8_t *)&siphash, &ctx);
 539
 540         return (siphash);
 541 }
 542
 543 static uint64_t
 544 tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len)
 545 {
 546         SIPHASH_CTX ctx;
 547         uint64_t psk_cookie;
 548
 549         SipHash24_Init(&ctx);
 550         SipHash_SetKey(&ctx, psk);
 551         SipHash_Update(&ctx, cookie, cookie_len);
 552         SipHash_Final((u_int8_t *)&psk_cookie, &ctx);
 553
 554         return (psk_cookie);
 555 }
 556
 557 static int
 558 tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie)
 559 {
 560         unsigned int i, psk_index;
 561         uint64_t psk_cookie;
 562
 563         if (V_tcp_fastopen_psk_enable) {
 564                 psk_index = V_tcp_fastopen_keys.newest_psk;
 565                 for (i = 0; i < V_tcp_fastopen_numpsks; i++) {
 566                         psk_cookie =
 567                             tcp_fastopen_make_psk_cookie(
 568                                  V_tcp_fastopen_keys.psk[psk_index],
 569                                  (uint8_t *)cur_cookie,
 570                                  TCP_FASTOPEN_COOKIE_LEN);
 571
 572                         if (memcmp(wire_cookie, &psk_cookie,
 573                                    TCP_FASTOPEN_COOKIE_LEN) == 0)
 574                                 return (1);
 575
 576                         if (psk_index == 0)
 577                                 psk_index = TCP_FASTOPEN_MAX_PSKS - 1;
 578                         else
 579                                 psk_index--;
 580                 }
 581         } else if (memcmp(wire_cookie, cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0)
 582                 return (1);
 583
 584         return (0);
 585 }
 586
 587 /*
 588  * Return values:
 589  *      -1      the cookie is invalid and no valid cookie is available
 590  *       0      the cookie is invalid and the latest cookie has been returned
 591  *       1      the cookie is valid and the latest cookie has been returned
 592  */
 593 int
 594 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie,
 595     unsigned int len, uint64_t *latest_cookie)
 596 {
 597         struct rm_priotracker tracker;
 598         unsigned int i, key_index;
 599         int rv;
 600         uint64_t cur_cookie;
 601
 602         if (V_tcp_fastopen_acceptany) {
 603                 *latest_cookie = 0;
 604                 return (1);
 605         }
 606
 607         TCP_FASTOPEN_KEYS_RLOCK(&tracker);
 608         if (len != TCP_FASTOPEN_COOKIE_LEN) {
 609                 if (V_tcp_fastopen_numkeys > 0) {
 610                         *latest_cookie =
 611                             tcp_fastopen_make_cookie(
 612                                 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest],
 613                                 inc);
 614                         rv = 0;
 615                 } else
 616                         rv = -1;
 617                 goto out;
 618         }
 619
 620         /*
 621          * Check against each available key, from newest to oldest.
 622          */
 623         key_index = V_tcp_fastopen_keys.newest;
 624         for (i = 0; i < V_tcp_fastopen_numkeys; i++) {
 625                 cur_cookie =
 626                     tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index],
 627                         inc);
 628                 if (i == 0)
 629                         *latest_cookie = cur_cookie;
 630                 rv = tcp_fastopen_find_cookie_match_locked(cookie, &cur_cookie);
 631                 if (rv)
 632                         goto out;
 633                 if (key_index == 0)
 634                         key_index = TCP_FASTOPEN_MAX_KEYS - 1;
 635                 else
 636                         key_index--;
 637         }
 638         rv = 0;
 639
 640  out:
 641         TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
 642         return (rv);
 643 }
 644
 645 static int
 646 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
 647 {
 648         int error;
 649         unsigned int new;
 650
 651         new = V_tcp_fastopen_autokey;
 652         error = sysctl_handle_int(oidp, &new, 0, req);
 653         if (error == 0 && req->newptr) {
 654                 if (new > (INT_MAX / hz))
 655                         return (EINVAL);
 656
 657                 TCP_FASTOPEN_KEYS_WLOCK();
 658                 if (V_tcp_fastopen_server_enable) {
 659                         if (V_tcp_fastopen_autokey && !new)
 660                                 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
 661                         else if (new)
 662                                 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
 663                                     new * hz, tcp_fastopen_autokey_callout,
 664                                     &V_tcp_fastopen_autokey_ctx);
 665                 }
 666                 V_tcp_fastopen_autokey = new;
 667                 TCP_FASTOPEN_KEYS_WUNLOCK();
 668         }
 669
 670         return (error);
 671 }
 672
 673 static int
 674 sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)
 675 {
 676         int error;
 677         unsigned int new;
 678
 679         new = V_tcp_fastopen_psk_enable;
 680         error = sysctl_handle_int(oidp, &new, 0, req);
 681         if (error == 0 && req->newptr) {
 682                 if (V_tcp_fastopen_psk_enable && !new) {
 683                         /* enabled -> disabled */
 684                         TCP_FASTOPEN_KEYS_WLOCK();
 685                         V_tcp_fastopen_numpsks = 0;
 686                         V_tcp_fastopen_keys.newest_psk =
 687                             TCP_FASTOPEN_MAX_PSKS - 1;
 688                         V_tcp_fastopen_psk_enable = 0;
 689                         TCP_FASTOPEN_KEYS_WUNLOCK();
 690                 } else if (!V_tcp_fastopen_psk_enable && new) {
 691                         /* disabled -> enabled */
 692                         TCP_FASTOPEN_KEYS_WLOCK();
 693                         V_tcp_fastopen_psk_enable = 1;
 694                         TCP_FASTOPEN_KEYS_WUNLOCK();
 695                 }
 696         }
 697         return (error);
 698 }
 699
 700 static int
 701 sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)
 702 {
 703         int error;
 704         unsigned int new;
 705
 706         new = V_tcp_fastopen_server_enable;
 707         error = sysctl_handle_int(oidp, &new, 0, req);
 708         if (error == 0 && req->newptr) {
 709                 if (V_tcp_fastopen_server_enable && !new) {
 710                         /* enabled -> disabled */
 711                         TCP_FASTOPEN_KEYS_WLOCK();
 712                         V_tcp_fastopen_numkeys = 0;
 713                         V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
 714                         if (V_tcp_fastopen_autokey)
 715                                 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
 716                         V_tcp_fastopen_numpsks = 0;
 717                         V_tcp_fastopen_keys.newest_psk =
 718                             TCP_FASTOPEN_MAX_PSKS - 1;
 719                         V_tcp_fastopen_server_enable = 0;
 720                         TCP_FASTOPEN_KEYS_WUNLOCK();
 721                 } else if (!V_tcp_fastopen_server_enable && new) {
 722                         /* disabled -> enabled */
 723                         TCP_FASTOPEN_KEYS_WLOCK();
 724                         if (V_tcp_fastopen_autokey &&
 725                             (V_tcp_fastopen_numkeys == 0)) {
 726                                 tcp_fastopen_autokey_locked();
 727                                 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
 728                                     V_tcp_fastopen_autokey * hz,
 729                                     tcp_fastopen_autokey_callout,
 730                                     &V_tcp_fastopen_autokey_ctx);
 731                         }
 732                         V_tcp_fastopen_server_enable = 1;
 733                         TCP_FASTOPEN_KEYS_WUNLOCK();
 734                 }
 735         }
 736         return (error);
 737 }
 738
 739 static int
 740 sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)
 741 {
 742         int error;
 743         uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
 744
 745         if (req->oldptr != NULL || req->oldlen != 0)
 746                 return (EINVAL);
 747         if (req->newptr == NULL)
 748                 return (EPERM);
 749         if (req->newlen != sizeof(newkey))
 750                 return (EINVAL);
 751         error = SYSCTL_IN(req, newkey, sizeof(newkey));
 752         if (error)
 753                 return (error);
 754
 755         TCP_FASTOPEN_KEYS_WLOCK();
 756         tcp_fastopen_addkey_locked(newkey);
 757         TCP_FASTOPEN_KEYS_WUNLOCK();
 758
 759         return (0);
 760 }
 761
 762 static int
 763 sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)
 764 {
 765         int error;
 766         uint8_t newpsk[TCP_FASTOPEN_KEY_LEN];
 767
 768         if (req->oldptr != NULL || req->oldlen != 0)
 769                 return (EINVAL);
 770         if (req->newptr == NULL)
 771                 return (EPERM);
 772         if (req->newlen != sizeof(newpsk))
 773                 return (EINVAL);
 774         error = SYSCTL_IN(req, newpsk, sizeof(newpsk));
 775         if (error)
 776                 return (error);
 777
 778         TCP_FASTOPEN_KEYS_WLOCK();
 779         tcp_fastopen_addpsk_locked(newpsk);
 780         TCP_FASTOPEN_KEYS_WUNLOCK();
 781
 782         return (0);
 783 }
 784
 785 static int
 786 sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)
 787 {
 788         struct tcp_fastopen_ccache_bucket *ccb;
 789         int error;
 790         unsigned int new;
 791         unsigned int i;
 792
 793         new = V_tcp_fastopen_ccache.bucket_limit;
 794         error = sysctl_handle_int(oidp, &new, 0, req);
 795         if (error == 0 && req->newptr) {
 796                 if ((new == 0) || (new > INT_MAX))
 797                         error = EINVAL;
 798                 else {
 799                         if (new < V_tcp_fastopen_ccache.bucket_limit) {
 800                                 for (i = 0; i < V_tcp_fastopen_ccache.buckets;
 801                                      i++) {
 802                                         ccb = &V_tcp_fastopen_ccache.base[i];
 803                                         tcp_fastopen_ccache_bucket_trim(ccb, new);
 804                                 }
 805                         }
 806                         V_tcp_fastopen_ccache.bucket_limit = new;
 807                 }
 808
 809         }
 810         return (error);
 811 }
 812
 813 static int
 814 sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)
 815 {
 816         struct tcp_fastopen_ccache_bucket *ccb;
 817         int error;
 818         unsigned int new, i;
 819
 820         new = V_tcp_fastopen_client_enable;
 821         error = sysctl_handle_int(oidp, &new, 0, req);
 822         if (error == 0 && req->newptr) {
 823                 if (V_tcp_fastopen_client_enable && !new) {
 824                         /* enabled -> disabled */
 825                         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
 826                                 ccb = &V_tcp_fastopen_ccache.base[i];
 827                                 tcp_fastopen_ccache_bucket_trim(ccb, 0);
 828                         }
 829                         V_tcp_fastopen_client_enable = 0;
 830                 } else if (!V_tcp_fastopen_client_enable && new) {
 831                         /* disabled -> enabled */
 832                         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
 833                                 ccb = &V_tcp_fastopen_ccache.base[i];
 834                                 CCB_LOCK(ccb);
 835                                 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
 836                                     ("%s: ccb->ccb_entries not empty", __func__));
 837                                 KASSERT(ccb->ccb_num_entries == -1,
 838                                     ("%s: ccb->ccb_num_entries %d not -1", __func__,
 839                                         ccb->ccb_num_entries));
 840                                 ccb->ccb_num_entries = 0; /* enable bucket */
 841                                 CCB_UNLOCK(ccb);
 842                         }
 843                         V_tcp_fastopen_client_enable = 1;
 844                 }
 845         }
 846         return (error);
 847 }
 848
 849 void
 850 tcp_fastopen_connect(struct tcpcb *tp)
 851 {
 852         struct inpcb *inp;
 853         struct tcp_fastopen_ccache_bucket *ccb;
 854         struct tcp_fastopen_ccache_entry *cce;
 855         sbintime_t now;
 856         uint16_t server_mss;
 857         uint64_t psk_cookie;
 858
 859         psk_cookie = 0;
 860         inp = tp->t_inpcb;
 861         cce = tcp_fastopen_ccache_lookup(&inp->inp_inc, &ccb);
 862         if (cce) {
 863                 if (cce->disable_time == 0) {
 864                         if ((cce->cookie_len > 0) &&
 865                             (tp->t_tfo_client_cookie_len ==
 866                              TCP_FASTOPEN_PSK_LEN)) {
 867                                 psk_cookie =
 868                                     tcp_fastopen_make_psk_cookie(
 869                                         tp->t_tfo_cookie.client,
 870                                         cce->cookie, cce->cookie_len);
 871                         } else {
 872                                 tp->t_tfo_client_cookie_len = cce->cookie_len;
 873                                 memcpy(tp->t_tfo_cookie.client, cce->cookie,
 874                                     cce->cookie_len);
 875                         }
 876                         server_mss = cce->server_mss;
 877                         CCB_UNLOCK(ccb);
 878                         if (tp->t_tfo_client_cookie_len ==
 879                             TCP_FASTOPEN_PSK_LEN && psk_cookie) {
 880                                 tp->t_tfo_client_cookie_len =
 881                                     TCP_FASTOPEN_COOKIE_LEN;
 882                                 memcpy(tp->t_tfo_cookie.client, &psk_cookie,
 883                                     TCP_FASTOPEN_COOKIE_LEN);
 884                         }
 885                         tcp_mss(tp, server_mss ? server_mss : -1);
 886                         tp->snd_wnd = tp->t_maxseg;
 887                 } else {
 888                         /*
 889                          * The path is disabled.  Check the time and
 890                          * possibly re-enable.
 891                          */
 892                         now = getsbinuptime();
 893                         if (now - cce->disable_time >
 894                             ((sbintime_t)V_tcp_fastopen_path_disable_time << 32)) {
 895                                 /*
 896                                  * Re-enable path.  Force a TFO cookie
 897                                  * request.  Forget the old MSS as it may be
 898                                  * bogus now, and we will rediscover it in
 899                                  * the SYN|ACK.
 900                                  */
 901                                 cce->disable_time = 0;
 902                                 cce->server_mss = 0;
 903                                 cce->cookie_len = 0;
 904                                 /*
 905                                  * tp->t_tfo... cookie details are already
 906                                  * zero from the tcpcb init.
 907                                  */
 908                         } else {
 909                                 /*
 910                                  * Path is disabled, so disable TFO on this
 911                                  * connection.
 912                                  */
 913                                 tp->t_flags &= ~TF_FASTOPEN;
 914                         }
 915                         CCB_UNLOCK(ccb);
 916                         tcp_mss(tp, -1);
 917                         /*
 918                          * snd_wnd is irrelevant since we are either forcing
 919                          * a TFO cookie request or disabling TFO - either
 920                          * way, no data with the SYN.
 921                          */
 922                 }
 923         } else {
 924                 /*
 925                  * A new entry for this path will be created when a SYN|ACK
 926                  * comes back, or the attempt otherwise fails.
 927                  */
 928                 CCB_UNLOCK(ccb);
 929                 tcp_mss(tp, -1);
 930                 /*
 931                  * snd_wnd is irrelevant since we are forcing a TFO cookie
 932                  * request.
 933                  */
 934         }
 935 }
 936
 937 void
 938 tcp_fastopen_disable_path(struct tcpcb *tp)
 939 {
 940         struct in_conninfo *inc = &tp->t_inpcb->inp_inc;
 941         struct tcp_fastopen_ccache_bucket *ccb;
 942         struct tcp_fastopen_ccache_entry *cce;
 943
 944         cce = tcp_fastopen_ccache_lookup(inc, &ccb);
 945         if (cce) {
 946                 cce->server_mss = 0;
 947                 cce->cookie_len = 0;
 948                 /*
 949                  * Preserve the existing disable time if it is already
 950                  * disabled.
 951                  */
 952                 if (cce->disable_time == 0)
 953                         cce->disable_time = getsbinuptime();
 954         } else /* use invalid cookie len to create disabled entry */
 955                 tcp_fastopen_ccache_create(ccb, inc, 0,
 956                     TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL);
 957
 958         CCB_UNLOCK(ccb);
 959         tp->t_flags &= ~TF_FASTOPEN;
 960 }
 961
 962 void
 963 tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss,
 964     uint8_t cookie_len, uint8_t *cookie)
 965 {
 966         struct in_conninfo *inc = &tp->t_inpcb->inp_inc;
 967         struct tcp_fastopen_ccache_bucket *ccb;
 968         struct tcp_fastopen_ccache_entry *cce;
 969
 970         cce = tcp_fastopen_ccache_lookup(inc, &ccb);
 971         if (cce) {
 972                 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
 973                     (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
 974                     ((cookie_len & 0x1) == 0)) {
 975                         cce->server_mss = mss;
 976                         cce->cookie_len = cookie_len;
 977                         memcpy(cce->cookie, cookie, cookie_len);
 978                         cce->disable_time = 0;
 979                 } else {
 980                         /* invalid cookie length, disable entry */
 981                         cce->server_mss = 0;
 982                         cce->cookie_len = 0;
 983                         /*
 984                          * Preserve the existing disable time if it is
 985                          * already disabled.
 986                          */
 987                         if (cce->disable_time == 0)
 988                                 cce->disable_time = getsbinuptime();
 989                 }
 990         } else
 991                 tcp_fastopen_ccache_create(ccb, inc, mss, cookie_len, cookie);
 992
 993         CCB_UNLOCK(ccb);
 994 }
 995
 996 static struct tcp_fastopen_ccache_entry *
 997 tcp_fastopen_ccache_lookup(struct in_conninfo *inc,
 998     struct tcp_fastopen_ccache_bucket **ccbp)
 999 {
1000         struct tcp_fastopen_ccache_bucket *ccb;
1001         struct tcp_fastopen_ccache_entry *cce;
1002         uint32_t last_word;
1003         uint32_t hash;
1004
1005         hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependladdr, 4,
1006             V_tcp_fastopen_ccache.secret);
1007         hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependfaddr, 4,
1008             hash);
1009         last_word = inc->inc_fport;
1010         hash = jenkins_hash32(&last_word, 1, hash);
1011         ccb = &V_tcp_fastopen_ccache.base[hash & V_tcp_fastopen_ccache.mask];
1012         *ccbp = ccb;
1013         CCB_LOCK(ccb);
1014
1015         /*
1016          * Always returns with locked bucket.
1017          */
1018         TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link)
1019                 if ((!(cce->af == AF_INET6) == !(inc->inc_flags & INC_ISIPV6)) &&
1020                     (cce->server_port == inc->inc_ie.ie_fport) &&
1021                     (((cce->af == AF_INET) &&
1022                       (cce->cce_client_ip.v4.s_addr == inc->inc_laddr.s_addr) &&
1023                       (cce->cce_server_ip.v4.s_addr == inc->inc_faddr.s_addr)) ||
1024                      ((cce->af == AF_INET6) &&
1025                       IN6_ARE_ADDR_EQUAL(&cce->cce_client_ip.v6, &inc->inc6_laddr) &&
1026                       IN6_ARE_ADDR_EQUAL(&cce->cce_server_ip.v6, &inc->inc6_faddr))))
1027                         break;
1028
1029         return (cce);
1030 }
1031
1032 static struct tcp_fastopen_ccache_entry *
1033 tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb,
1034     struct in_conninfo *inc, uint16_t mss, uint8_t cookie_len, uint8_t *cookie)
1035 {
1036         struct tcp_fastopen_ccache_entry *cce;
1037
1038         /*
1039          * 1. Create a new entry, or
1040          * 2. Reclaim an existing entry, or
1041          * 3. Fail
1042          */
1043
1044         CCB_LOCK_ASSERT(ccb);
1045
1046         cce = NULL;
1047         if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit)
1048                 cce = uma_zalloc(V_tcp_fastopen_ccache.zone, M_NOWAIT);
1049
1050         if (cce == NULL) {
1051                 /*
1052                  * At bucket limit, or out of memory - reclaim last
1053                  * entry in bucket.
1054                  */
1055                 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries);
1056                 if (cce == NULL) {
1057                         /* XXX count this event */
1058                         return (NULL);
1059                 }
1060
1061                 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1062         } else
1063                 ccb->ccb_num_entries++;
1064
1065         TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link);
1066         cce->af = (inc->inc_flags & INC_ISIPV6) ? AF_INET6 : AF_INET;
1067         if (cce->af == AF_INET) {
1068                 cce->cce_client_ip.v4 = inc->inc_laddr;
1069                 cce->cce_server_ip.v4 = inc->inc_faddr;
1070         } else {
1071                 cce->cce_client_ip.v6 = inc->inc6_laddr;
1072                 cce->cce_server_ip.v6 = inc->inc6_faddr;
1073         }
1074         cce->server_port = inc->inc_fport;
1075         if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
1076             (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1077             ((cookie_len & 0x1) == 0)) {
1078                 cce->server_mss = mss;
1079                 cce->cookie_len = cookie_len;
1080                 memcpy(cce->cookie, cookie, cookie_len);
1081                 cce->disable_time = 0;
1082         } else {
1083                 /* invalid cookie length, disable cce */
1084                 cce->server_mss = 0;
1085                 cce->cookie_len = 0;
1086                 cce->disable_time = getsbinuptime();
1087         }
1088
1089         return (cce);
1090 }
1091
1092 static void
1093 tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *ccb,
1094     unsigned int limit)
1095 {
1096         struct tcp_fastopen_ccache_entry *cce, *cce_tmp;
1097         unsigned int entries;
1098
1099         CCB_LOCK(ccb);
1100         entries = 0;
1101         TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) {
1102                 entries++;
1103                 if (entries > limit)
1104                         tcp_fastopen_ccache_entry_drop(cce, ccb);
1105         }
1106         KASSERT(ccb->ccb_num_entries <= (int)limit,
1107             ("%s: ccb->ccb_num_entries %d exceeds limit %d", __func__,
1108                 ccb->ccb_num_entries, limit));
1109         if (limit == 0) {
1110                 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
1111                     ("%s: ccb->ccb_entries not empty", __func__));
1112                 ccb->ccb_num_entries = -1; /* disable bucket */
1113         }
1114         CCB_UNLOCK(ccb);
1115 }
1116
1117 static void
1118 tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce,
1119     struct tcp_fastopen_ccache_bucket *ccb)
1120 {
1121
1122         CCB_LOCK_ASSERT(ccb);
1123
1124         TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1125         ccb->ccb_num_entries--;
1126         uma_zfree(V_tcp_fastopen_ccache.zone, cce);
1127 }
1128