sys/netinet/tcp_fastopen.c

   1 /*-
   2  * Copyright (c) 2015-2017 Patrick Kelsey
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 /*
  28  * This is an implementation of TCP Fast Open (TFO) [RFC7413]. To include
  29  * this code, add the following line to your kernel config:
  30  *
  31  * options TCP_RFC7413
  32  *
  33  *
  34  * The generated TFO cookies are the 64-bit output of
  35  * SipHash24(key=<16-byte-key>, msg=<client-ip>).  Multiple concurrent valid
  36  * keys are supported so that time-based rolling cookie invalidation
  37  * policies can be implemented in the system.  The default number of
  38  * concurrent keys is 2.  This can be adjusted in the kernel config as
  39  * follows:
  40  *
  41  * options TCP_RFC7413_MAX_KEYS=<num-keys>
  42  *
  43  *
  44  * In addition to the facilities defined in RFC7413, this implementation
  45  * supports a pre-shared key (PSK) mode of operation in which the TFO server
  46  * requires the client to be in posession of a shared secret in order for
  47  * the client to be able to successfully open TFO connections with the
  48  * server.  This is useful, for example, in environments where TFO servers
  49  * are exposed to both internal and external clients and only wish to allow
  50  * TFO connections from internal clients.
  51  *
  52  * In the PSK mode of operation, the server generates and sends TFO cookies
  53  * to requesting clients as usual.  However, when validating cookies
  54  * received in TFO SYNs from clients, the server requires the
  55  * client-supplied cookie to equal SipHash24(key=<16-byte-psk>,
  56  * msg=<cookie-sent-to-client>).
  57  *
  58  * Multiple concurrent valid pre-shared keys are supported so that
  59  * time-based rolling PSK invalidation policies can be implemented in the
  60  * system.  The default number of concurrent pre-shared keys is 2.  This can
  61  * be adjusted in the kernel config as follows:
  62  *
  63  * options TCP_RFC7413_MAX_PSKS=<num-psks>
  64  *
  65  *
  66  * The following TFO-specific sysctls are defined:
  67  *
  68  * net.inet.tcp.fastopen.acceptany (RW, default 0)
  69  *     When non-zero, all client-supplied TFO cookies will be considered to
  70  *     be valid.
  71  *
  72  * net.inet.tcp.fastopen.autokey (RW, default 120)
  73  *     When this and net.inet.tcp.fastopen.server_enable are non-zero, a new
  74  *     key will be automatically generated after this many seconds.
  75  *
  76  * net.inet.tcp.fastopen.ccache_bucket_limit
  77  *                     (RWTUN, default TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT)
  78  *     The maximum number of entries in a client cookie cache bucket.
  79  *
  80  * net.inet.tcp.fastopen.ccache_buckets
  81  *                          (RDTUN, default TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT)
  82  *     The number of client cookie cache buckets.
  83  *
  84  * net.inet.tcp.fastopen.ccache_list (RO)
  85  *     Print the client cookie cache.
  86  *
  87  * net.inet.tcp.fastopen.client_enable (RW, default 0)
  88  *     When zero, no new active (i.e., client) TFO connections can be
  89  *     created.  On the transition from enabled to disabled, the client
  90  *     cookie cache is cleared and disabled.  The transition from enabled to
  91  *     disabled does not affect any active TFO connections in progress; it
  92  *     only prevents new ones from being made.
  93  *
  94  * net.inet.tcp.fastopen.keylen (RD)
  95  *     The key length in bytes.
  96  *
  97  * net.inet.tcp.fastopen.maxkeys (RD)
  98  *     The maximum number of keys supported.
  99  *
 100  * net.inet.tcp.fastopen.maxpsks (RD)
 101  *     The maximum number of pre-shared keys supported.
 102  *
 103  * net.inet.tcp.fastopen.numkeys (RD)
 104  *     The current number of keys installed.
 105  *
 106  * net.inet.tcp.fastopen.numpsks (RD)
 107  *     The current number of pre-shared keys installed.
 108  *
 109  * net.inet.tcp.fastopen.path_disable_time
 110  *                          (RW, default TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT)
 111  *     When a failure occurs while trying to create a new active (i.e.,
 112  *     client) TFO connection, new active connections on the same path, as
 113  *     determined by the tuple {client_ip, server_ip, server_port}, will be
 114  *     forced to be non-TFO for this many seconds.  Note that the path
 115  *     disable mechanism relies on state stored in client cookie cache
 116  *     entries, so it is possible for the disable time for a given path to
 117  *     be reduced if the corresponding client cookie cache entry is reused
 118  *     due to resource pressure before the disable period has elapsed.
 119  *
 120  * net.inet.tcp.fastopen.psk_enable (RW, default 0)
 121  *     When non-zero, pre-shared key (PSK) mode is enabled for all TFO
 122  *     servers.  On the transition from enabled to disabled, all installed
 123  *     pre-shared keys are removed.
 124  *
 125  * net.inet.tcp.fastopen.server_enable (RW, default 0)
 126  *     When zero, no new passive (i.e., server) TFO connections can be
 127  *     created.  On the transition from enabled to disabled, all installed
 128  *     keys and pre-shared keys are removed.  On the transition from
 129  *     disabled to enabled, if net.inet.tcp.fastopen.autokey is non-zero and
 130  *     there are no keys installed, a new key will be generated immediately.
 131  *     The transition from enabled to disabled does not affect any passive
 132  *     TFO connections in progress; it only prevents new ones from being
 133  *     made.
 134  *
 135  * net.inet.tcp.fastopen.setkey (WR)
 136  *     Install a new key by writing net.inet.tcp.fastopen.keylen bytes to
 137  *     this sysctl.
 138  *
 139  * net.inet.tcp.fastopen.setpsk (WR)
 140  *     Install a new pre-shared key by writing net.inet.tcp.fastopen.keylen
 141  *     bytes to this sysctl.
 142  *
 143  * In order for TFO connections to be created via a listen socket, that
 144  * socket must have the TCP_FASTOPEN socket option set on it.  This option
 145  * can be set on the socket either before or after the listen() is invoked.
 146  * Clearing this option on a listen socket after it has been set has no
 147  * effect on existing TFO connections or TFO connections in progress; it
 148  * only prevents new TFO connections from being made.
 149  *
 150  * For passively-created sockets, the TCP_FASTOPEN socket option can be
 151  * queried to determine whether the connection was established using TFO.
 152  * Note that connections that are established via a TFO SYN, but that fall
 153  * back to using a non-TFO SYN|ACK will have the TCP_FASTOPEN socket option
 154  * set.
 155  *
 156  * Per the RFC, this implementation limits the number of TFO connections
 157  * that can be in the SYN_RECEIVED state on a per listen-socket basis.
 158  * Whenever this limit is exceeded, requests for new TFO connections are
 159  * serviced as non-TFO requests.  Without such a limit, given a valid TFO
 160  * cookie, an attacker could keep the listen queue in an overflow condition
 161  * using a TFO SYN flood.  This implementation sets the limit at half the
 162  * configured listen backlog.
 163  *
 164  */
 165
 166 #include <sys/cdefs.h>
 167 __FBSDID("$FreeBSD$");
 168
 169 #include "opt_inet.h"
 170
 171 #include <sys/param.h>
 172 #include <sys/jail.h>
 173 #include <sys/kernel.h>
 174 #include <sys/hash.h>
 175 #include <sys/limits.h>
 176 #include <sys/lock.h>
 177 #include <sys/proc.h>
 178 #include <sys/rmlock.h>
 179 #include <sys/sbuf.h>
 180 #include <sys/socket.h>
 181 #include <sys/socketvar.h>
 182 #include <sys/sysctl.h>
 183 #include <sys/systm.h>
 184
 185 #include <crypto/siphash/siphash.h>
 186
 187 #include <net/vnet.h>
 188
 189 #include <netinet/in.h>
 190 #include <netinet/in_pcb.h>
 191 #include <netinet/tcp_var.h>
 192 #include <netinet/tcp_fastopen.h>
 193
 194
 195 #define TCP_FASTOPEN_KEY_LEN    SIPHASH_KEY_LENGTH
 196
 197 #if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN
 198 #error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN
 199 #endif
 200
 201 /*
 202  * Because a PSK-mode setsockopt() uses tcpcb.t_tfo_cookie.client to hold
 203  * the PSK until the connect occurs.
 204  */
 205 #if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN
 206 #error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN
 207 #endif
 208
 209 #define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT        16
 210 #define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT             2048 /* must be power of 2 */
 211
 212 #define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT          900 /* seconds */
 213
 214 #if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1)
 215 #define TCP_FASTOPEN_MAX_KEYS   2
 216 #else
 217 #define TCP_FASTOPEN_MAX_KEYS   TCP_RFC7413_MAX_KEYS
 218 #endif
 219
 220 #if TCP_FASTOPEN_MAX_KEYS > 10
 221 #undef TCP_FASTOPEN_MAX_KEYS
 222 #define TCP_FASTOPEN_MAX_KEYS   10
 223 #endif
 224
 225 #if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1)
 226 #define TCP_FASTOPEN_MAX_PSKS   2
 227 #else
 228 #define TCP_FASTOPEN_MAX_PSKS   TCP_RFC7413_MAX_PSKS
 229 #endif
 230
 231 #if TCP_FASTOPEN_MAX_PSKS > 10
 232 #undef TCP_FASTOPEN_MAX_PSKS
 233 #define TCP_FASTOPEN_MAX_PSKS   10
 234 #endif
 235
 236 struct tcp_fastopen_keylist {
 237         unsigned int newest;
 238         unsigned int newest_psk;
 239         uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN];
 240         uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN];
 241 };
 242
 243 struct tcp_fastopen_callout {
 244         struct callout c;
 245         struct vnet *v;
 246 };
 247
 248 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup(
 249     struct in_conninfo *, struct tcp_fastopen_ccache_bucket **);
 250 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create(
 251     struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t,
 252     uint8_t *);
 253 static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *,
 254     unsigned int);
 255 static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *,
 256     struct tcp_fastopen_ccache_bucket *);
 257
 258 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 259     "TCP Fast Open");
 260
 261 VNET_DEFINE_STATIC(int, tcp_fastopen_acceptany) = 0;
 262 #define V_tcp_fastopen_acceptany        VNET(tcp_fastopen_acceptany)
 263 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany,
 264     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0,
 265     "Accept any non-empty cookie");
 266
 267 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_autokey) = 120;
 268 #define V_tcp_fastopen_autokey  VNET(tcp_fastopen_autokey)
 269 static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS);
 270 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey,
 271     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 272     NULL, 0, &sysctl_net_inet_tcp_fastopen_autokey, "IU",
 273     "Number of seconds between auto-generation of a new key; zero disables");
 274
 275 static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS);
 276 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_bucket_limit,
 277     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT,
 278     NULL, 0, &sysctl_net_inet_tcp_fastopen_ccache_bucket_limit, "IU",
 279     "Max entries per bucket in client cookie cache");
 280
 281 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_ccache_buckets) =
 282     TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
 283 #define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets)
 284 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets,
 285     CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0,
 286     "Client cookie cache number of buckets (power of 2)");
 287
 288 VNET_DEFINE(unsigned int, tcp_fastopen_client_enable) = 1;
 289 static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS);
 290 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, client_enable,
 291     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 292     NULL, 0, &sysctl_net_inet_tcp_fastopen_client_enable, "IU",
 293     "Enable/disable TCP Fast Open client functionality");
 294
 295 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, keylen,
 296     CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_KEY_LEN,
 297     "Key length in bytes");
 298
 299 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxkeys,
 300     CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_KEYS,
 301     "Maximum number of keys supported");
 302
 303 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxpsks,
 304     CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_PSKS,
 305     "Maximum number of pre-shared keys supported");
 306
 307 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numkeys) = 0;
 308 #define V_tcp_fastopen_numkeys  VNET(tcp_fastopen_numkeys)
 309 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numkeys,
 310     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0,
 311     "Number of keys installed");
 312
 313 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numpsks) = 0;
 314 #define V_tcp_fastopen_numpsks  VNET(tcp_fastopen_numpsks)
 315 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numpsks,
 316     CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0,
 317     "Number of pre-shared keys installed");
 318
 319 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_path_disable_time) =
 320     TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT;
 321 #define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time)
 322 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, path_disable_time,
 323     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0,
 324     "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path");
 325
 326 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_psk_enable) = 0;
 327 #define V_tcp_fastopen_psk_enable       VNET(tcp_fastopen_psk_enable)
 328 static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS);
 329 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, psk_enable,
 330     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 331     NULL, 0, &sysctl_net_inet_tcp_fastopen_psk_enable, "IU",
 332     "Enable/disable TCP Fast Open server pre-shared key mode");
 333
 334 VNET_DEFINE(unsigned int, tcp_fastopen_server_enable) = 0;
 335 static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS);
 336 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, server_enable,
 337     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 338     NULL, 0, &sysctl_net_inet_tcp_fastopen_server_enable, "IU",
 339     "Enable/disable TCP Fast Open server functionality");
 340
 341 static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS);
 342 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setkey,
 343     CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
 344     NULL, 0, &sysctl_net_inet_tcp_fastopen_setkey, "",
 345     "Install a new key");
 346
 347 static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS);
 348 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setpsk,
 349     CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
 350     NULL, 0, &sysctl_net_inet_tcp_fastopen_setpsk, "",
 351     "Install a new pre-shared key");
 352
 353 static int sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS);
 354 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_list,
 355     CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE,
 356     NULL, 0, sysctl_net_inet_tcp_fastopen_ccache_list, "A",
 357     "List of all client cookie cache entries");
 358
 359 VNET_DEFINE_STATIC(struct rmlock, tcp_fastopen_keylock);
 360 #define V_tcp_fastopen_keylock  VNET(tcp_fastopen_keylock)
 361
 362 #define TCP_FASTOPEN_KEYS_RLOCK(t)      rm_rlock(&V_tcp_fastopen_keylock, (t))
 363 #define TCP_FASTOPEN_KEYS_RUNLOCK(t)    rm_runlock(&V_tcp_fastopen_keylock, (t))
 364 #define TCP_FASTOPEN_KEYS_WLOCK()       rm_wlock(&V_tcp_fastopen_keylock)
 365 #define TCP_FASTOPEN_KEYS_WUNLOCK()     rm_wunlock(&V_tcp_fastopen_keylock)
 366
 367 VNET_DEFINE_STATIC(struct tcp_fastopen_keylist, tcp_fastopen_keys);
 368 #define V_tcp_fastopen_keys     VNET(tcp_fastopen_keys)
 369
 370 VNET_DEFINE_STATIC(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx);
 371 #define V_tcp_fastopen_autokey_ctx      VNET(tcp_fastopen_autokey_ctx)
 372
 373 VNET_DEFINE_STATIC(uma_zone_t, counter_zone);
 374 #define V_counter_zone                  VNET(counter_zone)
 375
 376 static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets");
 377
 378 VNET_DEFINE_STATIC(struct tcp_fastopen_ccache, tcp_fastopen_ccache);
 379 #define V_tcp_fastopen_ccache   VNET(tcp_fastopen_ccache)
 380
 381 #define CCB_LOCK(ccb)           mtx_lock(&(ccb)->ccb_mtx)
 382 #define CCB_UNLOCK(ccb)         mtx_unlock(&(ccb)->ccb_mtx)
 383 #define CCB_LOCK_ASSERT(ccb)    mtx_assert(&(ccb)->ccb_mtx, MA_OWNED)
 384
 385
 386 void
 387 tcp_fastopen_init(void)
 388 {
 389         unsigned int i;
 390
 391         V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int),
 392             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 393         rm_init(&V_tcp_fastopen_keylock, "tfo_keylock");
 394         callout_init_rm(&V_tcp_fastopen_autokey_ctx.c,
 395             &V_tcp_fastopen_keylock, 0);
 396         V_tcp_fastopen_autokey_ctx.v = curvnet;
 397         V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
 398         V_tcp_fastopen_keys.newest_psk = TCP_FASTOPEN_MAX_PSKS - 1;
 399
 400         /* May already be non-zero if kernel tunable was set */
 401         if (V_tcp_fastopen_ccache.bucket_limit == 0)
 402                 V_tcp_fastopen_ccache.bucket_limit =
 403                     TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT;
 404
 405         /* May already be non-zero if kernel tunable was set */
 406         if ((V_tcp_fastopen_ccache_buckets == 0) ||
 407             !powerof2(V_tcp_fastopen_ccache_buckets))
 408                 V_tcp_fastopen_ccache.buckets =
 409                         TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
 410         else
 411                 V_tcp_fastopen_ccache.buckets = V_tcp_fastopen_ccache_buckets;
 412
 413         V_tcp_fastopen_ccache.mask = V_tcp_fastopen_ccache.buckets - 1;
 414         V_tcp_fastopen_ccache.secret = arc4random();
 415
 416         V_tcp_fastopen_ccache.base = malloc(V_tcp_fastopen_ccache.buckets *
 417             sizeof(struct tcp_fastopen_ccache_bucket), M_TCP_FASTOPEN_CCACHE,
 418             M_WAITOK | M_ZERO);
 419
 420         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
 421                 TAILQ_INIT(&V_tcp_fastopen_ccache.base[i].ccb_entries);
 422                 mtx_init(&V_tcp_fastopen_ccache.base[i].ccb_mtx, "tfo_ccache_bucket",
 423                          NULL, MTX_DEF);
 424                 if (V_tcp_fastopen_client_enable) {
 425                         /* enable bucket */
 426                         V_tcp_fastopen_ccache.base[i].ccb_num_entries = 0;
 427                 } else {
 428                         /* disable bucket */
 429                         V_tcp_fastopen_ccache.base[i].ccb_num_entries = -1;
 430                 }
 431                 V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache;
 432         }
 433
 434         /*
 435          * Note that while the total number of entries in the cookie cache
 436          * is limited by the table management logic to
 437          * V_tcp_fastopen_ccache.buckets *
 438          * V_tcp_fastopen_ccache.bucket_limit, the total number of items in
 439          * this zone can exceed that amount by the number of CPUs in the
 440          * system times the maximum number of unallocated items that can be
 441          * present in each UMA per-CPU cache for this zone.
 442          */
 443         V_tcp_fastopen_ccache.zone = uma_zcreate("tfo_ccache_entries",
 444             sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL,
 445             UMA_ALIGN_CACHE, 0);
 446 }
 447
 448 void
 449 tcp_fastopen_destroy(void)
 450 {
 451         struct tcp_fastopen_ccache_bucket *ccb;
 452         unsigned int i;
 453
 454         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
 455                 ccb = &V_tcp_fastopen_ccache.base[i];
 456                 tcp_fastopen_ccache_bucket_trim(ccb, 0);
 457                 mtx_destroy(&ccb->ccb_mtx);
 458         }
 459
 460         KASSERT(uma_zone_get_cur(V_tcp_fastopen_ccache.zone) == 0,
 461             ("%s: TFO ccache zone allocation count not 0", __func__));
 462         uma_zdestroy(V_tcp_fastopen_ccache.zone);
 463         free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE);
 464
 465         callout_drain(&V_tcp_fastopen_autokey_ctx.c);
 466         rm_destroy(&V_tcp_fastopen_keylock);
 467         uma_zdestroy(V_counter_zone);
 468 }
 469
 470 unsigned int *
 471 tcp_fastopen_alloc_counter(void)
 472 {
 473         unsigned int *counter;
 474         counter = uma_zalloc(V_counter_zone, M_NOWAIT);
 475         if (counter)
 476                 *counter = 1;
 477         return (counter);
 478 }
 479
 480 void
 481 tcp_fastopen_decrement_counter(unsigned int *counter)
 482 {
 483         if (*counter == 1)
 484                 uma_zfree(V_counter_zone, counter);
 485         else
 486                 atomic_subtract_int(counter, 1);
 487 }
 488
 489 static void
 490 tcp_fastopen_addkey_locked(uint8_t *key)
 491 {
 492
 493         V_tcp_fastopen_keys.newest++;
 494         if (V_tcp_fastopen_keys.newest == TCP_FASTOPEN_MAX_KEYS)
 495                 V_tcp_fastopen_keys.newest = 0;
 496         memcpy(V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], key,
 497             TCP_FASTOPEN_KEY_LEN);
 498         if (V_tcp_fastopen_numkeys < TCP_FASTOPEN_MAX_KEYS)
 499                 V_tcp_fastopen_numkeys++;
 500 }
 501
 502 static void
 503 tcp_fastopen_addpsk_locked(uint8_t *psk)
 504 {
 505
 506         V_tcp_fastopen_keys.newest_psk++;
 507         if (V_tcp_fastopen_keys.newest_psk == TCP_FASTOPEN_MAX_PSKS)
 508                 V_tcp_fastopen_keys.newest_psk = 0;
 509         memcpy(V_tcp_fastopen_keys.psk[V_tcp_fastopen_keys.newest_psk], psk,
 510             TCP_FASTOPEN_KEY_LEN);
 511         if (V_tcp_fastopen_numpsks < TCP_FASTOPEN_MAX_PSKS)
 512                 V_tcp_fastopen_numpsks++;
 513 }
 514
 515 static void
 516 tcp_fastopen_autokey_locked(void)
 517 {
 518         uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
 519
 520         arc4rand(newkey, TCP_FASTOPEN_KEY_LEN, 0);
 521         tcp_fastopen_addkey_locked(newkey);
 522 }
 523
 524 static void
 525 tcp_fastopen_autokey_callout(void *arg)
 526 {
 527         struct tcp_fastopen_callout *ctx = arg;
 528
 529         CURVNET_SET(ctx->v);
 530         tcp_fastopen_autokey_locked();
 531         callout_reset(&ctx->c, V_tcp_fastopen_autokey * hz,
 532                       tcp_fastopen_autokey_callout, ctx);
 533         CURVNET_RESTORE();
 534 }
 535
 536
 537 static uint64_t
 538 tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc)
 539 {
 540         SIPHASH_CTX ctx;
 541         uint64_t siphash;
 542
 543         SipHash24_Init(&ctx);
 544         SipHash_SetKey(&ctx, key);
 545         switch (inc->inc_flags & INC_ISIPV6) {
 546 #ifdef INET
 547         case 0:
 548                 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(inc->inc_faddr));
 549                 break;
 550 #endif
 551 #ifdef INET6
 552         case INC_ISIPV6:
 553                 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(inc->inc6_faddr));
 554                 break;
 555 #endif
 556         }
 557         SipHash_Final((u_int8_t *)&siphash, &ctx);
 558
 559         return (siphash);
 560 }
 561
 562 static uint64_t
 563 tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len)
 564 {
 565         SIPHASH_CTX ctx;
 566         uint64_t psk_cookie;
 567
 568         SipHash24_Init(&ctx);
 569         SipHash_SetKey(&ctx, psk);
 570         SipHash_Update(&ctx, cookie, cookie_len);
 571         SipHash_Final((u_int8_t *)&psk_cookie, &ctx);
 572
 573         return (psk_cookie);
 574 }
 575
 576 static int
 577 tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie)
 578 {
 579         unsigned int i, psk_index;
 580         uint64_t psk_cookie;
 581
 582         if (V_tcp_fastopen_psk_enable) {
 583                 psk_index = V_tcp_fastopen_keys.newest_psk;
 584                 for (i = 0; i < V_tcp_fastopen_numpsks; i++) {
 585                         psk_cookie =
 586                             tcp_fastopen_make_psk_cookie(
 587                                  V_tcp_fastopen_keys.psk[psk_index],
 588                                  (uint8_t *)cur_cookie,
 589                                  TCP_FASTOPEN_COOKIE_LEN);
 590
 591                         if (memcmp(wire_cookie, &psk_cookie,
 592                                    TCP_FASTOPEN_COOKIE_LEN) == 0)
 593                                 return (1);
 594
 595                         if (psk_index == 0)
 596                                 psk_index = TCP_FASTOPEN_MAX_PSKS - 1;
 597                         else
 598                                 psk_index--;
 599                 }
 600         } else if (memcmp(wire_cookie, cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0)
 601                 return (1);
 602
 603         return (0);
 604 }
 605
 606 /*
 607  * Return values:
 608  *      -1      the cookie is invalid and no valid cookie is available
 609  *       0      the cookie is invalid and the latest cookie has been returned
 610  *       1      the cookie is valid and the latest cookie has been returned
 611  */
 612 int
 613 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie,
 614     unsigned int len, uint64_t *latest_cookie)
 615 {
 616         struct rm_priotracker tracker;
 617         unsigned int i, key_index;
 618         int rv;
 619         uint64_t cur_cookie;
 620
 621         if (V_tcp_fastopen_acceptany) {
 622                 *latest_cookie = 0;
 623                 return (1);
 624         }
 625
 626         TCP_FASTOPEN_KEYS_RLOCK(&tracker);
 627         if (len != TCP_FASTOPEN_COOKIE_LEN) {
 628                 if (V_tcp_fastopen_numkeys > 0) {
 629                         *latest_cookie =
 630                             tcp_fastopen_make_cookie(
 631                                 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest],
 632                                 inc);
 633                         rv = 0;
 634                 } else
 635                         rv = -1;
 636                 goto out;
 637         }
 638
 639         /*
 640          * Check against each available key, from newest to oldest.
 641          */
 642         key_index = V_tcp_fastopen_keys.newest;
 643         for (i = 0; i < V_tcp_fastopen_numkeys; i++) {
 644                 cur_cookie =
 645                     tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index],
 646                         inc);
 647                 if (i == 0)
 648                         *latest_cookie = cur_cookie;
 649                 rv = tcp_fastopen_find_cookie_match_locked(cookie, &cur_cookie);
 650                 if (rv)
 651                         goto out;
 652                 if (key_index == 0)
 653                         key_index = TCP_FASTOPEN_MAX_KEYS - 1;
 654                 else
 655                         key_index--;
 656         }
 657         rv = 0;
 658
 659  out:
 660         TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
 661         return (rv);
 662 }
 663
 664 static int
 665 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
 666 {
 667         int error;
 668         unsigned int new;
 669
 670         new = V_tcp_fastopen_autokey;
 671         error = sysctl_handle_int(oidp, &new, 0, req);
 672         if (error == 0 && req->newptr) {
 673                 if (new > (INT_MAX / hz))
 674                         return (EINVAL);
 675
 676                 TCP_FASTOPEN_KEYS_WLOCK();
 677                 if (V_tcp_fastopen_server_enable) {
 678                         if (V_tcp_fastopen_autokey && !new)
 679                                 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
 680                         else if (new)
 681                                 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
 682                                     new * hz, tcp_fastopen_autokey_callout,
 683                                     &V_tcp_fastopen_autokey_ctx);
 684                 }
 685                 V_tcp_fastopen_autokey = new;
 686                 TCP_FASTOPEN_KEYS_WUNLOCK();
 687         }
 688
 689         return (error);
 690 }
 691
 692 static int
 693 sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)
 694 {
 695         int error;
 696         unsigned int new;
 697
 698         new = V_tcp_fastopen_psk_enable;
 699         error = sysctl_handle_int(oidp, &new, 0, req);
 700         if (error == 0 && req->newptr) {
 701                 if (V_tcp_fastopen_psk_enable && !new) {
 702                         /* enabled -> disabled */
 703                         TCP_FASTOPEN_KEYS_WLOCK();
 704                         V_tcp_fastopen_numpsks = 0;
 705                         V_tcp_fastopen_keys.newest_psk =
 706                             TCP_FASTOPEN_MAX_PSKS - 1;
 707                         V_tcp_fastopen_psk_enable = 0;
 708                         TCP_FASTOPEN_KEYS_WUNLOCK();
 709                 } else if (!V_tcp_fastopen_psk_enable && new) {
 710                         /* disabled -> enabled */
 711                         TCP_FASTOPEN_KEYS_WLOCK();
 712                         V_tcp_fastopen_psk_enable = 1;
 713                         TCP_FASTOPEN_KEYS_WUNLOCK();
 714                 }
 715         }
 716         return (error);
 717 }
 718
 719 static int
 720 sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)
 721 {
 722         int error;
 723         unsigned int new;
 724
 725         new = V_tcp_fastopen_server_enable;
 726         error = sysctl_handle_int(oidp, &new, 0, req);
 727         if (error == 0 && req->newptr) {
 728                 if (V_tcp_fastopen_server_enable && !new) {
 729                         /* enabled -> disabled */
 730                         TCP_FASTOPEN_KEYS_WLOCK();
 731                         V_tcp_fastopen_numkeys = 0;
 732                         V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
 733                         if (V_tcp_fastopen_autokey)
 734                                 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
 735                         V_tcp_fastopen_numpsks = 0;
 736                         V_tcp_fastopen_keys.newest_psk =
 737                             TCP_FASTOPEN_MAX_PSKS - 1;
 738                         V_tcp_fastopen_server_enable = 0;
 739                         TCP_FASTOPEN_KEYS_WUNLOCK();
 740                 } else if (!V_tcp_fastopen_server_enable && new) {
 741                         /* disabled -> enabled */
 742                         TCP_FASTOPEN_KEYS_WLOCK();
 743                         if (V_tcp_fastopen_autokey &&
 744                             (V_tcp_fastopen_numkeys == 0)) {
 745                                 tcp_fastopen_autokey_locked();
 746                                 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
 747                                     V_tcp_fastopen_autokey * hz,
 748                                     tcp_fastopen_autokey_callout,
 749                                     &V_tcp_fastopen_autokey_ctx);
 750                         }
 751                         V_tcp_fastopen_server_enable = 1;
 752                         TCP_FASTOPEN_KEYS_WUNLOCK();
 753                 }
 754         }
 755         return (error);
 756 }
 757
 758 static int
 759 sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)
 760 {
 761         int error;
 762         uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
 763
 764         if (req->oldptr != NULL || req->oldlen != 0)
 765                 return (EINVAL);
 766         if (req->newptr == NULL)
 767                 return (EPERM);
 768         if (req->newlen != sizeof(newkey))
 769                 return (EINVAL);
 770         error = SYSCTL_IN(req, newkey, sizeof(newkey));
 771         if (error)
 772                 return (error);
 773
 774         TCP_FASTOPEN_KEYS_WLOCK();
 775         tcp_fastopen_addkey_locked(newkey);
 776         TCP_FASTOPEN_KEYS_WUNLOCK();
 777
 778         return (0);
 779 }
 780
 781 static int
 782 sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)
 783 {
 784         int error;
 785         uint8_t newpsk[TCP_FASTOPEN_KEY_LEN];
 786
 787         if (req->oldptr != NULL || req->oldlen != 0)
 788                 return (EINVAL);
 789         if (req->newptr == NULL)
 790                 return (EPERM);
 791         if (req->newlen != sizeof(newpsk))
 792                 return (EINVAL);
 793         error = SYSCTL_IN(req, newpsk, sizeof(newpsk));
 794         if (error)
 795                 return (error);
 796
 797         TCP_FASTOPEN_KEYS_WLOCK();
 798         tcp_fastopen_addpsk_locked(newpsk);
 799         TCP_FASTOPEN_KEYS_WUNLOCK();
 800
 801         return (0);
 802 }
 803
 804 static int
 805 sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)
 806 {
 807         struct tcp_fastopen_ccache_bucket *ccb;
 808         int error;
 809         unsigned int new;
 810         unsigned int i;
 811
 812         new = V_tcp_fastopen_ccache.bucket_limit;
 813         error = sysctl_handle_int(oidp, &new, 0, req);
 814         if (error == 0 && req->newptr) {
 815                 if ((new == 0) || (new > INT_MAX))
 816                         error = EINVAL;
 817                 else {
 818                         if (new < V_tcp_fastopen_ccache.bucket_limit) {
 819                                 for (i = 0; i < V_tcp_fastopen_ccache.buckets;
 820                                      i++) {
 821                                         ccb = &V_tcp_fastopen_ccache.base[i];
 822                                         tcp_fastopen_ccache_bucket_trim(ccb, new);
 823                                 }
 824                         }
 825                         V_tcp_fastopen_ccache.bucket_limit = new;
 826                 }
 827
 828         }
 829         return (error);
 830 }
 831
 832 static int
 833 sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)
 834 {
 835         struct tcp_fastopen_ccache_bucket *ccb;
 836         int error;
 837         unsigned int new, i;
 838
 839         new = V_tcp_fastopen_client_enable;
 840         error = sysctl_handle_int(oidp, &new, 0, req);
 841         if (error == 0 && req->newptr) {
 842                 if (V_tcp_fastopen_client_enable && !new) {
 843                         /* enabled -> disabled */
 844                         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
 845                                 ccb = &V_tcp_fastopen_ccache.base[i];
 846                                 KASSERT(ccb->ccb_num_entries > -1,
 847                                     ("%s: ccb->ccb_num_entries %d is negative",
 848                                         __func__, ccb->ccb_num_entries));
 849                                 tcp_fastopen_ccache_bucket_trim(ccb, 0);
 850                         }
 851                         V_tcp_fastopen_client_enable = 0;
 852                 } else if (!V_tcp_fastopen_client_enable && new) {
 853                         /* disabled -> enabled */
 854                         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
 855                                 ccb = &V_tcp_fastopen_ccache.base[i];
 856                                 CCB_LOCK(ccb);
 857                                 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
 858                                     ("%s: ccb->ccb_entries not empty", __func__));
 859                                 KASSERT(ccb->ccb_num_entries == -1,
 860                                     ("%s: ccb->ccb_num_entries %d not -1", __func__,
 861                                         ccb->ccb_num_entries));
 862                                 ccb->ccb_num_entries = 0; /* enable bucket */
 863                                 CCB_UNLOCK(ccb);
 864                         }
 865                         V_tcp_fastopen_client_enable = 1;
 866                 }
 867         }
 868         return (error);
 869 }
 870
 871 void
 872 tcp_fastopen_connect(struct tcpcb *tp)
 873 {
 874         struct inpcb *inp;
 875         struct tcp_fastopen_ccache_bucket *ccb;
 876         struct tcp_fastopen_ccache_entry *cce;
 877         sbintime_t now;
 878         uint16_t server_mss;
 879         uint64_t psk_cookie;
 880
 881         psk_cookie = 0;
 882         inp = tp->t_inpcb;
 883         cce = tcp_fastopen_ccache_lookup(&inp->inp_inc, &ccb);
 884         if (cce) {
 885                 if (cce->disable_time == 0) {
 886                         if ((cce->cookie_len > 0) &&
 887                             (tp->t_tfo_client_cookie_len ==
 888                              TCP_FASTOPEN_PSK_LEN)) {
 889                                 psk_cookie =
 890                                     tcp_fastopen_make_psk_cookie(
 891                                         tp->t_tfo_cookie.client,
 892                                         cce->cookie, cce->cookie_len);
 893                         } else {
 894                                 tp->t_tfo_client_cookie_len = cce->cookie_len;
 895                                 memcpy(tp->t_tfo_cookie.client, cce->cookie,
 896                                     cce->cookie_len);
 897                         }
 898                         server_mss = cce->server_mss;
 899                         CCB_UNLOCK(ccb);
 900                         if (tp->t_tfo_client_cookie_len ==
 901                             TCP_FASTOPEN_PSK_LEN && psk_cookie) {
 902                                 tp->t_tfo_client_cookie_len =
 903                                     TCP_FASTOPEN_COOKIE_LEN;
 904                                 memcpy(tp->t_tfo_cookie.client, &psk_cookie,
 905                                     TCP_FASTOPEN_COOKIE_LEN);
 906                         }
 907                         tcp_mss(tp, server_mss ? server_mss : -1);
 908                         tp->snd_wnd = tp->t_maxseg;
 909                 } else {
 910                         /*
 911                          * The path is disabled.  Check the time and
 912                          * possibly re-enable.
 913                          */
 914                         now = getsbinuptime();
 915                         if (now - cce->disable_time >
 916                             ((sbintime_t)V_tcp_fastopen_path_disable_time << 32)) {
 917                                 /*
 918                                  * Re-enable path.  Force a TFO cookie
 919                                  * request.  Forget the old MSS as it may be
 920                                  * bogus now, and we will rediscover it in
 921                                  * the SYN|ACK.
 922                                  */
 923                                 cce->disable_time = 0;
 924                                 cce->server_mss = 0;
 925                                 cce->cookie_len = 0;
 926                                 /*
 927                                  * tp->t_tfo... cookie details are already
 928                                  * zero from the tcpcb init.
 929                                  */
 930                         } else {
 931                                 /*
 932                                  * Path is disabled, so disable TFO on this
 933                                  * connection.
 934                                  */
 935                                 tp->t_flags &= ~TF_FASTOPEN;
 936                         }
 937                         CCB_UNLOCK(ccb);
 938                         tcp_mss(tp, -1);
 939                         /*
 940                          * snd_wnd is irrelevant since we are either forcing
 941                          * a TFO cookie request or disabling TFO - either
 942                          * way, no data with the SYN.
 943                          */
 944                 }
 945         } else {
 946                 /*
 947                  * A new entry for this path will be created when a SYN|ACK
 948                  * comes back, or the attempt otherwise fails.
 949                  */
 950                 CCB_UNLOCK(ccb);
 951                 tcp_mss(tp, -1);
 952                 /*
 953                  * snd_wnd is irrelevant since we are forcing a TFO cookie
 954                  * request.
 955                  */
 956         }
 957 }
 958
 959 void
 960 tcp_fastopen_disable_path(struct tcpcb *tp)
 961 {
 962         struct in_conninfo *inc = &tp->t_inpcb->inp_inc;
 963         struct tcp_fastopen_ccache_bucket *ccb;
 964         struct tcp_fastopen_ccache_entry *cce;
 965
 966         cce = tcp_fastopen_ccache_lookup(inc, &ccb);
 967         if (cce) {
 968                 cce->server_mss = 0;
 969                 cce->cookie_len = 0;
 970                 /*
 971                  * Preserve the existing disable time if it is already
 972                  * disabled.
 973                  */
 974                 if (cce->disable_time == 0)
 975                         cce->disable_time = getsbinuptime();
 976         } else /* use invalid cookie len to create disabled entry */
 977                 tcp_fastopen_ccache_create(ccb, inc, 0,
 978                     TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL);
 979
 980         CCB_UNLOCK(ccb);
 981         tp->t_flags &= ~TF_FASTOPEN;
 982 }
 983
 984 void
 985 tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss,
 986     uint8_t cookie_len, uint8_t *cookie)
 987 {
 988         struct in_conninfo *inc = &tp->t_inpcb->inp_inc;
 989         struct tcp_fastopen_ccache_bucket *ccb;
 990         struct tcp_fastopen_ccache_entry *cce;
 991
 992         cce = tcp_fastopen_ccache_lookup(inc, &ccb);
 993         if (cce) {
 994                 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
 995                     (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
 996                     ((cookie_len & 0x1) == 0)) {
 997                         cce->server_mss = mss;
 998                         cce->cookie_len = cookie_len;
 999                         memcpy(cce->cookie, cookie, cookie_len);
1000                         cce->disable_time = 0;
1001                 } else {
1002                         /* invalid cookie length, disable entry */
1003                         cce->server_mss = 0;
1004                         cce->cookie_len = 0;
1005                         /*
1006                          * Preserve the existing disable time if it is
1007                          * already disabled.
1008                          */
1009                         if (cce->disable_time == 0)
1010                                 cce->disable_time = getsbinuptime();
1011                 }
1012         } else
1013                 tcp_fastopen_ccache_create(ccb, inc, mss, cookie_len, cookie);
1014
1015         CCB_UNLOCK(ccb);
1016 }
1017
1018 static struct tcp_fastopen_ccache_entry *
1019 tcp_fastopen_ccache_lookup(struct in_conninfo *inc,
1020     struct tcp_fastopen_ccache_bucket **ccbp)
1021 {
1022         struct tcp_fastopen_ccache_bucket *ccb;
1023         struct tcp_fastopen_ccache_entry *cce;
1024         uint32_t last_word;
1025         uint32_t hash;
1026
1027         hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependladdr, 4,
1028             V_tcp_fastopen_ccache.secret);
1029         hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependfaddr, 4,
1030             hash);
1031         last_word = inc->inc_fport;
1032         hash = jenkins_hash32(&last_word, 1, hash);
1033         ccb = &V_tcp_fastopen_ccache.base[hash & V_tcp_fastopen_ccache.mask];
1034         *ccbp = ccb;
1035         CCB_LOCK(ccb);
1036
1037         /*
1038          * Always returns with locked bucket.
1039          */
1040         TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link)
1041                 if ((!(cce->af == AF_INET6) == !(inc->inc_flags & INC_ISIPV6)) &&
1042                     (cce->server_port == inc->inc_ie.ie_fport) &&
1043                     (((cce->af == AF_INET) &&
1044                       (cce->cce_client_ip.v4.s_addr == inc->inc_laddr.s_addr) &&
1045                       (cce->cce_server_ip.v4.s_addr == inc->inc_faddr.s_addr)) ||
1046                      ((cce->af == AF_INET6) &&
1047                       IN6_ARE_ADDR_EQUAL(&cce->cce_client_ip.v6, &inc->inc6_laddr) &&
1048                       IN6_ARE_ADDR_EQUAL(&cce->cce_server_ip.v6, &inc->inc6_faddr))))
1049                         break;
1050
1051         return (cce);
1052 }
1053
1054 static struct tcp_fastopen_ccache_entry *
1055 tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb,
1056     struct in_conninfo *inc, uint16_t mss, uint8_t cookie_len, uint8_t *cookie)
1057 {
1058         struct tcp_fastopen_ccache_entry *cce;
1059
1060         /*
1061          * 1. Create a new entry, or
1062          * 2. Reclaim an existing entry, or
1063          * 3. Fail
1064          */
1065
1066         CCB_LOCK_ASSERT(ccb);
1067
1068         cce = NULL;
1069         if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit)
1070                 cce = uma_zalloc(V_tcp_fastopen_ccache.zone, M_NOWAIT);
1071
1072         if (cce == NULL) {
1073                 /*
1074                  * At bucket limit, or out of memory - reclaim last
1075                  * entry in bucket.
1076                  */
1077                 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries);
1078                 if (cce == NULL) {
1079                         /* XXX count this event */
1080                         return (NULL);
1081                 }
1082
1083                 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1084         } else
1085                 ccb->ccb_num_entries++;
1086
1087         TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link);
1088         cce->af = (inc->inc_flags & INC_ISIPV6) ? AF_INET6 : AF_INET;
1089         if (cce->af == AF_INET) {
1090                 cce->cce_client_ip.v4 = inc->inc_laddr;
1091                 cce->cce_server_ip.v4 = inc->inc_faddr;
1092         } else {
1093                 cce->cce_client_ip.v6 = inc->inc6_laddr;
1094                 cce->cce_server_ip.v6 = inc->inc6_faddr;
1095         }
1096         cce->server_port = inc->inc_fport;
1097         if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
1098             (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1099             ((cookie_len & 0x1) == 0)) {
1100                 cce->server_mss = mss;
1101                 cce->cookie_len = cookie_len;
1102                 memcpy(cce->cookie, cookie, cookie_len);
1103                 cce->disable_time = 0;
1104         } else {
1105                 /* invalid cookie length, disable cce */
1106                 cce->server_mss = 0;
1107                 cce->cookie_len = 0;
1108                 cce->disable_time = getsbinuptime();
1109         }
1110
1111         return (cce);
1112 }
1113
1114 static void
1115 tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *ccb,
1116     unsigned int limit)
1117 {
1118         struct tcp_fastopen_ccache_entry *cce, *cce_tmp;
1119         unsigned int entries;
1120
1121         CCB_LOCK(ccb);
1122         entries = 0;
1123         TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) {
1124                 entries++;
1125                 if (entries > limit)
1126                         tcp_fastopen_ccache_entry_drop(cce, ccb);
1127         }
1128         KASSERT(ccb->ccb_num_entries <= (int)limit,
1129             ("%s: ccb->ccb_num_entries %d exceeds limit %d", __func__,
1130                 ccb->ccb_num_entries, limit));
1131         if (limit == 0) {
1132                 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
1133                     ("%s: ccb->ccb_entries not empty", __func__));
1134                 ccb->ccb_num_entries = -1; /* disable bucket */
1135         }
1136         CCB_UNLOCK(ccb);
1137 }
1138
1139 static void
1140 tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce,
1141     struct tcp_fastopen_ccache_bucket *ccb)
1142 {
1143
1144         CCB_LOCK_ASSERT(ccb);
1145
1146         TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1147         ccb->ccb_num_entries--;
1148         uma_zfree(V_tcp_fastopen_ccache.zone, cce);
1149 }
1150
1151 static int
1152 sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS)
1153 {
1154         struct sbuf sb;
1155         struct tcp_fastopen_ccache_bucket *ccb;
1156         struct tcp_fastopen_ccache_entry *cce;
1157         sbintime_t now, duration, limit;
1158         const int linesize = 128;
1159         int i, error, num_entries;
1160         unsigned int j;
1161 #ifdef INET6
1162         char clt_buf[INET6_ADDRSTRLEN], srv_buf[INET6_ADDRSTRLEN];
1163 #else
1164         char clt_buf[INET_ADDRSTRLEN], srv_buf[INET_ADDRSTRLEN];
1165 #endif
1166
1167         if (jailed_without_vnet(curthread->td_ucred) != 0)
1168                 return (EPERM);
1169
1170         /* Only allow root to read the client cookie cache */
1171         if (curthread->td_ucred->cr_uid != 0)
1172                 return (EPERM);
1173
1174         num_entries = 0;
1175         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1176                 ccb = &V_tcp_fastopen_ccache.base[i];
1177                 CCB_LOCK(ccb);
1178                 if (ccb->ccb_num_entries > 0)
1179                         num_entries += ccb->ccb_num_entries;
1180                 CCB_UNLOCK(ccb);
1181         }
1182         sbuf_new(&sb, NULL, linesize * (num_entries + 1), SBUF_INCLUDENUL);
1183
1184         sbuf_printf(&sb,
1185                     "\nLocal IP address     Remote IP address     Port   MSS"
1186                     " Disabled Cookie\n");
1187
1188         now = getsbinuptime();
1189         limit = (sbintime_t)V_tcp_fastopen_path_disable_time << 32;
1190         for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1191                 ccb = &V_tcp_fastopen_ccache.base[i];
1192                 CCB_LOCK(ccb);
1193                 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) {
1194                         if (cce->disable_time != 0) {
1195                                 duration = now - cce->disable_time;
1196                                 if (limit >= duration)
1197                                         duration = limit - duration;
1198                                 else
1199                                         duration = 0;
1200                         } else
1201                                 duration = 0;
1202                         sbuf_printf(&sb,
1203                                     "%-20s %-20s %5u %5u ",
1204                                     inet_ntop(cce->af, &cce->cce_client_ip,
1205                                         clt_buf, sizeof(clt_buf)),
1206                                     inet_ntop(cce->af, &cce->cce_server_ip,
1207                                         srv_buf, sizeof(srv_buf)),
1208                                     ntohs(cce->server_port),
1209                                     cce->server_mss);
1210                         if (duration > 0)
1211                                 sbuf_printf(&sb, "%7ds ", sbintime_getsec(duration));
1212                         else
1213                                 sbuf_printf(&sb, "%8s ", "No");
1214                         for (j = 0; j < cce->cookie_len; j++)
1215                                 sbuf_printf(&sb, "%02x", cce->cookie[j]);
1216                         sbuf_putc(&sb, '\n');
1217                 }
1218                 CCB_UNLOCK(ccb);
1219         }
1220         error = sbuf_finish(&sb);
1221         if (error == 0)
1222                 error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
1223         sbuf_delete(&sb);
1224         return (error);
1225 }
1226