2 * Copyright (c) 2015-2017 Patrick Kelsey
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * This is an implementation of TCP Fast Open (TFO) [RFC7413]. To include
29 * this code, add the following line to your kernel config:
34 * The generated TFO cookies are the 64-bit output of
35 * SipHash24(key=<16-byte-key>, msg=<client-ip>). Multiple concurrent valid
36 * keys are supported so that time-based rolling cookie invalidation
37 * policies can be implemented in the system. The default number of
38 * concurrent keys is 2. This can be adjusted in the kernel config as
41 * options TCP_RFC7413_MAX_KEYS=<num-keys>
44 * In addition to the facilities defined in RFC7413, this implementation
45 * supports a pre-shared key (PSK) mode of operation in which the TFO server
46 * requires the client to be in posession of a shared secret in order for
47 * the client to be able to successfully open TFO connections with the
48 * server. This is useful, for example, in environments where TFO servers
49 * are exposed to both internal and external clients and only wish to allow
50 * TFO connections from internal clients.
52 * In the PSK mode of operation, the server generates and sends TFO cookies
53 * to requesting clients as usual. However, when validating cookies
54 * received in TFO SYNs from clients, the server requires the
55 * client-supplied cookie to equal SipHash24(key=<16-byte-psk>,
56 * msg=<cookie-sent-to-client>).
58 * Multiple concurrent valid pre-shared keys are supported so that
59 * time-based rolling PSK invalidation policies can be implemented in the
60 * system. The default number of concurrent pre-shared keys is 2. This can
61 * be adjusted in the kernel config as follows:
63 * options TCP_RFC7413_MAX_PSKS=<num-psks>
66 * The following TFO-specific sysctls are defined:
68 * net.inet.tcp.fastopen.acceptany (RW, default 0)
69 * When non-zero, all client-supplied TFO cookies will be considered to
72 * net.inet.tcp.fastopen.autokey (RW, default 120)
73 * When this and net.inet.tcp.fastopen.server_enable are non-zero, a new
74 * key will be automatically generated after this many seconds.
76 * net.inet.tcp.fastopen.ccache_bucket_limit
77 * (RWTUN, default TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT)
78 * The maximum number of entries in a client cookie cache bucket.
80 * net.inet.tcp.fastopen.ccache_buckets
81 * (RDTUN, default TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT)
82 * The number of client cookie cache buckets.
84 * net.inet.tcp.fastopen.ccache_list (RO)
85 * Print the client cookie cache.
87 * net.inet.tcp.fastopen.client_enable (RW, default 0)
88 * When zero, no new active (i.e., client) TFO connections can be
89 * created. On the transition from enabled to disabled, the client
90 * cookie cache is cleared and disabled. The transition from enabled to
91 * disabled does not affect any active TFO connections in progress; it
92 * only prevents new ones from being made.
94 * net.inet.tcp.fastopen.keylen (RD)
95 * The key length in bytes.
97 * net.inet.tcp.fastopen.maxkeys (RD)
98 * The maximum number of keys supported.
100 * net.inet.tcp.fastopen.maxpsks (RD)
101 * The maximum number of pre-shared keys supported.
103 * net.inet.tcp.fastopen.numkeys (RD)
104 * The current number of keys installed.
106 * net.inet.tcp.fastopen.numpsks (RD)
107 * The current number of pre-shared keys installed.
109 * net.inet.tcp.fastopen.path_disable_time
110 * (RW, default TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT)
111 * When a failure occurs while trying to create a new active (i.e.,
112 * client) TFO connection, new active connections on the same path, as
113 * determined by the tuple {client_ip, server_ip, server_port}, will be
114 * forced to be non-TFO for this many seconds. Note that the path
115 * disable mechanism relies on state stored in client cookie cache
116 * entries, so it is possible for the disable time for a given path to
117 * be reduced if the corresponding client cookie cache entry is reused
118 * due to resource pressure before the disable period has elapsed.
120 * net.inet.tcp.fastopen.psk_enable (RW, default 0)
121 * When non-zero, pre-shared key (PSK) mode is enabled for all TFO
122 * servers. On the transition from enabled to disabled, all installed
123 * pre-shared keys are removed.
125 * net.inet.tcp.fastopen.server_enable (RW, default 0)
126 * When zero, no new passive (i.e., server) TFO connections can be
127 * created. On the transition from enabled to disabled, all installed
128 * keys and pre-shared keys are removed. On the transition from
129 * disabled to enabled, if net.inet.tcp.fastopen.autokey is non-zero and
130 * there are no keys installed, a new key will be generated immediately.
131 * The transition from enabled to disabled does not affect any passive
132 * TFO connections in progress; it only prevents new ones from being
135 * net.inet.tcp.fastopen.setkey (WR)
136 * Install a new key by writing net.inet.tcp.fastopen.keylen bytes to
139 * net.inet.tcp.fastopen.setpsk (WR)
140 * Install a new pre-shared key by writing net.inet.tcp.fastopen.keylen
141 * bytes to this sysctl.
143 * In order for TFO connections to be created via a listen socket, that
144 * socket must have the TCP_FASTOPEN socket option set on it. This option
145 * can be set on the socket either before or after the listen() is invoked.
146 * Clearing this option on a listen socket after it has been set has no
147 * effect on existing TFO connections or TFO connections in progress; it
148 * only prevents new TFO connections from being made.
150 * For passively-created sockets, the TCP_FASTOPEN socket option can be
151 * queried to determine whether the connection was established using TFO.
152 * Note that connections that are established via a TFO SYN, but that fall
153 * back to using a non-TFO SYN|ACK will have the TCP_FASTOPEN socket option
156 * Per the RFC, this implementation limits the number of TFO connections
157 * that can be in the SYN_RECEIVED state on a per listen-socket basis.
158 * Whenever this limit is exceeded, requests for new TFO connections are
159 * serviced as non-TFO requests. Without such a limit, given a valid TFO
160 * cookie, an attacker could keep the listen queue in an overflow condition
161 * using a TFO SYN flood. This implementation sets the limit at half the
162 * configured listen backlog.
166 #include <sys/cdefs.h>
167 __FBSDID("$FreeBSD$");
169 #include "opt_inet.h"
171 #include <sys/param.h>
172 #include <sys/jail.h>
173 #include <sys/kernel.h>
174 #include <sys/hash.h>
175 #include <sys/limits.h>
176 #include <sys/lock.h>
177 #include <sys/proc.h>
178 #include <sys/rmlock.h>
179 #include <sys/sbuf.h>
180 #include <sys/socket.h>
181 #include <sys/socketvar.h>
182 #include <sys/sysctl.h>
183 #include <sys/systm.h>
185 #include <crypto/siphash/siphash.h>
187 #include <net/vnet.h>
189 #include <netinet/in.h>
190 #include <netinet/in_pcb.h>
191 #include <netinet/tcp_var.h>
192 #include <netinet/tcp_fastopen.h>
195 #define TCP_FASTOPEN_KEY_LEN SIPHASH_KEY_LENGTH
197 #if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN
198 #error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN
202 * Because a PSK-mode setsockopt() uses tcpcb.t_tfo_cookie.client to hold
203 * the PSK until the connect occurs.
205 #if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN
206 #error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN
209 #define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT 16
210 #define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT 2048 /* must be power of 2 */
212 #define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT 900 /* seconds */
214 #if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1)
215 #define TCP_FASTOPEN_MAX_KEYS 2
217 #define TCP_FASTOPEN_MAX_KEYS TCP_RFC7413_MAX_KEYS
220 #if TCP_FASTOPEN_MAX_KEYS > 10
221 #undef TCP_FASTOPEN_MAX_KEYS
222 #define TCP_FASTOPEN_MAX_KEYS 10
225 #if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1)
226 #define TCP_FASTOPEN_MAX_PSKS 2
228 #define TCP_FASTOPEN_MAX_PSKS TCP_RFC7413_MAX_PSKS
231 #if TCP_FASTOPEN_MAX_PSKS > 10
232 #undef TCP_FASTOPEN_MAX_PSKS
233 #define TCP_FASTOPEN_MAX_PSKS 10
236 struct tcp_fastopen_keylist {
238 unsigned int newest_psk;
239 uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN];
240 uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN];
243 struct tcp_fastopen_callout {
248 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup(
249 struct in_conninfo *, struct tcp_fastopen_ccache_bucket **);
250 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create(
251 struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t,
253 static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *,
255 static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *,
256 struct tcp_fastopen_ccache_bucket *);
258 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
261 VNET_DEFINE_STATIC(int, tcp_fastopen_acceptany) = 0;
262 #define V_tcp_fastopen_acceptany VNET(tcp_fastopen_acceptany)
263 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany,
264 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0,
265 "Accept any non-empty cookie");
267 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_autokey) = 120;
268 #define V_tcp_fastopen_autokey VNET(tcp_fastopen_autokey)
269 static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS);
270 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey,
271 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
272 NULL, 0, &sysctl_net_inet_tcp_fastopen_autokey, "IU",
273 "Number of seconds between auto-generation of a new key; zero disables");
275 static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS);
276 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_bucket_limit,
277 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT,
278 NULL, 0, &sysctl_net_inet_tcp_fastopen_ccache_bucket_limit, "IU",
279 "Max entries per bucket in client cookie cache");
281 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_ccache_buckets) =
282 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
283 #define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets)
284 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets,
285 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0,
286 "Client cookie cache number of buckets (power of 2)");
288 VNET_DEFINE(unsigned int, tcp_fastopen_client_enable) = 1;
289 static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS);
290 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, client_enable,
291 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
292 NULL, 0, &sysctl_net_inet_tcp_fastopen_client_enable, "IU",
293 "Enable/disable TCP Fast Open client functionality");
295 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, keylen,
296 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_KEY_LEN,
297 "Key length in bytes");
299 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxkeys,
300 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_KEYS,
301 "Maximum number of keys supported");
303 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxpsks,
304 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_PSKS,
305 "Maximum number of pre-shared keys supported");
307 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numkeys) = 0;
308 #define V_tcp_fastopen_numkeys VNET(tcp_fastopen_numkeys)
309 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numkeys,
310 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0,
311 "Number of keys installed");
313 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numpsks) = 0;
314 #define V_tcp_fastopen_numpsks VNET(tcp_fastopen_numpsks)
315 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numpsks,
316 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0,
317 "Number of pre-shared keys installed");
319 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_path_disable_time) =
320 TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT;
321 #define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time)
322 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, path_disable_time,
323 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0,
324 "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path");
326 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_psk_enable) = 0;
327 #define V_tcp_fastopen_psk_enable VNET(tcp_fastopen_psk_enable)
328 static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS);
329 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, psk_enable,
330 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
331 NULL, 0, &sysctl_net_inet_tcp_fastopen_psk_enable, "IU",
332 "Enable/disable TCP Fast Open server pre-shared key mode");
334 VNET_DEFINE(unsigned int, tcp_fastopen_server_enable) = 0;
335 static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS);
336 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, server_enable,
337 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
338 NULL, 0, &sysctl_net_inet_tcp_fastopen_server_enable, "IU",
339 "Enable/disable TCP Fast Open server functionality");
341 static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS);
342 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setkey,
343 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
344 NULL, 0, &sysctl_net_inet_tcp_fastopen_setkey, "",
345 "Install a new key");
347 static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS);
348 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setpsk,
349 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
350 NULL, 0, &sysctl_net_inet_tcp_fastopen_setpsk, "",
351 "Install a new pre-shared key");
353 static int sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS);
354 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_list,
355 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE,
356 NULL, 0, sysctl_net_inet_tcp_fastopen_ccache_list, "A",
357 "List of all client cookie cache entries");
359 VNET_DEFINE_STATIC(struct rmlock, tcp_fastopen_keylock);
360 #define V_tcp_fastopen_keylock VNET(tcp_fastopen_keylock)
362 #define TCP_FASTOPEN_KEYS_RLOCK(t) rm_rlock(&V_tcp_fastopen_keylock, (t))
363 #define TCP_FASTOPEN_KEYS_RUNLOCK(t) rm_runlock(&V_tcp_fastopen_keylock, (t))
364 #define TCP_FASTOPEN_KEYS_WLOCK() rm_wlock(&V_tcp_fastopen_keylock)
365 #define TCP_FASTOPEN_KEYS_WUNLOCK() rm_wunlock(&V_tcp_fastopen_keylock)
367 VNET_DEFINE_STATIC(struct tcp_fastopen_keylist, tcp_fastopen_keys);
368 #define V_tcp_fastopen_keys VNET(tcp_fastopen_keys)
370 VNET_DEFINE_STATIC(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx);
371 #define V_tcp_fastopen_autokey_ctx VNET(tcp_fastopen_autokey_ctx)
373 VNET_DEFINE_STATIC(uma_zone_t, counter_zone);
374 #define V_counter_zone VNET(counter_zone)
376 static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets");
378 VNET_DEFINE_STATIC(struct tcp_fastopen_ccache, tcp_fastopen_ccache);
379 #define V_tcp_fastopen_ccache VNET(tcp_fastopen_ccache)
381 #define CCB_LOCK(ccb) mtx_lock(&(ccb)->ccb_mtx)
382 #define CCB_UNLOCK(ccb) mtx_unlock(&(ccb)->ccb_mtx)
383 #define CCB_LOCK_ASSERT(ccb) mtx_assert(&(ccb)->ccb_mtx, MA_OWNED)
387 tcp_fastopen_init(void)
391 V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int),
392 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
393 rm_init(&V_tcp_fastopen_keylock, "tfo_keylock");
394 callout_init_rm(&V_tcp_fastopen_autokey_ctx.c,
395 &V_tcp_fastopen_keylock, 0);
396 V_tcp_fastopen_autokey_ctx.v = curvnet;
397 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
398 V_tcp_fastopen_keys.newest_psk = TCP_FASTOPEN_MAX_PSKS - 1;
400 /* May already be non-zero if kernel tunable was set */
401 if (V_tcp_fastopen_ccache.bucket_limit == 0)
402 V_tcp_fastopen_ccache.bucket_limit =
403 TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT;
405 /* May already be non-zero if kernel tunable was set */
406 if ((V_tcp_fastopen_ccache_buckets == 0) ||
407 !powerof2(V_tcp_fastopen_ccache_buckets))
408 V_tcp_fastopen_ccache.buckets =
409 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
411 V_tcp_fastopen_ccache.buckets = V_tcp_fastopen_ccache_buckets;
413 V_tcp_fastopen_ccache.mask = V_tcp_fastopen_ccache.buckets - 1;
414 V_tcp_fastopen_ccache.secret = arc4random();
416 V_tcp_fastopen_ccache.base = malloc(V_tcp_fastopen_ccache.buckets *
417 sizeof(struct tcp_fastopen_ccache_bucket), M_TCP_FASTOPEN_CCACHE,
420 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
421 TAILQ_INIT(&V_tcp_fastopen_ccache.base[i].ccb_entries);
422 mtx_init(&V_tcp_fastopen_ccache.base[i].ccb_mtx, "tfo_ccache_bucket",
424 if (V_tcp_fastopen_client_enable) {
426 V_tcp_fastopen_ccache.base[i].ccb_num_entries = 0;
429 V_tcp_fastopen_ccache.base[i].ccb_num_entries = -1;
431 V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache;
435 * Note that while the total number of entries in the cookie cache
436 * is limited by the table management logic to
437 * V_tcp_fastopen_ccache.buckets *
438 * V_tcp_fastopen_ccache.bucket_limit, the total number of items in
439 * this zone can exceed that amount by the number of CPUs in the
440 * system times the maximum number of unallocated items that can be
441 * present in each UMA per-CPU cache for this zone.
443 V_tcp_fastopen_ccache.zone = uma_zcreate("tfo_ccache_entries",
444 sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL,
449 tcp_fastopen_destroy(void)
451 struct tcp_fastopen_ccache_bucket *ccb;
454 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
455 ccb = &V_tcp_fastopen_ccache.base[i];
456 tcp_fastopen_ccache_bucket_trim(ccb, 0);
457 mtx_destroy(&ccb->ccb_mtx);
460 KASSERT(uma_zone_get_cur(V_tcp_fastopen_ccache.zone) == 0,
461 ("%s: TFO ccache zone allocation count not 0", __func__));
462 uma_zdestroy(V_tcp_fastopen_ccache.zone);
463 free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE);
465 callout_drain(&V_tcp_fastopen_autokey_ctx.c);
466 rm_destroy(&V_tcp_fastopen_keylock);
467 uma_zdestroy(V_counter_zone);
471 tcp_fastopen_alloc_counter(void)
473 unsigned int *counter;
474 counter = uma_zalloc(V_counter_zone, M_NOWAIT);
481 tcp_fastopen_decrement_counter(unsigned int *counter)
484 uma_zfree(V_counter_zone, counter);
486 atomic_subtract_int(counter, 1);
490 tcp_fastopen_addkey_locked(uint8_t *key)
493 V_tcp_fastopen_keys.newest++;
494 if (V_tcp_fastopen_keys.newest == TCP_FASTOPEN_MAX_KEYS)
495 V_tcp_fastopen_keys.newest = 0;
496 memcpy(V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], key,
497 TCP_FASTOPEN_KEY_LEN);
498 if (V_tcp_fastopen_numkeys < TCP_FASTOPEN_MAX_KEYS)
499 V_tcp_fastopen_numkeys++;
503 tcp_fastopen_addpsk_locked(uint8_t *psk)
506 V_tcp_fastopen_keys.newest_psk++;
507 if (V_tcp_fastopen_keys.newest_psk == TCP_FASTOPEN_MAX_PSKS)
508 V_tcp_fastopen_keys.newest_psk = 0;
509 memcpy(V_tcp_fastopen_keys.psk[V_tcp_fastopen_keys.newest_psk], psk,
510 TCP_FASTOPEN_KEY_LEN);
511 if (V_tcp_fastopen_numpsks < TCP_FASTOPEN_MAX_PSKS)
512 V_tcp_fastopen_numpsks++;
516 tcp_fastopen_autokey_locked(void)
518 uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
520 arc4rand(newkey, TCP_FASTOPEN_KEY_LEN, 0);
521 tcp_fastopen_addkey_locked(newkey);
525 tcp_fastopen_autokey_callout(void *arg)
527 struct tcp_fastopen_callout *ctx = arg;
530 tcp_fastopen_autokey_locked();
531 callout_reset(&ctx->c, V_tcp_fastopen_autokey * hz,
532 tcp_fastopen_autokey_callout, ctx);
538 tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc)
543 SipHash24_Init(&ctx);
544 SipHash_SetKey(&ctx, key);
545 switch (inc->inc_flags & INC_ISIPV6) {
548 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(inc->inc_faddr));
553 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(inc->inc6_faddr));
557 SipHash_Final((u_int8_t *)&siphash, &ctx);
563 tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len)
568 SipHash24_Init(&ctx);
569 SipHash_SetKey(&ctx, psk);
570 SipHash_Update(&ctx, cookie, cookie_len);
571 SipHash_Final((u_int8_t *)&psk_cookie, &ctx);
577 tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie)
579 unsigned int i, psk_index;
582 if (V_tcp_fastopen_psk_enable) {
583 psk_index = V_tcp_fastopen_keys.newest_psk;
584 for (i = 0; i < V_tcp_fastopen_numpsks; i++) {
586 tcp_fastopen_make_psk_cookie(
587 V_tcp_fastopen_keys.psk[psk_index],
588 (uint8_t *)cur_cookie,
589 TCP_FASTOPEN_COOKIE_LEN);
591 if (memcmp(wire_cookie, &psk_cookie,
592 TCP_FASTOPEN_COOKIE_LEN) == 0)
596 psk_index = TCP_FASTOPEN_MAX_PSKS - 1;
600 } else if (memcmp(wire_cookie, cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0)
608 * -1 the cookie is invalid and no valid cookie is available
609 * 0 the cookie is invalid and the latest cookie has been returned
610 * 1 the cookie is valid and the latest cookie has been returned
613 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie,
614 unsigned int len, uint64_t *latest_cookie)
616 struct rm_priotracker tracker;
617 unsigned int i, key_index;
621 if (V_tcp_fastopen_acceptany) {
626 TCP_FASTOPEN_KEYS_RLOCK(&tracker);
627 if (len != TCP_FASTOPEN_COOKIE_LEN) {
628 if (V_tcp_fastopen_numkeys > 0) {
630 tcp_fastopen_make_cookie(
631 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest],
640 * Check against each available key, from newest to oldest.
642 key_index = V_tcp_fastopen_keys.newest;
643 for (i = 0; i < V_tcp_fastopen_numkeys; i++) {
645 tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index],
648 *latest_cookie = cur_cookie;
649 rv = tcp_fastopen_find_cookie_match_locked(cookie, &cur_cookie);
653 key_index = TCP_FASTOPEN_MAX_KEYS - 1;
660 TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
665 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
670 new = V_tcp_fastopen_autokey;
671 error = sysctl_handle_int(oidp, &new, 0, req);
672 if (error == 0 && req->newptr) {
673 if (new > (INT_MAX / hz))
676 TCP_FASTOPEN_KEYS_WLOCK();
677 if (V_tcp_fastopen_server_enable) {
678 if (V_tcp_fastopen_autokey && !new)
679 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
681 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
682 new * hz, tcp_fastopen_autokey_callout,
683 &V_tcp_fastopen_autokey_ctx);
685 V_tcp_fastopen_autokey = new;
686 TCP_FASTOPEN_KEYS_WUNLOCK();
693 sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)
698 new = V_tcp_fastopen_psk_enable;
699 error = sysctl_handle_int(oidp, &new, 0, req);
700 if (error == 0 && req->newptr) {
701 if (V_tcp_fastopen_psk_enable && !new) {
702 /* enabled -> disabled */
703 TCP_FASTOPEN_KEYS_WLOCK();
704 V_tcp_fastopen_numpsks = 0;
705 V_tcp_fastopen_keys.newest_psk =
706 TCP_FASTOPEN_MAX_PSKS - 1;
707 V_tcp_fastopen_psk_enable = 0;
708 TCP_FASTOPEN_KEYS_WUNLOCK();
709 } else if (!V_tcp_fastopen_psk_enable && new) {
710 /* disabled -> enabled */
711 TCP_FASTOPEN_KEYS_WLOCK();
712 V_tcp_fastopen_psk_enable = 1;
713 TCP_FASTOPEN_KEYS_WUNLOCK();
720 sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)
725 new = V_tcp_fastopen_server_enable;
726 error = sysctl_handle_int(oidp, &new, 0, req);
727 if (error == 0 && req->newptr) {
728 if (V_tcp_fastopen_server_enable && !new) {
729 /* enabled -> disabled */
730 TCP_FASTOPEN_KEYS_WLOCK();
731 V_tcp_fastopen_numkeys = 0;
732 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
733 if (V_tcp_fastopen_autokey)
734 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
735 V_tcp_fastopen_numpsks = 0;
736 V_tcp_fastopen_keys.newest_psk =
737 TCP_FASTOPEN_MAX_PSKS - 1;
738 V_tcp_fastopen_server_enable = 0;
739 TCP_FASTOPEN_KEYS_WUNLOCK();
740 } else if (!V_tcp_fastopen_server_enable && new) {
741 /* disabled -> enabled */
742 TCP_FASTOPEN_KEYS_WLOCK();
743 if (V_tcp_fastopen_autokey &&
744 (V_tcp_fastopen_numkeys == 0)) {
745 tcp_fastopen_autokey_locked();
746 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
747 V_tcp_fastopen_autokey * hz,
748 tcp_fastopen_autokey_callout,
749 &V_tcp_fastopen_autokey_ctx);
751 V_tcp_fastopen_server_enable = 1;
752 TCP_FASTOPEN_KEYS_WUNLOCK();
759 sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)
762 uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
764 if (req->oldptr != NULL || req->oldlen != 0)
766 if (req->newptr == NULL)
768 if (req->newlen != sizeof(newkey))
770 error = SYSCTL_IN(req, newkey, sizeof(newkey));
774 TCP_FASTOPEN_KEYS_WLOCK();
775 tcp_fastopen_addkey_locked(newkey);
776 TCP_FASTOPEN_KEYS_WUNLOCK();
782 sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)
785 uint8_t newpsk[TCP_FASTOPEN_KEY_LEN];
787 if (req->oldptr != NULL || req->oldlen != 0)
789 if (req->newptr == NULL)
791 if (req->newlen != sizeof(newpsk))
793 error = SYSCTL_IN(req, newpsk, sizeof(newpsk));
797 TCP_FASTOPEN_KEYS_WLOCK();
798 tcp_fastopen_addpsk_locked(newpsk);
799 TCP_FASTOPEN_KEYS_WUNLOCK();
805 sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)
807 struct tcp_fastopen_ccache_bucket *ccb;
812 new = V_tcp_fastopen_ccache.bucket_limit;
813 error = sysctl_handle_int(oidp, &new, 0, req);
814 if (error == 0 && req->newptr) {
815 if ((new == 0) || (new > INT_MAX))
818 if (new < V_tcp_fastopen_ccache.bucket_limit) {
819 for (i = 0; i < V_tcp_fastopen_ccache.buckets;
821 ccb = &V_tcp_fastopen_ccache.base[i];
822 tcp_fastopen_ccache_bucket_trim(ccb, new);
825 V_tcp_fastopen_ccache.bucket_limit = new;
833 sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)
835 struct tcp_fastopen_ccache_bucket *ccb;
839 new = V_tcp_fastopen_client_enable;
840 error = sysctl_handle_int(oidp, &new, 0, req);
841 if (error == 0 && req->newptr) {
842 if (V_tcp_fastopen_client_enable && !new) {
843 /* enabled -> disabled */
844 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
845 ccb = &V_tcp_fastopen_ccache.base[i];
846 KASSERT(ccb->ccb_num_entries > -1,
847 ("%s: ccb->ccb_num_entries %d is negative",
848 __func__, ccb->ccb_num_entries));
849 tcp_fastopen_ccache_bucket_trim(ccb, 0);
851 V_tcp_fastopen_client_enable = 0;
852 } else if (!V_tcp_fastopen_client_enable && new) {
853 /* disabled -> enabled */
854 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
855 ccb = &V_tcp_fastopen_ccache.base[i];
857 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
858 ("%s: ccb->ccb_entries not empty", __func__));
859 KASSERT(ccb->ccb_num_entries == -1,
860 ("%s: ccb->ccb_num_entries %d not -1", __func__,
861 ccb->ccb_num_entries));
862 ccb->ccb_num_entries = 0; /* enable bucket */
865 V_tcp_fastopen_client_enable = 1;
872 tcp_fastopen_connect(struct tcpcb *tp)
875 struct tcp_fastopen_ccache_bucket *ccb;
876 struct tcp_fastopen_ccache_entry *cce;
883 cce = tcp_fastopen_ccache_lookup(&inp->inp_inc, &ccb);
885 if (cce->disable_time == 0) {
886 if ((cce->cookie_len > 0) &&
887 (tp->t_tfo_client_cookie_len ==
888 TCP_FASTOPEN_PSK_LEN)) {
890 tcp_fastopen_make_psk_cookie(
891 tp->t_tfo_cookie.client,
892 cce->cookie, cce->cookie_len);
894 tp->t_tfo_client_cookie_len = cce->cookie_len;
895 memcpy(tp->t_tfo_cookie.client, cce->cookie,
898 server_mss = cce->server_mss;
900 if (tp->t_tfo_client_cookie_len ==
901 TCP_FASTOPEN_PSK_LEN && psk_cookie) {
902 tp->t_tfo_client_cookie_len =
903 TCP_FASTOPEN_COOKIE_LEN;
904 memcpy(tp->t_tfo_cookie.client, &psk_cookie,
905 TCP_FASTOPEN_COOKIE_LEN);
907 tcp_mss(tp, server_mss ? server_mss : -1);
908 tp->snd_wnd = tp->t_maxseg;
911 * The path is disabled. Check the time and
912 * possibly re-enable.
914 now = getsbinuptime();
915 if (now - cce->disable_time >
916 ((sbintime_t)V_tcp_fastopen_path_disable_time << 32)) {
918 * Re-enable path. Force a TFO cookie
919 * request. Forget the old MSS as it may be
920 * bogus now, and we will rediscover it in
923 cce->disable_time = 0;
927 * tp->t_tfo... cookie details are already
928 * zero from the tcpcb init.
932 * Path is disabled, so disable TFO on this
935 tp->t_flags &= ~TF_FASTOPEN;
940 * snd_wnd is irrelevant since we are either forcing
941 * a TFO cookie request or disabling TFO - either
942 * way, no data with the SYN.
947 * A new entry for this path will be created when a SYN|ACK
948 * comes back, or the attempt otherwise fails.
953 * snd_wnd is irrelevant since we are forcing a TFO cookie
960 tcp_fastopen_disable_path(struct tcpcb *tp)
962 struct in_conninfo *inc = &tp->t_inpcb->inp_inc;
963 struct tcp_fastopen_ccache_bucket *ccb;
964 struct tcp_fastopen_ccache_entry *cce;
966 cce = tcp_fastopen_ccache_lookup(inc, &ccb);
971 * Preserve the existing disable time if it is already
974 if (cce->disable_time == 0)
975 cce->disable_time = getsbinuptime();
976 } else /* use invalid cookie len to create disabled entry */
977 tcp_fastopen_ccache_create(ccb, inc, 0,
978 TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL);
981 tp->t_flags &= ~TF_FASTOPEN;
985 tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss,
986 uint8_t cookie_len, uint8_t *cookie)
988 struct in_conninfo *inc = &tp->t_inpcb->inp_inc;
989 struct tcp_fastopen_ccache_bucket *ccb;
990 struct tcp_fastopen_ccache_entry *cce;
992 cce = tcp_fastopen_ccache_lookup(inc, &ccb);
994 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
995 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
996 ((cookie_len & 0x1) == 0)) {
997 cce->server_mss = mss;
998 cce->cookie_len = cookie_len;
999 memcpy(cce->cookie, cookie, cookie_len);
1000 cce->disable_time = 0;
1002 /* invalid cookie length, disable entry */
1003 cce->server_mss = 0;
1004 cce->cookie_len = 0;
1006 * Preserve the existing disable time if it is
1009 if (cce->disable_time == 0)
1010 cce->disable_time = getsbinuptime();
1013 tcp_fastopen_ccache_create(ccb, inc, mss, cookie_len, cookie);
1018 static struct tcp_fastopen_ccache_entry *
1019 tcp_fastopen_ccache_lookup(struct in_conninfo *inc,
1020 struct tcp_fastopen_ccache_bucket **ccbp)
1022 struct tcp_fastopen_ccache_bucket *ccb;
1023 struct tcp_fastopen_ccache_entry *cce;
1027 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependladdr, 4,
1028 V_tcp_fastopen_ccache.secret);
1029 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependfaddr, 4,
1031 last_word = inc->inc_fport;
1032 hash = jenkins_hash32(&last_word, 1, hash);
1033 ccb = &V_tcp_fastopen_ccache.base[hash & V_tcp_fastopen_ccache.mask];
1038 * Always returns with locked bucket.
1040 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link)
1041 if ((!(cce->af == AF_INET6) == !(inc->inc_flags & INC_ISIPV6)) &&
1042 (cce->server_port == inc->inc_ie.ie_fport) &&
1043 (((cce->af == AF_INET) &&
1044 (cce->cce_client_ip.v4.s_addr == inc->inc_laddr.s_addr) &&
1045 (cce->cce_server_ip.v4.s_addr == inc->inc_faddr.s_addr)) ||
1046 ((cce->af == AF_INET6) &&
1047 IN6_ARE_ADDR_EQUAL(&cce->cce_client_ip.v6, &inc->inc6_laddr) &&
1048 IN6_ARE_ADDR_EQUAL(&cce->cce_server_ip.v6, &inc->inc6_faddr))))
1054 static struct tcp_fastopen_ccache_entry *
1055 tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb,
1056 struct in_conninfo *inc, uint16_t mss, uint8_t cookie_len, uint8_t *cookie)
1058 struct tcp_fastopen_ccache_entry *cce;
1061 * 1. Create a new entry, or
1062 * 2. Reclaim an existing entry, or
1066 CCB_LOCK_ASSERT(ccb);
1069 if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit)
1070 cce = uma_zalloc(V_tcp_fastopen_ccache.zone, M_NOWAIT);
1074 * At bucket limit, or out of memory - reclaim last
1077 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries);
1079 /* XXX count this event */
1083 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1085 ccb->ccb_num_entries++;
1087 TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link);
1088 cce->af = (inc->inc_flags & INC_ISIPV6) ? AF_INET6 : AF_INET;
1089 if (cce->af == AF_INET) {
1090 cce->cce_client_ip.v4 = inc->inc_laddr;
1091 cce->cce_server_ip.v4 = inc->inc_faddr;
1093 cce->cce_client_ip.v6 = inc->inc6_laddr;
1094 cce->cce_server_ip.v6 = inc->inc6_faddr;
1096 cce->server_port = inc->inc_fport;
1097 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
1098 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1099 ((cookie_len & 0x1) == 0)) {
1100 cce->server_mss = mss;
1101 cce->cookie_len = cookie_len;
1102 memcpy(cce->cookie, cookie, cookie_len);
1103 cce->disable_time = 0;
1105 /* invalid cookie length, disable cce */
1106 cce->server_mss = 0;
1107 cce->cookie_len = 0;
1108 cce->disable_time = getsbinuptime();
1115 tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *ccb,
1118 struct tcp_fastopen_ccache_entry *cce, *cce_tmp;
1119 unsigned int entries;
1123 TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) {
1125 if (entries > limit)
1126 tcp_fastopen_ccache_entry_drop(cce, ccb);
1128 KASSERT(ccb->ccb_num_entries <= (int)limit,
1129 ("%s: ccb->ccb_num_entries %d exceeds limit %d", __func__,
1130 ccb->ccb_num_entries, limit));
1132 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
1133 ("%s: ccb->ccb_entries not empty", __func__));
1134 ccb->ccb_num_entries = -1; /* disable bucket */
1140 tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce,
1141 struct tcp_fastopen_ccache_bucket *ccb)
1144 CCB_LOCK_ASSERT(ccb);
1146 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1147 ccb->ccb_num_entries--;
1148 uma_zfree(V_tcp_fastopen_ccache.zone, cce);
1152 sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS)
1155 struct tcp_fastopen_ccache_bucket *ccb;
1156 struct tcp_fastopen_ccache_entry *cce;
1157 sbintime_t now, duration, limit;
1158 const int linesize = 128;
1159 int i, error, num_entries;
1162 char clt_buf[INET6_ADDRSTRLEN], srv_buf[INET6_ADDRSTRLEN];
1164 char clt_buf[INET_ADDRSTRLEN], srv_buf[INET_ADDRSTRLEN];
1167 if (jailed_without_vnet(curthread->td_ucred) != 0)
1170 /* Only allow root to read the client cookie cache */
1171 if (curthread->td_ucred->cr_uid != 0)
1175 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1176 ccb = &V_tcp_fastopen_ccache.base[i];
1178 if (ccb->ccb_num_entries > 0)
1179 num_entries += ccb->ccb_num_entries;
1182 sbuf_new(&sb, NULL, linesize * (num_entries + 1), SBUF_INCLUDENUL);
1185 "\nLocal IP address Remote IP address Port MSS"
1186 " Disabled Cookie\n");
1188 now = getsbinuptime();
1189 limit = (sbintime_t)V_tcp_fastopen_path_disable_time << 32;
1190 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1191 ccb = &V_tcp_fastopen_ccache.base[i];
1193 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) {
1194 if (cce->disable_time != 0) {
1195 duration = now - cce->disable_time;
1196 if (limit >= duration)
1197 duration = limit - duration;
1203 "%-20s %-20s %5u %5u ",
1204 inet_ntop(cce->af, &cce->cce_client_ip,
1205 clt_buf, sizeof(clt_buf)),
1206 inet_ntop(cce->af, &cce->cce_server_ip,
1207 srv_buf, sizeof(srv_buf)),
1208 ntohs(cce->server_port),
1211 sbuf_printf(&sb, "%7ds ", sbintime_getsec(duration));
1213 sbuf_printf(&sb, "%8s ", "No");
1214 for (j = 0; j < cce->cookie_len; j++)
1215 sbuf_printf(&sb, "%02x", cce->cookie[j]);
1216 sbuf_putc(&sb, '\n');
1220 error = sbuf_finish(&sb);
1222 error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));