2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate)
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include "opt_inet6.h"
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/buf_ring.h>
34 #include <sys/epoch.h>
36 #include <sys/filedesc.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
40 #include <sys/module.h>
43 #include <sys/protosw.h>
44 #include <sys/rmlock.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49 #include <sys/sockio.h>
50 #include <sys/sysctl.h>
53 #include <machine/atomic.h>
57 #include <net/if_clone.h>
58 #include <net/if_types.h>
59 #include <net/if_var.h>
60 #include <net/if_private.h>
61 #include <net/netisr.h>
62 #include <net/route/nhop.h>
64 #include <netinet/in.h>
65 #include <netinet/in_fib.h>
66 #include <netinet/ip.h>
67 #include <netinet/ip6.h>
68 #include <netinet/ip_var.h>
69 #include <netinet/udp.h>
70 #include <netinet/udp_var.h>
72 #include <netinet6/ip6_var.h>
73 #include <netinet6/in6_fib.h>
75 #include <machine/in_cksum.h>
77 #include <opencrypto/cryptodev.h>
81 struct ovpn_kkey_dir {
87 enum ovpn_key_cipher cipher;
88 crypto_session_t cryptoid;
90 struct mtx replay_mtx;
92 * Last seen gapless sequence number. New rx seq numbers must be
93 * strictly higher than this.
98 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */
103 struct ovpn_kkey_dir *encrypt;
104 struct ovpn_kkey_dir *decrypt;
109 struct ovpn_keepalive {
114 struct ovpn_wire_header {
115 uint32_t opcode; /* opcode, key id, peer id */
117 uint8_t auth_tag[16];
120 struct ovpn_peer_counters {
126 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t))
128 struct ovpn_notification {
129 enum ovpn_notif_type type;
132 /* Delete notification */
133 enum ovpn_del_reason del_reason;
134 struct ovpn_peer_counters counters;
140 RB_ENTRY(ovpn_kpeer) tree;
144 struct ovpn_softc *sc;
145 struct sockaddr_storage local;
146 struct sockaddr_storage remote;
149 struct in6_addr vpn6;
151 struct ovpn_kkey keys[2];
153 enum ovpn_del_reason del_reason;
154 struct ovpn_keepalive keepalive;
155 uint32_t *last_active;
156 struct callout ping_send;
157 struct callout ping_rcv;
159 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE];
162 struct ovpn_counters {
163 uint64_t lost_ctrl_pkts_in;
164 uint64_t lost_ctrl_pkts_out;
165 uint64_t lost_data_pkts_in;
166 uint64_t lost_data_pkts_out;
167 uint64_t nomem_data_pkts_in;
168 uint64_t nomem_data_pkts_out;
169 uint64_t received_ctrl_pkts;
170 uint64_t received_data_pkts;
171 uint64_t sent_ctrl_pkts;
172 uint64_t sent_data_pkts;
174 uint64_t transport_bytes_sent;
175 uint64_t transport_bytes_received;
176 uint64_t tunnel_bytes_sent;
177 uint64_t tunnel_bytes_received;
179 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t))
181 RB_HEAD(ovpn_kpeers, ovpn_kpeer);
189 struct ovpn_kpeers peers;
191 /* Pending notification */
192 struct buf_ring *notifring;
194 counter_u64_t counters[OVPN_COUNTER_SIZE];
196 struct epoch_context epoch_ctx;
199 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t);
200 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *,
201 const struct sockaddr *, void *);
202 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *,
203 struct ovpn_kpeer *, struct rm_priotracker *);
204 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *);
205 static int ovpn_get_af(struct mbuf *);
206 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *);
207 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t);
208 static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *);
210 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare);
211 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare);
213 #define OVPN_MTU_MIN 576
214 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \
215 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header))
217 #define OVPN_OP_DATA_V2 0x09
218 #define OVPN_OP_SHIFT 3
219 #define OVPN_SEQ_ROTATE 0x80000000
221 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner);
222 #define V_ovpn_cloner VNET(ovpn_cloner)
224 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \
225 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker
226 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp)
227 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp)
228 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock)
229 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock)
230 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED)
231 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED)
232 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED)
233 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED)
235 #define OVPN_COUNTER(sc, name) \
236 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)])
237 #define OVPN_PEER_COUNTER(peer, name) \
238 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \
241 #define OVPN_COUNTER_ADD(sc, name, val) \
242 counter_u64_add(OVPN_COUNTER(sc, name), val)
243 #define OVPN_PEER_COUNTER_ADD(p, name, val) \
244 counter_u64_add(OVPN_PEER_COUNTER(p, name), val)
246 #define TO_IN(x) ((struct sockaddr_in *)(x))
247 #define TO_IN6(x) ((struct sockaddr_in6 *)(x))
249 SDT_PROVIDER_DEFINE(if_ovpn);
250 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *");
251 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *");
252 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *");
254 static const char ovpnname[] = "ovpn";
255 static const char ovpngroupname[] = "openvpn";
257 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface");
259 SYSCTL_DECL(_net_link);
260 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
261 "OpenVPN DCO Interface");
262 VNET_DEFINE_STATIC(int, replay_protection) = 0;
263 #define V_replay_protection VNET(replay_protection)
264 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW,
265 &VNET_NAME(replay_protection), 0, "Validate sequence numbers");
267 VNET_DEFINE_STATIC(int, async_crypto);
268 #define V_async_crypto VNET(async_crypto)
269 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto,
270 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0,
271 "Use asynchronous mode to parallelize crypto jobs.");
273 VNET_DEFINE_STATIC(int, async_netisr_queue);
274 #define V_async_netisr_queue VNET(async_netisr_queue)
275 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue,
276 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0,
277 "Use netisr_queue() rather than netisr_dispatch().");
280 ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b)
282 return (a->peerid - b->peerid);
285 static struct ovpn_kpeer *
286 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid)
294 return (RB_FIND(ovpn_kpeers, &sc->peers, &p));
297 static struct ovpn_kpeer *
298 ovpn_find_only_peer(struct ovpn_softc *sc)
302 return (RB_ROOT(&sc->peers));
306 ovpn_get_port(struct sockaddr_storage *s)
308 switch (s->ss_family) {
310 struct sockaddr_in *in = (struct sockaddr_in *)s;
311 return (in->sin_port);
314 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s;
315 return (in6->sin6_port);
318 panic("Unsupported address family %d", s->ss_family);
323 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa)
327 if (! nvlist_exists_number(nvl, "af"))
329 if (! nvlist_exists_binary(nvl, "address"))
331 if (! nvlist_exists_number(nvl, "port"))
334 af = nvlist_get_number(nvl, "af");
339 struct sockaddr_in *in = (struct sockaddr_in *)sa;
341 const void *addr = nvlist_get_binary(nvl, "address", &len);
343 if (len != sizeof(in->sin_addr))
346 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr));
347 in->sin_port = nvlist_get_number(nvl, "port");
353 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa;
355 const void *addr = nvlist_get_binary(nvl, "address", &len);
356 in6->sin6_family = af;
357 if (len != sizeof(in6->sin6_addr))
360 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr));
361 in6->sin6_port = nvlist_get_number(nvl, "port");
373 ovpn_has_peers(struct ovpn_softc *sc)
377 return (sc->peercount > 0);
381 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
390 has_peers = ovpn_has_peers(sc);
392 /* Only remove the tunnel function if we're releasing the socket for
395 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL);
404 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
406 struct ovpn_notification *n;
410 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT);
414 n->peerid = peer->peerid;
415 n->type = OVPN_NOTIF_DEL_PEER;
416 n->del_reason = peer->del_reason;
418 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in));
419 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out));
420 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in));
421 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out));
423 if (buf_ring_enqueue(sc->notifring, n) != 0) {
425 } else if (sc->so != NULL) {
426 /* Wake up userspace */
427 sc->so->so_error = EAGAIN;
434 ovpn_notify_key_rotation(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
436 struct ovpn_notification *n;
438 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT | M_ZERO);
442 n->peerid = peer->peerid;
443 n->type = OVPN_NOTIF_ROTATE_KEY;
445 if (buf_ring_enqueue(sc->notifring, n) != 0) {
447 } else if (sc->so != NULL) {
448 /* Wake up userspace */
449 sc->so->so_error = EAGAIN;
456 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked)
458 struct ovpn_softc *sc;
460 CURVNET_ASSERT_SET();
462 atomic_add_int(&peer->refcount, -1);
464 if (atomic_load_int(&peer->refcount) > 0)
472 /* Might have changed before we acquired the lock. */
473 if (atomic_load_int(&peer->refcount) > 0) {
481 /* The peer should have been removed from the list already. */
482 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL);
484 ovpn_notify_del_peer(sc, peer);
486 for (int i = 0; i < 2; i++) {
487 ovpn_free_kkey_dir(peer->keys[i].encrypt);
488 ovpn_free_kkey_dir(peer->keys[i].decrypt);
491 ovpn_rele_so(sc, peer);
493 callout_stop(&peer->ping_send);
494 callout_stop(&peer->ping_rcv);
495 uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
503 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl)
506 struct epoch_tracker et;
508 struct sockaddr_storage remote;
509 struct ovpn_kpeer *peer = NULL;
510 struct file *fp = NULL;
511 struct sockaddr *name = NULL;
512 struct ovpn_softc *sc = ifp->if_softc;
513 struct thread *td = curthread;
514 struct socket *so = NULL;
522 if (! nvlist_exists_number(nvl, "peerid"))
525 if (! nvlist_exists_number(nvl, "fd"))
528 if (! nvlist_exists_nvlist(nvl, "remote"))
531 peerid = nvlist_get_number(nvl, "peerid");
533 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"),
538 fd = nvlist_get_number(nvl, "fd");
540 /* Look up the userspace process and use the fd to find the socket. */
541 ret = getsock(td, fd, &cap_connect_rights, &fp);
547 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO);
548 peer->peerid = peerid;
551 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO);
552 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK);
554 if (nvlist_exists_binary(nvl, "vpn_ipv4")) {
556 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len);
557 if (len != sizeof(peer->vpn4)) {
561 memcpy(&peer->vpn4, addr, len);
564 if (nvlist_exists_binary(nvl, "vpn_ipv6")) {
566 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len);
567 if (len != sizeof(peer->vpn6)) {
571 memcpy(&peer->vpn6, addr, len);
574 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK);
575 callout_init_rm(&peer->ping_rcv, &sc->lock, 0);
577 ret = so->so_proto->pr_sockaddr(so, &name);
581 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) {
585 if (name->sa_family != remote.ss_family) {
590 memcpy(&peer->local, name, name->sa_len);
591 memcpy(&peer->remote, &remote, sizeof(remote));
592 free(name, M_SONAME);
595 if (peer->local.ss_family == AF_INET6 &&
596 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) {
597 /* V4 mapped address, so treat this as v4, not v6. */
598 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local);
599 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote);
603 if (peer->local.ss_family == AF_INET6 &&
604 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) {
606 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum,
607 &TO_IN6(&peer->remote)->sin6_addr,
608 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL);
617 /* Disallow peer id re-use. */
618 if (ovpn_find_peer(sc, peerid) != NULL) {
623 /* Make sure this is really a UDP socket. */
624 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) {
629 /* Must be the same socket as for other peers on this interface. */
630 if (sc->so != NULL && so != sc->so)
636 /* Insert the peer into the list. */
637 RB_INSERT(ovpn_kpeers, &sc->peers, peer);
641 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc);
643 /* Fine, another peer already set the input function. */
647 RB_REMOVE(ovpn_kpeers, &sc->peers, peer);
659 free(name, M_SONAME);
660 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE);
661 uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
671 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
673 struct ovpn_kpeer *tmp __diagused;
676 CURVNET_ASSERT_SET();
678 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer);
680 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer);
685 ovpn_peer_release_ref(peer, true);
691 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl)
693 struct ovpn_softc *sc = ifp->if_softc;
694 struct ovpn_kpeer *peer;
703 if (! nvlist_exists_number(nvl, "peerid"))
706 peerid = nvlist_get_number(nvl, "peerid");
708 peer = ovpn_find_peer(sc, peerid);
712 peer->del_reason = OVPN_DEL_REASON_REQUESTED;
713 ret = _ovpn_del_peer(sc, peer);
719 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp,
722 struct crypto_session_params csp;
723 struct ovpn_kkey_dir *kdir;
724 const char *ciphername;
725 enum ovpn_key_cipher cipher;
726 const void *key, *iv;
727 size_t keylen = 0, ivlen = 0;
730 if (! nvlist_exists_string(nvl, "cipher"))
732 ciphername = nvlist_get_string(nvl, "cipher");
734 if (strcmp(ciphername, "none") == 0)
735 cipher = OVPN_CIPHER_ALG_NONE;
736 else if (strcmp(ciphername, "AES-256-GCM") == 0 ||
737 strcmp(ciphername, "AES-192-GCM") == 0 ||
738 strcmp(ciphername, "AES-128-GCM") == 0)
739 cipher = OVPN_CIPHER_ALG_AES_GCM;
740 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0)
741 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305;
745 if (cipher != OVPN_CIPHER_ALG_NONE) {
746 if (! nvlist_exists_binary(nvl, "key"))
748 key = nvlist_get_binary(nvl, "key", &keylen);
749 if (keylen > sizeof(kdir->key))
752 if (! nvlist_exists_binary(nvl, "iv"))
754 iv = nvlist_get_binary(nvl, "iv", &ivlen);
759 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN,
762 kdir->cipher = cipher;
763 kdir->keylen = keylen;
765 memcpy(kdir->key, key, keylen);
766 kdir->noncelen = ivlen;
767 memcpy(kdir->nonce, iv, ivlen);
769 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) {
771 bzero(&csp, sizeof(csp));
772 csp.csp_mode = CSP_MODE_AEAD;
774 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305)
775 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
777 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
779 csp.csp_flags |= CSP_F_SEPARATE_AAD;
781 csp.csp_cipher_klen = kdir->keylen;
782 csp.csp_cipher_key = kdir->key;
783 csp.csp_ivlen = 96 / 8;
785 error = crypto_newsession(&kdir->cryptoid, &csp,
786 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
793 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF);
800 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir)
805 mtx_destroy(&kdir->replay_mtx);
807 crypto_freesession(kdir->cryptoid);
812 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl)
814 struct ovpn_softc *sc = ifp->if_softc;
815 struct ovpn_kkey_dir *enc, *dec;
816 struct ovpn_kpeer *peer;
817 int slot, keyid, peerid;
823 if (! nvlist_exists_number(nvl, "slot"))
825 slot = nvlist_get_number(nvl, "slot");
827 if (! nvlist_exists_number(nvl, "keyid"))
829 keyid = nvlist_get_number(nvl, "keyid");
831 if (! nvlist_exists_number(nvl, "peerid"))
833 peerid = nvlist_get_number(nvl, "peerid");
835 if (slot != OVPN_KEY_SLOT_PRIMARY &&
836 slot != OVPN_KEY_SLOT_SECONDARY)
839 if (! nvlist_exists_nvlist(nvl, "encrypt") ||
840 ! nvlist_exists_nvlist(nvl, "decrypt"))
843 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt"));
847 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt"));
849 ovpn_free_kkey_dir(enc);
855 peer = ovpn_find_peer(sc, peerid);
857 ovpn_free_kkey_dir(dec);
858 ovpn_free_kkey_dir(enc);
863 ovpn_free_kkey_dir(peer->keys[slot].encrypt);
864 ovpn_free_kkey_dir(peer->keys[slot].decrypt);
866 peer->keys[slot].encrypt = enc;
867 peer->keys[slot].decrypt = dec;
869 peer->keys[slot].keyid = keyid;
870 peer->keys[slot].peerid = peerid;
878 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot)
882 if (peer->keys[slot].encrypt == NULL)
885 if (peer->keys[slot].decrypt == NULL)
892 ovpn_start(struct ifnet *ifp)
894 struct ovpn_softc *sc = ifp->if_softc;
898 ifp->if_flags |= IFF_UP;
899 ifp->if_drv_flags |= IFF_DRV_RUNNING;
900 if_link_state_change(ifp, LINK_STATE_UP);
908 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl)
910 struct ovpn_softc *sc = ifp->if_softc;
911 struct ovpn_kpeer *peer;
912 struct ovpn_kkey tmpkey;
918 if (! nvlist_exists_number(nvl, "peerid"))
923 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
929 /* Check that we have a second key to swap to. */
930 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY);
936 tmpkey = peer->keys[0];
937 peer->keys[0] = peer->keys[1];
938 peer->keys[1] = tmpkey;
946 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl)
948 enum ovpn_key_slot slot;
949 struct ovpn_kpeer *peer;
950 struct ovpn_softc *sc = ifp->if_softc;
955 if (! nvlist_exists_number(nvl, "peerid"))
958 if (! nvlist_exists_number(nvl, "slot"))
960 slot = nvlist_get_number(nvl, "slot");
962 if (slot != OVPN_KEY_SLOT_PRIMARY &&
963 slot != OVPN_KEY_SLOT_SECONDARY)
968 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
974 ovpn_free_kkey_dir(peer->keys[slot].encrypt);
975 ovpn_free_kkey_dir(peer->keys[slot].decrypt);
977 peer->keys[slot].encrypt = NULL;
978 peer->keys[slot].decrypt = NULL;
980 peer->keys[slot].keyid = 0;
981 peer->keys[slot].peerid = 0;
989 ovpn_send_ping(void *arg)
991 static const uint8_t ping_str[] = {
992 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb,
993 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48
996 struct epoch_tracker et;
997 struct ovpn_kpeer *peer = arg;
998 struct ovpn_softc *sc = peer->sc;
1003 /* Ensure we repeat! */
1004 callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1005 ovpn_send_ping, peer);
1007 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR);
1011 m_copyback(m, 0, sizeof(ping_str), ping_str);
1012 m->m_len = m->m_pkthdr.len = sizeof(ping_str);
1014 CURVNET_SET(sc->ifp->if_vnet);
1015 NET_EPOCH_ENTER(et);
1016 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL);
1022 ovpn_timeout(void *arg)
1024 struct ovpn_kpeer *peer = arg;
1025 struct ovpn_softc *sc = peer->sc;
1026 uint32_t last, _last_active;
1034 _last_active = *zpcpu_get_cpu(peer->last_active, cpu);
1035 if (_last_active > last)
1036 last = _last_active;
1039 if (last + peer->keepalive.timeout > time_uptime) {
1040 callout_reset(&peer->ping_rcv,
1041 (peer->keepalive.timeout - (time_uptime - last)) * hz,
1042 ovpn_timeout, peer);
1046 CURVNET_SET(sc->ifp->if_vnet);
1047 peer->del_reason = OVPN_DEL_REASON_TIMEOUT;
1048 ret = _ovpn_del_peer(sc, peer);
1054 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl)
1056 struct ovpn_softc *sc = ifp->if_softc;
1057 struct ovpn_kpeer *peer;
1062 if (! nvlist_exists_number(nvl, "interval") ||
1063 ! nvlist_exists_number(nvl, "timeout") ||
1064 ! nvlist_exists_number(nvl, "peerid"))
1069 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
1075 peer->keepalive.interval = nvlist_get_number(nvl, "interval");
1076 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout");
1078 if (peer->keepalive.interval > 0)
1079 callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1080 ovpn_send_ping, peer);
1081 if (peer->keepalive.timeout > 0)
1082 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz,
1083 ovpn_timeout, peer);
1091 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl)
1093 struct ovpn_softc *sc = ifp->if_softc;
1099 if (! nvlist_exists_number(nvl, "ifmode") )
1102 ifmode = nvlist_get_number(nvl, "ifmode");
1106 /* deny this if UP */
1107 if (ifp->if_flags & IFF_UP) {
1112 switch (ifmode & ~IFF_MULTICAST) {
1113 case IFF_POINTOPOINT:
1116 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST);
1117 ifp->if_flags |= ifmode;
1130 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd)
1132 struct ovpn_softc *sc = ifp->if_softc;
1133 uint8_t *buf = NULL;
1134 nvlist_t *nvl = NULL;
1137 if (ifd->ifd_len != 0) {
1138 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE)
1141 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK);
1143 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len);
1149 nvl = nvlist_unpack(buf, ifd->ifd_len, 0);
1156 switch (ifd->ifd_cmd) {
1158 ret = ovpn_new_peer(ifp, nvl);
1162 ret = ovpn_del_peer(ifp, nvl);
1166 ret = ovpn_set_key(ifp, nvl);
1168 case OVPN_START_VPN:
1169 ret = ovpn_start(ifp);
1171 case OVPN_SWAP_KEYS:
1172 ret = ovpn_swap_keys(ifp, nvl);
1175 ret = ovpn_del_key(ifp, nvl);
1178 ret = ovpn_set_peer(ifp, nvl);
1180 case OVPN_SET_IFMODE:
1181 ret = ovpn_set_ifmode(ifp, nvl);
1187 nvlist_destroy(nvl);
1192 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in,
1197 nvl = nvlist_create(0);
1201 nvlist_add_number(nvl, "in", counter_u64_fetch(in));
1202 nvlist_add_number(nvl, "out", counter_u64_fetch(out));
1204 nvlist_add_nvlist(parent, name, nvl);
1206 nvlist_destroy(nvl);
1212 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl)
1217 nvl = nvlist_create(0);
1221 #define OVPN_COUNTER_OUT(name, in, out) \
1223 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \
1224 OVPN_COUNTER(sc, out)); \
1229 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out);
1230 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out);
1231 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in,
1232 nomem_data_pkts_out);
1233 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts);
1234 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts);
1235 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received,
1236 tunnel_bytes_received);
1237 OVPN_COUNTER_OUT("transport", transport_bytes_received,
1238 transport_bytes_received);
1239 #undef OVPN_COUNTER_OUT
1246 nvlist_destroy(nvl);
1251 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl)
1253 struct ovpn_kpeer *peer;
1254 nvlist_t *nvpeer = NULL;
1259 *nvl = nvlist_create(0);
1263 #define OVPN_PEER_COUNTER_OUT(name, in, out) \
1265 ret = ovpn_add_counters(nvpeer, name, \
1266 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \
1272 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1273 nvpeer = nvlist_create(0);
1274 if (nvpeer == NULL) {
1276 nvlist_destroy(*nvl);
1281 nvlist_add_number(nvpeer, "peerid", peer->peerid);
1283 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out);
1284 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out);
1286 nvlist_append_nvlist_array(*nvl, "peers", nvpeer);
1287 nvlist_destroy(nvpeer);
1289 #undef OVPN_PEER_COUNTER_OUT
1295 nvlist_destroy(nvpeer);
1296 nvlist_destroy(*nvl);
1302 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1306 nvl = nvlist_create(0);
1310 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring));
1318 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n)
1322 nvl = nvlist_create(0);
1326 nvlist_add_number(nvl, "in", n->counters.pkt_in);
1327 nvlist_add_number(nvl, "out", n->counters.pkt_out);
1329 nvlist_add_nvlist(parent, "packets", nvl);
1330 nvlist_destroy(nvl);
1332 nvl = nvlist_create(0);
1336 nvlist_add_number(nvl, "in", n->counters.bytes_in);
1337 nvlist_add_number(nvl, "out", n->counters.bytes_out);
1339 nvlist_add_nvlist(parent, "bytes", nvl);
1340 nvlist_destroy(nvl);
1344 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1346 struct ovpn_notification *n;
1349 /* Check if we have notifications pending. */
1350 n = buf_ring_dequeue_mc(sc->notifring);
1354 nvl = nvlist_create(0);
1359 nvlist_add_number(nvl, "peerid", n->peerid);
1360 nvlist_add_number(nvl, "notification", n->type);
1361 if (n->type == OVPN_NOTIF_DEL_PEER) {
1362 nvlist_add_number(nvl, "del_reason", n->del_reason);
1364 /* No error handling, because we want to send the notification
1365 * even if we can't attach the counters. */
1366 ovpn_notif_add_counters(nvl, n);
1376 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd)
1378 struct ovpn_softc *sc = ifp->if_softc;
1379 nvlist_t *nvl = NULL;
1382 switch (ifd->ifd_cmd) {
1383 case OVPN_GET_STATS:
1384 error = ovpn_get_stats(sc, &nvl);
1386 case OVPN_GET_PEER_STATS:
1387 error = ovpn_get_peer_stats(sc, &nvl);
1390 error = ovpn_poll_pkt(sc, &nvl);
1393 error = opvn_get_pkt(sc, &nvl);
1401 void *packed = NULL;
1406 packed = nvlist_pack(nvl, &len);
1408 nvlist_destroy(nvl);
1412 if (len > ifd->ifd_len) {
1413 free(packed, M_NVLIST);
1414 nvlist_destroy(nvl);
1418 error = copyout(packed, ifd->ifd_data, len);
1421 free(packed, M_NVLIST);
1422 nvlist_destroy(nvl);
1429 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1434 CURVNET_ASSERT_SET();
1439 error = priv_check(curthread, PRIV_NET_OVPN);
1447 ifd = (struct ifdrv *)data;
1448 error = ovpn_ioctl_set(ifp, ifd);
1451 ifd = (struct ifdrv *)data;
1452 error = ovpn_ioctl_get(ifp, ifd);
1455 struct ifreq *ifr = (struct ifreq *)data;
1456 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX)
1459 ifp->if_mtu = ifr->ifr_mtu;
1476 ovpn_encrypt_tx_cb(struct cryptop *crp)
1478 struct epoch_tracker et;
1479 struct ovpn_kpeer *peer = crp->crp_opaque;
1480 struct ovpn_softc *sc = peer->sc;
1481 struct mbuf *m = crp->crp_buf.cb_mbuf;
1485 CURVNET_SET(sc->ifp->if_vnet);
1486 NET_EPOCH_ENTER(et);
1488 if (crp->crp_etype != 0) {
1489 crypto_freereq(crp);
1490 ovpn_peer_release_ref(peer, false);
1493 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1498 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1500 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header);
1501 ret = ovpn_encap(sc, peer->peerid, m);
1503 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1504 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1507 crypto_freereq(crp);
1508 ovpn_peer_release_ref(peer, false);
1517 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m,
1518 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq,
1519 struct rm_priotracker *_ovpn_lock_trackerp)
1526 /* Replay protection. */
1527 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) {
1529 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1535 *zpcpu_get(peer->last_active) = time_uptime;
1540 OVPN_COUNTER_ADD(sc, received_data_pkts, 1);
1541 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len);
1542 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1);
1543 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len);
1545 /* Receive the packet on our interface. */
1546 m->m_pkthdr.rcvif = sc->ifp;
1548 /* Clear checksum flags in case the real hardware set them. */
1549 m->m_pkthdr.csum_flags = 0;
1551 /* Clear mbuf tags & flags */
1552 m_tag_delete_nonpersistent(m);
1555 /* Ensure we can read the first byte. */
1558 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
1563 * Check for address family, and disregard any control packets (e.g.
1566 af = ovpn_get_af(m);
1568 BPF_MTAP2(sc->ifp, &af, sizeof(af), m);
1569 if (V_async_netisr_queue)
1570 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m);
1572 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m);
1574 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1579 static struct ovpn_kkey *
1580 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer,
1581 const struct ovpn_wire_header *ohdr)
1583 struct ovpn_kkey *key = NULL;
1588 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07;
1590 if (peer->keys[0].keyid == keyid)
1591 key = &peer->keys[0];
1592 else if (peer->keys[1].keyid == keyid)
1593 key = &peer->keys[1];
1599 ovpn_decrypt_rx_cb(struct cryptop *crp)
1601 struct epoch_tracker et;
1602 struct ovpn_softc *sc = crp->crp_opaque;
1603 struct mbuf *m = crp->crp_buf.cb_mbuf;
1604 struct ovpn_kkey *key;
1605 struct ovpn_kpeer *peer;
1606 struct ovpn_wire_header *ohdr;
1613 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1615 if (crp->crp_etype != 0) {
1616 crypto_freereq(crp);
1617 atomic_add_int(&sc->refcount, -1);
1618 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1624 CURVNET_SET(sc->ifp->if_vnet);
1626 ohdr = mtodo(m, sizeof(struct udphdr));
1628 peerid = ntohl(ohdr->opcode) & 0x00ffffff;
1629 peer = ovpn_find_peer(sc, peerid);
1631 /* No such peer. Drop packet. */
1632 crypto_freereq(crp);
1633 atomic_add_int(&sc->refcount, -1);
1635 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1641 key = ovpn_find_key(sc, peer, ohdr);
1643 crypto_freereq(crp);
1644 atomic_add_int(&sc->refcount, -1);
1646 * Has this key been removed between us starting the decrypt
1650 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1656 /* Now remove the outer headers */
1657 m_adj_decap(m, sizeof(struct udphdr) +
1658 sizeof(struct ovpn_wire_header));
1660 NET_EPOCH_ENTER(et);
1661 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp);
1663 OVPN_UNLOCK_ASSERT(sc);
1667 crypto_freereq(crp);
1668 atomic_add_int(&sc->refcount, -1);
1674 ovpn_get_af(struct mbuf *m)
1677 struct ip6_hdr *ip6;
1680 * We should pullup, but we're only interested in the first byte, so
1681 * that'll always be contiguous.
1683 ip = mtod(m, struct ip *);
1684 if (ip->ip_v == IPVERSION)
1687 ip6 = mtod(m, struct ip6_hdr *);
1688 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION)
1695 static struct ovpn_kpeer *
1696 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr)
1698 struct ovpn_kpeer *peer = NULL;
1702 /* TODO: Add a second RB so we can look up by IP. */
1703 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1704 if (addr.s_addr == peer->vpn4.s_addr)
1713 static struct ovpn_kpeer *
1714 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr)
1716 struct ovpn_kpeer *peer = NULL;
1720 /* TODO: Add a third RB so we can look up by IPv6 address. */
1721 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1722 if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0)
1730 static struct ovpn_kpeer *
1731 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0,
1732 const struct sockaddr *dst)
1734 struct ovpn_kpeer *peer = NULL;
1740 /* Shortcut if we're a client (or are a server and have only one client). */
1741 if (sc->peercount == 1)
1742 return (ovpn_find_only_peer(sc));
1745 af = dst->sa_family;
1747 af = ovpn_get_af(*m0);
1752 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst;
1753 struct nhop_object *nh;
1754 const struct in_addr *ip_dst;
1757 ip_dst = &sa->sin_addr;
1761 *m0 = m_pullup(*m0, sizeof(struct ip));
1764 ip = mtod(*m0, struct ip *);
1765 ip_dst = &ip->ip_dst;
1768 peer = ovpn_find_peer_by_ip(sc, *ip_dst);
1769 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer);
1771 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0,
1773 if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1774 peer = ovpn_find_peer_by_ip(sc,
1775 nh->gw4_sa.sin_addr);
1776 SDT_PROBE2(if_ovpn, tx, route, ip4,
1777 &nh->gw4_sa.sin_addr, peer);
1785 const struct sockaddr_in6 *sa6 =
1786 (const struct sockaddr_in6 *)dst;
1787 struct nhop_object *nh;
1788 const struct in6_addr *ip6_dst;
1791 ip6_dst = &sa6->sin6_addr;
1793 struct ip6_hdr *ip6;
1795 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr));
1798 ip6 = mtod(*m0, struct ip6_hdr *);
1799 ip6_dst = &ip6->ip6_dst;
1802 peer = ovpn_find_peer_by_ip6(sc, ip6_dst);
1803 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer);
1805 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0,
1807 if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1808 peer = ovpn_find_peer_by_ip6(sc,
1809 &nh->gw6_sa.sin6_addr);
1810 SDT_PROBE2(if_ovpn, tx, route, ip6,
1811 &nh->gw6_sa.sin6_addr, peer);
1823 ovpn_transmit(struct ifnet *ifp, struct mbuf *m)
1825 return (ifp->if_output(ifp, m, NULL, NULL));
1829 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m,
1830 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp)
1832 struct ovpn_wire_header *ohdr;
1833 struct ovpn_kkey *key;
1834 struct ovpn_softc *sc;
1835 struct cryptop *crp;
1838 size_t len, ovpn_hdr_len;
1846 tunnel_len = m->m_pkthdr.len;
1848 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY];
1849 if (key->encrypt == NULL) {
1850 if (_ovpn_lock_trackerp != NULL)
1856 af = ovpn_get_af(m);
1857 /* Don't capture control packets. */
1859 BPF_MTAP2(ifp, &af, sizeof(af), m);
1861 len = m->m_pkthdr.len;
1862 MPASS(len <= ifp->if_mtu);
1864 ovpn_hdr_len = sizeof(struct ovpn_wire_header);
1865 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE)
1866 ovpn_hdr_len -= 16; /* No auth tag. */
1868 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT);
1870 if (_ovpn_lock_trackerp != NULL)
1872 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1875 ohdr = mtod(m, struct ovpn_wire_header *);
1876 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid;
1877 ohdr->opcode <<= 24;
1878 ohdr->opcode |= key->peerid;
1879 ohdr->opcode = htonl(ohdr->opcode);
1881 seq64 = atomic_fetchadd_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1);
1882 if (seq64 == OVPN_SEQ_ROTATE) {
1883 ovpn_notify_key_rotation(sc, peer);
1884 } else if (seq64 > UINT32_MAX) {
1885 /* We've wrapped, give up on this packet. */
1886 if (_ovpn_lock_trackerp != NULL)
1888 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1890 /* Let's avoid (very unlikely, but still) wraparounds of the
1891 * 64-bit counter taking us back to 0. */
1892 atomic_store_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq,
1898 seq = htonl(seq64 & UINT32_MAX);
1901 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1);
1902 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len);
1904 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) {
1905 ret = ovpn_encap(sc, peer->peerid, m);
1906 if (_ovpn_lock_trackerp != NULL)
1909 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1910 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1915 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT);
1917 if (_ovpn_lock_trackerp != NULL)
1919 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1924 /* Encryption covers only the payload, not the header. */
1925 crp->crp_payload_start = sizeof(*ohdr);
1926 crp->crp_payload_length = len;
1927 crp->crp_op = CRYPTO_OP_ENCRYPT;
1930 * AAD data covers the ovpn_wire_header minus the auth
1933 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
1934 crp->crp_aad = ohdr;
1935 crp->crp_aad_start = 0;
1936 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST;
1937 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag);
1939 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1940 memcpy(crp->crp_iv, &seq, sizeof(seq));
1941 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce,
1942 key->encrypt->noncelen);
1944 crypto_use_mbuf(crp, m);
1945 crp->crp_flags |= CRYPTO_F_CBIFSYNC;
1946 crp->crp_callback = ovpn_encrypt_tx_cb;
1947 crp->crp_opaque = peer;
1949 atomic_add_int(&peer->refcount, 1);
1950 if (_ovpn_lock_trackerp != NULL)
1953 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED);
1955 ret = crypto_dispatch(crp);
1957 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1964 * Note: Expects to hold the read lock on entry, and will release it itself.
1967 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m)
1970 struct ovpn_kpeer *peer;
1978 peer = ovpn_find_peer(sc, peerid);
1979 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) {
1981 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1986 len = m->m_pkthdr.len;
1988 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT);
1991 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1995 udp = mtod(m, struct udphdr *);
1997 MPASS(peer->local.ss_family == peer->remote.ss_family);
1999 udp->uh_sport = ovpn_get_port(&peer->local);
2000 udp->uh_dport = ovpn_get_port(&peer->remote);
2001 udp->uh_ulen = htons(sizeof(struct udphdr) + len);
2003 switch (peer->remote.ss_family) {
2006 struct sockaddr_in *in_local = TO_IN(&peer->local);
2007 struct sockaddr_in *in_remote = TO_IN(&peer->remote);
2011 * This requires knowing the source IP, which we don't. Happily
2012 * we're allowed to keep this at 0, and the checksum won't do
2013 * anything the crypto won't already do.
2017 /* Set the checksum flags so we recalculate checksums. */
2018 m->m_pkthdr.csum_flags |= CSUM_IP;
2019 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2021 M_PREPEND(m, sizeof(struct ip), M_NOWAIT);
2024 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2027 ip = mtod(m, struct ip *);
2030 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) +
2033 ip->ip_ttl = V_ip_defttl;
2034 ip->ip_p = IPPROTO_UDP;
2036 if (in_local->sin_port != 0)
2037 ip->ip_src = in_local->sin_addr;
2039 ip->ip_src.s_addr = INADDR_ANY;
2040 ip->ip_dst = in_remote->sin_addr;
2043 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
2045 return (ip_output(m, NULL, NULL, 0, NULL, NULL));
2050 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local);
2051 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote);
2052 struct ip6_hdr *ip6;
2054 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT);
2057 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2060 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp));
2063 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2067 ip6 = mtod(m, struct ip6_hdr *);
2069 ip6->ip6_vfc = IPV6_VERSION;
2070 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK;
2071 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) +
2073 ip6->ip6_nxt = IPPROTO_UDP;
2074 ip6->ip6_hlim = V_ip6_defhlim;
2076 memcpy(&ip6->ip6_src, &in6_local->sin6_addr,
2077 sizeof(ip6->ip6_src));
2078 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr,
2079 sizeof(ip6->ip6_dst));
2081 udp = mtodo(m, sizeof(*ip6));
2082 udp->uh_sum = in6_cksum_pseudo(ip6,
2083 m->m_pkthdr.len - sizeof(struct ip6_hdr),
2086 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6;
2087 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2090 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
2092 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL,
2097 panic("Unsupported address family %d",
2098 peer->remote.ss_family);
2103 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
2106 struct ovpn_softc *sc;
2107 struct ovpn_kpeer *peer;
2115 SDT_PROBE1(if_ovpn, tx, transmit, start, m);
2117 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) {
2118 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2125 * Only obey 'dst' (i.e. the gateway) if no route is supplied.
2126 * That's our indication that we're being called through pf's route-to,
2127 * and we should route according to 'dst' instead. We can't do so
2128 * consistently, because the usual openvpn configuration sets the first
2129 * non-server IP in the subnet as the gateway. If we always use that
2130 * one we'd end up routing all traffic to the first client.
2131 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but
2132 * only then, we should treat 'dst' as the destination. */
2133 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL);
2135 /* No destination. */
2136 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2142 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp));
2146 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq)
2150 mtx_lock(&key->replay_mtx);
2152 /* Sequence number must be strictly greater than rx_seq */
2153 if (seq <= key->rx_seq) {
2154 mtx_unlock(&key->replay_mtx);
2158 /* Large jump. The packet authenticated okay, so just accept that. */
2159 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) {
2162 mtx_unlock(&key->replay_mtx);
2167 if ((seq == key->rx_seq + 1) && key->rx_window == 0) {
2169 mtx_unlock(&key->replay_mtx);
2173 d = seq - key->rx_seq - 1;
2175 if (key->rx_window & ((uint64_t)1 << d)) {
2177 mtx_unlock(&key->replay_mtx);
2181 key->rx_window |= (uint64_t)1 << d;
2183 while (key->rx_window & 1) {
2185 key->rx_window >>= 1;
2188 mtx_unlock(&key->replay_mtx);
2193 static struct ovpn_kpeer *
2194 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off)
2196 struct ovpn_wire_header ohdr;
2198 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag);
2202 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen))
2205 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr);
2207 peerid = ntohl(ohdr.opcode) & 0x00ffffff;
2209 return (ovpn_find_peer(sc, peerid));
2213 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp,
2214 const struct sockaddr *sa, void *ctx)
2216 struct ovpn_softc *sc = ctx;
2217 struct ovpn_wire_header tmphdr;
2218 struct ovpn_wire_header *ohdr;
2219 struct udphdr *uhdr;
2220 struct ovpn_kkey *key;
2221 struct cryptop *crp;
2222 struct ovpn_kpeer *peer;
2231 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off);
2233 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2237 peer = ovpn_peer_from_mbuf(sc, m, off);
2243 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) {
2249 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr);
2251 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT;
2252 if (op != OVPN_OP_DATA_V2) {
2253 /* Control packet? */
2258 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen);
2261 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2266 * Simplify things by getting rid of the preceding headers, we don't
2269 m_adj_decap(m, off);
2272 ohdr = mtodo(m, sizeof(*uhdr));
2274 key = ovpn_find_key(sc, peer, ohdr);
2275 if (key == NULL || key->decrypt == NULL) {
2277 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2282 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) {
2283 /* Now remove the outer headers */
2284 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen);
2286 ohdr = mtodo(m, sizeof(*uhdr));
2288 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq),
2289 _ovpn_lock_trackerp);
2290 OVPN_UNLOCK_ASSERT(sc);
2294 ohdrlen += sizeof(ohdr->auth_tag);
2296 m = m_pullup(m, sizeof(*uhdr) + ohdrlen);
2299 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2303 ohdr = mtodo(m, sizeof(*uhdr));
2306 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT);
2308 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2314 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr);
2315 crp->crp_payload_length = ntohs(uhdr->uh_ulen) -
2316 sizeof(*uhdr) - sizeof(*ohdr);
2317 crp->crp_op = CRYPTO_OP_DECRYPT;
2319 /* AAD validation. */
2320 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2321 crp->crp_aad = ohdr;
2322 crp->crp_aad_start = 0;
2323 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST;
2324 crp->crp_digest_start = sizeof(struct udphdr) +
2325 offsetof(struct ovpn_wire_header, auth_tag);
2327 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
2328 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq));
2329 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce,
2330 key->decrypt->noncelen);
2332 crypto_use_mbuf(crp, m);
2333 crp->crp_flags |= CRYPTO_F_CBIFSYNC;
2334 crp->crp_callback = ovpn_decrypt_rx_cb;
2335 crp->crp_opaque = sc;
2337 atomic_add_int(&sc->refcount, 1);
2340 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED);
2342 ret = crypto_dispatch(crp);
2344 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2351 ovpn_qflush(struct ifnet *ifp __unused)
2357 ovpn_flush_rxring(struct ovpn_softc *sc)
2359 struct ovpn_notification *n;
2363 while (! buf_ring_empty(sc->notifring)) {
2364 n = buf_ring_dequeue_sc(sc->notifring);
2371 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused,
2372 char *unused __unused)
2374 struct ovpn_softc *sc = ifp->if_softc;
2375 struct ovpn_kpeer *peer, *tmppeer;
2380 /* Flush keys & configuration. */
2381 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) {
2382 peer->del_reason = OVPN_DEL_REASON_REQUESTED;
2383 ret = _ovpn_del_peer(sc, peer);
2387 ovpn_flush_rxring(sc);
2394 ovpn_clone_match(struct if_clone *ifc, const char *name)
2397 * Allow all names that start with 'ovpn', specifically because pfSense
2398 * uses ovpnc1 / ovpns2
2400 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0);
2404 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len,
2405 struct ifc_data *ifd, struct ifnet **ifpp)
2407 struct ovpn_softc *sc;
2410 int error, unit, wildcard;
2412 /* Try to see if a special unit was requested. */
2413 error = ifc_name2unit(name, &unit);
2416 wildcard = (unit < 0);
2418 error = ifc_alloc_unit(ifc, &unit);
2423 * If no unit had been given, we need to adjust the ifName.
2425 for (dp = name; *dp != '\0'; dp++);
2427 error = snprintf(dp, len - (dp - name), "%d", unit);
2428 if (error > len - (dp - name)) {
2429 /* ifName too long. */
2430 ifc_free_unit(ifc, unit);
2436 /* Make sure it doesn't already exist. */
2437 if (ifunit(name) != NULL)
2440 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO);
2441 sc->ifp = if_alloc(IFT_ENC);
2442 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE);
2445 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
2447 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK);
2451 strlcpy(ifp->if_xname, name, IFNAMSIZ);
2452 ifp->if_dname = ovpngroupname;
2453 ifp->if_dunit = unit;
2455 ifp->if_addrlen = 0;
2457 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
2458 ifp->if_ioctl = ovpn_ioctl;
2459 ifp->if_transmit = ovpn_transmit;
2460 ifp->if_output = ovpn_output;
2461 ifp->if_qflush = ovpn_qflush;
2463 ifp->if_reassign = ovpn_reassign;
2465 ifp->if_capabilities |= IFCAP_LINKSTATE;
2466 ifp->if_capenable |= IFCAP_LINKSTATE;
2469 bpfattach(ifp, DLT_NULL, sizeof(uint32_t));
2476 ovpn_clone_destroy_cb(struct epoch_context *ctx)
2478 struct ovpn_softc *sc;
2480 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx);
2482 MPASS(sc->peercount == 0);
2483 MPASS(RB_EMPTY(&sc->peers));
2485 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE);
2492 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags)
2494 struct ovpn_softc *sc;
2495 struct ovpn_kpeer *peer, *tmppeer;
2500 unit = ifp->if_dunit;
2504 if (atomic_load_int(&sc->refcount) > 0) {
2509 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) {
2510 peer->del_reason = OVPN_DEL_REASON_REQUESTED;
2511 ret = _ovpn_del_peer(sc, peer);
2515 ovpn_flush_rxring(sc);
2516 buf_ring_free(sc->notifring, M_OVPN);
2522 ifp->if_softc = NULL;
2524 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx);
2526 if (unit != IF_DUNIT_NONE)
2527 ifc_free_unit(ifc, unit);
2529 NET_EPOCH_DRAIN_CALLBACKS();
2535 vnet_ovpn_init(const void *unused __unused)
2537 struct if_clone_addreq req = {
2538 .match_f = ovpn_clone_match,
2539 .create_f = ovpn_clone_create,
2540 .destroy_f = ovpn_clone_destroy,
2542 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req);
2544 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
2545 vnet_ovpn_init, NULL);
2548 vnet_ovpn_uninit(const void *unused __unused)
2550 if_clone_detach(V_ovpn_cloner);
2552 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
2553 vnet_ovpn_uninit, NULL);
2556 ovpnmodevent(module_t mod, int type, void *data)
2560 /* Done in vnet_ovpn_init() */
2563 /* Done in vnet_ovpn_uninit() */
2566 return (EOPNOTSUPP);
2572 static moduledata_t ovpn_mod = {
2578 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
2579 MODULE_VERSION(if_ovpn, 1);