2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate)
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include "opt_inet6.h"
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/buf_ring.h>
34 #include <sys/epoch.h>
36 #include <sys/filedesc.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
40 #include <sys/module.h>
43 #include <sys/protosw.h>
44 #include <sys/rmlock.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49 #include <sys/sockio.h>
50 #include <sys/sysctl.h>
53 #include <machine/atomic.h>
57 #include <net/if_clone.h>
58 #include <net/if_types.h>
59 #include <net/if_var.h>
60 #include <net/netisr.h>
61 #include <net/route/nhop.h>
63 #include <netinet/in.h>
64 #include <netinet/in_fib.h>
65 #include <netinet/ip.h>
66 #include <netinet/ip6.h>
67 #include <netinet/ip_var.h>
68 #include <netinet/udp.h>
69 #include <netinet/udp_var.h>
71 #include <netinet6/ip6_var.h>
72 #include <netinet6/in6_fib.h>
74 #include <machine/in_cksum.h>
76 #include <opencrypto/cryptodev.h>
80 struct ovpn_kkey_dir {
86 enum ovpn_key_cipher cipher;
87 crypto_session_t cryptoid;
89 struct mtx replay_mtx;
91 * Last seen gapless sequence number. New rx seq numbers must be
92 * strictly higher than this.
95 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */
100 struct ovpn_kkey_dir *encrypt;
101 struct ovpn_kkey_dir *decrypt;
106 struct ovpn_keepalive {
111 struct ovpn_wire_header {
112 uint32_t opcode; /* opcode, key id, peer id */
114 uint8_t auth_tag[16];
117 struct ovpn_notification {
118 enum ovpn_notif_type type;
128 struct ovpn_softc *sc;
129 struct sockaddr_storage local;
130 struct sockaddr_storage remote;
133 struct in6_addr vpn6;
135 struct ovpn_kkey keys[2];
138 struct ovpn_keepalive keepalive;
139 uint32_t *last_active;
140 struct callout ping_send;
141 struct callout ping_rcv;
144 #define OVPN_MAX_PEERS 128
146 struct ovpn_counters {
147 uint64_t lost_ctrl_pkts_in;
148 uint64_t lost_ctrl_pkts_out;
149 uint64_t lost_data_pkts_in;
150 uint64_t lost_data_pkts_out;
151 uint64_t nomem_data_pkts_in;
152 uint64_t nomem_data_pkts_out;
153 uint64_t received_ctrl_pkts;
154 uint64_t received_data_pkts;
155 uint64_t sent_ctrl_pkts;
156 uint64_t sent_data_pkts;
158 uint64_t transport_bytes_sent;
159 uint64_t transport_bytes_received;
160 uint64_t tunnel_bytes_sent;
161 uint64_t tunnel_bytes_received;
163 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t))
171 struct ovpn_kpeer *peers[OVPN_MAX_PEERS]; /* XXX Hard limit for now? */
173 /* Pending packets */
174 struct buf_ring *rxring;
175 struct buf_ring *notifring;
177 counter_u64_t counters[OVPN_COUNTER_SIZE];
179 struct epoch_context epoch_ctx;
182 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t);
183 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *,
184 const struct sockaddr *, void *);
185 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *,
186 struct ovpn_kpeer *, struct rm_priotracker *);
187 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *);
188 static int ovpn_get_af(struct mbuf *);
189 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *);
190 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t);
192 #define OVPN_MTU_MIN 576
193 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \
194 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header))
196 #define OVPN_OP_DATA_V2 0x09
197 #define OVPN_OP_SHIFT 3
199 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner);
200 #define V_ovpn_cloner VNET(ovpn_cloner)
202 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \
203 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker
204 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp)
205 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp)
206 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock)
207 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock)
208 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED)
209 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED)
210 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED)
211 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED)
213 #define OVPN_COUNTER_ADD(sc, name, val) \
214 counter_u64_add(sc->counters[offsetof(struct ovpn_counters, name) / \
215 sizeof(uint64_t)], val)
217 #define TO_IN(x) ((struct sockaddr_in *)(x))
218 #define TO_IN6(x) ((struct sockaddr_in6 *)(x))
220 SDT_PROVIDER_DEFINE(if_ovpn);
221 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *");
222 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *");
223 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *");
225 static const char ovpnname[] = "ovpn";
226 static const char ovpngroupname[] = "openvpn";
228 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface");
230 SYSCTL_DECL(_net_link);
231 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
232 "OpenVPN DCO Interface");
233 VNET_DEFINE_STATIC(int, replay_protection) = 0;
234 #define V_replay_protection VNET(replay_protection)
235 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW,
236 &VNET_NAME(replay_protection), 0, "Validate sequence numbers");
238 static struct ovpn_kpeer *
239 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid)
241 struct ovpn_kpeer *p = NULL;
245 for (int i = 0; i < OVPN_MAX_PEERS; i++) {
250 if (p->peerid == peerid) {
259 static struct ovpn_kpeer *
260 ovpn_find_only_peer(struct ovpn_softc *sc)
264 for (int i = 0; i < OVPN_MAX_PEERS; i++) {
265 if (sc->peers[i] == NULL)
267 return (sc->peers[i]);
276 ovpn_get_port(struct sockaddr_storage *s)
278 switch (s->ss_family) {
280 struct sockaddr_in *in = (struct sockaddr_in *)s;
281 return (in->sin_port);
284 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s;
285 return (in6->sin6_port);
288 panic("Unsupported address family %d", s->ss_family);
293 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa)
297 if (! nvlist_exists_number(nvl, "af"))
299 if (! nvlist_exists_binary(nvl, "address"))
301 if (! nvlist_exists_number(nvl, "port"))
304 af = nvlist_get_number(nvl, "af");
309 struct sockaddr_in *in = (struct sockaddr_in *)sa;
311 const void *addr = nvlist_get_binary(nvl, "address", &len);
313 if (len != sizeof(in->sin_addr))
316 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr));
317 in->sin_port = nvlist_get_number(nvl, "port");
323 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa;
325 const void *addr = nvlist_get_binary(nvl, "address", &len);
326 in6->sin6_family = af;
327 if (len != sizeof(in6->sin6_addr))
330 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr));
331 in6->sin6_port = nvlist_get_number(nvl, "port");
343 ovpn_has_peers(struct ovpn_softc *sc)
347 return (sc->peercount > 0);
351 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
360 has_peers = ovpn_has_peers(sc);
362 /* Only remove the tunnel function if we're releasing the socket for
365 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL);
374 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
376 struct ovpn_notification *n;
380 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT);
384 n->peerid = peer->peerid;
385 n->type = OVPN_NOTIF_DEL_PEER;
386 if (buf_ring_enqueue(sc->notifring, n) != 0) {
388 } else if (sc->so != NULL) {
389 /* Wake up userspace */
390 sc->so->so_error = EAGAIN;
397 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked)
399 struct ovpn_softc *sc;
401 atomic_add_int(&peer->refcount, -1);
403 if (atomic_load_int(&peer->refcount) > 0)
411 /* Might have changed before we acquired the lock. */
412 if (atomic_load_int(&peer->refcount) > 0) {
418 /* The peer should have been removed from the list already. */
419 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL);
421 ovpn_notify_del_peer(sc, peer);
423 for (int i = 0; i < 2; i++) {
424 ovpn_free_kkey_dir(peer->keys[i].encrypt);
425 ovpn_free_kkey_dir(peer->keys[i].decrypt);
428 ovpn_rele_so(sc, peer);
430 callout_stop(&peer->ping_send);
431 callout_stop(&peer->ping_rcv);
432 uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
440 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl)
443 struct epoch_tracker et;
445 struct sockaddr_storage remote;
446 struct ovpn_kpeer *peer = NULL;
447 struct file *fp = NULL;
448 struct sockaddr *name = NULL;
449 struct ovpn_softc *sc = ifp->if_softc;
450 struct thread *td = curthread;
451 struct socket *so = NULL;
460 if (! nvlist_exists_number(nvl, "peerid"))
463 if (! nvlist_exists_number(nvl, "fd"))
466 if (! nvlist_exists_nvlist(nvl, "remote"))
469 peerid = nvlist_get_number(nvl, "peerid");
471 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"),
476 fd = nvlist_get_number(nvl, "fd");
478 /* Look up the userspace process and use the fd to find the socket. */
479 ret = getsock_cap(td, fd, &cap_connect_rights, &fp,
486 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO);
487 peer->peerid = peerid;
491 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO);
493 if (nvlist_exists_binary(nvl, "vpn_ipv4")) {
495 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len);
496 if (len != sizeof(peer->vpn4)) {
500 memcpy(&peer->vpn4, addr, len);
503 if (nvlist_exists_binary(nvl, "vpn_ipv6")) {
505 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len);
506 if (len != sizeof(peer->vpn6)) {
510 memcpy(&peer->vpn6, addr, len);
513 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK);
514 callout_init_rm(&peer->ping_rcv, &sc->lock, 0);
516 ret = so->so_proto->pr_sockaddr(so, &name);
520 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) {
524 if (name->sa_family != remote.ss_family) {
529 memcpy(&peer->local, name, name->sa_len);
530 memcpy(&peer->remote, &remote, sizeof(remote));
531 free(name, M_SONAME);
534 if (peer->local.ss_family == AF_INET6 &&
535 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) {
536 /* V4 mapped address, so treat this as v4, not v6. */
537 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local);
538 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote);
542 if (peer->local.ss_family == AF_INET6 &&
543 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) {
545 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum,
546 &TO_IN6(&peer->remote)->sin6_addr,
547 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL);
556 /* Disallow peer id re-use. */
557 if (ovpn_find_peer(sc, peerid) != NULL) {
562 /* Make sure this is really a UDP socket. */
563 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) {
568 /* Must be the same socket as for other peers on this interface. */
569 if (sc->so != NULL && so != sc->so)
575 /* Insert the peer into the list. */
576 for (i = 0; i < OVPN_MAX_PEERS; i++) {
577 if (sc->peers[i] != NULL)
580 MPASS(sc->peers[i] == NULL);
586 if (i == OVPN_MAX_PEERS) {
591 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc);
593 /* Fine, another peer already set the input function. */
609 free(name, M_SONAME);
610 uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
620 _ovpn_del_peer(struct ovpn_softc *sc, uint32_t peerid)
622 struct ovpn_kpeer *peer;
627 for (i = 0; i < OVPN_MAX_PEERS; i++) {
628 if (sc->peers[i] == NULL)
630 if (sc->peers[i]->peerid != peerid)
636 if (i == OVPN_MAX_PEERS)
642 ovpn_peer_release_ref(peer, true);
648 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl)
650 struct ovpn_softc *sc = ifp->if_softc;
659 if (! nvlist_exists_number(nvl, "peerid"))
662 peerid = nvlist_get_number(nvl, "peerid");
664 ret = _ovpn_del_peer(sc, peerid);
670 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp,
673 struct crypto_session_params csp;
674 struct ovpn_kkey_dir *kdir;
675 const char *ciphername;
676 enum ovpn_key_cipher cipher;
677 const void *key, *iv;
678 size_t keylen = 0, ivlen = 0;
681 if (! nvlist_exists_string(nvl, "cipher"))
683 ciphername = nvlist_get_string(nvl, "cipher");
685 if (strcmp(ciphername, "none") == 0)
686 cipher = OVPN_CIPHER_ALG_NONE;
687 else if (strcmp(ciphername, "AES-256-GCM") == 0)
688 cipher = OVPN_CIPHER_ALG_AES_GCM;
689 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0)
690 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305;
694 if (cipher != OVPN_CIPHER_ALG_NONE) {
695 if (! nvlist_exists_binary(nvl, "key"))
697 key = nvlist_get_binary(nvl, "key", &keylen);
698 if (keylen > sizeof(kdir->key))
701 if (! nvlist_exists_binary(nvl, "iv"))
703 iv = nvlist_get_binary(nvl, "iv", &ivlen);
708 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN,
711 kdir->cipher = cipher;
712 kdir->keylen = keylen;
713 memcpy(kdir->key, key, keylen);
714 kdir->noncelen = ivlen;
715 memcpy(kdir->nonce, iv, ivlen);
717 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) {
719 bzero(&csp, sizeof(csp));
720 csp.csp_mode = CSP_MODE_AEAD;
722 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305)
723 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
725 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
727 csp.csp_flags |= CSP_F_SEPARATE_AAD;
729 csp.csp_cipher_klen = kdir->keylen;
730 csp.csp_cipher_key = kdir->key;
731 csp.csp_ivlen = 96 / 8;
733 error = crypto_newsession(&kdir->cryptoid, &csp,
734 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
741 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF);
748 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir)
753 mtx_destroy(&kdir->replay_mtx);
755 crypto_freesession(kdir->cryptoid);
760 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl)
762 struct ovpn_softc *sc = ifp->if_softc;
763 struct ovpn_kkey_dir *enc, *dec;
764 struct ovpn_kpeer *peer;
765 int slot, keyid, peerid;
771 if (! nvlist_exists_number(nvl, "slot"))
773 slot = nvlist_get_number(nvl, "slot");
775 if (! nvlist_exists_number(nvl, "keyid"))
777 keyid = nvlist_get_number(nvl, "keyid");
779 if (! nvlist_exists_number(nvl, "peerid"))
781 peerid = nvlist_get_number(nvl, "peerid");
783 if (slot != OVPN_KEY_SLOT_PRIMARY &&
784 slot != OVPN_KEY_SLOT_SECONDARY)
787 if (! nvlist_exists_nvlist(nvl, "encrypt") ||
788 ! nvlist_exists_nvlist(nvl, "decrypt"))
791 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt"));
795 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt"));
797 ovpn_free_kkey_dir(enc);
803 peer = ovpn_find_peer(sc, peerid);
805 ovpn_free_kkey_dir(dec);
806 ovpn_free_kkey_dir(enc);
811 ovpn_free_kkey_dir(peer->keys[slot].encrypt);
812 ovpn_free_kkey_dir(peer->keys[slot].decrypt);
814 peer->keys[slot].encrypt = enc;
815 peer->keys[slot].decrypt = dec;
817 peer->keys[slot].keyid = keyid;
818 peer->keys[slot].peerid = peerid;
826 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot)
830 if (peer->keys[slot].encrypt == NULL)
833 if (peer->keys[slot].decrypt == NULL)
840 ovpn_start(struct ifnet *ifp)
842 struct ovpn_softc *sc = ifp->if_softc;
846 ifp->if_flags |= IFF_UP;
847 ifp->if_drv_flags |= IFF_DRV_RUNNING;
848 if_link_state_change(ifp, LINK_STATE_UP);
856 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl)
858 struct ovpn_softc *sc = ifp->if_softc;
859 struct ovpn_kpeer *peer;
860 struct ovpn_kkey tmpkey;
866 if (! nvlist_exists_number(nvl, "peerid"))
871 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
877 /* Check that we have a second key to swap to. */
878 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY);
884 tmpkey = peer->keys[0];
885 peer->keys[0] = peer->keys[1];
886 peer->keys[1] = tmpkey;
894 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl)
896 enum ovpn_key_slot slot;
897 struct ovpn_kpeer *peer;
898 struct ovpn_softc *sc = ifp->if_softc;
903 if (! nvlist_exists_number(nvl, "peerid"))
906 if (! nvlist_exists_number(nvl, "slot"))
908 slot = nvlist_get_number(nvl, "slot");
910 if (slot != OVPN_KEY_SLOT_PRIMARY &&
911 slot != OVPN_KEY_SLOT_SECONDARY)
916 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
922 ovpn_free_kkey_dir(peer->keys[slot].encrypt);
923 ovpn_free_kkey_dir(peer->keys[slot].decrypt);
925 peer->keys[slot].encrypt = NULL;
926 peer->keys[slot].decrypt = NULL;
928 peer->keys[slot].keyid = 0;
929 peer->keys[slot].peerid = 0;
937 ovpn_send_pkt(struct ifnet *ifp, const nvlist_t *nvl)
939 struct epoch_tracker et;
940 struct ovpn_softc *sc = ifp->if_softc;
950 if (! nvlist_exists_binary(nvl, "packet"))
952 pkt = nvlist_get_binary(nvl, "packet", &pktlen);
954 if (! nvlist_exists_number(nvl, "peerid"))
957 peerid = nvlist_get_number(nvl, "peerid");
960 * Check that userspace isn't giving us a data packet. That might lead
961 * to IV re-use, which would be bad.
963 if ((pkt[0] >> OVPN_OP_SHIFT) == OVPN_OP_DATA_V2)
966 m = m_get2(pktlen, M_WAITOK, MT_DATA, M_PKTHDR);
970 m->m_len = m->m_pkthdr.len = pktlen;
971 m_copyback(m, 0, m->m_len, pkt);
973 /* Now prepend IP/UDP headers and transmit the mbuf. */
975 ret = ovpn_encap(sc, peerid, m);
978 OVPN_COUNTER_ADD(sc, sent_ctrl_pkts, 1);
980 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_out, 1);
986 ovpn_send_ping(void *arg)
988 static const uint8_t ping_str[] = {
989 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb,
990 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48
993 struct epoch_tracker et;
994 struct ovpn_kpeer *peer = arg;
995 struct ovpn_softc *sc = peer->sc;
1000 /* Ensure we repeat! */
1001 callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1002 ovpn_send_ping, peer);
1004 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR);
1008 m_copyback(m, 0, sizeof(ping_str), ping_str);
1009 m->m_len = m->m_pkthdr.len = sizeof(ping_str);
1011 CURVNET_SET(sc->ifp->if_vnet);
1012 NET_EPOCH_ENTER(et);
1013 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL);
1019 ovpn_timeout(void *arg)
1021 struct ovpn_kpeer *peer = arg;
1022 struct ovpn_softc *sc = peer->sc;
1023 uint32_t last, _last_active;
1031 _last_active = *zpcpu_get_cpu(peer->last_active, cpu);
1032 if (_last_active > last)
1033 last = _last_active;
1036 if (last + peer->keepalive.timeout > time_uptime) {
1037 callout_reset(&peer->ping_rcv,
1038 (peer->keepalive.timeout - (time_uptime - last)) * hz,
1039 ovpn_timeout, peer);
1043 CURVNET_SET(sc->ifp->if_vnet);
1044 ret = _ovpn_del_peer(sc, peer->peerid);
1050 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl)
1052 struct ovpn_softc *sc = ifp->if_softc;
1053 struct ovpn_kpeer *peer;
1058 if (! nvlist_exists_number(nvl, "interval") ||
1059 ! nvlist_exists_number(nvl, "timeout") ||
1060 ! nvlist_exists_number(nvl, "peerid"))
1065 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
1071 peer->keepalive.interval = nvlist_get_number(nvl, "interval");
1072 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout");
1074 if (peer->keepalive.interval > 0)
1075 callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1076 ovpn_send_ping, peer);
1077 if (peer->keepalive.timeout > 0)
1078 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz,
1079 ovpn_timeout, peer);
1087 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd)
1089 struct ovpn_softc *sc = ifp->if_softc;
1090 uint8_t *buf = NULL;
1091 nvlist_t *nvl = NULL;
1094 if (ifd->ifd_len != 0) {
1095 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE)
1098 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK);
1100 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len);
1106 nvl = nvlist_unpack(buf, ifd->ifd_len, 0);
1113 switch (ifd->ifd_cmd) {
1115 ret = ovpn_new_peer(ifp, nvl);
1119 ret = ovpn_del_peer(ifp, nvl);
1123 ret = ovpn_set_key(ifp, nvl);
1125 case OVPN_START_VPN:
1126 ret = ovpn_start(ifp);
1128 case OVPN_SWAP_KEYS:
1129 ret = ovpn_swap_keys(ifp, nvl);
1132 ret = ovpn_del_key(ifp, nvl);
1135 ret = ovpn_send_pkt(ifp, nvl);
1138 ret = ovpn_set_peer(ifp, nvl);
1144 nvlist_destroy(nvl);
1149 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in,
1154 nvl = nvlist_create(0);
1158 nvlist_add_number(nvl, "in", counter_u64_fetch(in));
1159 nvlist_add_number(nvl, "out", counter_u64_fetch(out));
1161 nvlist_add_nvlist(parent, name, nvl);
1163 nvlist_destroy(nvl);
1169 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl)
1174 nvl = nvlist_create(0);
1178 #define OVPN_COUNTER_OUT(name, in, out) \
1180 ret = ovpn_add_counters(nvl, name, \
1181 sc->counters[offsetof(struct ovpn_counters, in) / \
1182 sizeof(uint64_t)], \
1183 sc->counters[offsetof(struct ovpn_counters, out) / \
1184 sizeof(uint64_t)]); \
1189 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out);
1190 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out);
1191 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in,
1192 nomem_data_pkts_out);
1193 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts);
1194 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts);
1195 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received,
1196 tunnel_bytes_received);
1197 OVPN_COUNTER_OUT("transport", transport_bytes_received,
1198 transport_bytes_received);
1199 #undef OVPN_COUNTER_OUT
1206 nvlist_destroy(nvl);
1211 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1215 nvl = nvlist_create(0);
1219 nvlist_add_number(nvl, "pending",
1220 buf_ring_count(sc->rxring) + buf_ring_count(sc->notifring));
1228 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1230 struct ovpn_notification *n;
1231 struct ovpn_wire_header *ohdr;
1238 /* Check if we have notifications pending. */
1239 n = buf_ring_dequeue_mc(sc->notifring);
1241 nvl = nvlist_create(0);
1246 nvlist_add_number(nvl, "peerid", n->peerid);
1247 nvlist_add_number(nvl, "notification", n->type);
1255 /* Queued packet. */
1256 m = buf_ring_dequeue_mc(sc->rxring);
1260 mlength = m_length(m, NULL);
1261 buf = malloc(mlength, M_NVLIST, M_WAITOK);
1262 m_copydata(m, 0, mlength, buf);
1263 ohdr = (struct ovpn_wire_header *)buf;
1264 peerid = ntohl(ohdr->opcode) & 0x00ffffff;
1266 nvl = nvlist_create(0);
1268 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1);
1270 free(buf, M_NVLIST);
1274 nvlist_move_binary(nvl, "packet", buf, mlength);
1276 nvlist_add_number(nvl, "peerid", peerid);
1286 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd)
1288 struct ovpn_softc *sc = ifp->if_softc;
1289 nvlist_t *nvl = NULL;
1292 switch (ifd->ifd_cmd) {
1293 case OVPN_GET_STATS:
1294 error = ovpn_get_stats(sc, &nvl);
1297 error = ovpn_poll_pkt(sc, &nvl);
1300 error = opvn_get_pkt(sc, &nvl);
1308 void *packed = NULL;
1313 packed = nvlist_pack(nvl, &len);
1315 nvlist_destroy(nvl);
1319 if (len > ifd->ifd_len) {
1320 free(packed, M_NVLIST);
1321 nvlist_destroy(nvl);
1325 error = copyout(packed, ifd->ifd_data, len);
1328 free(packed, M_NVLIST);
1329 nvlist_destroy(nvl);
1336 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1344 error = priv_check(curthread, PRIV_NET_OVPN);
1352 ifd = (struct ifdrv *)data;
1353 error = ovpn_ioctl_set(ifp, ifd);
1356 ifd = (struct ifdrv *)data;
1357 error = ovpn_ioctl_get(ifp, ifd);
1360 struct ifreq *ifr = (struct ifreq *)data;
1361 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX)
1364 ifp->if_mtu = ifr->ifr_mtu;
1381 ovpn_encrypt_tx_cb(struct cryptop *crp)
1383 struct ovpn_kpeer *peer = crp->crp_opaque;
1384 struct ovpn_softc *sc = peer->sc;
1385 struct mbuf *m = crp->crp_buf.cb_mbuf;
1388 if (crp->crp_etype != 0) {
1389 crypto_freereq(crp);
1390 ovpn_peer_release_ref(peer, false);
1391 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1396 CURVNET_SET(sc->ifp->if_vnet);
1398 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1400 ret = ovpn_encap(sc, peer->peerid, m);
1402 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1403 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, m->m_pkthdr.len -
1404 sizeof(struct ovpn_wire_header));
1409 crypto_freereq(crp);
1410 ovpn_peer_release_ref(peer, false);
1416 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m,
1417 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq,
1418 struct rm_priotracker *_ovpn_lock_trackerp)
1425 /* Replay protection. */
1426 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) {
1428 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1434 *zpcpu_get(peer->last_active) = time_uptime;
1439 OVPN_COUNTER_ADD(sc, received_data_pkts, 1);
1440 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len);
1442 /* Receive the packet on our interface. */
1443 m->m_pkthdr.rcvif = sc->ifp;
1445 /* Clear checksum flags in case the real hardware set them. */
1446 m->m_pkthdr.csum_flags = 0;
1448 /* Ensure we can read the first byte. */
1451 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
1456 * Check for address family, and disregard any control packets (e.g.
1459 af = ovpn_get_af(m);
1461 BPF_MTAP2(sc->ifp, &af, sizeof(af), m);
1462 ret = netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6,
1466 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1471 static struct ovpn_kkey *
1472 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer,
1473 const struct ovpn_wire_header *ohdr)
1475 struct ovpn_kkey *key = NULL;
1480 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07;
1482 if (peer->keys[0].keyid == keyid)
1483 key = &peer->keys[0];
1484 else if (peer->keys[1].keyid == keyid)
1485 key = &peer->keys[1];
1491 ovpn_decrypt_rx_cb(struct cryptop *crp)
1493 struct ovpn_softc *sc = crp->crp_opaque;
1494 struct mbuf *m = crp->crp_buf.cb_mbuf;
1495 struct ovpn_kkey *key;
1496 struct ovpn_kpeer *peer;
1497 struct ovpn_wire_header *ohdr;
1504 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1506 if (crp->crp_etype != 0) {
1507 crypto_freereq(crp);
1508 atomic_add_int(&sc->refcount, -1);
1509 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1515 CURVNET_SET(sc->ifp->if_vnet);
1517 ohdr = mtodo(m, sizeof(struct udphdr));
1519 peerid = ntohl(ohdr->opcode) & 0x00ffffff;
1520 peer = ovpn_find_peer(sc, peerid);
1522 /* No such peer. Drop packet. */
1523 crypto_freereq(crp);
1524 atomic_add_int(&sc->refcount, -1);
1526 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1532 key = ovpn_find_key(sc, peer, ohdr);
1534 crypto_freereq(crp);
1535 atomic_add_int(&sc->refcount, -1);
1537 * Has this key been removed between us starting the decrypt
1541 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1547 /* Now remove the outer headers */
1548 m_adj_decap(m, sizeof(struct udphdr) +
1549 sizeof(struct ovpn_wire_header));
1551 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp);
1552 OVPN_UNLOCK_ASSERT(sc);
1556 crypto_freereq(crp);
1557 atomic_add_int(&sc->refcount, -1);
1562 static uint8_t EMPTY_BUFFER[AES_BLOCK_LEN];
1565 ovpn_get_af(struct mbuf *m)
1568 struct ip6_hdr *ip6;
1571 * We should pullup, but we're only interested in the first byte, so
1572 * that'll always be contiguous.
1574 ip = mtod(m, struct ip *);
1575 if (ip->ip_v == IPVERSION)
1578 ip6 = mtod(m, struct ip6_hdr *);
1579 if (ip6->ip6_vfc == IPV6_VERSION)
1586 static struct ovpn_kpeer *
1587 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr)
1589 struct ovpn_kpeer *peer = NULL;
1593 for (int i = 0; i < OVPN_MAX_PEERS; i++) {
1594 if (sc->peers[i] == NULL)
1596 if (addr.s_addr == sc->peers[i]->vpn4.s_addr) {
1597 peer = sc->peers[i];
1607 static struct ovpn_kpeer *
1608 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr)
1610 struct ovpn_kpeer *peer = NULL;
1614 for (int i = 0; i < OVPN_MAX_PEERS; i++) {
1615 if (sc->peers[i] == NULL)
1617 if (memcmp(addr, &sc->peers[i]->vpn6, sizeof(*addr)) == 0) {
1618 peer = sc->peers[i];
1627 static struct ovpn_kpeer *
1628 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0,
1629 const struct sockaddr *dst)
1631 struct ovpn_kpeer *peer = NULL;
1637 /* Shortcut if we're a client (or are a server and have only one client). */
1638 if (sc->peercount == 1)
1639 return (ovpn_find_only_peer(sc));
1642 af = dst->sa_family;
1644 af = ovpn_get_af(*m0);
1649 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst;
1650 struct nhop_object *nh;
1651 const struct in_addr *ip_dst;
1654 ip_dst = &sa->sin_addr;
1658 *m0 = m_pullup(*m0, sizeof(struct ip));
1661 ip = mtod(*m0, struct ip *);
1662 ip_dst = &ip->ip_dst;
1665 peer = ovpn_find_peer_by_ip(sc, *ip_dst);
1666 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer);
1668 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0,
1670 if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1671 peer = ovpn_find_peer_by_ip(sc,
1672 nh->gw4_sa.sin_addr);
1673 SDT_PROBE2(if_ovpn, tx, route, ip4,
1674 &nh->gw4_sa.sin_addr, peer);
1682 const struct sockaddr_in6 *sa6 =
1683 (const struct sockaddr_in6 *)dst;
1684 struct nhop_object *nh;
1685 const struct in6_addr *ip6_dst;
1688 ip6_dst = &sa6->sin6_addr;
1690 struct ip6_hdr *ip6;
1692 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr));
1695 ip6 = mtod(*m0, struct ip6_hdr *);
1696 ip6_dst = &ip6->ip6_dst;
1699 peer = ovpn_find_peer_by_ip6(sc, ip6_dst);
1700 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer);
1702 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0,
1704 if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1705 peer = ovpn_find_peer_by_ip6(sc,
1706 &nh->gw6_sa.sin6_addr);
1707 SDT_PROBE2(if_ovpn, tx, route, ip6,
1708 &nh->gw6_sa.sin6_addr, peer);
1720 ovpn_transmit(struct ifnet *ifp, struct mbuf *m)
1722 return (ifp->if_output(ifp, m, NULL, NULL));
1726 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m,
1727 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp)
1729 struct ovpn_wire_header *ohdr;
1730 struct ovpn_kkey *key;
1731 struct ovpn_softc *sc;
1732 struct cryptop *crp;
1734 size_t len, real_len, ovpn_hdr_len;
1742 tunnel_len = m->m_pkthdr.len;
1744 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY];
1745 if (key->encrypt == NULL) {
1746 if (_ovpn_lock_trackerp != NULL)
1752 af = ovpn_get_af(m);
1753 /* Don't capture control packets. */
1755 BPF_MTAP2(ifp, &af, sizeof(af), m);
1757 real_len = len = m->m_pkthdr.len;
1758 MPASS(real_len <= ifp->if_mtu);
1760 ovpn_hdr_len = sizeof(struct ovpn_wire_header);
1761 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE)
1762 ovpn_hdr_len -= 16; /* No auth tag. */
1764 /* Round up the len to a multiple of our block size. */
1765 len = roundup2(real_len, AES_BLOCK_LEN);
1767 /* Now extend the mbuf. */
1768 m_append(m, len - real_len, EMPTY_BUFFER);
1771 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT);
1773 if (_ovpn_lock_trackerp != NULL)
1775 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1778 ohdr = mtod(m, struct ovpn_wire_header *);
1779 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid;
1780 ohdr->opcode <<= 24;
1781 ohdr->opcode |= key->peerid;
1782 ohdr->opcode = htonl(ohdr->opcode);
1784 seq = atomic_fetchadd_32(&peer->tx_seq, 1);
1788 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) {
1789 ret = ovpn_encap(sc, peer->peerid, m);
1790 if (_ovpn_lock_trackerp != NULL)
1793 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1794 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1799 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT);
1801 if (_ovpn_lock_trackerp != NULL)
1803 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1808 /* Encryption covers only the payload, not the header. */
1809 crp->crp_payload_start = sizeof(*ohdr);
1810 crp->crp_payload_length = len;
1811 crp->crp_op = CRYPTO_OP_ENCRYPT;
1814 * AAD data covers the ovpn_wire_header minus the auth
1817 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
1818 crp->crp_aad = ohdr;
1819 crp->crp_aad_start = 0;
1820 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST;
1821 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag);
1823 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1824 memcpy(crp->crp_iv, &seq, sizeof(seq));
1825 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce,
1826 key->encrypt->noncelen);
1828 crypto_use_mbuf(crp, m);
1829 crp->crp_flags |= CRYPTO_F_CBIFSYNC;
1830 crp->crp_callback = ovpn_encrypt_tx_cb;
1831 crp->crp_opaque = peer;
1833 atomic_add_int(&peer->refcount, 1);
1834 if (_ovpn_lock_trackerp != NULL)
1836 ret = crypto_dispatch(crp);
1838 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1845 * Note: Expects to hold the read lock on entry, and will release it itself.
1848 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m)
1851 struct ovpn_kpeer *peer;
1859 peer = ovpn_find_peer(sc, peerid);
1860 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) {
1862 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1867 len = m->m_pkthdr.len;
1869 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT);
1872 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1876 udp = mtod(m, struct udphdr *);
1878 MPASS(peer->local.ss_family == peer->remote.ss_family);
1880 udp->uh_sport = ovpn_get_port(&peer->local);
1881 udp->uh_dport = ovpn_get_port(&peer->remote);
1882 udp->uh_ulen = htons(sizeof(struct udphdr) + len);
1884 switch (peer->remote.ss_family) {
1887 struct sockaddr_in *in_local = TO_IN(&peer->local);
1888 struct sockaddr_in *in_remote = TO_IN(&peer->remote);
1892 * This requires knowing the source IP, which we don't. Happily
1893 * we're allowed to keep this at 0, and the checksum won't do
1894 * anything the crypto won't already do.
1898 /* Set the checksum flags so we recalculate checksums. */
1899 m->m_pkthdr.csum_flags |= CSUM_IP;
1900 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
1902 M_PREPEND(m, sizeof(struct ip), M_NOWAIT);
1905 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1908 ip = mtod(m, struct ip *);
1911 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) +
1914 ip->ip_ttl = V_ip_defttl;
1915 ip->ip_p = IPPROTO_UDP;
1917 if (in_local->sin_port != 0)
1918 ip->ip_src = in_local->sin_addr;
1920 ip->ip_src.s_addr = INADDR_ANY;
1921 ip->ip_dst = in_remote->sin_addr;
1924 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
1926 return (ip_output(m, NULL, NULL, 0, NULL, NULL));
1931 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local);
1932 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote);
1933 struct ip6_hdr *ip6;
1935 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT);
1938 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1941 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp));
1944 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1948 ip6 = mtod(m, struct ip6_hdr *);
1950 ip6->ip6_vfc = IPV6_VERSION;
1951 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK;
1952 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) +
1954 ip6->ip6_nxt = IPPROTO_UDP;
1955 ip6->ip6_hlim = V_ip6_defhlim;
1957 memcpy(&ip6->ip6_src, &in6_local->sin6_addr,
1958 sizeof(ip6->ip6_src));
1959 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr,
1960 sizeof(ip6->ip6_dst));
1962 udp = mtodo(m, sizeof(*ip6));
1963 udp->uh_sum = in6_cksum_pseudo(ip6,
1964 m->m_pkthdr.len - sizeof(struct ip6_hdr),
1967 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6;
1968 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
1971 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
1973 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL,
1978 panic("Unsupported address family %d",
1979 peer->remote.ss_family);
1984 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
1987 struct ovpn_softc *sc;
1988 struct ovpn_kpeer *peer;
1996 SDT_PROBE1(if_ovpn, tx, transmit, start, m);
1998 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) {
1999 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2006 * Only obey 'dst' (i.e. the gateway) if no route is supplied.
2007 * That's our indication that we're being called through pf's route-to,
2008 * and we should route according to 'dst' instead. We can't do so
2009 * consistently, because the usual openvpn configuration sets the first
2010 * non-server IP in the subnet as the gateway. If we always use that
2011 * one we'd end up routing all traffic to the first client.
2012 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but
2013 * only then, we should treat 'dst' as the destination. */
2014 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL);
2016 /* No destination. */
2017 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2023 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp));
2027 ovpn_rcv_ctrl(struct ovpn_softc *sc, struct mbuf *m, int off)
2029 /* Lop off the IP and UDP headers */
2030 m_adj_decap(m, off);
2032 /* Keep in the local ring until userspace fetches it. */
2033 if (buf_ring_enqueue(sc->rxring, m) != 0) {
2034 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1);
2039 OVPN_COUNTER_ADD(sc, received_ctrl_pkts, 1);
2043 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq)
2047 mtx_lock(&key->replay_mtx);
2049 /* Sequence number must be strictly greater than rx_seq */
2050 if (seq <= key->rx_seq) {
2051 mtx_unlock(&key->replay_mtx);
2055 /* Large jump. The packet authenticated okay, so just accept that. */
2056 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) {
2059 mtx_unlock(&key->replay_mtx);
2064 if ((seq == key->rx_seq + 1) && key->rx_window == 0) {
2066 mtx_unlock(&key->replay_mtx);
2070 d = seq - key->rx_seq - 1;
2072 if (key->rx_window & ((uint64_t)1 << d)) {
2074 mtx_unlock(&key->replay_mtx);
2078 key->rx_window |= (uint64_t)1 << d;
2080 while (key->rx_window & 1) {
2082 key->rx_window >>= 1;
2085 mtx_unlock(&key->replay_mtx);
2090 static struct ovpn_kpeer *
2091 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off)
2093 struct ovpn_wire_header ohdr;
2095 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag);
2099 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen))
2102 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr);
2104 peerid = ntohl(ohdr.opcode) & 0x00ffffff;
2106 return (ovpn_find_peer(sc, peerid));
2110 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp,
2111 const struct sockaddr *sa, void *ctx)
2113 struct ovpn_softc *sc = ctx;
2114 struct ovpn_wire_header *ohdr;
2115 struct udphdr *uhdr;
2116 struct ovpn_kkey *key;
2117 struct cryptop *crp;
2118 struct ovpn_kpeer *peer;
2127 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off);
2129 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2133 peer = ovpn_peer_from_mbuf(sc, m, off);
2139 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen);
2142 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2145 uhdr = mtodo(m, off);
2146 ohdr = mtodo(m, off + sizeof(*uhdr));
2148 op = ntohl(ohdr->opcode) >> 24 >> OVPN_OP_SHIFT;
2151 * Simplify things by getting rid of the preceding headers, we don't
2154 m_adj_decap(m, off);
2157 ohdr = mtodo(m, sizeof(*uhdr));
2159 if (op != OVPN_OP_DATA_V2) {
2161 ovpn_rcv_ctrl(sc, m, sizeof(struct udphdr));
2163 udp_notify(inp, EAGAIN);
2168 key = ovpn_find_key(sc, peer, ohdr);
2169 if (key == NULL || key->decrypt == NULL) {
2171 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2176 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) {
2177 /* Now remove the outer headers */
2178 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen);
2180 ohdr = mtodo(m, sizeof(*uhdr));
2182 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq),
2183 _ovpn_lock_trackerp);
2184 OVPN_UNLOCK_ASSERT(sc);
2188 ohdrlen += sizeof(ohdr->auth_tag);
2190 m = m_pullup(m, sizeof(*uhdr) + ohdrlen);
2193 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2197 ohdr = mtodo(m, sizeof(*uhdr));
2200 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT);
2202 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2208 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr);
2209 crp->crp_payload_length = ntohs(uhdr->uh_ulen) -
2210 sizeof(*uhdr) - sizeof(*ohdr);
2211 crp->crp_op = CRYPTO_OP_DECRYPT;
2213 /* AAD validation. */
2214 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2215 crp->crp_aad = ohdr;
2216 crp->crp_aad_start = 0;
2217 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST;
2218 crp->crp_digest_start = sizeof(struct udphdr) +
2219 offsetof(struct ovpn_wire_header, auth_tag);
2221 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
2222 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq));
2223 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce,
2224 key->decrypt->noncelen);
2226 crypto_use_mbuf(crp, m);
2227 crp->crp_flags |= CRYPTO_F_CBIFSYNC;
2228 crp->crp_callback = ovpn_decrypt_rx_cb;
2229 crp->crp_opaque = sc;
2231 atomic_add_int(&sc->refcount, 1);
2233 ret = crypto_dispatch(crp);
2235 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2242 ovpn_qflush(struct ifnet *ifp __unused)
2248 ovpn_flush_rxring(struct ovpn_softc *sc)
2251 struct ovpn_notification *n;
2255 while (! buf_ring_empty(sc->rxring)) {
2256 m = buf_ring_dequeue_sc(sc->rxring);
2260 while (! buf_ring_empty(sc->notifring)) {
2261 n = buf_ring_dequeue_sc(sc->notifring);
2268 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused,
2269 char *unused __unused)
2271 struct ovpn_softc *sc = ifp->if_softc;
2279 /* Flush keys & configuration. */
2281 if (sc->peers[i] != NULL) {
2282 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid);
2286 } while (i < OVPN_MAX_PEERS);
2288 ovpn_flush_rxring(sc);
2295 ovpn_clone_match(struct if_clone *ifc, const char *name)
2298 * Allow all names that start with 'ovpn', specifically because pfSense
2299 * uses ovpnc1 / ovpns2
2301 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0);
2305 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params)
2307 struct ovpn_softc *sc;
2310 int error, unit, wildcard;
2312 /* Try to see if a special unit was requested. */
2313 error = ifc_name2unit(name, &unit);
2316 wildcard = (unit < 0);
2318 error = ifc_alloc_unit(ifc, &unit);
2323 * If no unit had been given, we need to adjust the ifName.
2325 for (dp = name; *dp != '\0'; dp++);
2327 error = snprintf(dp, len - (dp - name), "%d", unit);
2328 if (error > len - (dp - name)) {
2329 /* ifName too long. */
2330 ifc_free_unit(ifc, unit);
2336 /* Make sure it doesn't already exist. */
2337 if (ifunit(name) != NULL)
2340 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO);
2341 sc->ifp = if_alloc(IFT_ENC);
2342 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE);
2345 sc->rxring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
2346 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
2348 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK);
2352 strlcpy(ifp->if_xname, name, IFNAMSIZ);
2353 ifp->if_dname = ovpngroupname;
2354 ifp->if_dunit = unit;
2356 ifp->if_addrlen = 0;
2358 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
2359 ifp->if_ioctl = ovpn_ioctl;
2360 ifp->if_transmit = ovpn_transmit;
2361 ifp->if_output = ovpn_output;
2362 ifp->if_qflush = ovpn_qflush;
2364 ifp->if_reassign = ovpn_reassign;
2366 ifp->if_capabilities |= IFCAP_LINKSTATE;
2367 ifp->if_capenable |= IFCAP_LINKSTATE;
2370 bpfattach(ifp, DLT_NULL, sizeof(uint32_t));
2376 ovpn_clone_destroy_cb(struct epoch_context *ctx)
2378 struct ovpn_softc *sc;
2380 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx);
2382 MPASS(sc->peercount == 0);
2383 for (int i = 0; i < OVPN_MAX_PEERS; i++) {
2384 MPASS(sc->peers[i] == NULL);
2387 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE);
2394 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp)
2396 struct ovpn_softc *sc;
2402 unit = ifp->if_dunit;
2406 if (atomic_load_int(&sc->refcount) > 0) {
2413 if (sc->peers[i] != NULL) {
2414 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid);
2418 } while (i < OVPN_MAX_PEERS);
2420 ovpn_flush_rxring(sc);
2421 buf_ring_free(sc->rxring, M_OVPN);
2422 buf_ring_free(sc->notifring, M_OVPN);
2428 ifp->if_softc = NULL;
2430 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx);
2432 if (unit != IF_DUNIT_NONE)
2433 ifc_free_unit(ifc, unit);
2439 vnet_ovpn_init(const void *unused __unused)
2441 V_ovpn_cloner = if_clone_advanced(ovpngroupname, 0, ovpn_clone_match,
2442 ovpn_clone_create, ovpn_clone_destroy);
2444 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
2445 vnet_ovpn_init, NULL);
2448 vnet_ovpn_uninit(const void *unused __unused)
2450 if_clone_detach(V_ovpn_cloner);
2452 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
2453 vnet_ovpn_uninit, NULL);
2456 ovpnmodevent(module_t mod, int type, void *data)
2460 /* Done in vnet_ovpn_init() */
2463 /* Done in vnet_ovpn_uninit() */
2466 return (EOPNOTSUPP);
2472 static moduledata_t ovpn_mod = {
2478 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
2479 MODULE_VERSION(if_ovpn, 1);