2 * Copyright (C) 2013-2016 Universita` di Pisa
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * This module implements the VALE switch for netmap
33 NMG_LOCK() serializes all modifications to switches and ports.
34 A switch cannot be deleted until all ports are gone.
36 For each switch, an SX lock (RWlock on linux) protects
37 deletion of ports. When configuring or deleting a new port, the
38 lock is acquired in exclusive mode (after holding NMG_LOCK).
39 When forwarding, the lock is acquired in shared mode (without NMG_LOCK).
40 The lock is held throughout the entire forwarding cycle,
41 during which the thread may incur in a page fault.
42 Hence it is important that sleepable shared locks are used.
44 On the rx ring, the per-port lock is grabbed initially to reserve
45 a number of slot in the ring, then the lock is released,
46 packets are copied from source to destination, and then
47 the lock is acquired again and the receive ring is updated.
48 (A similar thing is done on the tx ring for NIC and host stack
49 ports attached to the switch)
54 * OS-specific code that is used only within this file.
55 * Other OS-specific code that must be accessed by drivers
56 * is present in netmap_kern.h
59 #if defined(__FreeBSD__)
60 #include <sys/cdefs.h> /* prerequisite */
61 __FBSDID("$FreeBSD$");
63 #include <sys/types.h>
64 #include <sys/errno.h>
65 #include <sys/param.h> /* defines used in kernel.h */
66 #include <sys/kernel.h> /* types used in module initialization */
67 #include <sys/conf.h> /* cdevsw struct, UID, GID */
68 #include <sys/sockio.h>
69 #include <sys/socketvar.h> /* struct socket */
70 #include <sys/malloc.h>
72 #include <sys/rwlock.h>
73 #include <sys/socket.h> /* sockaddrs */
74 #include <sys/selinfo.h>
75 #include <sys/sysctl.h>
77 #include <net/if_var.h>
78 #include <net/bpf.h> /* BIOCIMMEDIATE */
79 #include <machine/bus.h> /* bus_dmamap_* */
80 #include <sys/endian.h>
81 #include <sys/refcount.h>
89 #elif defined(__APPLE__)
91 #warning OSX support is only partial
99 #error Unsupported platform
101 #endif /* unsupported */
107 #include <net/netmap.h>
108 #include <dev/netmap/netmap_kern.h>
109 #include <dev/netmap/netmap_mem2.h>
111 #include <dev/netmap/netmap_bdg.h>
114 netmap_bdg_name(struct netmap_vp_adapter *vp)
116 struct nm_bridge *b = vp->na_bdg;
119 return b->bdg_basename;
123 #ifndef CONFIG_NET_NS
125 * XXX in principle nm_bridges could be created dynamically
126 * Right now we have a static array and deletions are protected
127 * by an exclusive lock.
129 struct nm_bridge *nm_bridges;
130 #endif /* !CONFIG_NET_NS */
134 nm_is_id_char(const char c)
136 return (c >= 'a' && c <= 'z') ||
137 (c >= 'A' && c <= 'Z') ||
138 (c >= '0' && c <= '9') ||
142 /* Validate the name of a bdg port and return the
143 * position of the ":" character. */
145 nm_bdg_name_validate(const char *name, size_t prefixlen)
150 if (!name || strlen(name) < prefixlen) {
154 for (i = 0; i < NM_BDG_IFNAMSIZ && name[i]; i++) {
155 if (name[i] == ':') {
158 } else if (!nm_is_id_char(name[i])) {
163 if (strlen(name) - colon_pos > IFNAMSIZ) {
164 /* interface name too long */
172 * locate a bridge among the existing ones.
173 * MUST BE CALLED WITH NMG_LOCK()
175 * a ':' in the name terminates the bridge name. Otherwise, just NM_NAME.
176 * We assume that this is called with a name of at least NM_NAME chars.
179 nm_find_bridge(const char *name, int create, struct netmap_bdg_ops *ops)
182 struct nm_bridge *b = NULL, *bridges;
187 netmap_bns_getbridges(&bridges, &num_bridges);
189 namelen = nm_bdg_name_validate(name,
190 (ops != NULL ? strlen(ops->name) : 0));
192 nm_prerr("invalid bridge name %s", name ? name : NULL);
196 /* lookup the name, remember empty slot if there is one */
197 for (i = 0; i < num_bridges; i++) {
198 struct nm_bridge *x = bridges + i;
200 if ((x->bdg_flags & NM_BDG_ACTIVE) + x->bdg_active_ports == 0) {
201 if (create && b == NULL)
202 b = x; /* record empty slot */
203 } else if (x->bdg_namelen != namelen) {
205 } else if (strncmp(name, x->bdg_basename, namelen) == 0) {
206 nm_prdis("found '%.*s' at %d", namelen, name, i);
211 if (i == num_bridges && b) { /* name not found, can create entry */
212 /* initialize the bridge */
213 nm_prdis("create new bridge %s with ports %d", b->bdg_basename,
214 b->bdg_active_ports);
215 b->ht = nm_os_malloc(sizeof(struct nm_hash_ent) * NM_BDG_HASH);
217 nm_prerr("failed to allocate hash table");
220 strncpy(b->bdg_basename, name, namelen);
221 b->bdg_namelen = namelen;
222 b->bdg_active_ports = 0;
223 for (i = 0; i < NM_BDG_MAXPORTS; i++)
224 b->bdg_port_index[i] = i;
225 /* set the default function */
226 b->bdg_ops = b->bdg_saved_ops = *ops;
227 b->private_data = b->ht;
236 netmap_bdg_free(struct nm_bridge *b)
238 if ((b->bdg_flags & NM_BDG_ACTIVE) + b->bdg_active_ports != 0) {
242 nm_prdis("marking bridge %s as free", b->bdg_basename);
244 memset(&b->bdg_ops, 0, sizeof(b->bdg_ops));
245 memset(&b->bdg_saved_ops, 0, sizeof(b->bdg_saved_ops));
251 /* Called by external kernel modules (e.g., Openvswitch).
252 * to modify the private data previously given to regops().
253 * 'name' may be just bridge's name (including ':' if it
254 * is not just NM_BDG_NAME).
255 * Called without NMG_LOCK.
258 netmap_bdg_update_private_data(const char *name, bdg_update_private_data_fn_t callback,
259 void *callback_data, void *auth_token)
261 void *private_data = NULL;
266 b = nm_find_bridge(name, 0 /* don't create */, NULL);
269 goto unlock_update_priv;
271 if (!nm_bdg_valid_auth_token(b, auth_token)) {
273 goto unlock_update_priv;
276 private_data = callback(b->private_data, callback_data, &error);
277 b->private_data = private_data;
287 /* remove from bridge b the ports in slots hw and sw
288 * (sw can be -1 if not needed)
291 netmap_bdg_detach_common(struct nm_bridge *b, int hw, int sw)
293 int s_hw = hw, s_sw = sw;
294 int i, lim =b->bdg_active_ports;
295 uint32_t *tmp = b->tmp_bdg_port_index;
299 make a copy of bdg_port_index;
300 lookup NA(ifp)->bdg_port and SWNA(ifp)->bdg_port
301 in the array of bdg_port_index, replacing them with
302 entries from the bottom of the array;
303 decrement bdg_active_ports;
304 acquire BDG_WLOCK() and copy back the array.
307 if (netmap_debug & NM_DEBUG_BDG)
308 nm_prinf("detach %d and %d (lim %d)", hw, sw, lim);
309 /* make a copy of the list of active ports, update it,
310 * and then copy back within BDG_WLOCK().
312 memcpy(b->tmp_bdg_port_index, b->bdg_port_index, sizeof(b->tmp_bdg_port_index));
313 for (i = 0; (hw >= 0 || sw >= 0) && i < lim; ) {
314 if (hw >= 0 && tmp[i] == hw) {
315 nm_prdis("detach hw %d at %d", hw, i);
316 lim--; /* point to last active port */
317 tmp[i] = tmp[lim]; /* swap with i */
318 tmp[lim] = hw; /* now this is inactive */
320 } else if (sw >= 0 && tmp[i] == sw) {
321 nm_prdis("detach sw %d at %d", sw, i);
330 if (hw >= 0 || sw >= 0) {
331 nm_prerr("delete failed hw %d sw %d, should panic...", hw, sw);
336 b->bdg_ops.dtor(b->bdg_ports[s_hw]);
337 b->bdg_ports[s_hw] = NULL;
339 b->bdg_ports[s_sw] = NULL;
341 memcpy(b->bdg_port_index, b->tmp_bdg_port_index, sizeof(b->tmp_bdg_port_index));
342 b->bdg_active_ports = lim;
345 nm_prdis("now %d active ports", lim);
350 /* nm_bdg_ctl callback for VALE ports */
352 netmap_vp_bdg_ctl(struct nmreq_header *hdr, struct netmap_adapter *na)
354 struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter *)na;
355 struct nm_bridge *b = vpna->na_bdg;
357 if (hdr->nr_reqtype == NETMAP_REQ_VALE_ATTACH) {
358 return 0; /* nothing to do */
361 netmap_set_all_rings(na, 0 /* disable */);
362 netmap_bdg_detach_common(b, vpna->bdg_port, -1);
364 netmap_set_all_rings(na, 1 /* enable */);
366 /* I have took reference just for attach */
367 netmap_adapter_put(na);
372 netmap_default_bdg_attach(const char *name, struct netmap_adapter *na,
375 return NM_NEED_BWRAP;
378 /* Try to get a reference to a netmap adapter attached to a VALE switch.
379 * If the adapter is found (or is created), this function returns 0, a
380 * non NULL pointer is returned into *na, and the caller holds a
381 * reference to the adapter.
382 * If an adapter is not found, then no reference is grabbed and the
383 * function returns an error code, or 0 if there is just a VALE prefix
384 * mismatch. Therefore the caller holds a reference when
385 * (*na != NULL && return == 0).
388 netmap_get_bdg_na(struct nmreq_header *hdr, struct netmap_adapter **na,
389 struct netmap_mem_d *nmd, int create, struct netmap_bdg_ops *ops)
391 char *nr_name = hdr->nr_name;
393 struct ifnet *ifp = NULL;
395 struct netmap_vp_adapter *vpna, *hostna = NULL;
398 uint32_t cand = NM_BDG_NOPORT, cand2 = NM_BDG_NOPORT;
401 *na = NULL; /* default return value */
403 /* first try to see if this is a bridge port. */
405 if (strncmp(nr_name, ops->name, strlen(ops->name) - 1)) {
406 return 0; /* no error, but no VALE prefix */
409 b = nm_find_bridge(nr_name, create, ops);
411 nm_prdis("no bridges available for '%s'", nr_name);
412 return (create ? ENOMEM : ENXIO);
414 if (strlen(nr_name) < b->bdg_namelen) /* impossible */
417 /* Now we are sure that name starts with the bridge's name,
418 * lookup the port in the bridge. We need to scan the entire
419 * list. It is not important to hold a WLOCK on the bridge
420 * during the search because NMG_LOCK already guarantees
421 * that there are no other possible writers.
424 /* lookup in the local list of ports */
425 for (j = 0; j < b->bdg_active_ports; j++) {
426 i = b->bdg_port_index[j];
427 vpna = b->bdg_ports[i];
428 nm_prdis("checking %s", vpna->up.name);
429 if (!strcmp(vpna->up.name, nr_name)) {
430 netmap_adapter_get(&vpna->up);
431 nm_prdis("found existing if %s refs %d", nr_name)
436 /* not found, should we create it? */
439 /* yes we should, see if we have space to attach entries */
440 needed = 2; /* in some cases we only need 1 */
441 if (b->bdg_active_ports + needed >= NM_BDG_MAXPORTS) {
442 nm_prerr("bridge full %d, cannot create new port", b->bdg_active_ports);
445 /* record the next two ports available, but do not allocate yet */
446 cand = b->bdg_port_index[b->bdg_active_ports];
447 cand2 = b->bdg_port_index[b->bdg_active_ports + 1];
448 nm_prdis("+++ bridge %s port %s used %d avail %d %d",
449 b->bdg_basename, ifname, b->bdg_active_ports, cand, cand2);
452 * try see if there is a matching NIC with this name
453 * (after the bridge's name)
455 ifname = nr_name + b->bdg_namelen + 1;
456 ifp = ifunit_ref(ifname);
458 /* Create an ephemeral virtual port.
459 * This block contains all the ephemeral-specific logic.
462 if (hdr->nr_reqtype != NETMAP_REQ_REGISTER) {
467 /* bdg_netmap_attach creates a struct netmap_adapter */
468 error = b->bdg_ops.vp_create(hdr, NULL, nmd, &vpna);
470 if (netmap_debug & NM_DEBUG_BDG)
471 nm_prerr("error %d", error);
474 /* shortcut - we can skip get_hw_na(),
475 * ownership check and nm_bdg_attach()
479 struct netmap_adapter *hw;
481 /* the vale:nic syntax is only valid for some commands */
482 switch (hdr->nr_reqtype) {
483 case NETMAP_REQ_VALE_ATTACH:
484 case NETMAP_REQ_VALE_DETACH:
485 case NETMAP_REQ_VALE_POLLING_ENABLE:
486 case NETMAP_REQ_VALE_POLLING_DISABLE:
493 error = netmap_get_hw_na(ifp, nmd, &hw);
494 if (error || hw == NULL)
497 /* host adapter might not be created */
498 error = hw->nm_bdg_attach(nr_name, hw, b);
499 if (error == NM_NEED_BWRAP) {
500 error = b->bdg_ops.bwrap_attach(nr_name, hw);
505 hostna = hw->na_hostvp;
506 if (hdr->nr_reqtype == NETMAP_REQ_VALE_ATTACH) {
507 /* Check if we need to skip the host rings. */
508 struct nmreq_vale_attach *areq =
509 (struct nmreq_vale_attach *)(uintptr_t)hdr->nr_body;
510 if (areq->reg.nr_mode != NR_REG_NIC_SW) {
517 vpna->bdg_port = cand;
518 nm_prdis("NIC %p to bridge port %d", vpna, cand);
519 /* bind the port to the bridge (virtual ports are not active) */
520 b->bdg_ports[cand] = vpna;
522 b->bdg_active_ports++;
523 if (hostna != NULL) {
524 /* also bind the host stack to the bridge */
525 b->bdg_ports[cand2] = hostna;
526 hostna->bdg_port = cand2;
528 b->bdg_active_ports++;
529 nm_prdis("host %p to bridge port %d", hostna, cand2);
531 nm_prdis("if %s refs %d", ifname, vpna->up.na_refcount);
534 netmap_adapter_get(*na);
545 nm_is_bwrap(struct netmap_adapter *na)
547 return na->nm_register == netmap_bwrap_reg;
551 struct nm_bdg_polling_state;
557 struct nm_bdg_polling_state *bps;
560 struct nm_bdg_polling_state {
563 struct netmap_bwrap_adapter *bna;
569 struct nm_bdg_kthread *kthreads;
573 netmap_bwrap_polling(void *data)
575 struct nm_bdg_kthread *nbk = data;
576 struct netmap_bwrap_adapter *bna;
577 u_int qfirst, qlast, i;
578 struct netmap_kring **kring0, *kring;
582 qfirst = nbk->qfirst;
585 kring0 = NMR(bna->hwna, NR_RX);
587 for (i = qfirst; i < qlast; i++) {
589 kring->nm_notify(kring, 0);
594 nm_bdg_create_kthreads(struct nm_bdg_polling_state *bps)
596 struct nm_kctx_cfg kcfg;
599 bps->kthreads = nm_os_malloc(sizeof(struct nm_bdg_kthread) * bps->ncpus);
600 if (bps->kthreads == NULL)
603 bzero(&kcfg, sizeof(kcfg));
604 kcfg.worker_fn = netmap_bwrap_polling;
605 for (i = 0; i < bps->ncpus; i++) {
606 struct nm_bdg_kthread *t = bps->kthreads + i;
607 int all = (bps->ncpus == 1 &&
608 bps->mode == NETMAP_POLLING_MODE_SINGLE_CPU);
609 int affinity = bps->cpu_from + i;
612 t->qfirst = all ? bps->qfirst /* must be 0 */: affinity;
613 t->qlast = all ? bps->qlast : t->qfirst + 1;
615 nm_prinf("kthread %d a:%u qf:%u ql:%u", i, affinity, t->qfirst,
619 kcfg.worker_private = t;
620 t->nmk = nm_os_kctx_create(&kcfg, NULL);
621 if (t->nmk == NULL) {
624 nm_os_kctx_worker_setaff(t->nmk, affinity);
629 for (j = 0; j < i; j++) {
630 struct nm_bdg_kthread *t = bps->kthreads + i;
631 nm_os_kctx_destroy(t->nmk);
633 nm_os_free(bps->kthreads);
637 /* A variant of ptnetmap_start_kthreads() */
639 nm_bdg_polling_start_kthreads(struct nm_bdg_polling_state *bps)
644 nm_prerr("polling is not configured");
647 bps->stopped = false;
649 for (i = 0; i < bps->ncpus; i++) {
650 struct nm_bdg_kthread *t = bps->kthreads + i;
651 error = nm_os_kctx_worker_start(t->nmk);
653 nm_prerr("error in nm_kthread_start(): %d", error);
660 for (j = 0; j < i; j++) {
661 struct nm_bdg_kthread *t = bps->kthreads + i;
662 nm_os_kctx_worker_stop(t->nmk);
669 nm_bdg_polling_stop_delete_kthreads(struct nm_bdg_polling_state *bps)
676 for (i = 0; i < bps->ncpus; i++) {
677 struct nm_bdg_kthread *t = bps->kthreads + i;
678 nm_os_kctx_worker_stop(t->nmk);
679 nm_os_kctx_destroy(t->nmk);
685 get_polling_cfg(struct nmreq_vale_polling *req, struct netmap_adapter *na,
686 struct nm_bdg_polling_state *bps)
688 unsigned int avail_cpus, core_from;
689 unsigned int qfirst, qlast;
690 uint32_t i = req->nr_first_cpu_id;
691 uint32_t req_cpus = req->nr_num_polling_cpus;
693 avail_cpus = nm_os_ncpus();
696 nm_prerr("req_cpus must be > 0");
698 } else if (req_cpus >= avail_cpus) {
699 nm_prerr("Cannot use all the CPUs in the system");
703 if (req->nr_mode == NETMAP_POLLING_MODE_MULTI_CPU) {
704 /* Use a separate core for each ring. If nr_num_polling_cpus>1
705 * more consecutive rings are polled.
706 * For example, if nr_first_cpu_id=2 and nr_num_polling_cpus=2,
707 * ring 2 and 3 are polled by core 2 and 3, respectively. */
708 if (i + req_cpus > nma_get_nrings(na, NR_RX)) {
709 nm_prerr("Rings %u-%u not in range (have %d rings)",
710 i, i + req_cpus, nma_get_nrings(na, NR_RX));
714 qlast = qfirst + req_cpus;
717 } else if (req->nr_mode == NETMAP_POLLING_MODE_SINGLE_CPU) {
718 /* Poll all the rings using a core specified by nr_first_cpu_id.
719 * the number of cores must be 1. */
721 nm_prerr("ncpus must be 1 for NETMAP_POLLING_MODE_SINGLE_CPU "
722 "(was %d)", req_cpus);
726 qlast = nma_get_nrings(na, NR_RX);
729 nm_prerr("Invalid polling mode");
733 bps->mode = req->nr_mode;
734 bps->qfirst = qfirst;
736 bps->cpu_from = core_from;
737 bps->ncpus = req_cpus;
738 nm_prinf("%s qfirst %u qlast %u cpu_from %u ncpus %u",
739 req->nr_mode == NETMAP_POLLING_MODE_MULTI_CPU ?
741 qfirst, qlast, core_from, req_cpus);
746 nm_bdg_ctl_polling_start(struct nmreq_vale_polling *req, struct netmap_adapter *na)
748 struct nm_bdg_polling_state *bps;
749 struct netmap_bwrap_adapter *bna;
752 bna = (struct netmap_bwrap_adapter *)na;
753 if (bna->na_polling_state) {
754 nm_prerr("ERROR adapter already in polling mode");
758 bps = nm_os_malloc(sizeof(*bps));
761 bps->configured = false;
764 if (get_polling_cfg(req, na, bps)) {
769 if (nm_bdg_create_kthreads(bps)) {
774 bps->configured = true;
775 bna->na_polling_state = bps;
778 /* disable interrupts if possible */
779 nma_intr_enable(bna->hwna, 0);
780 /* start kthread now */
781 error = nm_bdg_polling_start_kthreads(bps);
783 nm_prerr("ERROR nm_bdg_polling_start_kthread()");
784 nm_os_free(bps->kthreads);
786 bna->na_polling_state = NULL;
787 nma_intr_enable(bna->hwna, 1);
793 nm_bdg_ctl_polling_stop(struct netmap_adapter *na)
795 struct netmap_bwrap_adapter *bna = (struct netmap_bwrap_adapter *)na;
796 struct nm_bdg_polling_state *bps;
798 if (!bna->na_polling_state) {
799 nm_prerr("ERROR adapter is not in polling mode");
802 bps = bna->na_polling_state;
803 nm_bdg_polling_stop_delete_kthreads(bna->na_polling_state);
804 bps->configured = false;
806 bna->na_polling_state = NULL;
807 /* reenable interrupts */
808 nma_intr_enable(bna->hwna, 1);
813 nm_bdg_polling(struct nmreq_header *hdr)
815 struct nmreq_vale_polling *req =
816 (struct nmreq_vale_polling *)(uintptr_t)hdr->nr_body;
817 struct netmap_adapter *na = NULL;
821 error = netmap_get_vale_na(hdr, &na, NULL, /*create=*/0);
823 if (!nm_is_bwrap(na)) {
825 } else if (hdr->nr_reqtype == NETMAP_BDG_POLLING_ON) {
826 error = nm_bdg_ctl_polling_start(req, na);
828 netmap_adapter_get(na);
830 error = nm_bdg_ctl_polling_stop(na);
832 netmap_adapter_put(na);
834 netmap_adapter_put(na);
835 } else if (!na && !error) {
844 /* Called by external kernel modules (e.g., Openvswitch).
845 * to set configure/lookup/dtor functions of a VALE instance.
846 * Register callbacks to the given bridge. 'name' may be just
847 * bridge's name (including ':' if it is not just NM_BDG_NAME).
849 * Called without NMG_LOCK.
853 netmap_bdg_regops(const char *name, struct netmap_bdg_ops *bdg_ops, void *private_data, void *auth_token)
859 b = nm_find_bridge(name, 0 /* don't create */, NULL);
864 if (!nm_bdg_valid_auth_token(b, auth_token)) {
871 /* resetting the bridge */
872 bzero(b->ht, sizeof(struct nm_hash_ent) * NM_BDG_HASH);
873 b->bdg_ops = b->bdg_saved_ops;
874 b->private_data = b->ht;
876 /* modifying the bridge */
877 b->private_data = private_data;
878 #define nm_bdg_override(m) if (bdg_ops->m) b->bdg_ops.m = bdg_ops->m
879 nm_bdg_override(lookup);
880 nm_bdg_override(config);
881 nm_bdg_override(dtor);
882 nm_bdg_override(vp_create);
883 nm_bdg_override(bwrap_attach);
884 #undef nm_bdg_override
896 netmap_bdg_config(struct nm_ifreq *nr)
902 b = nm_find_bridge(nr->nifr_name, 0, NULL);
908 /* Don't call config() with NMG_LOCK() held */
910 if (b->bdg_ops.config != NULL)
911 error = b->bdg_ops.config(nr);
917 /* nm_register callback for VALE ports */
919 netmap_vp_reg(struct netmap_adapter *na, int onoff)
921 struct netmap_vp_adapter *vpna =
922 (struct netmap_vp_adapter*)na;
924 /* persistent ports may be put in netmap mode
925 * before being attached to a bridge
928 BDG_WLOCK(vpna->na_bdg);
930 netmap_krings_mode_commit(na, onoff);
931 if (na->active_fds == 0)
932 na->na_flags |= NAF_NETMAP_ON;
933 /* XXX on FreeBSD, persistent VALE ports should also
934 * toggle IFCAP_NETMAP in na->ifp (2014-03-16)
937 if (na->active_fds == 0)
938 na->na_flags &= ~NAF_NETMAP_ON;
939 netmap_krings_mode_commit(na, onoff);
942 BDG_WUNLOCK(vpna->na_bdg);
947 /* rxsync code used by VALE ports nm_rxsync callback and also
948 * internally by the brwap
951 netmap_vp_rxsync_locked(struct netmap_kring *kring, int flags)
953 struct netmap_adapter *na = kring->na;
954 struct netmap_ring *ring = kring->ring;
955 u_int nm_i, lim = kring->nkr_num_slots - 1;
956 u_int head = kring->rhead;
960 nm_prerr("ouch dangerous reset!!!");
961 n = netmap_ring_reinit(kring);
965 /* First part, import newly received packets. */
966 /* actually nothing to do here, they are already in the kring */
968 /* Second part, skip past packets that userspace has released. */
969 nm_i = kring->nr_hwcur;
971 /* consistency check, but nothing really important here */
972 for (n = 0; likely(nm_i != head); n++) {
973 struct netmap_slot *slot = &ring->slot[nm_i];
974 void *addr = NMB(na, slot);
976 if (addr == NETMAP_BUF_BASE(kring->na)) { /* bad buf */
977 nm_prerr("bad buffer index %d, ignore ?",
980 slot->flags &= ~NS_BUF_CHANGED;
981 nm_i = nm_next(nm_i, lim);
983 kring->nr_hwcur = head;
992 * nm_rxsync callback for VALE ports
993 * user process reading from a VALE switch.
994 * Already protected against concurrent calls from userspace,
995 * but we must acquire the queue's lock to protect against
996 * writers on the same queue.
999 netmap_vp_rxsync(struct netmap_kring *kring, int flags)
1003 mtx_lock(&kring->q_lock);
1004 n = netmap_vp_rxsync_locked(kring, flags);
1005 mtx_unlock(&kring->q_lock);
1010 netmap_bwrap_attach(const char *nr_name, struct netmap_adapter *hwna,
1011 struct netmap_bdg_ops *ops)
1013 return ops->bwrap_attach(nr_name, hwna);
1017 /* Bridge wrapper code (bwrap).
1018 * This is used to connect a non-VALE-port netmap_adapter (hwna) to a
1020 * The main task is to swap the meaning of tx and rx rings to match the
1021 * expectations of the VALE switch code (see nm_bdg_flush).
1023 * The bwrap works by interposing a netmap_bwrap_adapter between the
1024 * rest of the system and the hwna. The netmap_bwrap_adapter looks like
1025 * a netmap_vp_adapter to the rest the system, but, internally, it
1026 * translates all callbacks to what the hwna expects.
1028 * Note that we have to intercept callbacks coming from two sides:
1030 * - callbacks coming from the netmap module are intercepted by
1031 * passing around the netmap_bwrap_adapter instead of the hwna
1033 * - callbacks coming from outside of the netmap module only know
1034 * about the hwna. This, however, only happens in interrupt
1035 * handlers, where only the hwna->nm_notify callback is called.
1036 * What the bwrap does is to overwrite the hwna->nm_notify callback
1037 * with its own netmap_bwrap_intr_notify.
1038 * XXX This assumes that the hwna->nm_notify callback was the
1039 * standard netmap_notify(), as it is the case for nic adapters.
1040 * Any additional action performed by hwna->nm_notify will not be
1041 * performed by netmap_bwrap_intr_notify.
1043 * Additionally, the bwrap can optionally attach the host rings pair
1044 * of the wrapped adapter to a different port of the switch.
1049 netmap_bwrap_dtor(struct netmap_adapter *na)
1051 struct netmap_bwrap_adapter *bna = (struct netmap_bwrap_adapter*)na;
1052 struct netmap_adapter *hwna = bna->hwna;
1053 struct nm_bridge *b = bna->up.na_bdg,
1054 *bh = bna->host.na_bdg;
1056 if (bna->host.up.nm_mem)
1057 netmap_mem_put(bna->host.up.nm_mem);
1060 netmap_bdg_detach_common(b, bna->up.bdg_port,
1061 (bh ? bna->host.bdg_port : -1));
1064 nm_prdis("na %p", na);
1066 bna->host.up.ifp = NULL;
1067 hwna->na_vp = bna->saved_na_vp;
1068 hwna->na_hostvp = NULL;
1069 hwna->na_private = NULL;
1070 hwna->na_flags &= ~NAF_BUSY;
1071 netmap_adapter_put(hwna);
1077 * Intr callback for NICs connected to a bridge.
1078 * Simply ignore tx interrupts (maybe we could try to recover space ?)
1079 * and pass received packets from nic to the bridge.
1081 * XXX TODO check locking: this is called from the interrupt
1082 * handler so we should make sure that the interface is not
1083 * disconnected while passing down an interrupt.
1085 * Note, no user process can access this NIC or the host stack.
1086 * The only part of the ring that is significant are the slots,
1087 * and head/cur/tail are set from the kring as needed
1088 * (part as a receive ring, part as a transmit ring).
1090 * callback that overwrites the hwna notify callback.
1091 * Packets come from the outside or from the host stack and are put on an
1093 * The bridge wrapper then sends the packets through the bridge.
1096 netmap_bwrap_intr_notify(struct netmap_kring *kring, int flags)
1098 struct netmap_adapter *na = kring->na;
1099 struct netmap_bwrap_adapter *bna = na->na_private;
1100 struct netmap_kring *bkring;
1101 struct netmap_vp_adapter *vpna = &bna->up;
1102 u_int ring_nr = kring->ring_id;
1103 int ret = NM_IRQ_COMPLETED;
1106 if (netmap_debug & NM_DEBUG_RXINTR)
1107 nm_prinf("%s %s 0x%x", na->name, kring->name, flags);
1109 bkring = vpna->up.tx_rings[ring_nr];
1111 /* make sure the ring is not disabled */
1112 if (nm_kr_tryget(kring, 0 /* can't sleep */, NULL)) {
1116 if (netmap_debug & NM_DEBUG_RXINTR)
1117 nm_prinf("%s head %d cur %d tail %d", na->name,
1118 kring->rhead, kring->rcur, kring->rtail);
1120 /* simulate a user wakeup on the rx ring
1121 * fetch packets that have arrived.
1123 error = kring->nm_sync(kring, 0);
1126 if (kring->nr_hwcur == kring->nr_hwtail) {
1128 nm_prlim(1, "interrupt with no packets on %s",
1133 /* new packets are kring->rcur to kring->nr_hwtail, and the bkring
1134 * had hwcur == bkring->rhead. So advance bkring->rhead to kring->nr_hwtail
1135 * to push all packets out.
1137 bkring->rhead = bkring->rcur = kring->nr_hwtail;
1139 bkring->nm_sync(bkring, flags);
1141 /* mark all buffers as released on this ring */
1142 kring->rhead = kring->rcur = kring->rtail = kring->nr_hwtail;
1143 /* another call to actually release the buffers */
1144 error = kring->nm_sync(kring, 0);
1146 /* The second rxsync may have further advanced hwtail. If this happens,
1147 * return NM_IRQ_RESCHED, otherwise just return NM_IRQ_COMPLETED. */
1148 if (kring->rcur != kring->nr_hwtail) {
1149 ret = NM_IRQ_RESCHED;
1154 return error ? error : ret;
1158 /* nm_register callback for bwrap */
1160 netmap_bwrap_reg(struct netmap_adapter *na, int onoff)
1162 struct netmap_bwrap_adapter *bna =
1163 (struct netmap_bwrap_adapter *)na;
1164 struct netmap_adapter *hwna = bna->hwna;
1165 struct netmap_vp_adapter *hostna = &bna->host;
1169 nm_prdis("%s %s", na->name, onoff ? "on" : "off");
1172 /* netmap_do_regif has been called on the bwrap na.
1173 * We need to pass the information about the
1174 * memory allocator down to the hwna before
1175 * putting it in netmap mode
1177 hwna->na_lut = na->na_lut;
1179 if (hostna->na_bdg) {
1180 /* if the host rings have been attached to switch,
1181 * we need to copy the memory allocator information
1182 * in the hostna also
1184 hostna->up.na_lut = na->na_lut;
1189 /* pass down the pending ring state information */
1191 for (i = 0; i < netmap_all_rings(na, t); i++) {
1192 NMR(hwna, nm_txrx_swap(t))[i]->nr_pending_mode =
1193 NMR(na, t)[i]->nr_pending_mode;
1197 /* forward the request to the hwna */
1198 error = hwna->nm_register(hwna, onoff);
1202 /* copy up the current ring state information */
1204 for (i = 0; i < netmap_all_rings(na, t); i++) {
1205 struct netmap_kring *kring = NMR(hwna, nm_txrx_swap(t))[i];
1206 NMR(na, t)[i]->nr_mode = kring->nr_mode;
1210 /* impersonate a netmap_vp_adapter */
1211 netmap_vp_reg(na, onoff);
1213 netmap_vp_reg(&hostna->up, onoff);
1217 /* intercept the hwna nm_nofify callback on the hw rings */
1218 for (i = 0; i < hwna->num_rx_rings; i++) {
1219 hwna->rx_rings[i]->save_notify = hwna->rx_rings[i]->nm_notify;
1220 hwna->rx_rings[i]->nm_notify = netmap_bwrap_intr_notify;
1222 i = hwna->num_rx_rings; /* for safety */
1223 /* save the host ring notify unconditionally */
1224 for (; i < netmap_real_rings(hwna, NR_RX); i++) {
1225 hwna->rx_rings[i]->save_notify =
1226 hwna->rx_rings[i]->nm_notify;
1227 if (hostna->na_bdg) {
1228 /* also intercept the host ring notify */
1229 hwna->rx_rings[i]->nm_notify =
1230 netmap_bwrap_intr_notify;
1231 na->tx_rings[i]->nm_sync = na->nm_txsync;
1234 if (na->active_fds == 0)
1235 na->na_flags |= NAF_NETMAP_ON;
1239 if (na->active_fds == 0)
1240 na->na_flags &= ~NAF_NETMAP_ON;
1242 /* reset all notify callbacks (including host ring) */
1243 for (i = 0; i < netmap_all_rings(hwna, NR_RX); i++) {
1244 hwna->rx_rings[i]->nm_notify =
1245 hwna->rx_rings[i]->save_notify;
1246 hwna->rx_rings[i]->save_notify = NULL;
1248 hwna->na_lut.lut = NULL;
1249 hwna->na_lut.plut = NULL;
1250 hwna->na_lut.objtotal = 0;
1251 hwna->na_lut.objsize = 0;
1253 /* pass ownership of the netmap rings to the hwna */
1255 for (i = 0; i < netmap_all_rings(na, t); i++) {
1256 NMR(na, t)[i]->ring = NULL;
1259 /* reset the number of host rings to default */
1261 nma_set_host_nrings(hwna, t, 1);
1269 /* nm_config callback for bwrap */
1271 netmap_bwrap_config(struct netmap_adapter *na, struct nm_config_info *info)
1273 struct netmap_bwrap_adapter *bna =
1274 (struct netmap_bwrap_adapter *)na;
1275 struct netmap_adapter *hwna = bna->hwna;
1278 /* Forward the request to the hwna. It may happen that nobody
1279 * registered hwna yet, so netmap_mem_get_lut() may have not
1280 * been called yet. */
1281 error = netmap_mem_get_lut(hwna->nm_mem, &hwna->na_lut);
1284 netmap_update_config(hwna);
1285 /* swap the results and propagate */
1286 info->num_tx_rings = hwna->num_rx_rings;
1287 info->num_tx_descs = hwna->num_rx_desc;
1288 info->num_rx_rings = hwna->num_tx_rings;
1289 info->num_rx_descs = hwna->num_tx_desc;
1290 info->rx_buf_maxsize = hwna->rx_buf_maxsize;
1296 /* nm_krings_create callback for bwrap */
1298 netmap_bwrap_krings_create_common(struct netmap_adapter *na)
1300 struct netmap_bwrap_adapter *bna =
1301 (struct netmap_bwrap_adapter *)na;
1302 struct netmap_adapter *hwna = bna->hwna;
1303 struct netmap_adapter *hostna = &bna->host.up;
1307 /* also create the hwna krings */
1308 error = hwna->nm_krings_create(hwna);
1313 /* increment the usage counter for all the hwna krings */
1315 for (i = 0; i < netmap_all_rings(hwna, t); i++) {
1316 NMR(hwna, t)[i]->users++;
1320 /* now create the actual rings */
1321 error = netmap_mem_rings_create(hwna);
1326 /* cross-link the netmap rings
1327 * The original number of rings comes from hwna,
1328 * rx rings on one side equals tx rings on the other.
1331 enum txrx r = nm_txrx_swap(t); /* swap NR_TX <-> NR_RX */
1332 for (i = 0; i < netmap_all_rings(hwna, r); i++) {
1333 NMR(na, t)[i]->nkr_num_slots = NMR(hwna, r)[i]->nkr_num_slots;
1334 NMR(na, t)[i]->ring = NMR(hwna, r)[i]->ring;
1338 if (na->na_flags & NAF_HOST_RINGS) {
1339 /* the hostna rings are the host rings of the bwrap.
1340 * The corresponding krings must point back to the
1343 hostna->tx_rings = &na->tx_rings[na->num_tx_rings];
1344 hostna->rx_rings = &na->rx_rings[na->num_rx_rings];
1346 for (i = 0; i < nma_get_nrings(hostna, t); i++) {
1347 NMR(hostna, t)[i]->na = hostna;
1356 for (i = 0; i < netmap_all_rings(hwna, t); i++) {
1357 NMR(hwna, t)[i]->users--;
1360 hwna->nm_krings_delete(hwna);
1366 netmap_bwrap_krings_delete_common(struct netmap_adapter *na)
1368 struct netmap_bwrap_adapter *bna =
1369 (struct netmap_bwrap_adapter *)na;
1370 struct netmap_adapter *hwna = bna->hwna;
1374 nm_prdis("%s", na->name);
1376 /* decrement the usage counter for all the hwna krings */
1378 for (i = 0; i < netmap_all_rings(hwna, t); i++) {
1379 NMR(hwna, t)[i]->users--;
1383 /* delete any netmap rings that are no longer needed */
1384 netmap_mem_rings_delete(hwna);
1385 hwna->nm_krings_delete(hwna);
1389 /* notify method for the bridge-->hwna direction */
1391 netmap_bwrap_notify(struct netmap_kring *kring, int flags)
1393 struct netmap_adapter *na = kring->na;
1394 struct netmap_bwrap_adapter *bna = na->na_private;
1395 struct netmap_adapter *hwna = bna->hwna;
1396 u_int ring_n = kring->ring_id;
1397 u_int lim = kring->nkr_num_slots - 1;
1398 struct netmap_kring *hw_kring;
1401 nm_prdis("%s: na %s hwna %s",
1402 (kring ? kring->name : "NULL!"),
1403 (na ? na->name : "NULL!"),
1404 (hwna ? hwna->name : "NULL!"));
1405 hw_kring = hwna->tx_rings[ring_n];
1407 if (nm_kr_tryget(hw_kring, 0, NULL)) {
1411 /* first step: simulate a user wakeup on the rx ring */
1412 netmap_vp_rxsync(kring, flags);
1413 nm_prdis("%s[%d] PRE rx(c%3d t%3d l%3d) ring(h%3d c%3d t%3d) tx(c%3d ht%3d t%3d)",
1415 kring->nr_hwcur, kring->nr_hwtail, kring->nkr_hwlease,
1416 kring->rhead, kring->rcur, kring->rtail,
1417 hw_kring->nr_hwcur, hw_kring->nr_hwtail, hw_kring->rtail);
1418 /* second step: the new packets are sent on the tx ring
1419 * (which is actually the same ring)
1421 hw_kring->rhead = hw_kring->rcur = kring->nr_hwtail;
1422 error = hw_kring->nm_sync(hw_kring, flags);
1426 /* third step: now we are back the rx ring */
1427 /* claim ownership on all hw owned bufs */
1428 kring->rhead = kring->rcur = nm_next(hw_kring->nr_hwtail, lim); /* skip past reserved slot */
1430 /* fourth step: the user goes to sleep again, causing another rxsync */
1431 netmap_vp_rxsync(kring, flags);
1432 nm_prdis("%s[%d] PST rx(c%3d t%3d l%3d) ring(h%3d c%3d t%3d) tx(c%3d ht%3d t%3d)",
1434 kring->nr_hwcur, kring->nr_hwtail, kring->nkr_hwlease,
1435 kring->rhead, kring->rcur, kring->rtail,
1436 hw_kring->nr_hwcur, hw_kring->nr_hwtail, hw_kring->rtail);
1438 nm_kr_put(hw_kring);
1440 return error ? error : NM_IRQ_COMPLETED;
1444 /* nm_bdg_ctl callback for the bwrap.
1445 * Called on bridge-attach and detach, as an effect of valectl -[ahd].
1446 * On attach, it needs to provide a fake netmap_priv_d structure and
1447 * perform a netmap_do_regif() on the bwrap. This will put both the
1448 * bwrap and the hwna in netmap mode, with the netmap rings shared
1449 * and cross linked. Moroever, it will start intercepting interrupts
1453 netmap_bwrap_bdg_ctl(struct nmreq_header *hdr, struct netmap_adapter *na)
1455 struct netmap_priv_d *npriv;
1456 struct netmap_bwrap_adapter *bna = (struct netmap_bwrap_adapter*)na;
1459 if (hdr->nr_reqtype == NETMAP_REQ_VALE_ATTACH) {
1460 struct nmreq_vale_attach *req =
1461 (struct nmreq_vale_attach *)(uintptr_t)hdr->nr_body;
1462 if (req->reg.nr_ringid != 0 ||
1463 (req->reg.nr_mode != NR_REG_ALL_NIC &&
1464 req->reg.nr_mode != NR_REG_NIC_SW)) {
1465 /* We only support attaching all the NIC rings
1466 * and/or the host stack. */
1469 if (NETMAP_OWNED_BY_ANY(na)) {
1472 if (bna->na_kpriv) {
1476 npriv = netmap_priv_new();
1479 npriv->np_ifp = na->ifp; /* let the priv destructor release the ref */
1480 error = netmap_do_regif(npriv, na, req->reg.nr_mode,
1481 req->reg.nr_ringid, req->reg.nr_flags);
1483 netmap_priv_delete(npriv);
1486 bna->na_kpriv = npriv;
1487 na->na_flags |= NAF_BUSY;
1489 if (na->active_fds == 0) /* not registered */
1491 netmap_priv_delete(bna->na_kpriv);
1492 bna->na_kpriv = NULL;
1493 na->na_flags &= ~NAF_BUSY;
1499 /* attach a bridge wrapper to the 'real' device */
1501 netmap_bwrap_attach_common(struct netmap_adapter *na,
1502 struct netmap_adapter *hwna)
1504 struct netmap_bwrap_adapter *bna;
1505 struct netmap_adapter *hostna = NULL;
1509 /* make sure the NIC is not already in use */
1510 if (NETMAP_OWNED_BY_ANY(hwna)) {
1511 nm_prerr("NIC %s busy, cannot attach to bridge", hwna->name);
1515 bna = (struct netmap_bwrap_adapter *)na;
1516 /* make bwrap ifp point to the real ifp */
1517 na->ifp = hwna->ifp;
1519 na->na_private = bna;
1520 /* fill the ring data for the bwrap adapter with rx/tx meanings
1521 * swapped. The real cross-linking will be done during register,
1522 * when all the krings will have been created.
1525 enum txrx r = nm_txrx_swap(t); /* swap NR_TX <-> NR_RX */
1526 nma_set_nrings(na, t, nma_get_nrings(hwna, r));
1527 nma_set_ndesc(na, t, nma_get_ndesc(hwna, r));
1529 na->nm_dtor = netmap_bwrap_dtor;
1530 na->nm_config = netmap_bwrap_config;
1531 na->nm_bdg_ctl = netmap_bwrap_bdg_ctl;
1532 na->pdev = hwna->pdev;
1533 na->nm_mem = netmap_mem_get(hwna->nm_mem);
1534 na->virt_hdr_len = hwna->virt_hdr_len;
1535 na->rx_buf_maxsize = hwna->rx_buf_maxsize;
1538 netmap_adapter_get(hwna);
1539 hwna->na_private = bna; /* weak reference */
1540 bna->saved_na_vp = hwna->na_vp;
1541 hwna->na_vp = &bna->up;
1542 bna->up.up.na_vp = &(bna->up);
1544 if (hwna->na_flags & NAF_HOST_RINGS) {
1545 if (hwna->na_flags & NAF_SW_ONLY)
1546 na->na_flags |= NAF_SW_ONLY;
1547 na->na_flags |= NAF_HOST_RINGS;
1548 hostna = &bna->host.up;
1550 /* limit the number of host rings to that of hw */
1551 nm_bound_var(&hostna->num_tx_rings, 1, 1,
1552 nma_get_nrings(hwna, NR_TX), NULL);
1553 nm_bound_var(&hostna->num_rx_rings, 1, 1,
1554 nma_get_nrings(hwna, NR_RX), NULL);
1556 snprintf(hostna->name, sizeof(hostna->name), "%s^", na->name);
1557 hostna->ifp = hwna->ifp;
1559 enum txrx r = nm_txrx_swap(t);
1560 u_int nr = nma_get_nrings(hostna, t);
1562 nma_set_nrings(hostna, t, nr);
1563 nma_set_host_nrings(na, t, nr);
1564 if (nma_get_host_nrings(hwna, t) < nr) {
1565 nma_set_host_nrings(hwna, t, nr);
1567 nma_set_ndesc(hostna, t, nma_get_ndesc(hwna, r));
1569 // hostna->nm_txsync = netmap_bwrap_host_txsync;
1570 // hostna->nm_rxsync = netmap_bwrap_host_rxsync;
1571 hostna->nm_mem = netmap_mem_get(na->nm_mem);
1572 hostna->na_private = bna;
1573 hostna->na_vp = &bna->up;
1574 na->na_hostvp = hwna->na_hostvp =
1575 hostna->na_hostvp = &bna->host;
1576 hostna->na_flags = NAF_BUSY; /* prevent NIOCREGIF */
1577 hostna->rx_buf_maxsize = hwna->rx_buf_maxsize;
1579 if (hwna->na_flags & NAF_MOREFRAG)
1580 na->na_flags |= NAF_MOREFRAG;
1582 nm_prdis("%s<->%s txr %d txd %d rxr %d rxd %d",
1583 na->name, ifp->if_xname,
1584 na->num_tx_rings, na->num_tx_desc,
1585 na->num_rx_rings, na->num_rx_desc);
1587 error = netmap_attach_common(na);
1591 hwna->na_flags |= NAF_BUSY;
1595 hwna->na_vp = hwna->na_hostvp = NULL;
1596 netmap_adapter_put(hwna);
1602 netmap_init_bridges2(u_int n)
1605 struct nm_bridge *b;
1607 b = nm_os_malloc(sizeof(struct nm_bridge) * n);
1610 for (i = 0; i < n; i++)
1616 netmap_uninit_bridges2(struct nm_bridge *b, u_int n)
1623 for (i = 0; i < n; i++)
1624 BDG_RWDESTROY(&b[i]);
1629 netmap_init_bridges(void)
1631 #ifdef CONFIG_NET_NS
1632 return netmap_bns_register();
1634 nm_bridges = netmap_init_bridges2(NM_BRIDGES);
1635 if (nm_bridges == NULL)
1642 netmap_uninit_bridges(void)
1644 #ifdef CONFIG_NET_NS
1645 netmap_bns_unregister();
1647 netmap_uninit_bridges2(nm_bridges, NM_BRIDGES);