2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (C) 2014-2016 Giuseppe Lettieri
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #if defined(__FreeBSD__)
32 #include <sys/cdefs.h> /* prerequisite */
34 #include <sys/types.h>
35 #include <sys/errno.h>
36 #include <sys/param.h> /* defines used in kernel.h */
37 #include <sys/kernel.h> /* types used in module initialization */
38 #include <sys/malloc.h>
41 #include <sys/rwlock.h>
42 #include <sys/selinfo.h>
43 #include <sys/sysctl.h>
44 #include <sys/socket.h> /* sockaddrs */
46 #include <net/if_var.h>
47 #include <machine/bus.h> /* bus_dmamap_* */
48 #include <sys/refcount.h>
55 #elif defined(__APPLE__)
57 #warning OSX support is only partial
65 #error Unsupported platform
67 #endif /* unsupported */
73 #include <net/netmap.h>
74 #include <dev/netmap/netmap_kern.h>
75 #include <dev/netmap/netmap_mem2.h>
79 #define NM_PIPE_MAXSLOTS 4096
81 static int netmap_default_pipes = 0; /* ignored, kept for compatibility */
83 SYSCTL_DECL(_dev_netmap);
84 SYSCTL_INT(_dev_netmap, OID_AUTO, default_pipes, CTLFLAG_RW, &netmap_default_pipes, 0 , "");
87 /* allocate the pipe array in the parent adapter */
89 nm_pipe_alloc(struct netmap_adapter *na, u_int npipes)
92 struct netmap_pipe_adapter **npa;
94 if (npipes <= na->na_max_pipes)
95 /* we already have more entries that requested */
98 if (npipes < na->na_next_pipe || npipes > NM_MAXPIPES)
101 old_len = sizeof(struct netmap_pipe_adapter *)*na->na_max_pipes;
102 len = sizeof(struct netmap_pipe_adapter *) * npipes;
103 npa = nm_os_realloc(na->na_pipes, len, old_len);
108 na->na_max_pipes = npipes;
113 /* deallocate the parent array in the parent adapter */
115 netmap_pipe_dealloc(struct netmap_adapter *na)
118 if (na->na_next_pipe > 0) {
119 D("freeing not empty pipe array for %s (%d dangling pipes)!", na->name,
122 nm_os_free(na->na_pipes);
124 na->na_max_pipes = 0;
125 na->na_next_pipe = 0;
129 /* find a pipe endpoint with the given id among the parent's pipes */
130 static struct netmap_pipe_adapter *
131 netmap_pipe_find(struct netmap_adapter *parent, u_int pipe_id)
134 struct netmap_pipe_adapter *na;
136 for (i = 0; i < parent->na_next_pipe; i++) {
137 na = parent->na_pipes[i];
138 if (na->id == pipe_id) {
145 /* add a new pipe endpoint to the parent array */
147 netmap_pipe_add(struct netmap_adapter *parent, struct netmap_pipe_adapter *na)
149 if (parent->na_next_pipe >= parent->na_max_pipes) {
150 u_int npipes = parent->na_max_pipes ? 2*parent->na_max_pipes : 2;
151 int error = nm_pipe_alloc(parent, npipes);
156 parent->na_pipes[parent->na_next_pipe] = na;
157 na->parent_slot = parent->na_next_pipe;
158 parent->na_next_pipe++;
162 /* remove the given pipe endpoint from the parent array */
164 netmap_pipe_remove(struct netmap_adapter *parent, struct netmap_pipe_adapter *na)
167 n = --parent->na_next_pipe;
168 if (n != na->parent_slot) {
169 struct netmap_pipe_adapter **p =
170 &parent->na_pipes[na->parent_slot];
171 *p = parent->na_pipes[n];
172 (*p)->parent_slot = na->parent_slot;
174 parent->na_pipes[n] = NULL;
178 netmap_pipe_txsync(struct netmap_kring *txkring, int flags)
180 struct netmap_kring *rxkring = txkring->pipe;
181 u_int limit; /* slots to transfer */
182 u_int j, k, lim_tx = txkring->nkr_num_slots - 1,
183 lim_rx = rxkring->nkr_num_slots - 1;
186 ND("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name);
187 ND(2, "before: hwcur %d hwtail %d cur %d head %d tail %d", txkring->nr_hwcur, txkring->nr_hwtail,
188 txkring->rcur, txkring->rhead, txkring->rtail);
190 j = rxkring->nr_hwtail; /* RX */
191 k = txkring->nr_hwcur; /* TX */
192 m = txkring->rhead - txkring->nr_hwcur; /* new slots */
194 m += txkring->nkr_num_slots;
196 m = lim_rx; /* max avail space on destination */
197 busy = j - rxkring->nr_hwcur; /* busy slots */
199 busy += rxkring->nkr_num_slots;
200 m -= busy; /* subtract busy slots */
201 ND(2, "m %d limit %d", m, limit);
206 /* either the rxring is full, or nothing to send */
210 while (limit-- > 0) {
211 struct netmap_slot *rs = &rxkring->ring->slot[j];
212 struct netmap_slot *ts = &txkring->ring->slot[k];
213 struct netmap_slot tmp;
220 /* report the buffer change */
221 ts->flags |= NS_BUF_CHANGED;
222 rs->flags |= NS_BUF_CHANGED;
224 j = nm_next(j, lim_rx);
225 k = nm_next(k, lim_tx);
228 mb(); /* make sure the slots are updated before publishing them */
229 rxkring->nr_hwtail = j;
230 txkring->nr_hwcur = k;
231 txkring->nr_hwtail = nm_prev(k, lim_tx);
233 ND(2, "after: hwcur %d hwtail %d cur %d head %d tail %d j %d", txkring->nr_hwcur, txkring->nr_hwtail,
234 txkring->rcur, txkring->rhead, txkring->rtail, j);
236 mb(); /* make sure rxkring->nr_hwtail is updated before notifying */
237 rxkring->nm_notify(rxkring, 0);
243 netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags)
245 struct netmap_kring *txkring = rxkring->pipe;
246 uint32_t oldhwcur = rxkring->nr_hwcur;
248 ND("%s %x <- %s", rxkring->name, flags, txkring->name);
249 rxkring->nr_hwcur = rxkring->rhead; /* recover user-relased slots */
250 ND(5, "hwcur %d hwtail %d cur %d head %d tail %d", rxkring->nr_hwcur, rxkring->nr_hwtail,
251 rxkring->rcur, rxkring->rhead, rxkring->rtail);
252 mb(); /* paired with the first mb() in txsync */
254 if (oldhwcur != rxkring->nr_hwcur) {
255 /* we have released some slots, notify the other end */
256 mb(); /* make sure nr_hwcur is updated before notifying */
257 txkring->nm_notify(txkring, 0);
262 /* Pipe endpoints are created and destroyed together, so that endopoints do not
263 * have to check for the existence of their peer at each ?xsync.
265 * To play well with the existing netmap infrastructure (refcounts etc.), we
266 * adopt the following strategy:
268 * 1) The first endpoint that is created also creates the other endpoint and
269 * grabs a reference to it.
271 * state A) user1 --> endpoint1 --> endpoint2
273 * 2) If, starting from state A, endpoint2 is then registered, endpoint1 gives
274 * its reference to the user:
276 * state B) user1 --> endpoint1 endpoint2 <--- user2
278 * 3) Assume that, starting from state B endpoint2 is closed. In the unregister
279 * callback endpoint2 notes that endpoint1 is still active and adds a reference
280 * from endpoint1 to itself. When user2 then releases her own reference,
281 * endpoint2 is not destroyed and we are back to state A. A symmetrical state
282 * would be reached if endpoint1 were released instead.
284 * 4) If, starting from state A, endpoint1 is closed, the destructor notes that
285 * it owns a reference to endpoint2 and releases it.
287 * Something similar goes on for the creation and destruction of the krings.
291 /* netmap_pipe_krings_create.
293 * There are two cases:
299 * and we are e1. We have to create both sets
306 * and we are e2. e1 is certainly registered and our
307 * krings already exist. Nothing to do.
310 netmap_pipe_krings_create(struct netmap_adapter *na)
312 struct netmap_pipe_adapter *pna =
313 (struct netmap_pipe_adapter *)na;
314 struct netmap_adapter *ona = &pna->peer->up;
322 ND("%p: case 1, create both ends", na);
323 error = netmap_krings_create(na, 0);
327 /* create the krings of the other end */
328 error = netmap_krings_create(ona, 0);
332 /* cross link the krings */
334 enum txrx r = nm_txrx_swap(t); /* swap NR_TX <-> NR_RX */
335 for (i = 0; i < nma_get_nrings(na, t); i++) {
336 NMR(na, t)[i].pipe = NMR(ona, r) + i;
337 NMR(ona, r)[i].pipe = NMR(na, t) + i;
345 netmap_krings_delete(na);
352 * There are two cases on registration (onoff==1)
358 * and we are e1. Create the needed rings of the
363 * usr1 --> e1 --> e2 <-- usr2
365 * and we are e2. Drop the ref e1 is holding.
367 * There are two additional cases on unregister (onoff==0)
373 * and we are e1. Nothing special to do, e2 will
374 * be cleaned up by the destructor of e1.
378 * usr1 --> e1 e2 <-- usr2
380 * and we are either e1 or e2. Add a ref from the
381 * other end and hide our rings.
384 netmap_pipe_reg(struct netmap_adapter *na, int onoff)
386 struct netmap_pipe_adapter *pna =
387 (struct netmap_pipe_adapter *)na;
388 struct netmap_adapter *ona = &pna->peer->up;
392 ND("%p: onoff %d", na, onoff);
395 for (i = 0; i < nma_get_nrings(na, t); i++) {
396 struct netmap_kring *kring = &NMR(na, t)[i];
398 if (nm_kring_pending_on(kring)) {
399 /* mark the peer ring as needed */
400 kring->pipe->nr_kflags |= NKR_NEEDRING;
405 /* create all missing needed rings on the other end */
406 error = netmap_mem_rings_create(ona);
410 /* In case of no error we put our rings in netmap mode */
412 for (i = 0; i < nma_get_nrings(na, t) + 1; i++) {
413 struct netmap_kring *kring = &NMR(na, t)[i];
415 if (nm_kring_pending_on(kring)) {
416 kring->nr_mode = NKR_NETMAP_ON;
420 if (na->active_fds == 0)
421 na->na_flags |= NAF_NETMAP_ON;
423 if (na->active_fds == 0)
424 na->na_flags &= ~NAF_NETMAP_ON;
426 for (i = 0; i < nma_get_nrings(na, t) + 1; i++) {
427 struct netmap_kring *kring = &NMR(na, t)[i];
429 if (nm_kring_pending_off(kring)) {
430 kring->nr_mode = NKR_NETMAP_OFF;
431 /* mark the peer ring as no longer needed by us
432 * (it may still be kept if sombody else is using it)
435 kring->pipe->nr_kflags &= ~NKR_NEEDRING;
440 /* delete all the peer rings that are no longer needed */
441 netmap_mem_rings_delete(ona);
444 if (na->active_fds) {
445 ND("active_fds %d", na->active_fds);
450 ND("%p: case 1.a or 2.a, nothing to do", na);
454 ND("%p: case 1.b, drop peer", na);
455 pna->peer->peer_ref = 0;
456 netmap_adapter_put(na);
458 ND("%p: case 2.b, grab peer", na);
459 netmap_adapter_get(na);
460 pna->peer->peer_ref = 1;
465 /* netmap_pipe_krings_delete.
467 * There are two cases:
473 * and we are e1 (e2 is not registered, so krings_delete cannot be
478 * usr1 --> e1 e2 <-- usr2
480 * and we are either e1 or e2.
482 * In the former case we have to also delete the krings of e2;
483 * in the latter case we do nothing (note that our krings
484 * have already been hidden in the unregister callback).
487 netmap_pipe_krings_delete(struct netmap_adapter *na)
489 struct netmap_pipe_adapter *pna =
490 (struct netmap_pipe_adapter *)na;
491 struct netmap_adapter *ona; /* na of the other end */
493 if (!pna->peer_ref) {
494 ND("%p: case 2, kept alive by peer", na);
498 ND("%p: case 1, deleting everything", na);
499 netmap_krings_delete(na); /* also zeroes tx_rings etc. */
500 ona = &pna->peer->up;
501 if (ona->tx_rings == NULL) {
502 /* already deleted, we must be on an
503 * cleanup-after-error path */
506 netmap_krings_delete(ona);
511 netmap_pipe_dtor(struct netmap_adapter *na)
513 struct netmap_pipe_adapter *pna =
514 (struct netmap_pipe_adapter *)na;
515 ND("%p %p", na, pna->parent_ifp);
517 ND("%p: clean up peer", na);
519 netmap_adapter_put(&pna->peer->up);
521 if (pna->role == NR_REG_PIPE_MASTER)
522 netmap_pipe_remove(pna->parent, pna);
524 if_rele(pna->parent_ifp);
525 netmap_adapter_put(pna->parent);
530 netmap_get_pipe_na(struct nmreq *nmr, struct netmap_adapter **na,
531 struct netmap_mem_d *nmd, int create)
534 struct netmap_adapter *pna; /* parent adapter */
535 struct netmap_pipe_adapter *mna, *sna, *req;
536 struct ifnet *ifp = NULL;
538 int role = nmr->nr_flags & NR_REG_MASK;
539 int error, retries = 0;
541 ND("flags %x", nmr->nr_flags);
543 if (role != NR_REG_PIPE_MASTER && role != NR_REG_PIPE_SLAVE) {
547 role = nmr->nr_flags & NR_REG_MASK;
549 /* first, try to find the parent adapter */
550 bzero(&pnmr, sizeof(pnmr));
551 memcpy(&pnmr.nr_name, nmr->nr_name, IFNAMSIZ);
552 /* pass to parent the requested number of pipes */
553 pnmr.nr_arg1 = nmr->nr_arg1;
557 error = netmap_get_na(&pnmr, &pna, &ifp, nmd, create);
560 if (error != ENXIO || retries++) {
561 ND("parent lookup failed: %d", error);
564 ND("try to create a persistent vale port");
565 /* create a persistent vale port and try again */
567 create_error = netmap_vi_create(&pnmr, 1 /* autodelete */);
569 if (create_error && create_error != EEXIST) {
570 if (create_error != EOPNOTSUPP) {
571 D("failed to create a persistent vale port: %d", create_error);
577 if (NETMAP_OWNED_BY_KERN(pna)) {
583 /* next, lookup the pipe id in the parent list */
585 pipe_id = nmr->nr_ringid & NETMAP_RING_MASK;
586 mna = netmap_pipe_find(pna, pipe_id);
588 if (mna->role == role) {
589 ND("found %d directly at %d", pipe_id, mna->parent_slot);
592 ND("found %d indirectly at %d", pipe_id, mna->parent_slot);
595 /* the pipe we have found already holds a ref to the parent,
596 * so we need to drop the one we got from netmap_get_na()
598 netmap_unget_na(pna, ifp);
601 ND("pipe %d not found, create %d", pipe_id, create);
606 /* we create both master and slave.
607 * The endpoint we were asked for holds a reference to
610 mna = nm_os_malloc(sizeof(*mna));
615 snprintf(mna->up.name, sizeof(mna->up.name), "%s{%d", pna->name, pipe_id);
618 mna->role = NR_REG_PIPE_MASTER;
620 mna->parent_ifp = ifp;
622 mna->up.nm_txsync = netmap_pipe_txsync;
623 mna->up.nm_rxsync = netmap_pipe_rxsync;
624 mna->up.nm_register = netmap_pipe_reg;
625 mna->up.nm_dtor = netmap_pipe_dtor;
626 mna->up.nm_krings_create = netmap_pipe_krings_create;
627 mna->up.nm_krings_delete = netmap_pipe_krings_delete;
628 mna->up.nm_mem = netmap_mem_get(pna->nm_mem);
629 mna->up.na_flags |= NAF_MEM_OWNER;
630 mna->up.na_lut = pna->na_lut;
632 mna->up.num_tx_rings = 1;
633 mna->up.num_rx_rings = 1;
634 mna->up.num_tx_desc = nmr->nr_tx_slots;
635 nm_bound_var(&mna->up.num_tx_desc, pna->num_tx_desc,
636 1, NM_PIPE_MAXSLOTS, NULL);
637 mna->up.num_rx_desc = nmr->nr_rx_slots;
638 nm_bound_var(&mna->up.num_rx_desc, pna->num_rx_desc,
639 1, NM_PIPE_MAXSLOTS, NULL);
640 error = netmap_attach_common(&mna->up);
643 /* register the master with the parent */
644 error = netmap_pipe_add(pna, mna);
648 /* create the slave */
649 sna = nm_os_malloc(sizeof(*mna));
654 /* most fields are the same, copy from master and then fix */
656 sna->up.nm_mem = netmap_mem_get(mna->up.nm_mem);
657 snprintf(sna->up.name, sizeof(sna->up.name), "%s}%d", pna->name, pipe_id);
658 sna->role = NR_REG_PIPE_SLAVE;
659 error = netmap_attach_common(&sna->up);
663 /* join the two endpoints */
667 /* we already have a reference to the parent, but we
668 * need another one for the other endpoint we created
670 netmap_adapter_get(pna);
671 /* likewise for the ifp, if any */
675 if (role == NR_REG_PIPE_MASTER) {
678 netmap_adapter_get(&sna->up);
682 netmap_adapter_get(&mna->up);
684 ND("created master %p and slave %p", mna, sna);
687 ND("pipe %d %s at %p", pipe_id,
688 (req->role == NR_REG_PIPE_MASTER ? "master" : "slave"), req);
690 netmap_adapter_get(*na);
692 /* keep the reference to the parent.
693 * It will be released by the req destructor
701 netmap_pipe_remove(pna, mna);
705 netmap_unget_na(pna, ifp);
710 #endif /* WITH_PIPES */