2 * Copyright (C) 2014 Giuseppe Lettieri. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #if defined(__FreeBSD__)
29 #include <sys/cdefs.h> /* prerequisite */
31 #include <sys/types.h>
32 #include <sys/errno.h>
33 #include <sys/param.h> /* defines used in kernel.h */
34 #include <sys/kernel.h> /* types used in module initialization */
35 #include <sys/malloc.h>
38 #include <sys/rwlock.h>
39 #include <sys/selinfo.h>
40 #include <sys/sysctl.h>
41 #include <sys/socket.h> /* sockaddrs */
43 #include <net/if_var.h>
44 #include <machine/bus.h> /* bus_dmamap_* */
45 #include <sys/refcount.h>
52 #elif defined(__APPLE__)
54 #warning OSX support is only partial
59 #error Unsupported platform
61 #endif /* unsupported */
67 #include <net/netmap.h>
68 #include <dev/netmap/netmap_kern.h>
69 #include <dev/netmap/netmap_mem2.h>
73 #define NM_PIPE_MAXSLOTS 4096
75 int netmap_default_pipes = 0; /* default number of pipes for each nic */
76 SYSCTL_DECL(_dev_netmap);
77 SYSCTL_INT(_dev_netmap, OID_AUTO, default_pipes, CTLFLAG_RW, &netmap_default_pipes, 0 , "");
79 /* allocate the pipe array in the parent adapter */
81 netmap_pipe_alloc(struct netmap_adapter *na, struct nmreq *nmr)
84 int mode = nmr->nr_flags & NR_REG_MASK;
87 if (mode == NR_REG_PIPE_MASTER || mode == NR_REG_PIPE_SLAVE) {
88 /* this is for our parent, not for us */
92 /* TODO: we can resize the array if the new
93 * request can accomodate the already existing pipes
96 nmr->nr_arg1 = na->na_max_pipes;
100 npipes = nmr->nr_arg1;
102 npipes = netmap_default_pipes;
103 nm_bound_var(&npipes, 0, 0, NM_MAXPIPES, NULL);
106 /* really zero, nothing to alloc */
110 len = sizeof(struct netmap_pipe_adapter *) * npipes;
111 na->na_pipes = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO);
112 if (na->na_pipes == NULL)
115 na->na_max_pipes = npipes;
116 na->na_next_pipe = 0;
119 nmr->nr_arg1 = npipes;
124 /* deallocate the parent array in the parent adapter */
126 netmap_pipe_dealloc(struct netmap_adapter *na)
129 ND("freeing pipes for %s", na->name);
130 free(na->na_pipes, M_DEVBUF);
132 na->na_max_pipes = 0;
133 na->na_next_pipe = 0;
137 /* find a pipe endpoint with the given id among the parent's pipes */
138 static struct netmap_pipe_adapter *
139 netmap_pipe_find(struct netmap_adapter *parent, u_int pipe_id)
142 struct netmap_pipe_adapter *na;
144 for (i = 0; i < parent->na_next_pipe; i++) {
145 na = parent->na_pipes[i];
146 if (na->id == pipe_id) {
153 /* add a new pipe endpoint to the parent array */
155 netmap_pipe_add(struct netmap_adapter *parent, struct netmap_pipe_adapter *na)
157 if (parent->na_next_pipe >= parent->na_max_pipes) {
158 D("%s: no space left for pipes", parent->name);
162 parent->na_pipes[parent->na_next_pipe] = na;
163 na->parent_slot = parent->na_next_pipe;
164 parent->na_next_pipe++;
168 /* remove the given pipe endpoint from the parent array */
170 netmap_pipe_remove(struct netmap_adapter *parent, struct netmap_pipe_adapter *na)
173 n = --parent->na_next_pipe;
174 if (n != na->parent_slot) {
175 parent->na_pipes[na->parent_slot] =
178 parent->na_pipes[n] = NULL;
182 netmap_pipe_txsync(struct netmap_kring *txkring, int flags)
184 struct netmap_kring *rxkring = txkring->pipe;
185 u_int limit; /* slots to transfer */
186 u_int j, k, lim_tx = txkring->nkr_num_slots - 1,
187 lim_rx = rxkring->nkr_num_slots - 1;
190 ND("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name);
191 ND(2, "before: hwcur %d hwtail %d cur %d head %d tail %d", txkring->nr_hwcur, txkring->nr_hwtail,
192 txkring->rcur, txkring->rhead, txkring->rtail);
194 j = rxkring->nr_hwtail; /* RX */
195 k = txkring->nr_hwcur; /* TX */
196 m = txkring->rhead - txkring->nr_hwcur; /* new slots */
198 m += txkring->nkr_num_slots;
200 m = lim_rx; /* max avail space on destination */
201 busy = j - rxkring->nr_hwcur; /* busy slots */
203 busy += rxkring->nkr_num_slots;
204 m -= busy; /* subtract busy slots */
205 ND(2, "m %d limit %d", m, limit);
210 /* either the rxring is full, or nothing to send */
211 nm_txsync_finalize(txkring); /* actually useless */
215 while (limit-- > 0) {
216 struct netmap_slot *rs = &rxkring->save_ring->slot[j];
217 struct netmap_slot *ts = &txkring->ring->slot[k];
218 struct netmap_slot tmp;
225 /* no need to report the buffer change */
227 j = nm_next(j, lim_rx);
228 k = nm_next(k, lim_tx);
231 mb(); /* make sure the slots are updated before publishing them */
232 rxkring->nr_hwtail = j;
233 txkring->nr_hwcur = k;
234 txkring->nr_hwtail = nm_prev(k, lim_tx);
236 nm_txsync_finalize(txkring);
237 ND(2, "after: hwcur %d hwtail %d cur %d head %d tail %d j %d", txkring->nr_hwcur, txkring->nr_hwtail,
238 txkring->rcur, txkring->rhead, txkring->rtail, j);
240 mb(); /* make sure rxkring->nr_hwtail is updated before notifying */
241 rxkring->na->nm_notify(rxkring->na, rxkring->ring_id, NR_RX, 0);
247 netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags)
249 struct netmap_kring *txkring = rxkring->pipe;
250 uint32_t oldhwcur = rxkring->nr_hwcur;
252 ND("%s %x <- %s", rxkring->name, flags, txkring->name);
253 rxkring->nr_hwcur = rxkring->rhead; /* recover user-relased slots */
254 ND(5, "hwcur %d hwtail %d cur %d head %d tail %d", rxkring->nr_hwcur, rxkring->nr_hwtail,
255 rxkring->rcur, rxkring->rhead, rxkring->rtail);
256 mb(); /* paired with the first mb() in txsync */
257 nm_rxsync_finalize(rxkring);
259 if (oldhwcur != rxkring->nr_hwcur) {
260 /* we have released some slots, notify the other end */
261 mb(); /* make sure nr_hwcur is updated before notifying */
262 txkring->na->nm_notify(txkring->na, txkring->ring_id, NR_TX, 0);
267 /* Pipe endpoints are created and destroyed together, so that endopoints do not
268 * have to check for the existence of their peer at each ?xsync.
270 * To play well with the existing netmap infrastructure (refcounts etc.), we
271 * adopt the following strategy:
273 * 1) The first endpoint that is created also creates the other endpoint and
274 * grabs a reference to it.
276 * state A) user1 --> endpoint1 --> endpoint2
278 * 2) If, starting from state A, endpoint2 is then registered, endpoint1 gives
279 * its reference to the user:
281 * state B) user1 --> endpoint1 endpoint2 <--- user2
283 * 3) Assume that, starting from state B endpoint2 is closed. In the unregister
284 * callback endpoint2 notes that endpoint1 is still active and adds a reference
285 * from endpoint1 to itself. When user2 then releases her own reference,
286 * endpoint2 is not destroyed and we are back to state A. A symmetrical state
287 * would be reached if endpoint1 were released instead.
289 * 4) If, starting from state A, endpoint1 is closed, the destructor notes that
290 * it owns a reference to endpoint2 and releases it.
292 * Something similar goes on for the creation and destruction of the krings.
296 /* netmap_pipe_krings_delete.
298 * There are two cases:
304 * and we are e1. We have to create both sets
311 * and we are e2. e1 is certainly registered and our
312 * krings already exist, but they may be hidden.
315 netmap_pipe_krings_create(struct netmap_adapter *na)
317 struct netmap_pipe_adapter *pna =
318 (struct netmap_pipe_adapter *)na;
319 struct netmap_adapter *ona = &pna->peer->up;
325 D("%p: case 1, create everything", na);
326 error = netmap_krings_create(na, 0);
330 /* we also create all the rings, since we need to
331 * update the save_ring pointers.
332 * netmap_mem_rings_create (called by our caller)
333 * will not create the rings again
336 error = netmap_mem_rings_create(na);
340 /* update our hidden ring pointers */
341 for (i = 0; i < na->num_tx_rings + 1; i++)
342 na->tx_rings[i].save_ring = na->tx_rings[i].ring;
343 for (i = 0; i < na->num_rx_rings + 1; i++)
344 na->rx_rings[i].save_ring = na->rx_rings[i].ring;
346 /* now, create krings and rings of the other end */
347 error = netmap_krings_create(ona, 0);
351 error = netmap_mem_rings_create(ona);
355 for (i = 0; i < ona->num_tx_rings + 1; i++)
356 ona->tx_rings[i].save_ring = ona->tx_rings[i].ring;
357 for (i = 0; i < ona->num_rx_rings + 1; i++)
358 ona->rx_rings[i].save_ring = ona->rx_rings[i].ring;
360 /* cross link the krings */
361 for (i = 0; i < na->num_tx_rings; i++) {
362 na->tx_rings[i].pipe = pna->peer->up.rx_rings + i;
363 na->rx_rings[i].pipe = pna->peer->up.tx_rings + i;
364 pna->peer->up.tx_rings[i].pipe = na->rx_rings + i;
365 pna->peer->up.rx_rings[i].pipe = na->tx_rings + i;
370 /* recover the hidden rings */
371 ND("%p: case 2, hidden rings", na);
372 for (i = 0; i < na->num_tx_rings + 1; i++)
373 na->tx_rings[i].ring = na->tx_rings[i].save_ring;
374 for (i = 0; i < na->num_rx_rings + 1; i++)
375 na->rx_rings[i].ring = na->rx_rings[i].save_ring;
380 netmap_krings_delete(ona);
382 netmap_mem_rings_delete(na);
384 netmap_krings_delete(na);
391 * There are two cases on registration (onoff==1)
397 * and we are e1. Nothing special to do.
401 * usr1 --> e1 --> e2 <-- usr2
403 * and we are e2. Drop the ref e1 is holding.
405 * There are two additional cases on unregister (onoff==0)
411 * and we are e1. Nothing special to do, e2 will
412 * be cleaned up by the destructor of e1.
416 * usr1 --> e1 e2 <-- usr2
418 * and we are either e1 or e2. Add a ref from the
419 * other end and hide our rings.
422 netmap_pipe_reg(struct netmap_adapter *na, int onoff)
424 struct netmap_pipe_adapter *pna =
425 (struct netmap_pipe_adapter *)na;
426 ND("%p: onoff %d", na, onoff);
428 na->na_flags |= NAF_NETMAP_ON;
430 na->na_flags &= ~NAF_NETMAP_ON;
433 ND("%p: case 1.a or 2.a, nothing to do", na);
437 ND("%p: case 1.b, drop peer", na);
438 pna->peer->peer_ref = 0;
439 netmap_adapter_put(na);
442 ND("%p: case 2.b, grab peer", na);
443 netmap_adapter_get(na);
444 pna->peer->peer_ref = 1;
445 /* hide our rings from netmap_mem_rings_delete */
446 for (i = 0; i < na->num_tx_rings + 1; i++) {
447 na->tx_rings[i].ring = NULL;
449 for (i = 0; i < na->num_rx_rings + 1; i++) {
450 na->rx_rings[i].ring = NULL;
456 /* netmap_pipe_krings_delete.
458 * There are two cases:
464 * and we are e1 (e2 is not registered, so krings_delete cannot be
469 * usr1 --> e1 e2 <-- usr2
471 * and we are either e1 or e2.
473 * In the former case we have to also delete the krings of e2;
474 * in the latter case we do nothing (note that our krings
475 * have already been hidden in the unregister callback).
478 netmap_pipe_krings_delete(struct netmap_adapter *na)
480 struct netmap_pipe_adapter *pna =
481 (struct netmap_pipe_adapter *)na;
482 struct netmap_adapter *ona; /* na of the other end */
485 if (!pna->peer_ref) {
486 ND("%p: case 2, kept alive by peer", na);
490 ND("%p: case 1, deleting everyhing", na);
491 netmap_krings_delete(na); /* also zeroes tx_rings etc. */
492 /* restore the ring to be deleted on the peer */
493 ona = &pna->peer->up;
494 if (ona->tx_rings == NULL) {
495 /* already deleted, we must be on an
496 * cleanup-after-error path */
499 for (i = 0; i < ona->num_tx_rings + 1; i++)
500 ona->tx_rings[i].ring = ona->tx_rings[i].save_ring;
501 for (i = 0; i < ona->num_rx_rings + 1; i++)
502 ona->rx_rings[i].ring = ona->rx_rings[i].save_ring;
503 netmap_mem_rings_delete(ona);
504 netmap_krings_delete(ona);
509 netmap_pipe_dtor(struct netmap_adapter *na)
511 struct netmap_pipe_adapter *pna =
512 (struct netmap_pipe_adapter *)na;
515 ND("%p: clean up peer", na);
517 netmap_adapter_put(&pna->peer->up);
519 if (pna->role == NR_REG_PIPE_MASTER)
520 netmap_pipe_remove(pna->parent, pna);
521 netmap_adapter_put(pna->parent);
526 netmap_get_pipe_na(struct nmreq *nmr, struct netmap_adapter **na, int create)
529 struct netmap_adapter *pna; /* parent adapter */
530 struct netmap_pipe_adapter *mna, *sna, *req;
532 int role = nmr->nr_flags & NR_REG_MASK;
535 ND("flags %x", nmr->nr_flags);
537 if (role != NR_REG_PIPE_MASTER && role != NR_REG_PIPE_SLAVE) {
541 role = nmr->nr_flags & NR_REG_MASK;
543 /* first, try to find the parent adapter */
544 bzero(&pnmr, sizeof(pnmr));
545 memcpy(&pnmr.nr_name, nmr->nr_name, IFNAMSIZ);
546 /* pass to parent the requested number of pipes */
547 pnmr.nr_arg1 = nmr->nr_arg1;
548 error = netmap_get_na(&pnmr, &pna, create);
550 ND("parent lookup failed: %d", error);
553 ND("found parent: %s", na->name);
555 if (NETMAP_OWNED_BY_KERN(pna)) {
561 /* next, lookup the pipe id in the parent list */
563 pipe_id = nmr->nr_ringid & NETMAP_RING_MASK;
564 mna = netmap_pipe_find(pna, pipe_id);
566 if (mna->role == role) {
567 ND("found %d directly at %d", pipe_id, mna->parent_slot);
570 ND("found %d indirectly at %d", pipe_id, mna->parent_slot);
573 /* the pipe we have found already holds a ref to the parent,
574 * so we need to drop the one we got from netmap_get_na()
576 netmap_adapter_put(pna);
579 ND("pipe %d not found, create %d", pipe_id, create);
584 /* we create both master and slave.
585 * The endpoint we were asked for holds a reference to
588 mna = malloc(sizeof(*mna), M_DEVBUF, M_NOWAIT | M_ZERO);
593 snprintf(mna->up.name, sizeof(mna->up.name), "%s{%d", pna->name, pipe_id);
596 mna->role = NR_REG_PIPE_MASTER;
599 mna->up.nm_txsync = netmap_pipe_txsync;
600 mna->up.nm_rxsync = netmap_pipe_rxsync;
601 mna->up.nm_register = netmap_pipe_reg;
602 mna->up.nm_dtor = netmap_pipe_dtor;
603 mna->up.nm_krings_create = netmap_pipe_krings_create;
604 mna->up.nm_krings_delete = netmap_pipe_krings_delete;
605 mna->up.nm_mem = pna->nm_mem;
606 mna->up.na_lut = pna->na_lut;
607 mna->up.na_lut_objtotal = pna->na_lut_objtotal;
608 mna->up.na_lut_objsize = pna->na_lut_objsize;
610 mna->up.num_tx_rings = 1;
611 mna->up.num_rx_rings = 1;
612 mna->up.num_tx_desc = nmr->nr_tx_slots;
613 nm_bound_var(&mna->up.num_tx_desc, pna->num_tx_desc,
614 1, NM_PIPE_MAXSLOTS, NULL);
615 mna->up.num_rx_desc = nmr->nr_rx_slots;
616 nm_bound_var(&mna->up.num_rx_desc, pna->num_rx_desc,
617 1, NM_PIPE_MAXSLOTS, NULL);
618 error = netmap_attach_common(&mna->up);
621 /* register the master with the parent */
622 error = netmap_pipe_add(pna, mna);
626 /* create the slave */
627 sna = malloc(sizeof(*mna), M_DEVBUF, M_NOWAIT | M_ZERO);
632 /* most fields are the same, copy from master and then fix */
634 snprintf(sna->up.name, sizeof(sna->up.name), "%s}%d", pna->name, pipe_id);
635 sna->role = NR_REG_PIPE_SLAVE;
636 error = netmap_attach_common(&sna->up);
640 /* join the two endpoints */
644 /* we already have a reference to the parent, but we
645 * need another one for the other endpoint we created
647 netmap_adapter_get(pna);
649 if (role == NR_REG_PIPE_MASTER) {
652 netmap_adapter_get(&sna->up);
656 netmap_adapter_get(&mna->up);
658 ND("created master %p and slave %p", mna, sna);
661 ND("pipe %d %s at %p", pipe_id,
662 (req->role == NR_REG_PIPE_MASTER ? "master" : "slave"), req);
664 netmap_adapter_get(*na);
666 /* write the configuration back */
667 nmr->nr_tx_rings = req->up.num_tx_rings;
668 nmr->nr_rx_rings = req->up.num_rx_rings;
669 nmr->nr_tx_slots = req->up.num_tx_desc;
670 nmr->nr_rx_slots = req->up.num_rx_desc;
672 /* keep the reference to the parent.
673 * It will be released by the req destructor
683 netmap_adapter_put(pna);
688 #endif /* WITH_PIPES */