2 * Copyright (C) 2011-2012 Matteo Landi, Luigi Rizzo. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * $Id: netmap_kern.h 10602 2012-02-21 16:47:55Z luigi $
30 * The header contains the definitions of constants and function
31 * prototypes used only in kernelspace.
34 #ifndef _NET_NETMAP_KERN_H_
35 #define _NET_NETMAP_KERN_H_
37 #define NETMAP_MEM2 // use the new memory allocator
39 #if defined(__FreeBSD__)
40 #define NM_LOCK_T struct mtx
41 #define NM_SELINFO_T struct selinfo
42 #define MBUF_LEN(m) ((m)->m_pkthdr.len)
43 #define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m)
45 #define NM_LOCK_T spinlock_t
46 #define NM_SELINFO_T wait_queue_head_t
47 #define MBUF_LEN(m) ((m)->len)
48 #define NM_SEND_UP(ifp, m) netif_rx(m)
50 #error unsupported platform
54 MALLOC_DECLARE(M_NETMAP);
57 #define ND(format, ...)
58 #define D(format, ...) \
60 struct timeval __xxts; \
62 printf("%03d.%06d %s [%d] " format "\n", \
63 (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \
64 __FUNCTION__, __LINE__, ##__VA_ARGS__); \
67 struct netmap_adapter;
70 * private, kernel view of a ring. Keeps track of the status of
71 * a ring across system calls.
73 * nr_hwcur index of the next buffer to refill.
74 * It corresponds to ring->cur - ring->reserved
76 * nr_hwavail the number of slots "owned" by userspace.
77 * nr_hwavail =:= ring->avail + ring->reserved
79 * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots.
80 * This is so that, on a reset, buffers owned by userspace are not
81 * modified by the kernel. In particular:
82 * RX rings: the next empty buffer (hwcur + hwavail + hwofs) coincides with
83 * the next empty buffer as known by the hardware (next_to_check or so).
84 * TX rings: hwcur + hwofs coincides with next_to_send
87 struct netmap_ring *ring;
90 u_int nr_kflags; /* private driver flags */
91 #define NKR_PENDINTR 0x1 // Pending interrupt.
94 int nkr_hwofs; /* offset between NIC and netmap ring */
95 struct netmap_adapter *na;
96 NM_SELINFO_T si; /* poll/select wait queue */
97 NM_LOCK_T q_lock; /* used if no device lock available */
98 } __attribute__((__aligned__(64)));
101 * This struct extends the 'struct adapter' (or
102 * equivalent) device descriptor. It contains all fields needed to
103 * support netmap operation.
105 struct netmap_adapter {
106 int refcount; /* number of user-space descriptors using this
107 interface, which is equal to the number of
108 struct netmap_if objs in the mapped region. */
110 * The selwakeup in the interrupt thread can use per-ring
111 * and/or global wait queues. We track how many clients
112 * of each type we have so we can optimize the drivers,
113 * and especially avoid huge contention on the locks.
115 int na_single; /* threads attached to a single hw queue */
116 int na_multi; /* threads attached to multiple hw queues */
118 int separate_locks; /* set if the interface suports different
119 locks for rx, tx and core. */
121 u_int num_rx_rings; /* number of tx/rx ring pairs */
122 u_int num_tx_rings; // if nonzero, overrides num_rx_rings
124 u_int num_tx_desc; /* number of descriptor in each queue */
126 //u_int buff_size; // XXX deprecate, use NETMAP_BUF_SIZE
128 /* tx_rings and rx_rings are private but allocated
129 * as a contiguous chunk of memory. Each array has
130 * N+1 entries, for the adapter queues and for the host queue.
132 struct netmap_kring *tx_rings; /* array of TX rings. */
133 struct netmap_kring *rx_rings; /* array of RX rings. */
135 NM_SELINFO_T tx_si, rx_si; /* global wait queues */
137 /* copy of if_qflush and if_transmit pointers, to intercept
138 * packets from the network stack when netmap is active.
140 int (*if_transmit)(struct ifnet *, struct mbuf *);
142 /* references to the ifnet and device routines, used by
143 * the generic netmap functions.
145 struct ifnet *ifp; /* adapter is ifp->if_softc */
147 NM_LOCK_T core_lock; /* used if no device lock available */
149 int (*nm_register)(struct ifnet *, int onoff);
150 void (*nm_lock)(struct ifnet *, int what, u_int ringid);
151 int (*nm_txsync)(struct ifnet *, u_int ring, int lock);
152 int (*nm_rxsync)(struct ifnet *, u_int ring, int lock);
154 struct net_device_ops nm_ndo;
159 * The combination of "enable" (ifp->if_capabilities &IFCAP_NETMAP)
160 * and refcount gives the status of the interface, namely:
162 * enable refcount Status
164 * FALSE 0 normal operation
165 * FALSE != 0 -- (impossible)
167 * TRUE 0 being deleted.
170 #define NETMAP_DELETING(_na) ( ((_na)->refcount == 0) && \
171 ( (_na)->ifp->if_capenable & IFCAP_NETMAP) )
174 * parameters for (*nm_lock)(adapter, what, index)
178 NETMAP_CORE_LOCK, NETMAP_CORE_UNLOCK,
179 NETMAP_TX_LOCK, NETMAP_TX_UNLOCK,
180 NETMAP_RX_LOCK, NETMAP_RX_UNLOCK,
182 #define NETMAP_REG_LOCK NETMAP_CORE_LOCK
183 #define NETMAP_REG_UNLOCK NETMAP_CORE_UNLOCK
185 NETMAP_REG_LOCK, NETMAP_REG_UNLOCK
190 * The following are support routines used by individual drivers to
191 * support netmap operation.
193 * netmap_attach() initializes a struct netmap_adapter, allocating the
194 * struct netmap_ring's and the struct selinfo.
196 * netmap_detach() frees the memory allocated by netmap_attach().
198 * netmap_start() replaces the if_transmit routine of the interface,
199 * and is used to intercept packets coming from the stack.
201 * netmap_load_map/netmap_reload_map are helper routines to set/reset
202 * the dmamap for a packet buffer
204 * netmap_reset() is a helper routine to be called in the driver
205 * when reinitializing a ring.
207 int netmap_attach(struct netmap_adapter *, int);
208 void netmap_detach(struct ifnet *);
209 int netmap_start(struct ifnet *, struct mbuf *);
210 enum txrx { NR_RX = 0, NR_TX = 1 };
211 struct netmap_slot *netmap_reset(struct netmap_adapter *na,
212 enum txrx tx, int n, u_int new_cur);
213 int netmap_ring_reinit(struct netmap_kring *);
215 extern int netmap_buf_size;
216 #define NETMAP_BUF_SIZE netmap_buf_size
217 extern int netmap_mitigate;
218 extern int netmap_no_pendintr;
219 extern u_int netmap_total_buffers;
220 extern char *netmap_buffer_base;
221 extern int netmap_verbose; // XXX debugging
222 enum { /* verbose flags */
223 NM_VERB_ON = 1, /* generic verbose */
224 NM_VERB_HOST = 0x2, /* verbose host stack */
225 NM_VERB_RXSYNC = 0x10, /* verbose on rxsync/txsync */
226 NM_VERB_TXSYNC = 0x20,
227 NM_VERB_RXINTR = 0x100, /* verbose on rx/tx intr (driver) */
228 NM_VERB_TXINTR = 0x200,
229 NM_VERB_NIC_RXSYNC = 0x1000, /* verbose on rx/tx intr (driver) */
230 NM_VERB_NIC_TXSYNC = 0x2000,
234 * NA returns a pointer to the struct netmap adapter from the ifp,
235 * WNA is used to write it.
238 #define WNA(_ifp) (_ifp)->if_pspare[0]
240 #define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp))
243 /* Callback invoked by the dma machinery after a successfull dmamap_load */
244 static void netmap_dmamap_cb(__unused void *arg,
245 __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error)
249 /* bus_dmamap_load wrapper: call aforementioned function if map != NULL.
250 * XXX can we do it without a callback ?
253 netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
256 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE,
257 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
260 /* update the map when a buffer changes. */
262 netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
265 bus_dmamap_unload(tag, map);
266 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE,
267 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
272 * functions to map NIC to KRING indexes (n2k) and vice versa (k2n)
275 netmap_idx_n2k(struct netmap_kring *kr, int idx)
277 int n = kr->nkr_num_slots;
278 idx += kr->nkr_hwofs;
289 netmap_idx_k2n(struct netmap_kring *kr, int idx)
291 int n = kr->nkr_num_slots;
292 idx -= kr->nkr_hwofs;
303 /* Entries of the look-up table. */
305 void *vaddr; /* virtual address. */
306 vm_paddr_t paddr; /* phisical address. */
309 struct netmap_obj_pool;
310 extern struct lut_entry *netmap_buffer_lut;
311 #define NMB_VA(i) (netmap_buffer_lut[i].vaddr)
312 #define NMB_PA(i) (netmap_buffer_lut[i].paddr)
313 #else /* NETMAP_MEM1 */
314 #define NMB_VA(i) (netmap_buffer_base + (i * NETMAP_BUF_SIZE) )
315 #endif /* NETMAP_MEM2 */
318 * NMB return the virtual address of a buffer (buffer 0 on bad index)
319 * PNMB also fills the physical address
322 NMB(struct netmap_slot *slot)
324 uint32_t i = slot->buf_idx;
325 return (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i);
329 PNMB(struct netmap_slot *slot, uint64_t *pp)
331 uint32_t i = slot->buf_idx;
332 void *ret = (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i);
334 *pp = (i >= netmap_total_buffers) ? NMB_PA(0) : NMB_PA(i);
341 /* default functions to handle rx/tx interrupts */
342 int netmap_rx_irq(struct ifnet *, int, int *);
343 #define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL)
344 #endif /* _NET_NETMAP_KERN_H_ */