2 * Copyright (C) 2013 Intel Corporation
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
30 #include <sys/param.h>
31 #include <sys/kernel.h>
32 #include <sys/systm.h>
36 #include <sys/malloc.h>
37 #include <sys/module.h>
38 #include <sys/mutex.h>
39 #include <sys/queue.h>
40 #include <sys/socket.h>
41 #include <sys/sockio.h>
42 #include <sys/taskqueue.h>
44 #include <net/if_media.h>
45 #include <net/if_types.h>
46 #include <net/if_var.h>
48 #include <net/ethernet.h>
51 #include <machine/bus.h>
52 #include <machine/cpufunc.h>
53 #include <machine/pmap.h>
55 #include "../ntb_hw/ntb_hw.h"
58 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that
59 * allows you to connect two systems using a PCI-e link.
61 * This module contains a protocol for sending and receiving messages, and
62 * exposes that protocol through a simulated ethernet device called ntb.
64 * NOTE: Much of the code in this module is shared with Linux. Any patches may
65 * be picked up and redistributed in Linux with a dual GPL/BSD license.
68 /* TODO: These functions should really be part of the kernel */
69 #define test_bit(pos, bitmap_addr) (*(bitmap_addr) & 1UL << (pos))
70 #define set_bit(pos, bitmap_addr) *(bitmap_addr) |= 1UL << (pos)
71 #define clear_bit(pos, bitmap_addr) *(bitmap_addr) &= ~(1UL << (pos))
73 #define KTR_NTB KTR_SPARE3
75 #define NTB_TRANSPORT_VERSION 3
76 #define NTB_RX_MAX_PKTS 64
77 #define NTB_RXQ_SIZE 300
79 static unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN;
80 static unsigned int max_num_clients = 1;
82 STAILQ_HEAD(ntb_queue_list, ntb_queue_entry);
84 struct ntb_queue_entry {
85 /* ntb_queue list reference */
86 STAILQ_ENTRY(ntb_queue_entry) entry;
88 /* info on data to be transfered */
99 struct ntb_transport_qp {
100 struct ntb_netdev *transport;
101 struct ntb_softc *ntb;
107 uint8_t qp_num; /* Only 64 QPs are allowed. 0-63 */
109 struct ntb_rx_info *rx_info;
110 struct ntb_rx_info *remote_rx_info;
112 void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
113 void *data, int len);
114 struct ntb_queue_list tx_free_q;
115 struct mtx ntb_tx_free_q_lock;
118 uint64_t tx_max_entry;
119 uint64_t tx_max_frame;
121 void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
122 void *data, int len);
123 struct ntb_queue_list rx_pend_q;
124 struct ntb_queue_list rx_free_q;
125 struct mtx ntb_rx_pend_q_lock;
126 struct mtx ntb_rx_free_q_lock;
127 struct task rx_completion_task;
130 uint64_t rx_max_entry;
131 uint64_t rx_max_frame;
133 void (*event_handler) (void *data, int status);
134 struct callout link_work;
135 struct callout queue_full;
136 struct callout rx_full;
138 uint64_t last_rx_no_buf;
143 uint64_t rx_ring_empty;
144 uint64_t rx_err_no_buf;
145 uint64_t rx_err_oflow;
149 uint64_t tx_ring_full;
152 struct ntb_queue_handlers {
153 void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
154 void *data, int len);
155 void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
156 void *data, int len);
157 void (*event_handler) (void *data, int status);
161 struct ntb_transport_mw {
168 struct ntb_softc *ntb;
170 struct ntb_transport_mw mw[NTB_NUM_MW];
171 struct ntb_transport_qp *qps;
175 struct callout link_work;
176 struct ntb_transport_qp *qp;
178 u_char eaddr[ETHER_ADDR_LEN];
183 static struct ntb_netdev net_softc;
186 IF_NTB_DESC_DONE_FLAG = 1 << 0,
187 IF_NTB_LINK_DOWN_FLAG = 1 << 1,
190 struct ntb_payload_header {
205 #define QP_TO_MW(qp) ((qp) % NTB_NUM_MW)
206 #define NTB_QP_DEF_NUM_ENTRIES 100
207 #define NTB_LINK_DOWN_TIMEOUT 10
209 static int ntb_handle_module_events(struct module *m, int what, void *arg);
210 static int ntb_setup_interface(void);
211 static int ntb_teardown_interface(void);
212 static void ntb_net_init(void *arg);
213 static int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
214 static void ntb_start(struct ifnet *ifp);
215 static void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data,
216 void *data, int len);
217 static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
218 void *data, int len);
219 static void ntb_net_event_handler(void *data, int status);
220 static int ntb_transport_init(struct ntb_softc *ntb);
221 static void ntb_transport_free(void *transport);
222 static void ntb_transport_init_queue(struct ntb_netdev *nt,
223 unsigned int qp_num);
224 static void ntb_transport_free_queue(struct ntb_transport_qp *qp);
225 static struct ntb_transport_qp * ntb_transport_create_queue(void *data,
226 struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers);
227 static void ntb_transport_link_up(struct ntb_transport_qp *qp);
228 static int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb,
229 void *data, unsigned int len);
230 static int ntb_process_tx(struct ntb_transport_qp *qp,
231 struct ntb_queue_entry *entry);
232 static void ntb_tx_copy_task(struct ntb_transport_qp *qp,
233 struct ntb_queue_entry *entry, void *offset);
234 static void ntb_qp_full(void *arg);
235 static void ntb_transport_rxc_db(void *data, int db_num);
236 static void ntb_rx_pendq_full(void *arg);
237 static void ntb_transport_rx(struct ntb_transport_qp *qp);
238 static int ntb_process_rxc(struct ntb_transport_qp *qp);
239 static void ntb_rx_copy_task(struct ntb_transport_qp *qp,
240 struct ntb_queue_entry *entry, void *offset);
241 static void ntb_rx_completion_task(void *arg, int pending);
242 static void ntb_transport_event_callback(void *data, enum ntb_hw_event event);
243 static void ntb_transport_link_work(void *arg);
244 static int ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size);
245 static void ntb_transport_setup_qp_mw(struct ntb_netdev *nt,
246 unsigned int qp_num);
247 static void ntb_qp_link_work(void *arg);
248 static void ntb_transport_link_cleanup(struct ntb_netdev *nt);
249 static void ntb_qp_link_down(struct ntb_transport_qp *qp);
250 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp);
251 static void ntb_transport_link_down(struct ntb_transport_qp *qp);
252 static void ntb_send_link_down(struct ntb_transport_qp *qp);
253 static void ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry,
254 struct ntb_queue_list *list);
255 static struct ntb_queue_entry *ntb_list_rm(struct mtx *lock,
256 struct ntb_queue_list *list);
257 static void create_random_local_eui48(u_char *eaddr);
258 static unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp);
260 MALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver");
262 /* Module setup and teardown */
264 ntb_handle_module_events(struct module *m, int what, void *arg)
270 err = ntb_setup_interface();
273 err = ntb_teardown_interface();
282 static moduledata_t if_ntb_mod = {
284 ntb_handle_module_events,
288 DECLARE_MODULE(if_ntb, if_ntb_mod, SI_SUB_KLD, SI_ORDER_ANY);
289 MODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1);
292 ntb_setup_interface()
295 struct ntb_queue_handlers handlers = { ntb_net_rx_handler,
296 ntb_net_tx_handler, ntb_net_event_handler };
298 net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0);
299 if (net_softc.ntb == NULL) {
300 printf("ntb: Cannot find devclass\n");
304 ntb_transport_init(net_softc.ntb);
306 ifp = net_softc.ifp = if_alloc(IFT_ETHER);
308 printf("ntb: cannot allocate ifnet structure\n");
312 net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb,
314 if_initname(ifp, "ntb", 0);
315 ifp->if_init = ntb_net_init;
316 ifp->if_softc = &net_softc;
317 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX;
318 ifp->if_ioctl = ntb_ioctl;
319 ifp->if_start = ntb_start;
320 IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN);
321 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN;
322 IFQ_SET_READY(&ifp->if_snd);
323 create_random_local_eui48(net_softc.eaddr);
324 ether_ifattach(ifp, net_softc.eaddr);
325 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU;
326 ifp->if_capenable = ifp->if_capabilities;
328 ntb_transport_link_up(net_softc.qp);
329 net_softc.bufsize = ntb_transport_max_size(net_softc.qp) +
330 sizeof(struct ether_header);
335 ntb_teardown_interface()
338 if (net_softc.qp != NULL)
339 ntb_transport_link_down(net_softc.qp);
341 if (net_softc.ifp != NULL) {
342 ether_ifdetach(net_softc.ifp);
343 if_free(net_softc.ifp);
346 if (net_softc.qp != NULL) {
347 ntb_transport_free_queue(net_softc.qp);
348 ntb_transport_free(&net_softc);
354 /* Network device interface */
357 ntb_net_init(void *arg)
359 struct ntb_netdev *ntb_softc = arg;
360 struct ifnet *ifp = ntb_softc->ifp;
362 ifp->if_drv_flags |= IFF_DRV_RUNNING;
363 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
364 ifp->if_flags |= IFF_UP;
365 if_link_state_change(ifp, LINK_STATE_UP);
369 ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
371 struct ntb_netdev *nt = ifp->if_softc;
372 struct ifreq *ifr = (struct ifreq *)data;
378 if (ifr->ifr_mtu > ntb_transport_max_size(nt->qp) -
379 ETHER_HDR_LEN - ETHER_CRC_LEN) {
384 ifp->if_mtu = ifr->ifr_mtu;
388 error = ether_ioctl(ifp, command, data);
397 ntb_start(struct ifnet *ifp)
400 struct ntb_netdev *nt = ifp->if_softc;
403 mtx_lock(&nt->tx_lock);
404 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
405 CTR0(KTR_NTB, "TX: ntb_start");
406 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
407 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
408 CTR1(KTR_NTB, "TX: start mbuf %p", m_head);
409 rc = ntb_transport_tx_enqueue(nt->qp, m_head, m_head,
410 m_length(m_head, NULL));
413 "TX: could not tx mbuf %p. Returning to snd q",
416 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
417 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
418 callout_reset(&nt->qp->queue_full, hz / 1000,
425 mtx_unlock(&nt->tx_lock);
428 /* Network Device Callbacks */
430 ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data,
435 CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data);
439 ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data,
442 struct mbuf *m = data;
443 struct ifnet *ifp = qp_data;
445 CTR0(KTR_NTB, "RX: rx handler");
446 (*ifp->if_input)(ifp, m);
450 ntb_net_event_handler(void *data, int status)
455 /* Transport Init and teardown */
458 ntb_transport_init(struct ntb_softc *ntb)
460 struct ntb_netdev *nt = &net_softc;
463 nt->max_qps = max_num_clients;
464 ntb_register_transport(ntb, nt);
465 mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF);
466 mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF);
468 nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp),
469 M_NTB_IF, M_WAITOK|M_ZERO);
471 nt->qp_bitmap = ((uint64_t) 1 << nt->max_qps) - 1;
473 for (i = 0; i < nt->max_qps; i++)
474 ntb_transport_init_queue(nt, i);
476 callout_init(&nt->link_work, 0);
478 rc = ntb_register_event_callback(ntb,
479 ntb_transport_event_callback);
483 if (ntb_query_link_status(ntb)) {
485 device_printf(ntb_get_device(ntb), "link up\n");
486 callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
492 free(nt->qps, M_NTB_IF);
493 ntb_unregister_transport(ntb);
498 ntb_transport_free(void *transport)
500 struct ntb_netdev *nt = transport;
501 struct ntb_softc *ntb = nt->ntb;
504 nt->transport_link = NTB_LINK_DOWN;
506 callout_drain(&nt->link_work);
508 /* verify that all the qps are freed */
509 for (i = 0; i < nt->max_qps; i++)
510 if (!test_bit(i, &nt->qp_bitmap))
511 ntb_transport_free_queue(&nt->qps[i]);
514 ntb_unregister_event_callback(ntb);
516 for (i = 0; i < NTB_NUM_MW; i++)
517 if (nt->mw[i].virt_addr != NULL)
518 contigfree(nt->mw[i].virt_addr, nt->mw[i].size,
521 free(nt->qps, M_NTB_IF);
522 ntb_unregister_transport(ntb);
526 ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num)
528 struct ntb_transport_qp *qp;
529 unsigned int num_qps_mw, tx_size;
530 uint8_t mw_num = QP_TO_MW(qp_num);
532 qp = &nt->qps[qp_num];
536 qp->qp_link = NTB_LINK_DOWN;
537 qp->client_ready = NTB_LINK_DOWN;
538 qp->event_handler = NULL;
540 if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
541 num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
543 num_qps_mw = nt->max_qps / NTB_NUM_MW;
545 tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw;
546 qp->rx_info = (struct ntb_rx_info *)
547 ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) +
548 (qp_num / NTB_NUM_MW * tx_size));
549 tx_size -= sizeof(struct ntb_rx_info);
551 qp->tx_mw = qp->rx_info + sizeof(struct ntb_rx_info);
552 qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
554 qp->tx_max_entry = tx_size / qp->tx_max_frame;
557 callout_init(&qp->link_work, 0);
558 callout_init(&qp->queue_full, CALLOUT_MPSAFE);
559 callout_init(&qp->rx_full, CALLOUT_MPSAFE);
561 mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN);
562 mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN);
563 mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN);
564 TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp);
566 STAILQ_INIT(&qp->rx_pend_q);
567 STAILQ_INIT(&qp->rx_free_q);
568 STAILQ_INIT(&qp->tx_free_q);
572 ntb_transport_free_queue(struct ntb_transport_qp *qp)
574 struct ntb_queue_entry *entry;
579 callout_drain(&qp->link_work);
581 ntb_unregister_db_callback(qp->ntb, qp->qp_num);
583 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
584 free(entry, M_NTB_IF);
586 while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q)))
587 free(entry, M_NTB_IF);
589 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
590 free(entry, M_NTB_IF);
592 set_bit(qp->qp_num, &qp->transport->qp_bitmap);
596 * ntb_transport_create_queue - Create a new NTB transport layer queue
597 * @rx_handler: receive callback function
598 * @tx_handler: transmit callback function
599 * @event_handler: event callback function
601 * Create a new NTB transport layer queue and provide the queue with a callback
602 * routine for both transmit and receive. The receive callback routine will be
603 * used to pass up data when the transport has received it on the queue. The
604 * transmit callback routine will be called when the transport has completed the
605 * transmission of the data on the queue and the data is ready to be freed.
607 * RETURNS: pointer to newly created ntb_queue, NULL on error.
609 static struct ntb_transport_qp *
610 ntb_transport_create_queue(void *data, struct ntb_softc *pdev,
611 const struct ntb_queue_handlers *handlers)
613 struct ntb_queue_entry *entry;
614 struct ntb_transport_qp *qp;
615 struct ntb_netdev *nt;
616 unsigned int free_queue;
619 nt = ntb_find_transport(pdev);
623 free_queue = ffs(nt->qp_bitmap);
627 /* decrement free_queue to make it zero based */
630 clear_bit(free_queue, &nt->qp_bitmap);
632 qp = &nt->qps[free_queue];
634 qp->rx_handler = handlers->rx_handler;
635 qp->tx_handler = handlers->tx_handler;
636 qp->event_handler = handlers->event_handler;
638 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
639 entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
641 entry->cb_data = nt->ifp;
643 entry->len = transport_mtu;
644 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
647 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
648 entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
650 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
653 rc = ntb_register_db_callback(qp->ntb, free_queue, qp,
654 ntb_transport_rxc_db);
661 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
662 free(entry, M_NTB_IF);
663 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
664 free(entry, M_NTB_IF);
665 set_bit(free_queue, &nt->qp_bitmap);
671 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
672 * @qp: NTB transport layer queue to be enabled
674 * Notify NTB transport layer of client readiness to use queue
677 ntb_transport_link_up(struct ntb_transport_qp *qp)
683 qp->client_ready = NTB_LINK_UP;
685 device_printf(ntb_get_device(qp->ntb), "qp client ready\n");
687 if (qp->transport->transport_link == NTB_LINK_UP)
688 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);
696 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
697 * @qp: NTB transport layer queue the entry is to be enqueued on
698 * @cb: per buffer pointer for callback function to use
699 * @data: pointer to data buffer that will be sent
700 * @len: length of the data buffer
702 * Enqueue a new transmit buffer onto the transport queue from which a NTB
703 * payload will be transmitted. This assumes that a lock is behing held to
704 * serialize access to the qp.
706 * RETURNS: An appropriate ERRNO error value on error, or zero for success.
709 ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
712 struct ntb_queue_entry *entry;
715 if (qp == NULL || qp->qp_link != NTB_LINK_UP || len == 0) {
716 CTR0(KTR_NTB, "TX: link not up");
720 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
722 CTR0(KTR_NTB, "TX: could not get entry from tx_free_q");
725 CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry);
732 rc = ntb_process_tx(qp, entry);
734 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
736 "TX: process_tx failed. Returning entry %p to tx_free_q",
743 ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry)
747 offset = (char *)qp->tx_mw + qp->tx_max_frame * qp->tx_index;
749 "TX: process_tx: tx_pkts=%u, tx_index=%u, remote entry=%u",
750 qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry);
751 if (qp->tx_index == qp->remote_rx_info->entry) {
752 CTR0(KTR_NTB, "TX: ring full");
757 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
758 if (qp->tx_handler != NULL)
759 qp->tx_handler(qp, qp->cb_data, entry->buf,
762 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
764 "TX: frame too big. returning entry %p to tx_free_q",
768 CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset);
769 ntb_tx_copy_task(qp, entry, offset);
772 qp->tx_index %= qp->tx_max_entry;
780 ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
783 struct ntb_payload_header *hdr;
785 CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);
786 if (entry->buf != NULL)
787 m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);
789 hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame -
790 sizeof(struct ntb_payload_header));
791 hdr->len = entry->len; /* TODO: replace with bus_space_write */
792 hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */
794 /* TODO: replace with bus_space_write */
795 hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG;
797 ntb_ring_sdb(qp->ntb, qp->qp_num);
800 * The entry length can only be zero if the packet is intended to be a
801 * "link down" or similar. Since no payload is being sent in these
802 * cases, there is nothing to add to the completion queue.
804 if (entry->len > 0) {
805 qp->tx_bytes += entry->len;
808 qp->tx_handler(qp, qp->cb_data, entry->cb_data,
813 "TX: entry %p sent. hdr->ver = %d, Returning to tx_free_q", entry,
815 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
819 ntb_qp_full(void *arg)
822 CTR0(KTR_NTB, "TX: qp_full callout");
828 ntb_transport_rxc_db(void *data, int db_num)
830 struct ntb_transport_qp *qp = data;
832 ntb_transport_rx(qp);
836 ntb_rx_pendq_full(void *arg)
839 CTR0(KTR_NTB, "RX: ntb_rx_pendq_full callout");
840 ntb_transport_rx(arg);
844 ntb_transport_rx(struct ntb_transport_qp *qp)
849 * Limit the number of packets processed in a single interrupt to
850 * provide fairness to others
852 mtx_lock(&qp->transport->rx_lock);
853 CTR0(KTR_NTB, "RX: transport_rx");
854 for (i = 0; i < NTB_RX_MAX_PKTS; i++) {
855 rc = ntb_process_rxc(qp);
857 CTR0(KTR_NTB, "RX: process_rxc failed");
861 mtx_unlock(&qp->transport->rx_lock);
865 ntb_process_rxc(struct ntb_transport_qp *qp)
867 struct ntb_payload_header *hdr;
868 struct ntb_queue_entry *entry;
872 ((char *)qp->rx_buff + qp->rx_max_frame * qp->rx_index);
874 ((char *)offset + qp->rx_max_frame -
875 sizeof(struct ntb_payload_header));
877 CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index);
878 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
881 CTR0(KTR_NTB, "RX: No entries in rx_pend_q");
884 callout_stop(&qp->rx_full);
885 CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry);
887 if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) {
889 "RX: hdr not done. Returning entry %p to rx_pend_q", entry);
890 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
895 if (hdr->ver != (uint32_t) qp->rx_pkts) {
896 CTR3(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). "
897 "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts,
899 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
904 if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) {
905 ntb_qp_link_down(qp);
907 "RX: link down. adding entry %p back to rx_pend_q", entry);
908 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
912 if (hdr->len <= entry->len) {
913 entry->len = hdr->len;
914 ntb_rx_copy_task(qp, entry, offset);
917 "RX: len too long. Returning entry %p to rx_pend_q", entry);
918 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
923 qp->rx_bytes += hdr->len;
925 CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);
929 /* Ensure that the data is globally visible before clearing the flag */
932 /* TODO: replace with bus_space_write */
933 qp->rx_info->entry = qp->rx_index;
936 qp->rx_index %= qp->rx_max_entry;
942 ntb_rx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
945 struct ifnet *ifp = entry->cb_data;
946 unsigned int len = entry->len;
949 CTR2(KTR_NTB, "RX: copying %d bytes from offset %p", len, offset);
950 m = m_devget(offset, len, 0, ifp, NULL);
951 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID;
953 entry->buf = (void *)m;
956 "RX: copied entry %p to mbuf %p. Adding entry to rx_free_q", entry,
958 ntb_list_add(&qp->ntb_rx_free_q_lock, entry, &qp->rx_free_q);
960 taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task);
964 ntb_rx_completion_task(void *arg, int pending)
966 struct ntb_transport_qp *qp = arg;
968 struct ntb_queue_entry *entry;
970 CTR0(KTR_NTB, "RX: rx_completion_task");
972 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) {
974 CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m);
975 if (qp->rx_handler && qp->client_ready == NTB_LINK_UP)
976 qp->rx_handler(qp, qp->cb_data, m, entry->len);
979 entry->len = qp->transport->bufsize;
981 CTR1(KTR_NTB,"RX: entry %p removed from rx_free_q "
982 "and added to rx_pend_q", entry);
983 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
984 if (qp->rx_err_no_buf > qp->last_rx_no_buf) {
985 qp->last_rx_no_buf = qp->rx_err_no_buf;
986 CTR0(KTR_NTB, "RX: could spawn rx task");
987 callout_reset(&qp->rx_full, hz / 1000, ntb_rx_pendq_full,
993 /* Link Event handler */
995 ntb_transport_event_callback(void *data, enum ntb_hw_event event)
997 struct ntb_netdev *nt = data;
1000 case NTB_EVENT_HW_LINK_UP:
1002 device_printf(ntb_get_device(nt->ntb), "HW link up\n");
1003 callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
1005 case NTB_EVENT_HW_LINK_DOWN:
1007 device_printf(ntb_get_device(nt->ntb), "HW link down\n");
1008 ntb_transport_link_cleanup(nt);
1011 panic("ntb: Unknown NTB event");
1017 ntb_transport_link_work(void *arg)
1019 struct ntb_netdev *nt = arg;
1020 struct ntb_softc *ntb = nt->ntb;
1021 struct ntb_transport_qp *qp;
1025 /* send the local info */
1026 rc = ntb_write_remote_spad(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
1030 rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ, ntb_get_mw_size(ntb, 0));
1034 rc = ntb_write_remote_spad(ntb, IF_NTB_MW1_SZ, ntb_get_mw_size(ntb, 1));
1038 rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_QPS, nt->max_qps);
1042 rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val);
1046 rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val);
1050 /* Query the remote side for its info */
1051 rc = ntb_read_local_spad(ntb, IF_NTB_VERSION, &val);
1055 if (val != NTB_TRANSPORT_VERSION)
1058 rc = ntb_read_local_spad(ntb, IF_NTB_NUM_QPS, &val);
1062 if (val != nt->max_qps)
1065 rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ, &val);
1072 rc = ntb_set_mw(nt, 0, val);
1076 rc = ntb_read_local_spad(ntb, IF_NTB_MW1_SZ, &val);
1083 rc = ntb_set_mw(nt, 1, val);
1087 nt->transport_link = NTB_LINK_UP;
1089 device_printf(ntb_get_device(ntb), "transport link up\n");
1091 for (i = 0; i < nt->max_qps; i++) {
1094 ntb_transport_setup_qp_mw(nt, i);
1096 if (qp->client_ready == NTB_LINK_UP)
1097 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);
1103 if (ntb_query_link_status(ntb))
1104 callout_reset(&nt->link_work,
1105 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt);
1109 ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size)
1111 struct ntb_transport_mw *mw = &nt->mw[num_mw];
1113 /* Alloc memory for receiving data. Must be 4k aligned */
1116 mw->virt_addr = contigmalloc(mw->size, M_NTB_IF, M_ZERO, 0,
1117 BUS_SPACE_MAXADDR, mw->size, 0);
1118 if (mw->virt_addr == NULL) {
1119 printf("ntb: Unable to allocate MW buffer of size %d\n",
1123 /* TODO: replace with bus_space_* functions */
1124 mw->dma_addr = vtophys(mw->virt_addr);
1126 /* Notify HW the memory location of the receive buffer */
1127 ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr);
1133 ntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num)
1135 struct ntb_transport_qp *qp = &nt->qps[qp_num];
1137 unsigned int rx_size, num_qps_mw;
1138 uint8_t mw_num = QP_TO_MW(qp_num);
1141 if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
1142 num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
1144 num_qps_mw = nt->max_qps / NTB_NUM_MW;
1146 rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw;
1147 qp->remote_rx_info = (void *)((uint8_t *)nt->mw[mw_num].virt_addr +
1148 (qp_num / NTB_NUM_MW * rx_size));
1149 rx_size -= sizeof(struct ntb_rx_info);
1151 qp->rx_buff = qp->remote_rx_info + sizeof(struct ntb_rx_info);
1152 qp->rx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
1154 qp->rx_max_entry = rx_size / qp->rx_max_frame;
1158 qp->remote_rx_info->entry = qp->rx_max_entry;
1160 /* setup the hdr offsets with 0's */
1161 for (i = 0; i < qp->rx_max_entry; i++) {
1162 offset = (void *)((uint8_t *)qp->rx_buff +
1163 qp->rx_max_frame * (i + 1) -
1164 sizeof(struct ntb_payload_header));
1165 memset(offset, 0, sizeof(struct ntb_payload_header));
1173 ntb_qp_link_work(void *arg)
1175 struct ntb_transport_qp *qp = arg;
1176 struct ntb_softc *ntb = qp->ntb;
1177 struct ntb_netdev *nt = qp->transport;
1181 rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val);
1185 rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num);
1187 /* query remote spad for qp ready bits */
1188 rc = ntb_read_local_spad(ntb, IF_NTB_QP_LINKS, &val);
1190 /* See if the remote side is up */
1191 if ((1 << qp->qp_num & val) != 0) {
1192 qp->qp_link = NTB_LINK_UP;
1193 if (qp->event_handler != NULL)
1194 qp->event_handler(qp->cb_data, NTB_LINK_UP);
1196 device_printf(ntb_get_device(ntb), "qp link up\n");
1197 } else if (nt->transport_link == NTB_LINK_UP) {
1198 callout_reset(&qp->link_work,
1199 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
1203 /* Link down event*/
1205 ntb_transport_link_cleanup(struct ntb_netdev *nt)
1209 if (nt->transport_link == NTB_LINK_DOWN)
1210 callout_drain(&nt->link_work);
1212 nt->transport_link = NTB_LINK_DOWN;
1214 /* Pass along the info to any clients */
1215 for (i = 0; i < nt->max_qps; i++)
1216 if (!test_bit(i, &nt->qp_bitmap))
1217 ntb_qp_link_down(&nt->qps[i]);
1220 * The scratchpad registers keep the values if the remote side
1221 * goes down, blast them now to give them a sane value the next
1222 * time they are accessed
1224 for (i = 0; i < IF_NTB_MAX_SPAD; i++)
1225 ntb_write_local_spad(nt->ntb, i, 0);
1230 ntb_qp_link_down(struct ntb_transport_qp *qp)
1233 ntb_qp_link_cleanup(qp);
1237 ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
1239 struct ntb_netdev *nt = qp->transport;
1241 if (qp->qp_link == NTB_LINK_DOWN) {
1242 callout_drain(&qp->link_work);
1246 if (qp->event_handler != NULL)
1247 qp->event_handler(qp->cb_data, NTB_LINK_DOWN);
1249 qp->qp_link = NTB_LINK_DOWN;
1251 if (nt->transport_link == NTB_LINK_UP)
1252 callout_reset(&qp->link_work,
1253 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
1256 /* Link commanded down */
1258 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
1259 * @qp: NTB transport layer queue to be disabled
1261 * Notify NTB transport layer of client's desire to no longer receive data on
1262 * transport queue specified. It is the client's responsibility to ensure all
1263 * entries on queue are purged or otherwise handled appropraitely.
1266 ntb_transport_link_down(struct ntb_transport_qp *qp)
1273 qp->client_ready = NTB_LINK_DOWN;
1275 rc = ntb_read_remote_spad(qp->ntb, IF_NTB_QP_LINKS, &val);
1279 rc = ntb_write_remote_spad(qp->ntb, IF_NTB_QP_LINKS,
1280 val & ~(1 << qp->qp_num));
1282 if (qp->qp_link == NTB_LINK_UP)
1283 ntb_send_link_down(qp);
1285 callout_drain(&qp->link_work);
1290 ntb_send_link_down(struct ntb_transport_qp *qp)
1292 struct ntb_queue_entry *entry;
1295 if (qp->qp_link == NTB_LINK_DOWN)
1298 qp->qp_link = NTB_LINK_DOWN;
1300 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) {
1301 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
1304 pause("NTB Wait for link down", hz / 10);
1310 entry->cb_data = NULL;
1313 entry->flags = IF_NTB_LINK_DOWN_FLAG;
1315 mtx_lock(&qp->transport->tx_lock);
1316 rc = ntb_process_tx(qp, entry);
1318 printf("ntb: Failed to send link down\n");
1319 mtx_unlock(&qp->transport->tx_lock);
1323 /* List Management */
1326 ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry,
1327 struct ntb_queue_list *list)
1330 mtx_lock_spin(lock);
1331 STAILQ_INSERT_TAIL(list, entry, entry);
1332 mtx_unlock_spin(lock);
1335 static struct ntb_queue_entry *
1336 ntb_list_rm(struct mtx *lock, struct ntb_queue_list *list)
1338 struct ntb_queue_entry *entry;
1340 mtx_lock_spin(lock);
1341 if (STAILQ_EMPTY(list)) {
1345 entry = STAILQ_FIRST(list);
1346 STAILQ_REMOVE_HEAD(list, entry);
1348 mtx_unlock_spin(lock);
1353 /* Helper functions */
1354 /* TODO: This too should really be part of the kernel */
1355 #define EUI48_MULTICAST 1 << 0
1356 #define EUI48_LOCALLY_ADMINISTERED 1 << 1
1358 create_random_local_eui48(u_char *eaddr)
1360 static uint8_t counter = 0;
1361 uint32_t seed = ticks;
1363 eaddr[0] = EUI48_LOCALLY_ADMINISTERED;
1364 memcpy(&eaddr[1], &seed, sizeof(uint32_t));
1365 eaddr[5] = counter++;
1369 * ntb_transport_max_size - Query the max payload size of a qp
1370 * @qp: NTB transport layer queue to be queried
1372 * Query the maximum payload size permissible on the given qp
1374 * RETURNS: the max payload size of a qp
1377 ntb_transport_max_size(struct ntb_transport_qp *qp)
1383 return (qp->tx_max_frame - sizeof(struct ntb_payload_header));