1 /*************************************************************************
2 Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are
10 * Redistributions of source code must retain the above copyright
11 notice, this list of conditions and the following disclaimer.
13 * Redistributions in binary form must reproduce the above
14 copyright notice, this list of conditions and the following
15 disclaimer in the documentation and/or other materials provided
16 with the distribution.
18 * Neither the name of Cavium Networks nor the names of
19 its contributors may be used to endorse or promote products
20 derived from this software without specific prior written
23 This Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries.
25 TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
26 AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
27 *************************************************************************/
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
33 #include <sys/systm.h>
36 #include <sys/endian.h>
37 #include <sys/kernel.h>
40 #include <sys/socket.h>
41 #include <sys/module.h>
43 #include <sys/taskqueue.h>
45 #include <net/ethernet.h>
47 #include <net/if_types.h>
49 #include "wrapper-cvmx-includes.h"
50 #include "ethernet-headers.h"
52 #include "octebusvar.h"
56 * Convert 0444 to tunables, 0644 to sysctls.
58 #if defined(CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS) && CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS
59 int num_packet_buffers = CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS;
61 int num_packet_buffers = 1024;
63 TUNABLE_INT("hw.octe.num_packet_buffers", &num_packet_buffers);
65 "\t\tNumber of packet buffers to allocate and store in the\n"
66 "\t\tFPA. By default, 1024 packet buffers are used unless\n"
67 "\t\tCONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS is defined." */
69 int pow_receive_group = 15;
70 TUNABLE_INT("hw.octe.pow_receive_group", &pow_receive_group);
72 "\t\tPOW group to receive packets from. All ethernet hardware\n"
73 "\t\twill be configured to send incomming packets to this POW\n"
74 "\t\tgroup. Also any other software can submit packets to this\n"
75 "\t\tgroup for the kernel to process." */
77 int pow_send_group = -1; /* XXX Should be a sysctl. */
78 TUNABLE_INT("hw.octe.pow_send_group", &pow_send_group);
80 "\t\tPOW group to send packets to other software on. This\n"
81 "\t\tcontrols the creation of the virtual device pow0.\n"
82 "\t\talways_use_pow also depends on this value." */
85 TUNABLE_INT("hw.octe.always_use_pow", &always_use_pow);
87 "\t\tWhen set, always send to the pow group. This will cause\n"
88 "\t\tpackets sent to real ethernet devices to be sent to the\n"
89 "\t\tPOW group instead of the hardware. Unless some other\n"
90 "\t\tapplication changes the config, packets will still be\n"
91 "\t\treceived from the low level hardware. Use this option\n"
92 "\t\tto allow a CVMX app to intercept all packets from the\n"
93 "\t\tlinux kernel. You must specify pow_send_group along with\n"
96 char pow_send_list[128] = "";
97 TUNABLE_STR("hw.octe.pow_send_list", pow_send_list, sizeof pow_send_list);
99 "\t\tComma separated list of ethernet devices that should use the\n"
100 "\t\tPOW for transmit instead of the actual ethernet hardware. This\n"
101 "\t\tis a per port version of always_use_pow. always_use_pow takes\n"
102 "\t\tprecedence over this list. For example, setting this to\n"
103 "\t\t\"eth2,spi3,spi7\" would cause these three devices to transmit\n"
104 "\t\tusing the pow_send_group." */
107 static int disable_core_queueing = 1;
108 TUNABLE_INT("hw.octe.disable_core_queueing", &disable_core_queueing);
110 "\t\tWhen set the networking core's tx_queue_len is set to zero. This\n"
111 "\t\tallows packets to be sent without lock contention in the packet scheduler\n"
112 "\t\tresulting in some cases in improved throughput.\n" */
114 extern int octeon_is_simulation(void);
117 * Exported from the kernel so we can determine board information. It is
118 * passed by the bootloader to the kernel.
120 extern cvmx_bootinfo_t *octeon_bootinfo;
123 * Periodic timer to check auto negotiation
125 static struct callout cvm_oct_poll_timer;
128 * Array of every ethernet device owned by this driver indexed by
129 * the ipd input port number.
131 struct ifnet *cvm_oct_device[TOTAL_NUMBER_OF_PORTS];
134 * Task to handle link status changes.
136 static struct taskqueue *cvm_oct_link_taskq;
139 * Function to update link status.
141 static void cvm_oct_update_link(void *context, int pending)
143 cvm_oct_private_t *priv = (cvm_oct_private_t *)context;
144 struct ifnet *ifp = priv->ifp;
145 cvmx_helper_link_info_t link_info;
147 link_info.u64 = priv->link_info;
149 if (link_info.s.link_up) {
150 if_link_state_change(ifp, LINK_STATE_UP);
151 if (priv->queue != -1)
152 DEBUGPRINT("%s: %u Mbps %s duplex, port %2d, queue %2d\n",
153 if_name(ifp), link_info.s.speed,
154 (link_info.s.full_duplex) ? "Full" : "Half",
155 priv->port, priv->queue);
157 DEBUGPRINT("%s: %u Mbps %s duplex, port %2d, POW\n",
158 if_name(ifp), link_info.s.speed,
159 (link_info.s.full_duplex) ? "Full" : "Half",
162 if_link_state_change(ifp, LINK_STATE_DOWN);
163 DEBUGPRINT("%s: Link down\n", if_name(ifp));
165 priv->need_link_update = 0;
169 * Periodic timer tick for slow management operations
171 * @param arg Device to check
173 static void cvm_do_timer(void *arg)
177 if (port < CVMX_PIP_NUM_INPUT_PORTS) {
178 if (cvm_oct_device[port]) {
181 cvm_oct_private_t *priv = (cvm_oct_private_t *)cvm_oct_device[port]->if_softc;
184 /* skip polling if we don't get the lock */
185 if (MDIO_TRYLOCK()) {
186 priv->poll(cvm_oct_device[port]);
189 if (priv->need_link_update) {
191 taskqueue_enqueue(cvm_oct_link_taskq, &priv->link_task);
196 queues_per_port = cvmx_pko_get_num_queues(port);
197 /* Drain any pending packets in the free list */
198 for (qos = 0; qos < queues_per_port; qos++) {
199 if (_IF_QLEN(&priv->tx_free_queue[qos]) > 0) {
200 IF_LOCK(&priv->tx_free_queue[qos]);
201 while (_IF_QLEN(&priv->tx_free_queue[qos]) > cvmx_fau_fetch_and_add32(priv->fau+qos*4, 0)) {
204 _IF_DEQUEUE(&priv->tx_free_queue[qos], m);
207 IF_UNLOCK(&priv->tx_free_queue[qos]);
212 priv->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
216 cvm_oct_device[port]->get_stats(cvm_oct_device[port]);
220 /* Poll the next port in a 50th of a second.
221 This spreads the polling of ports out a little bit */
222 callout_reset(&cvm_oct_poll_timer, hz / 50, cvm_do_timer, NULL);
225 /* If any updates were made in this run, continue iterating at
226 * 1/50th of a second, so that if a link has merely gone down
227 * temporarily (e.g. because of interface reinitialization) it
228 * will not be forced to stay down for an entire second.
232 callout_reset(&cvm_oct_poll_timer, hz / 50, cvm_do_timer, NULL);
234 /* All ports have been polled. Start the next iteration through
235 the ports in one second */
236 callout_reset(&cvm_oct_poll_timer, hz, cvm_do_timer, NULL);
243 * Configure common hardware for all interfaces
245 static void cvm_oct_configure_common_hw(device_t bus)
247 struct octebus_softc *sc;
251 sc = device_get_softc(bus);
255 cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, num_packet_buffers);
256 cvm_oct_mem_fill_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, num_packet_buffers);
257 if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL)
258 cvm_oct_mem_fill_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL, CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, 128);
261 cvmx_helper_setup_red(num_packet_buffers/4, num_packet_buffers/8);
263 /* Enable the MII interface */
264 if (!octeon_is_simulation())
265 cvmx_write_csr(CVMX_SMI_EN, 1);
267 /* Register an IRQ hander for to receive POW interrupts */
269 sc->sc_rx_irq = bus_alloc_resource(bus, SYS_RES_IRQ, &rid,
270 CVMX_IRQ_WORKQ0 + pow_receive_group,
271 CVMX_IRQ_WORKQ0 + pow_receive_group,
273 if (sc->sc_rx_irq == NULL) {
274 device_printf(bus, "could not allocate workq irq");
278 error = bus_setup_intr(bus, sc->sc_rx_irq, INTR_TYPE_NET | INTR_MPSAFE,
279 cvm_oct_do_interrupt, NULL, cvm_oct_device,
282 device_printf(bus, "could not setup workq irq");
288 if (USE_MULTICORE_RECEIVE) {
292 for (cpu = 0; cpu < mp_maxid; cpu++) {
293 if (!CPU_ABSENT(cpu) &&
294 (cpu != PCPU_GET(cpuid))) {
296 en.u64 = cvmx_read_csr(CVMX_CIU_INTX_EN0(cpu*2));
297 en.s.workq |= (1<<pow_receive_group);
298 cvmx_write_csr(CVMX_CIU_INTX_EN0(cpu*2), en.u64);
309 * Free a work queue entry received in a intercept callback.
311 * @param work_queue_entry
312 * Work queue entry to free
313 * @return Zero on success, Negative on failure.
315 int cvm_oct_free_work(void *work_queue_entry)
317 cvmx_wqe_t *work = work_queue_entry;
319 int segments = work->word2.s.bufs;
320 cvmx_buf_ptr_t segment_ptr = work->packet_ptr;
323 cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8);
324 if (__predict_false(!segment_ptr.s.i))
325 cvmx_fpa_free(cvm_oct_get_buffer_ptr(segment_ptr), segment_ptr.s.pool, DONT_WRITEBACK(CVMX_FPA_PACKET_POOL_SIZE/128));
326 segment_ptr = next_ptr;
328 cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, DONT_WRITEBACK(1));
335 * Module/ driver initialization. Creates the linux network
338 * @return Zero on success
340 int cvm_oct_init_module(device_t bus)
346 int fau = FAU_NUM_PACKET_BUFFERS_TO_FREE;
349 printf("cavium-ethernet: %s\n", OCTEON_SDK_VERSION_STRING);
352 cvm_oct_proc_initialize();
354 cvm_oct_rx_initialize();
355 cvm_oct_configure_common_hw(bus);
357 cvmx_helper_initialize_packet_io_global();
359 /* Change the input group for all ports before input is enabled */
360 num_interfaces = cvmx_helper_get_number_of_interfaces();
361 for (interface = 0; interface < num_interfaces; interface++) {
362 int num_ports = cvmx_helper_ports_on_interface(interface);
365 for (port = cvmx_helper_get_ipd_port(interface, 0); port < cvmx_helper_get_ipd_port(interface, num_ports); port++) {
366 cvmx_pip_prt_tagx_t pip_prt_tagx;
367 pip_prt_tagx.u64 = cvmx_read_csr(CVMX_PIP_PRT_TAGX(port));
368 pip_prt_tagx.s.grp = pow_receive_group;
369 cvmx_write_csr(CVMX_PIP_PRT_TAGX(port), pip_prt_tagx.u64);
373 cvmx_helper_ipd_and_packet_input_enable();
375 memset(cvm_oct_device, 0, sizeof(cvm_oct_device));
377 cvm_oct_link_taskq = taskqueue_create("octe link", M_NOWAIT,
378 taskqueue_thread_enqueue, &cvm_oct_link_taskq);
379 taskqueue_start_threads(&cvm_oct_link_taskq, 1, PI_NET,
382 /* Initialize the FAU used for counting packet buffers that need to be freed */
383 cvmx_fau_atomic_write32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
385 if ((pow_send_group != -1)) {
388 printf("\tConfiguring device for POW only access\n");
389 dev = BUS_ADD_CHILD(bus, 0, "pow", 0);
391 ifp = if_alloc(IFT_ETHER);
392 if (dev != NULL && ifp != NULL) {
393 /* Initialize the device private structure. */
394 cvm_oct_private_t *priv;
397 priv = device_get_softc(dev);
400 priv->init = cvm_oct_common_init;
401 priv->imode = CVMX_HELPER_INTERFACE_MODE_DISABLED;
402 priv->port = CVMX_PIP_NUM_INPUT_PORTS;
404 TASK_INIT(&priv->link_task, 0, cvm_oct_update_link, priv);
406 device_set_desc(dev, "Cavium Octeon POW Ethernet\n");
408 ifp->if_softc = priv;
410 if (priv->init(ifp) < 0) {
411 printf("\t\tFailed to register ethernet device for POW\n");
412 panic("%s: need to free ifp.", __func__);
414 cvm_oct_device[CVMX_PIP_NUM_INPUT_PORTS] = ifp;
415 printf("\t\t%s: POW send group %d, receive group %d\n",
416 if_name(ifp), pow_send_group, pow_receive_group);
419 printf("\t\tFailed to allocate ethernet device for POW\n");
424 num_interfaces = cvmx_helper_get_number_of_interfaces();
425 for (interface = 0; interface < num_interfaces; interface++) {
426 cvmx_helper_interface_mode_t imode = cvmx_helper_interface_get_mode(interface);
427 int num_ports = cvmx_helper_ports_on_interface(interface);
430 for (port = cvmx_helper_get_ipd_port(interface, 0); port < cvmx_helper_get_ipd_port(interface, num_ports); port++) {
431 cvm_oct_private_t *priv;
434 dev = BUS_ADD_CHILD(bus, 0, "octe", ifnum++);
436 ifp = if_alloc(IFT_ETHER);
437 if (dev == NULL || ifp == NULL) {
438 printf("\t\tFailed to allocate ethernet device for port %d\n", port);
441 /* XXX/juli set max send q len. */
443 if (disable_core_queueing)
444 ifp->tx_queue_len = 0;
447 /* Initialize the device private structure. */
449 priv = device_get_softc(dev);
454 priv->queue = cvmx_pko_get_base_queue(priv->port);
455 priv->fau = fau - cvmx_pko_get_num_queues(port) * 4;
456 for (qos = 0; qos < cvmx_pko_get_num_queues(port); qos++)
457 cvmx_fau_atomic_write32(priv->fau+qos*4, 0);
458 TASK_INIT(&priv->link_task, 0, cvm_oct_update_link, priv);
460 switch (priv->imode) {
462 /* These types don't support ports to IPD/PKO */
463 case CVMX_HELPER_INTERFACE_MODE_DISABLED:
464 case CVMX_HELPER_INTERFACE_MODE_PCIE:
465 case CVMX_HELPER_INTERFACE_MODE_PICMG:
468 case CVMX_HELPER_INTERFACE_MODE_NPI:
469 priv->init = cvm_oct_common_init;
470 priv->uninit = cvm_oct_common_uninit;
471 device_set_desc(dev, "Cavium Octeon NPI Ethernet");
474 case CVMX_HELPER_INTERFACE_MODE_XAUI:
475 priv->init = cvm_oct_xaui_init;
476 priv->uninit = cvm_oct_xaui_uninit;
477 device_set_desc(dev, "Cavium Octeon XAUI Ethernet");
480 case CVMX_HELPER_INTERFACE_MODE_LOOP:
481 priv->init = cvm_oct_common_init;
482 priv->uninit = cvm_oct_common_uninit;
483 device_set_desc(dev, "Cavium Octeon LOOP Ethernet");
486 case CVMX_HELPER_INTERFACE_MODE_SGMII:
487 priv->init = cvm_oct_sgmii_init;
488 priv->uninit = cvm_oct_sgmii_uninit;
489 device_set_desc(dev, "Cavium Octeon SGMII Ethernet");
492 case CVMX_HELPER_INTERFACE_MODE_SPI:
493 priv->init = cvm_oct_spi_init;
494 priv->uninit = cvm_oct_spi_uninit;
495 device_set_desc(dev, "Cavium Octeon SPI Ethernet");
498 case CVMX_HELPER_INTERFACE_MODE_RGMII:
499 priv->init = cvm_oct_rgmii_init;
500 priv->uninit = cvm_oct_rgmii_uninit;
501 device_set_desc(dev, "Cavium Octeon RGMII Ethernet");
504 case CVMX_HELPER_INTERFACE_MODE_GMII:
505 priv->init = cvm_oct_rgmii_init;
506 priv->uninit = cvm_oct_rgmii_uninit;
507 device_set_desc(dev, "Cavium Octeon GMII Ethernet");
511 ifp->if_softc = priv;
514 panic("%s: unsupported device type, need to free ifp.", __func__);
516 if (priv->init(ifp) < 0) {
517 printf("\t\tFailed to register ethernet device for interface %d, port %d\n",
518 interface, priv->port);
519 panic("%s: init failed, need to free ifp.", __func__);
521 cvm_oct_device[priv->port] = ifp;
522 fau -= cvmx_pko_get_num_queues(priv->port) * sizeof(uint32_t);
527 if (INTERRUPT_LIMIT) {
528 /* Set the POW timer rate to give an interrupt at most INTERRUPT_LIMIT times per second */
529 cvmx_write_csr(CVMX_POW_WQ_INT_PC, octeon_bootinfo->eclock_hz/(INTERRUPT_LIMIT*16*256)<<8);
531 /* Enable POW timer interrupt. It will count when there are packets available */
532 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0x1ful<<24);
534 /* Enable POW interrupt when our port has at least one packet */
535 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0x1001);
538 callout_init(&cvm_oct_poll_timer, CALLOUT_MPSAFE);
539 callout_reset(&cvm_oct_poll_timer, hz, cvm_do_timer, NULL);
546 * Module / driver shutdown
548 * @return Zero on success
550 void cvm_oct_cleanup_module(void)
554 /* Disable POW interrupt */
555 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0);
560 /* Free the interrupt handler */
561 free_irq(8 + pow_receive_group, cvm_oct_device);
564 callout_stop(&cvm_oct_poll_timer);
565 cvm_oct_rx_shutdown();
568 /* Free the ethernet devices */
569 for (port = 0; port < TOTAL_NUMBER_OF_PORTS; port++) {
570 if (cvm_oct_device[port]) {
571 cvm_oct_tx_shutdown(cvm_oct_device[port]);
573 unregister_netdev(cvm_oct_device[port]);
574 kfree(cvm_oct_device[port]);
576 panic("%s: need to detach and free interface.", __func__);
578 cvm_oct_device[port] = NULL;
584 cvm_oct_proc_shutdown();
589 /* Free the HW pools */
590 cvm_oct_mem_empty_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, num_packet_buffers);
591 cvm_oct_mem_empty_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, num_packet_buffers);
592 if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL)
593 cvm_oct_mem_empty_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL, CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, 128);