sys/mips/cavium/octe/ethernet-rx.c

   1 /*************************************************************************
   2 Copyright (c) 2003-2007  Cavium Networks (support@cavium.com). All rights
   3 reserved.
   4
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are
   8 met:
   9
  10     * Redistributions of source code must retain the above copyright
  11       notice, this list of conditions and the following disclaimer.
  12
  13     * Redistributions in binary form must reproduce the above
  14       copyright notice, this list of conditions and the following
  15       disclaimer in the documentation and/or other materials provided
  16       with the distribution.
  17
  18     * Neither the name of Cavium Networks nor the names of
  19       its contributors may be used to endorse or promote products
  20       derived from this software without specific prior written
  21       permission.
  22
  23 This Software, including technical data, may be subject to U.S. export  control laws, including the U.S. Export Administration Act and its  associated regulations, and may be subject to export or import  regulations in other countries.
  24
  25 TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  26 AND WITH ALL FAULTS AND CAVIUM  NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  27
  28 *************************************************************************/
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/bus.h>
  36 #include <sys/endian.h>
  37 #include <sys/kernel.h>
  38 #include <sys/mbuf.h>
  39 #include <sys/socket.h>
  40 #include <sys/proc.h>
  41 #include <sys/sched.h>
  42 #include <sys/smp.h>
  43 #include <sys/taskqueue.h>
  44
  45 #include <net/ethernet.h>
  46 #include <net/if.h>
  47
  48 #include "wrapper-cvmx-includes.h"
  49 #include "ethernet-headers.h"
  50
  51 extern int pow_receive_group;
  52 extern struct ifnet *cvm_oct_device[];
  53
  54 static struct task cvm_oct_task;
  55 static struct taskqueue *cvm_oct_taskq;
  56
  57 /**
  58  * Interrupt handler. The interrupt occurs whenever the POW
  59  * transitions from 0->1 packets in our group.
  60  *
  61  * @param cpl
  62  * @param dev_id
  63  * @param regs
  64  * @return
  65  */
  66 int cvm_oct_do_interrupt(void *dev_id)
  67 {
  68         /* Acknowledge the interrupt */
  69         if (INTERRUPT_LIMIT)
  70                 cvmx_write_csr(CVMX_POW_WQ_INT, 1<<pow_receive_group);
  71         else
  72                 cvmx_write_csr(CVMX_POW_WQ_INT, 0x10001<<pow_receive_group);
  73         taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task);
  74         return FILTER_HANDLED;
  75 }
  76
  77
  78 #ifdef CONFIG_NET_POLL_CONTROLLER
  79 /**
  80  * This is called when the kernel needs to manually poll the
  81  * device. For Octeon, this is simply calling the interrupt
  82  * handler. We actually poll all the devices, not just the
  83  * one supplied.
  84  *
  85  * @param dev    Device to poll. Unused
  86  */
  87 void cvm_oct_poll_controller(struct ifnet *ifp)
  88 {
  89         taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task);
  90 }
  91 #endif
  92
  93 /**
  94  * This is called on receive errors, and determines if the packet
  95  * can be dropped early-on in cvm_oct_tasklet_rx().
  96  *
  97  * @param work Work queue entry pointing to the packet.
  98  * @return Non-zero if the packet can be dropped, zero otherwise.
  99  */
 100 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work)
 101 {
 102         if ((work->word2.snoip.err_code == 10) && (work->len <= 64)) {
 103                 /* Ignore length errors on min size packets. Some equipment
 104                    incorrectly pads packets to 64+4FCS instead of 60+4FCS.
 105                    Note these packets still get counted as frame errors. */
 106         } else
 107         if (USE_10MBPS_PREAMBLE_WORKAROUND && ((work->word2.snoip.err_code == 5) || (work->word2.snoip.err_code == 7))) {
 108
 109                 /* We received a packet with either an alignment error or a
 110                    FCS error. This may be signalling that we are running
 111                    10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK} off. If this is the
 112                    case we need to parse the packet to determine if we can
 113                    remove a non spec preamble and generate a correct packet */
 114                 int interface = cvmx_helper_get_interface_num(work->ipprt);
 115                 int index = cvmx_helper_get_interface_index_num(work->ipprt);
 116                 cvmx_gmxx_rxx_frm_ctl_t gmxx_rxx_frm_ctl;
 117                 gmxx_rxx_frm_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface));
 118                 if (gmxx_rxx_frm_ctl.s.pre_chk == 0) {
 119
 120                         uint8_t *ptr = cvmx_phys_to_ptr(work->packet_ptr.s.addr);
 121                         int i = 0;
 122
 123                         while (i < work->len-1) {
 124                                 if (*ptr != 0x55)
 125                                         break;
 126                                 ptr++;
 127                                 i++;
 128                         }
 129
 130                         if (*ptr == 0xd5) {
 131                                 /*
 132                                 DEBUGPRINT("Port %d received 0xd5 preamble\n", work->ipprt);
 133                                 */
 134                                 work->packet_ptr.s.addr += i+1;
 135                                 work->len -= i+5;
 136                         } else
 137                         if ((*ptr & 0xf) == 0xd) {
 138                                 /*
 139                                 DEBUGPRINT("Port %d received 0x?d preamble\n", work->ipprt);
 140                                 */
 141                                 work->packet_ptr.s.addr += i;
 142                                 work->len -= i+4;
 143                                 for (i = 0; i < work->len; i++) {
 144                                         *ptr = ((*ptr&0xf0)>>4) | ((*(ptr+1)&0xf)<<4);
 145                                         ptr++;
 146                                 }
 147                         } else {
 148                                 DEBUGPRINT("Port %d unknown preamble, packet dropped\n", work->ipprt);
 149                                 /*
 150                                 cvmx_helper_dump_packet(work);
 151                                 */
 152                                 cvm_oct_free_work(work);
 153                                 return 1;
 154                         }
 155                 }
 156         } else {
 157                 DEBUGPRINT("Port %d receive error code %d, packet dropped\n", work->ipprt, work->word2.snoip.err_code);
 158                 cvm_oct_free_work(work);
 159                 return 1;
 160         }
 161
 162         return 0;
 163 }
 164
 165 /**
 166  * Tasklet function that is scheduled on a core when an interrupt occurs.
 167  *
 168  * @param unused
 169  */
 170 void cvm_oct_tasklet_rx(void *context, int pending)
 171 {
 172         int                 coreid;
 173         uint64_t            old_group_mask;
 174         uint64_t            old_scratch;
 175         int                 rx_count = 0;
 176         int                 number_to_free;
 177         int                 num_freed;
 178         int                 packet_not_copied;
 179
 180         sched_pin();
 181         coreid = cvmx_get_core_num();
 182
 183         /* Prefetch cvm_oct_device since we know we need it soon */
 184         CVMX_PREFETCH(cvm_oct_device, 0);
 185
 186         if (USE_ASYNC_IOBDMA) {
 187                 /* Save scratch in case userspace is using it */
 188                 CVMX_SYNCIOBDMA;
 189                 old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH);
 190         }
 191
 192         /* Only allow work for our group (and preserve priorities) */
 193         old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid));
 194         cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid),
 195                        (old_group_mask & ~0xFFFFull) | 1<<pow_receive_group);
 196
 197         if (USE_ASYNC_IOBDMA)
 198                 cvmx_pow_work_request_async(CVMX_SCR_SCRATCH, CVMX_POW_NO_WAIT);
 199
 200         while (1) {
 201                 struct mbuf *m = NULL;
 202                 int mbuf_in_hw;
 203                 cvmx_wqe_t *work;
 204
 205                 if (USE_ASYNC_IOBDMA) {
 206                         work = cvmx_pow_work_response_async(CVMX_SCR_SCRATCH);
 207                 } else {
 208                         if ((INTERRUPT_LIMIT == 0) || (rx_count < MAX_RX_PACKETS))
 209                                 work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT);
 210                         else
 211                                 work = NULL;
 212                 }
 213                 CVMX_PREFETCH(work, 0);
 214                 if (work == NULL)
 215                         break;
 216
 217                 /* Limit each core to processing MAX_RX_PACKETS packets without a break.
 218                    This way the RX can't starve the TX task. */
 219                 if (USE_ASYNC_IOBDMA) {
 220
 221                         if ((INTERRUPT_LIMIT == 0) || (rx_count < MAX_RX_PACKETS))
 222                                 cvmx_pow_work_request_async_nocheck(CVMX_SCR_SCRATCH, CVMX_POW_NO_WAIT);
 223                         else {
 224                                 cvmx_scratch_write64(CVMX_SCR_SCRATCH, 0x8000000000000000ull);
 225                                 cvmx_pow_tag_sw_null_nocheck();
 226                         }
 227                 }
 228
 229                 mbuf_in_hw = USE_MBUFS_IN_HW && work->word2.s.bufs == 1;
 230                 if ((mbuf_in_hw)) {
 231                         m = *(struct mbuf **)(cvm_oct_get_buffer_ptr(work->packet_ptr) - sizeof(void *));
 232                         CVMX_PREFETCH(m, offsetof(struct mbuf, m_data));
 233                         CVMX_PREFETCH(m, offsetof(struct mbuf, m_pkthdr));
 234                 }
 235                 CVMX_PREFETCH(cvm_oct_device[work->ipprt], 0);
 236                 //CVMX_PREFETCH(m, 0);
 237
 238
 239                 rx_count++;
 240                 /* Immediately throw away all packets with receive errors */
 241                 if ((work->word2.snoip.rcv_error)) {
 242                         if (cvm_oct_check_rcv_error(work))
 243                                 continue;
 244                 }
 245
 246                 /* We can only use the zero copy path if mbufs are in the FPA pool
 247                    and the packet fits in a single buffer */
 248                 if ((mbuf_in_hw)) {
 249                         CVMX_PREFETCH(m->m_data, 0);
 250
 251                         m->m_pkthdr.len = m->m_len = work->len;
 252
 253                         packet_not_copied = 1;
 254
 255                         /*
 256                          * Adjust the data pointer based on the offset
 257                          * of the packet within the buffer.
 258                          */
 259                         m->m_data += (work->packet_ptr.s.back << 7) + (work->packet_ptr.s.addr & 0x7f);
 260                 } else {
 261
 262                         /* We have to copy the packet. First allocate an
 263                            mbuf for it */
 264                         MGETHDR(m, M_DONTWAIT, MT_DATA);
 265                         if (m == NULL) {
 266                                 DEBUGPRINT("Port %d failed to allocate mbuf, packet dropped\n", work->ipprt);
 267                                 cvm_oct_free_work(work);
 268                                 continue;
 269                         }
 270
 271                         /* Check if we've received a packet that was entirely
 272                            stored in the work entry. This is untested */
 273                         if ((work->word2.s.bufs == 0)) {
 274                                 uint8_t *ptr = work->packet_data;
 275
 276                                 if (cvmx_likely(!work->word2.s.not_IP)) {
 277                                         /* The beginning of the packet moves
 278                                            for IP packets */
 279                                         if (work->word2.s.is_v6)
 280                                                 ptr += 2;
 281                                         else
 282                                                 ptr += 6;
 283                                 }
 284                                 panic("%s: not yet implemented; copy in small packet.", __func__);
 285                                 /* No packet buffers to free */
 286                         } else {
 287                                 int segments = work->word2.s.bufs;
 288                                 cvmx_buf_ptr_t segment_ptr = work->packet_ptr;
 289                                 int len = work->len;
 290
 291                                 while (segments--) {
 292                                         cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8);
 293                                         /* Octeon Errata PKI-100: The segment
 294                                            size is wrong. Until it is fixed,
 295                                            calculate the segment size based on
 296                                            the packet pool buffer size. When
 297                                            it is fixed, the following line
 298                                            should be replaced with this one:
 299                                         int segment_size = segment_ptr.s.size; */
 300                                         int segment_size = CVMX_FPA_PACKET_POOL_SIZE - (segment_ptr.s.addr - (((segment_ptr.s.addr >> 7) - segment_ptr.s.back) << 7));
 301                                         /* Don't copy more than what is left
 302                                            in the packet */
 303                                         if (segment_size > len)
 304                                                 segment_size = len;
 305                                         /* Copy the data into the packet */
 306                                         panic("%s: not yet implemented; copy in packet segments.", __func__);
 307 #if 0
 308                                         memcpy(m_put(m, segment_size), cvmx_phys_to_ptr(segment_ptr.s.addr), segment_size);
 309 #endif
 310                                         /* Reduce the amount of bytes left
 311                                            to copy */
 312                                         len -= segment_size;
 313                                         segment_ptr = next_ptr;
 314                                 }
 315                         }
 316                         packet_not_copied = 0;
 317                 }
 318
 319                 if (((work->ipprt < TOTAL_NUMBER_OF_PORTS) &&
 320                     cvm_oct_device[work->ipprt])) {
 321                         struct ifnet *ifp = cvm_oct_device[work->ipprt];
 322
 323                         /* Only accept packets for devices
 324                            that are currently up */
 325                         if ((ifp->if_flags & IFF_UP)) {
 326                                 m->m_pkthdr.rcvif = ifp;
 327
 328                                 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
 329                                         if ((work->word2.s.not_IP || work->word2.s.IP_exc || work->word2.s.L4_error))
 330                                                 m->m_pkthdr.csum_flags = 0; /* XXX */
 331                                         else {
 332                                                 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
 333                                                 m->m_pkthdr.csum_data = 0xffff;
 334                                         }
 335                                 } else {
 336                                         m->m_pkthdr.csum_flags = 0; /* XXX */
 337                                 }
 338
 339                                 ifp->if_ipackets++;
 340
 341                                 (*ifp->if_input)(ifp, m);
 342                         } else {
 343                                 /* Drop any packet received for a device that isn't up */
 344                                 /*
 345                                 DEBUGPRINT("%s: Device not up, packet dropped\n",
 346                                            if_name(ifp));
 347                                 */
 348                                 m_freem(m);
 349                         }
 350                 } else {
 351                         /* Drop any packet received for a device that
 352                            doesn't exist */
 353                         DEBUGPRINT("Port %d not controlled by Linux, packet dropped\n", work->ipprt);
 354                         m_freem(m);
 355                 }
 356
 357                 /* Check to see if the mbuf and work share
 358                    the same packet buffer */
 359                 if (USE_MBUFS_IN_HW && (packet_not_copied)) {
 360                         /* This buffer needs to be replaced, increment
 361                         the number of buffers we need to free by one */
 362                         cvmx_fau_atomic_add32(
 363                                 FAU_NUM_PACKET_BUFFERS_TO_FREE, 1);
 364
 365                         cvmx_fpa_free(work, CVMX_FPA_WQE_POOL,
 366                                       DONT_WRITEBACK(1));
 367                 } else
 368                         cvm_oct_free_work(work);
 369         }
 370
 371         /* Restore the original POW group mask */
 372         cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask);
 373         if (USE_ASYNC_IOBDMA) {
 374                 /* Restore the scratch area */
 375                 cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch);
 376         }
 377
 378         if (USE_MBUFS_IN_HW) {
 379                 /* Refill the packet buffer pool */
 380                 number_to_free =
 381                   cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
 382
 383                 if (number_to_free > 0) {
 384                         cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
 385                                               -number_to_free);
 386                         num_freed =
 387                                 cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL,
 388                                                      CVMX_FPA_PACKET_POOL_SIZE,
 389                                                      number_to_free);
 390                         if (num_freed != number_to_free) {
 391                                 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
 392                                                       number_to_free - num_freed);
 393                         }
 394                 }
 395         }
 396         sched_unpin();
 397 }
 398
 399
 400
 401 void cvm_oct_rx_initialize(void)
 402 {
 403         TASK_INIT(&cvm_oct_task, 0, cvm_oct_tasklet_rx, NULL);
 404
 405         cvm_oct_taskq = taskqueue_create_fast("oct_rx", M_NOWAIT,
 406                                               taskqueue_thread_enqueue,
 407                                               &cvm_oct_taskq);
 408         taskqueue_start_threads(&cvm_oct_taskq, min(mp_ncpus, MAXCPU),
 409                                 PI_NET, "octe taskq");
 410 }
 411
 412 void cvm_oct_rx_shutdown(void)
 413 {
 414         panic("%s: not yet implemented.", __func__);
 415 }
 416