sys/mips/cavium/octe/ethernet-rx.c

   1 /*************************************************************************
   2 Copyright (c) 2003-2007  Cavium Networks (support@cavium.com). All rights
   3 reserved.
   4
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are
   8 met:
   9
  10     * Redistributions of source code must retain the above copyright
  11       notice, this list of conditions and the following disclaimer.
  12
  13     * Redistributions in binary form must reproduce the above
  14       copyright notice, this list of conditions and the following
  15       disclaimer in the documentation and/or other materials provided
  16       with the distribution.
  17
  18     * Neither the name of Cavium Networks nor the names of
  19       its contributors may be used to endorse or promote products
  20       derived from this software without specific prior written
  21       permission.
  22
  23 This Software, including technical data, may be subject to U.S. export  control laws, including the U.S. Export Administration Act and its  associated regulations, and may be subject to export or import  regulations in other countries.
  24
  25 TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  26 AND WITH ALL FAULTS AND CAVIUM  NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  27
  28 *************************************************************************/
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/bus.h>
  36 #include <sys/endian.h>
  37 #include <sys/kernel.h>
  38 #include <sys/mbuf.h>
  39 #include <sys/socket.h>
  40 #include <sys/proc.h>
  41 #include <sys/sched.h>
  42 #include <sys/smp.h>
  43 #include <sys/taskqueue.h>
  44
  45 #include <net/ethernet.h>
  46 #include <net/if.h>
  47 #include <net/if_var.h>
  48
  49 #include "wrapper-cvmx-includes.h"
  50 #include "ethernet-headers.h"
  51
  52 extern int pow_receive_group;
  53 extern struct ifnet *cvm_oct_device[];
  54
  55 static struct task cvm_oct_task;
  56 static struct taskqueue *cvm_oct_taskq;
  57
  58 static int cvm_oct_rx_active;
  59
  60 /**
  61  * Interrupt handler. The interrupt occurs whenever the POW
  62  * transitions from 0->1 packets in our group.
  63  *
  64  * @param cpl
  65  * @param dev_id
  66  * @param regs
  67  * @return
  68  */
  69 int cvm_oct_do_interrupt(void *dev_id)
  70 {
  71         /* Acknowledge the interrupt */
  72         if (INTERRUPT_LIMIT)
  73                 cvmx_write_csr(CVMX_POW_WQ_INT, 1<<pow_receive_group);
  74         else
  75                 cvmx_write_csr(CVMX_POW_WQ_INT, 0x10001<<pow_receive_group);
  76
  77         /*
  78          * Schedule task if there isn't one running.
  79          */
  80         if (atomic_cmpset_int(&cvm_oct_rx_active, 0, 1))
  81                 taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task);
  82
  83         return FILTER_HANDLED;
  84 }
  85
  86
  87 /**
  88  * This is called on receive errors, and determines if the packet
  89  * can be dropped early-on in cvm_oct_tasklet_rx().
  90  *
  91  * @param work Work queue entry pointing to the packet.
  92  * @return Non-zero if the packet can be dropped, zero otherwise.
  93  */
  94 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work)
  95 {
  96         if ((work->word2.snoip.err_code == 10) && (work->word1.s.len <= 64)) {
  97                 /* Ignore length errors on min size packets. Some equipment
  98                    incorrectly pads packets to 64+4FCS instead of 60+4FCS.
  99                    Note these packets still get counted as frame errors. */
 100         } else
 101         if (USE_10MBPS_PREAMBLE_WORKAROUND && ((work->word2.snoip.err_code == 5) || (work->word2.snoip.err_code == 7))) {
 102
 103                 /* We received a packet with either an alignment error or a
 104                    FCS error. This may be signalling that we are running
 105                    10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK} off. If this is the
 106                    case we need to parse the packet to determine if we can
 107                    remove a non spec preamble and generate a correct packet */
 108                 int interface = cvmx_helper_get_interface_num(work->word1.cn38xx.ipprt);
 109                 int index = cvmx_helper_get_interface_index_num(work->word1.cn38xx.ipprt);
 110                 cvmx_gmxx_rxx_frm_ctl_t gmxx_rxx_frm_ctl;
 111                 gmxx_rxx_frm_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface));
 112                 if (gmxx_rxx_frm_ctl.s.pre_chk == 0) {
 113
 114                         uint8_t *ptr = cvmx_phys_to_ptr(work->packet_ptr.s.addr);
 115                         int i = 0;
 116
 117                         while (i < work->word1.s.len-1) {
 118                                 if (*ptr != 0x55)
 119                                         break;
 120                                 ptr++;
 121                                 i++;
 122                         }
 123
 124                         if (*ptr == 0xd5) {
 125                                 /*
 126                                 DEBUGPRINT("Port %d received 0xd5 preamble\n", work->word1.cn38xx.ipprt);
 127                                 */
 128                                 work->packet_ptr.s.addr += i+1;
 129                                 work->word1.s.len -= i+5;
 130                         } else
 131                         if ((*ptr & 0xf) == 0xd) {
 132                                 /*
 133                                 DEBUGPRINT("Port %d received 0x?d preamble\n", work->word1.cn38xx.ipprt);
 134                                 */
 135                                 work->packet_ptr.s.addr += i;
 136                                 work->word1.s.len -= i+4;
 137                                 for (i = 0; i < work->word1.s.len; i++) {
 138                                         *ptr = ((*ptr&0xf0)>>4) | ((*(ptr+1)&0xf)<<4);
 139                                         ptr++;
 140                                 }
 141                         } else {
 142                                 DEBUGPRINT("Port %d unknown preamble, packet dropped\n", work->word1.cn38xx.ipprt);
 143                                 /*
 144                                 cvmx_helper_dump_packet(work);
 145                                 */
 146                                 cvm_oct_free_work(work);
 147                                 return 1;
 148                         }
 149                 }
 150         } else {
 151                 DEBUGPRINT("Port %d receive error code %d, packet dropped\n", work->word1.cn38xx.ipprt, work->word2.snoip.err_code);
 152                 cvm_oct_free_work(work);
 153                 return 1;
 154         }
 155
 156         return 0;
 157 }
 158
 159 /**
 160  * Tasklet function that is scheduled on a core when an interrupt occurs.
 161  *
 162  * @param unused
 163  */
 164 void cvm_oct_tasklet_rx(void *context, int pending)
 165 {
 166         int                 coreid;
 167         uint64_t            old_group_mask;
 168         int                 rx_count = 0;
 169         int                 number_to_free;
 170         int                 num_freed;
 171         int                 packet_not_copied;
 172
 173         sched_pin();
 174         coreid = cvmx_get_core_num();
 175
 176         /* Prefetch cvm_oct_device since we know we need it soon */
 177         CVMX_PREFETCH(cvm_oct_device, 0);
 178
 179         /* Only allow work for our group (and preserve priorities) */
 180         old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid));
 181         cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid),
 182                        (old_group_mask & ~0xFFFFull) | 1<<pow_receive_group);
 183
 184         while (1) {
 185                 struct mbuf *m = NULL;
 186                 int mbuf_in_hw;
 187                 cvmx_wqe_t *work;
 188
 189                 if ((INTERRUPT_LIMIT == 0) || (rx_count < MAX_RX_PACKETS))
 190                         work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT);
 191                 else
 192                         work = NULL;
 193                 CVMX_PREFETCH(work, 0);
 194                 if (work == NULL)
 195                         break;
 196
 197                 mbuf_in_hw = work->word2.s.bufs == 1;
 198                 if ((mbuf_in_hw)) {
 199                         m = *(struct mbuf **)(cvm_oct_get_buffer_ptr(work->packet_ptr) - sizeof(void *));
 200                         CVMX_PREFETCH(m, offsetof(struct mbuf, m_data));
 201                         CVMX_PREFETCH(m, offsetof(struct mbuf, m_pkthdr));
 202                 }
 203                 CVMX_PREFETCH(cvm_oct_device[work->word1.cn38xx.ipprt], 0);
 204                 //CVMX_PREFETCH(m, 0);
 205
 206
 207                 rx_count++;
 208                 /* Immediately throw away all packets with receive errors */
 209                 if ((work->word2.snoip.rcv_error)) {
 210                         if (cvm_oct_check_rcv_error(work))
 211                                 continue;
 212                 }
 213
 214                 /* We can only use the zero copy path if mbufs are in the FPA pool
 215                    and the packet fits in a single buffer */
 216                 if ((mbuf_in_hw)) {
 217                         CVMX_PREFETCH(m->m_data, 0);
 218
 219                         m->m_pkthdr.len = m->m_len = work->word1.s.len;
 220
 221                         packet_not_copied = 1;
 222
 223                         /*
 224                          * Adjust the data pointer based on the offset
 225                          * of the packet within the buffer.
 226                          */
 227                         m->m_data += (work->packet_ptr.s.back << 7) + (work->packet_ptr.s.addr & 0x7f);
 228                 } else {
 229
 230                         /* We have to copy the packet. First allocate an
 231                            mbuf for it */
 232                         MGETHDR(m, M_NOWAIT, MT_DATA);
 233                         if (m == NULL) {
 234                                 DEBUGPRINT("Port %d failed to allocate mbuf, packet dropped\n", work->word1.cn38xx.ipprt);
 235                                 cvm_oct_free_work(work);
 236                                 continue;
 237                         }
 238
 239                         /* Check if we've received a packet that was entirely
 240                            stored in the work entry. This is untested */
 241                         if ((work->word2.s.bufs == 0)) {
 242                                 uint8_t *ptr = work->packet_data;
 243
 244                                 if (cvmx_likely(!work->word2.s.not_IP)) {
 245                                         /* The beginning of the packet moves
 246                                            for IP packets */
 247                                         if (work->word2.s.is_v6)
 248                                                 ptr += 2;
 249                                         else
 250                                                 ptr += 6;
 251                                 }
 252                                 panic("%s: not yet implemented; copy in small packet.", __func__);
 253                                 /* No packet buffers to free */
 254                         } else {
 255                                 int segments = work->word2.s.bufs;
 256                                 cvmx_buf_ptr_t segment_ptr = work->packet_ptr;
 257                                 int len = work->word1.s.len;
 258
 259                                 while (segments--) {
 260                                         cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8);
 261                                         /* Octeon Errata PKI-100: The segment
 262                                            size is wrong. Until it is fixed,
 263                                            calculate the segment size based on
 264                                            the packet pool buffer size. When
 265                                            it is fixed, the following line
 266                                            should be replaced with this one:
 267                                         int segment_size = segment_ptr.s.size; */
 268                                         int segment_size = CVMX_FPA_PACKET_POOL_SIZE - (segment_ptr.s.addr - (((segment_ptr.s.addr >> 7) - segment_ptr.s.back) << 7));
 269                                         /* Don't copy more than what is left
 270                                            in the packet */
 271                                         if (segment_size > len)
 272                                                 segment_size = len;
 273                                         /* Copy the data into the packet */
 274                                         panic("%s: not yet implemented; copy in packet segments.", __func__);
 275 #if 0
 276                                         memcpy(m_put(m, segment_size), cvmx_phys_to_ptr(segment_ptr.s.addr), segment_size);
 277 #endif
 278                                         /* Reduce the amount of bytes left
 279                                            to copy */
 280                                         len -= segment_size;
 281                                         segment_ptr = next_ptr;
 282                                 }
 283                         }
 284                         packet_not_copied = 0;
 285                 }
 286
 287                 if (((work->word1.cn38xx.ipprt < TOTAL_NUMBER_OF_PORTS) &&
 288                     cvm_oct_device[work->word1.cn38xx.ipprt])) {
 289                         struct ifnet *ifp = cvm_oct_device[work->word1.cn38xx.ipprt];
 290
 291                         /* Only accept packets for devices
 292                            that are currently up */
 293                         if ((ifp->if_flags & IFF_UP)) {
 294                                 m->m_pkthdr.rcvif = ifp;
 295
 296                                 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
 297                                         if ((work->word2.s.not_IP || work->word2.s.IP_exc || work->word2.s.L4_error))
 298                                                 m->m_pkthdr.csum_flags = 0; /* XXX */
 299                                         else {
 300                                                 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
 301                                                 m->m_pkthdr.csum_data = 0xffff;
 302                                         }
 303                                 } else {
 304                                         m->m_pkthdr.csum_flags = 0; /* XXX */
 305                                 }
 306
 307                                 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
 308
 309                                 (*ifp->if_input)(ifp, m);
 310                         } else {
 311                                 /* Drop any packet received for a device that isn't up */
 312                                 /*
 313                                 DEBUGPRINT("%s: Device not up, packet dropped\n",
 314                                            if_name(ifp));
 315                                 */
 316                                 m_freem(m);
 317                         }
 318                 } else {
 319                         /* Drop any packet received for a device that
 320                            doesn't exist */
 321                         DEBUGPRINT("Port %d not controlled by FreeBSD, packet dropped\n", work->word1.cn38xx.ipprt);
 322                         m_freem(m);
 323                 }
 324
 325                 /* Check to see if the mbuf and work share
 326                    the same packet buffer */
 327                 if ((packet_not_copied)) {
 328                         /* This buffer needs to be replaced, increment
 329                         the number of buffers we need to free by one */
 330                         cvmx_fau_atomic_add32(
 331                                 FAU_NUM_PACKET_BUFFERS_TO_FREE, 1);
 332
 333                         cvmx_fpa_free(work, CVMX_FPA_WQE_POOL,
 334                                       DONT_WRITEBACK(1));
 335                 } else
 336                         cvm_oct_free_work(work);
 337         }
 338
 339         /*
 340          * If we hit our limit, schedule another task while we clean up.
 341          */
 342         if (INTERRUPT_LIMIT != 0 && rx_count == MAX_RX_PACKETS) {
 343                 taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task);
 344         } else {
 345                 /*
 346                  * No more packets, all done.
 347                  */
 348                 if (!atomic_cmpset_int(&cvm_oct_rx_active, 1, 0))
 349                         panic("%s: inconsistent rx active state.", __func__);
 350         }
 351
 352         /* Restore the original POW group mask */
 353         cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask);
 354
 355         /* Refill the packet buffer pool */
 356         number_to_free =
 357           cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
 358
 359         if (number_to_free > 0) {
 360                 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
 361                                       -number_to_free);
 362                 num_freed =
 363                         cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL,
 364                                              CVMX_FPA_PACKET_POOL_SIZE,
 365                                              number_to_free);
 366                 if (num_freed != number_to_free) {
 367                         cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
 368                                               number_to_free - num_freed);
 369                 }
 370         }
 371         sched_unpin();
 372 }
 373
 374
 375
 376 void cvm_oct_rx_initialize(void)
 377 {
 378         TASK_INIT(&cvm_oct_task, 0, cvm_oct_tasklet_rx, NULL);
 379
 380         cvm_oct_taskq = taskqueue_create_fast("oct_rx", M_NOWAIT,
 381                                               taskqueue_thread_enqueue,
 382                                               &cvm_oct_taskq);
 383         taskqueue_start_threads(&cvm_oct_taskq, min(mp_ncpus, MAXCPU),
 384                                 PI_NET, "octe taskq");
 385 }
 386
 387 void cvm_oct_rx_shutdown(void)
 388 {
 389         panic("%s: not yet implemented.", __func__);
 390 }
 391