sys/dev/cxgb/ulp/tom/cxgb_tom.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2007, Chelsio Inc.
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are met:
   8
   9  1. Redistributions of source code must retain the above copyright notice,
  10     this list of conditions and the following disclaimer.
  11
  12  2. Neither the name of the Chelsio Corporation nor the names of its
  13     contributors may be used to endorse or promote products derived from
  14     this software without specific prior written permission.
  15
  16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  26 POSSIBILITY OF SUCH DAMAGE.
  27
  28 ***************************************************************************/
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/fcntl.h>
  37 #include <sys/ktr.h>
  38 #include <sys/limits.h>
  39 #include <sys/lock.h>
  40 #include <sys/eventhandler.h>
  41 #include <sys/mbuf.h>
  42 #include <sys/module.h>
  43 #include <sys/condvar.h>
  44 #include <sys/mutex.h>
  45 #include <sys/socket.h>
  46 #include <sys/sockopt.h>
  47 #include <sys/sockstate.h>
  48 #include <sys/sockbuf.h>
  49 #include <sys/syslog.h>
  50 #include <sys/taskqueue.h>
  51
  52 #include <net/if.h>
  53 #include <net/route.h>
  54
  55 #include <netinet/in.h>
  56 #include <netinet/in_pcb.h>
  57 #include <netinet/in_systm.h>
  58 #include <netinet/in_var.h>
  59
  60 #include <cxgb_osdep.h>
  61 #include <sys/mbufq.h>
  62
  63 #include <netinet/in_pcb.h>
  64
  65 #include <ulp/tom/cxgb_tcp_offload.h>
  66 #include <netinet/tcp.h>
  67 #include <netinet/tcp_var.h>
  68 #include <netinet/tcp_offload.h>
  69 #include <netinet/tcp_fsm.h>
  70
  71 #include <cxgb_include.h>
  72
  73 #include <net/if_vlan_var.h>
  74 #include <net/route.h>
  75
  76 #include <t3cdev.h>
  77 #include <common/cxgb_firmware_exports.h>
  78 #include <common/cxgb_tcb.h>
  79 #include <cxgb_include.h>
  80 #include <common/cxgb_ctl_defs.h>
  81 #include <common/cxgb_t3_cpl.h>
  82 #include <cxgb_offload.h>
  83 #include <ulp/toecore/cxgb_toedev.h>
  84 #include <ulp/tom/cxgb_l2t.h>
  85 #include <ulp/tom/cxgb_tom.h>
  86 #include <ulp/tom/cxgb_defs.h>
  87 #include <ulp/tom/cxgb_t3_ddp.h>
  88 #include <ulp/tom/cxgb_toepcb.h>
  89 #include <ulp/tom/cxgb_tcp.h>
  90
  91
  92 TAILQ_HEAD(, adapter) adapter_list;
  93 static struct rwlock adapter_list_lock;
  94
  95 static TAILQ_HEAD(, tom_data) cxgb_list;
  96 static struct mtx cxgb_list_lock;
  97 static const unsigned int MAX_ATIDS = 64 * 1024;
  98 static const unsigned int ATID_BASE = 0x100000;
  99
 100 static int t3_toe_attach(struct toedev *dev, const struct offload_id *entry);
 101 static void cxgb_register_listeners(void);
 102 static void t3c_tom_add(struct t3cdev *cdev);
 103
 104 /*
 105  * Handlers for each CPL opcode
 106  */
 107 static cxgb_cpl_handler_func tom_cpl_handlers[256];
 108
 109
 110 static eventhandler_tag listen_tag;
 111
 112 static struct offload_id t3_toe_id_tab[] = {
 113         { TOE_ID_CHELSIO_T3, 0 },
 114         { TOE_ID_CHELSIO_T3B, 0 },
 115         { TOE_ID_CHELSIO_T3C, 0 },
 116         { 0 }
 117 };
 118
 119 static struct tom_info t3_tom_info = {
 120         .ti_attach = t3_toe_attach,
 121         .ti_id_table = t3_toe_id_tab,
 122         .ti_name = "Chelsio-T3"
 123 };
 124
 125 struct cxgb_client t3c_tom_client = {
 126         .name = "tom_cxgb3",
 127         .add = t3c_tom_add,
 128         .remove = NULL,
 129         .handlers = tom_cpl_handlers,
 130         .redirect = NULL
 131 };
 132
 133 void
 134 cxgb_log_tcb(struct adapter *sc, unsigned int tid)
 135 {
 136
 137         char buf[TCB_SIZE];
 138         uint64_t *tcb = (uint64_t *)buf;
 139         int i, error;
 140         struct mc7 *mem = &sc->cm;
 141
 142         error = t3_mc7_bd_read(mem, tid*TCB_SIZE/8, TCB_SIZE/8, tcb);
 143         if (error)
 144                 printf("cxgb_tcb_log failed\n");
 145
 146
 147         CTR1(KTR_CXGB, "TCB tid=%u", tid);
 148         for (i = 0; i < TCB_SIZE / 32; i++) {
 149
 150                 CTR5(KTR_CXGB, "%1d: %08x %08x %08x %08x",
 151                     i, (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
 152                     (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
 153
 154                 tcb += 2;
 155                 CTR4(KTR_CXGB, "   %08x %08x %08x %08x",
 156                     (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
 157                     (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
 158                 tcb += 2;
 159         }
 160 }
 161
 162 /*
 163  * Add an skb to the deferred skb queue for processing from process context.
 164  */
 165 void
 166 t3_defer_reply(struct mbuf *m, struct toedev *dev, defer_handler_t handler)
 167 {
 168         struct tom_data *td = TOM_DATA(dev);
 169
 170         m_set_handler(m, handler);
 171         mtx_lock(&td->deferq.lock);
 172
 173         mbufq_tail(&td->deferq, m);
 174         if (mbufq_len(&td->deferq) == 1)
 175                 taskqueue_enqueue(td->tq, &td->deferq_task);
 176         mtx_lock(&td->deferq.lock);
 177 }
 178
 179 struct toepcb *
 180 toepcb_alloc(void)
 181 {
 182         struct toepcb *toep;
 183
 184         toep = malloc(sizeof(struct toepcb), M_CXGB, M_NOWAIT|M_ZERO);
 185
 186         if (toep == NULL)
 187                 return (NULL);
 188
 189         toepcb_init(toep);
 190         return (toep);
 191 }
 192
 193 void
 194 toepcb_init(struct toepcb *toep)
 195 {
 196         toep->tp_refcount = 1;
 197         cv_init(&toep->tp_cv, "toep cv");
 198 }
 199
 200 void
 201 toepcb_hold(struct toepcb *toep)
 202 {
 203         atomic_add_acq_int(&toep->tp_refcount, 1);
 204 }
 205
 206 void
 207 toepcb_release(struct toepcb *toep)
 208 {
 209         if (toep->tp_refcount == 1) {
 210                 free(toep, M_CXGB);
 211                 return;
 212         }
 213         atomic_add_acq_int(&toep->tp_refcount, -1);
 214 }
 215
 216
 217 /*
 218  * Add a T3 offload device to the list of devices we are managing.
 219  */
 220 static void
 221 t3cdev_add(struct tom_data *t)
 222 {
 223         mtx_lock(&cxgb_list_lock);
 224         TAILQ_INSERT_TAIL(&cxgb_list, t, entry);
 225         mtx_unlock(&cxgb_list_lock);
 226 }
 227
 228 static inline int
 229 cdev2type(struct t3cdev *cdev)
 230 {
 231         int type = 0;
 232
 233         switch (cdev->type) {
 234         case T3A:
 235                 type = TOE_ID_CHELSIO_T3;
 236                 break;
 237         case T3B:
 238                 type = TOE_ID_CHELSIO_T3B;
 239                 break;
 240         case T3C:
 241                 type = TOE_ID_CHELSIO_T3C;
 242                 break;
 243         }
 244         return (type);
 245 }
 246
 247 /*
 248  * Allocate and initialize the TID tables.  Returns 0 on success.
 249  */
 250 static int
 251 init_tid_tabs(struct tid_info *t, unsigned int ntids,
 252                          unsigned int natids, unsigned int nstids,
 253                          unsigned int atid_base, unsigned int stid_base)
 254 {
 255         unsigned long size = ntids * sizeof(*t->tid_tab) +
 256             natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab);
 257
 258         t->tid_tab = cxgb_alloc_mem(size);
 259         if (!t->tid_tab)
 260                 return (ENOMEM);
 261
 262         t->stid_tab = (union listen_entry *)&t->tid_tab[ntids];
 263         t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids];
 264         t->ntids = ntids;
 265         t->nstids = nstids;
 266         t->stid_base = stid_base;
 267         t->sfree = NULL;
 268         t->natids = natids;
 269         t->atid_base = atid_base;
 270         t->afree = NULL;
 271         t->stids_in_use = t->atids_in_use = 0;
 272         atomic_set_int(&t->tids_in_use, 0);
 273         mtx_init(&t->stid_lock, "stid", NULL, MTX_DUPOK|MTX_DEF);
 274         mtx_init(&t->atid_lock, "atid", NULL, MTX_DUPOK|MTX_DEF);
 275
 276         /*
 277          * Setup the free lists for stid_tab and atid_tab.
 278          */
 279         if (nstids) {
 280                 while (--nstids)
 281                         t->stid_tab[nstids - 1].next = &t->stid_tab[nstids];
 282                 t->sfree = t->stid_tab;
 283         }
 284         if (natids) {
 285                 while (--natids)
 286                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
 287                 t->afree = t->atid_tab;
 288         }
 289         return 0;
 290 }
 291
 292 static void
 293 free_tid_maps(struct tid_info *t)
 294 {
 295         mtx_destroy(&t->stid_lock);
 296         mtx_destroy(&t->atid_lock);
 297         cxgb_free_mem(t->tid_tab);
 298 }
 299
 300 static inline void
 301 add_adapter(adapter_t *adap)
 302 {
 303         rw_wlock(&adapter_list_lock);
 304         TAILQ_INSERT_TAIL(&adapter_list, adap, adapter_entry);
 305         rw_wunlock(&adapter_list_lock);
 306 }
 307
 308 static inline void
 309 remove_adapter(adapter_t *adap)
 310 {
 311         rw_wlock(&adapter_list_lock);
 312         TAILQ_REMOVE(&adapter_list, adap, adapter_entry);
 313         rw_wunlock(&adapter_list_lock);
 314 }
 315
 316 /*
 317  * Populate a TID_RELEASE WR.  The mbuf must be already propely sized.
 318  */
 319 static inline void
 320 mk_tid_release(struct mbuf *m, unsigned int tid)
 321 {
 322         struct cpl_tid_release *req;
 323
 324         m_set_priority(m, CPL_PRIORITY_SETUP);
 325         req = mtod(m, struct cpl_tid_release *);
 326         m->m_pkthdr.len = m->m_len = sizeof(*req);
 327         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 328         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
 329 }
 330
 331 static void
 332 t3_process_tid_release_list(void *data, int pending)
 333 {
 334         struct mbuf *m;
 335         struct t3cdev *tdev = data;
 336         struct t3c_data *td = T3C_DATA (tdev);
 337
 338         mtx_lock(&td->tid_release_lock);
 339         while (td->tid_release_list) {
 340                 struct toe_tid_entry *p = td->tid_release_list;
 341
 342                 td->tid_release_list = (struct toe_tid_entry *)p->ctx;
 343                 mtx_unlock(&td->tid_release_lock);
 344                 m = m_get(M_WAIT, MT_DATA);
 345                 mk_tid_release(m, p - td->tid_maps.tid_tab);
 346                 cxgb_ofld_send(tdev, m);
 347                 p->ctx = NULL;
 348                 mtx_lock(&td->tid_release_lock);
 349         }
 350         mtx_unlock(&td->tid_release_lock);
 351 }
 352
 353 int
 354 cxgb_offload_activate(struct adapter *adapter)
 355 {
 356         struct t3cdev *dev = &adapter->tdev;
 357         int natids, err;
 358         struct t3c_data *t;
 359         struct tid_range stid_range, tid_range;
 360         struct mtutab mtutab;
 361         unsigned int l2t_capacity;
 362
 363         t = malloc(sizeof(*t), M_CXGB, M_NOWAIT|M_ZERO);
 364         if (!t)
 365                 return (ENOMEM);
 366         dev->adapter = adapter;
 367
 368         err = (EOPNOTSUPP);
 369         if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 ||
 370             dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 ||
 371             dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 ||
 372             dev->ctl(dev, GET_MTUS, &mtutab) < 0 ||
 373             dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 ||
 374             dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0) {
 375                 device_printf(adapter->dev, "%s: dev->ctl check failed\n", __FUNCTION__);
 376                 goto out_free;
 377         }
 378
 379         err = (ENOMEM);
 380         L2DATA(dev) = t3_init_l2t(l2t_capacity);
 381         if (!L2DATA(dev)) {
 382                 device_printf(adapter->dev, "%s: t3_init_l2t failed\n", __FUNCTION__);
 383                 goto out_free;
 384         }
 385         natids = min(tid_range.num / 2, MAX_ATIDS);
 386         err = init_tid_tabs(&t->tid_maps, tid_range.num, natids,
 387                             stid_range.num, ATID_BASE, stid_range.base);
 388         if (err) {
 389                 device_printf(adapter->dev, "%s: init_tid_tabs failed\n", __FUNCTION__);
 390                 goto out_free_l2t;
 391         }
 392
 393         t->mtus = mtutab.mtus;
 394         t->nmtus = mtutab.size;
 395
 396         TASK_INIT(&t->tid_release_task, 0 /* XXX? */, t3_process_tid_release_list, dev);
 397         mtx_init(&t->tid_release_lock, "tid release", NULL, MTX_DUPOK|MTX_DEF);
 398         t->dev = dev;
 399
 400         T3C_DATA (dev) = t;
 401         dev->recv = process_rx;
 402         dev->arp_update = t3_l2t_update;
 403         /* Register netevent handler once */
 404         if (TAILQ_EMPTY(&adapter_list)) {
 405 #if defined(CONFIG_CHELSIO_T3_MODULE)
 406                 if (prepare_arp_with_t3core())
 407                         log(LOG_ERR, "Unable to set offload capabilities\n");
 408 #endif
 409         }
 410         CTR1(KTR_CXGB, "adding adapter %p", adapter);
 411         add_adapter(adapter);
 412         device_printf(adapter->dev, "offload started\n");
 413         adapter->flags |= CXGB_OFLD_INIT;
 414         return (0);
 415
 416 out_free_l2t:
 417         t3_free_l2t(L2DATA(dev));
 418         L2DATA(dev) = NULL;
 419 out_free:
 420         free(t, M_CXGB);
 421         return (err);
 422 }
 423
 424 void
 425 cxgb_offload_deactivate(struct adapter *adapter)
 426 {
 427         struct t3cdev *tdev = &adapter->tdev;
 428         struct t3c_data *t = T3C_DATA(tdev);
 429
 430         printf("removing adapter %p\n", adapter);
 431         remove_adapter(adapter);
 432         if (TAILQ_EMPTY(&adapter_list)) {
 433 #if defined(CONFIG_CHELSIO_T3_MODULE)
 434                 restore_arp_sans_t3core();
 435 #endif
 436         }
 437         free_tid_maps(&t->tid_maps);
 438         T3C_DATA(tdev) = NULL;
 439         t3_free_l2t(L2DATA(tdev));
 440         L2DATA(tdev) = NULL;
 441         mtx_destroy(&t->tid_release_lock);
 442         free(t, M_CXGB);
 443 }
 444
 445 /*
 446  * Sends an sk_buff to a T3C driver after dealing with any active network taps.
 447  */
 448 int
 449 cxgb_ofld_send(struct t3cdev *dev, struct mbuf *m)
 450 {
 451         int r;
 452
 453         r = dev->send(dev, m);
 454         return r;
 455 }
 456
 457 static struct ifnet *
 458 get_iff_from_mac(adapter_t *adapter, const uint8_t *mac, unsigned int vlan)
 459 {
 460         int i;
 461
 462         for_each_port(adapter, i) {
 463 #ifdef notyet
 464                 const struct vlan_group *grp;
 465 #endif
 466                 const struct port_info *p = &adapter->port[i];
 467                 struct ifnet *ifp = p->ifp;
 468
 469                 if (!memcmp(p->hw_addr, mac, ETHER_ADDR_LEN)) {
 470 #ifdef notyet
 471
 472                         if (vlan && vlan != EVL_VLID_MASK) {
 473                                 grp = p->vlan_grp;
 474                                 dev = grp ? grp->vlan_devices[vlan] : NULL;
 475                         } else
 476                                 while (dev->master)
 477                                         dev = dev->master;
 478 #endif
 479                         return (ifp);
 480                 }
 481         }
 482         return (NULL);
 483 }
 484
 485 static inline void
 486 failover_fixup(adapter_t *adapter, int port)
 487 {
 488         if (adapter->params.rev == 0) {
 489                 struct ifnet *ifp = adapter->port[port].ifp;
 490                 struct cmac *mac = &adapter->port[port].mac;
 491                 if (!(ifp->if_flags & IFF_UP)) {
 492                         /* Failover triggered by the interface ifdown */
 493                         t3_write_reg(adapter, A_XGM_TX_CTRL + mac->offset,
 494                                      F_TXEN);
 495                         t3_read_reg(adapter, A_XGM_TX_CTRL + mac->offset);
 496                 } else {
 497                         /* Failover triggered by the interface link down */
 498                         t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 0);
 499                         t3_read_reg(adapter, A_XGM_RX_CTRL + mac->offset);
 500                         t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset,
 501                                      F_RXEN);
 502                 }
 503         }
 504 }
 505
 506 static int
 507 cxgb_ulp_iscsi_ctl(adapter_t *adapter, unsigned int req, void *data)
 508 {
 509         int ret = 0;
 510         struct ulp_iscsi_info *uiip = data;
 511
 512         switch (req) {
 513         case ULP_ISCSI_GET_PARAMS:
 514                 uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT);
 515                 uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT);
 516                 uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK);
 517                 /*
 518                  * On tx, the iscsi pdu has to be <= tx page size and has to
 519                  * fit into the Tx PM FIFO.
 520                  */
 521                 uiip->max_txsz = min(adapter->params.tp.tx_pg_size,
 522                                      t3_read_reg(adapter, A_PM1_TX_CFG) >> 17);
 523                 /* on rx, the iscsi pdu has to be < rx page size and the
 524                    whole pdu + cpl headers has to fit into one sge buffer */
 525                 /* also check the max rx data length programmed in TP */
 526                 uiip->max_rxsz = min(uiip->max_rxsz,
 527                                      ((t3_read_reg(adapter, A_TP_PARA_REG2))
 528                                         >> S_MAXRXDATA) & M_MAXRXDATA);
 529                 break;
 530         case ULP_ISCSI_SET_PARAMS:
 531                 t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask);
 532                 break;
 533         default:
 534                 ret = (EOPNOTSUPP);
 535         }
 536         return ret;
 537 }
 538
 539 /* Response queue used for RDMA events. */
 540 #define ASYNC_NOTIF_RSPQ 0
 541
 542 static int
 543 cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data)
 544 {
 545         int ret = 0;
 546
 547         switch (req) {
 548         case RDMA_GET_PARAMS: {
 549                 struct rdma_info *req = data;
 550
 551                 req->udbell_physbase = rman_get_start(adapter->udbs_res);
 552                 req->udbell_len = rman_get_size(adapter->udbs_res);
 553                 req->tpt_base = t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT);
 554                 req->tpt_top  = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT);
 555                 req->pbl_base = t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT);
 556                 req->pbl_top  = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT);
 557                 req->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT);
 558                 req->rqt_top  = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT);
 559                 req->kdb_addr =  (void *)((unsigned long)rman_get_virtual(adapter->regs_res) + A_SG_KDOORBELL);         break;
 560         }
 561         case RDMA_CQ_OP: {
 562                 struct rdma_cq_op *req = data;
 563
 564                 /* may be called in any context */
 565                 mtx_lock_spin(&adapter->sge.reg_lock);
 566                 ret = t3_sge_cqcntxt_op(adapter, req->id, req->op,
 567                                         req->credits);
 568                 mtx_unlock_spin(&adapter->sge.reg_lock);
 569                 break;
 570         }
 571         case RDMA_GET_MEM: {
 572                 struct ch_mem_range *t = data;
 573                 struct mc7 *mem;
 574
 575                 if ((t->addr & 7) || (t->len & 7))
 576                         return (EINVAL);
 577                 if (t->mem_id == MEM_CM)
 578                         mem = &adapter->cm;
 579                 else if (t->mem_id == MEM_PMRX)
 580                         mem = &adapter->pmrx;
 581                 else if (t->mem_id == MEM_PMTX)
 582                         mem = &adapter->pmtx;
 583                 else
 584                         return (EINVAL);
 585
 586                 ret = t3_mc7_bd_read(mem, t->addr/8, t->len/8, (u64 *)t->buf);
 587                 if (ret)
 588                         return (ret);
 589                 break;
 590         }
 591         case RDMA_CQ_SETUP: {
 592                 struct rdma_cq_setup *req = data;
 593
 594                 mtx_lock_spin(&adapter->sge.reg_lock);
 595                 ret = t3_sge_init_cqcntxt(adapter, req->id, req->base_addr,
 596                                           req->size, ASYNC_NOTIF_RSPQ,
 597                                           req->ovfl_mode, req->credits,
 598                                           req->credit_thres);
 599                 mtx_unlock_spin(&adapter->sge.reg_lock);
 600                 break;
 601         }
 602         case RDMA_CQ_DISABLE:
 603                 mtx_lock_spin(&adapter->sge.reg_lock);
 604                 ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data);
 605                 mtx_unlock_spin(&adapter->sge.reg_lock);
 606                 break;
 607         case RDMA_CTRL_QP_SETUP: {
 608                 struct rdma_ctrlqp_setup *req = data;
 609
 610                 mtx_lock_spin(&adapter->sge.reg_lock);
 611                 ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0,
 612                                          SGE_CNTXT_RDMA, ASYNC_NOTIF_RSPQ,
 613                                          req->base_addr, req->size,
 614                                          FW_RI_TID_START, 1, 0);
 615                 mtx_unlock_spin(&adapter->sge.reg_lock);
 616                 break;
 617         }
 618         default:
 619                 ret = EOPNOTSUPP;
 620         }
 621         return (ret);
 622 }
 623
 624 static int
 625 cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data)
 626 {
 627         struct adapter *adapter = tdev2adap(tdev);
 628         struct tid_range *tid;
 629         struct mtutab *mtup;
 630         struct iff_mac *iffmacp;
 631         struct ddp_params *ddpp;
 632         struct adap_ports *ports;
 633         struct ofld_page_info *rx_page_info;
 634         struct tp_params *tp = &adapter->params.tp;
 635         int port;
 636
 637         switch (req) {
 638         case GET_MAX_OUTSTANDING_WR:
 639                 *(unsigned int *)data = FW_WR_NUM;
 640                 break;
 641         case GET_WR_LEN:
 642                 *(unsigned int *)data = WR_FLITS;
 643                 break;
 644         case GET_TX_MAX_CHUNK:
 645                 *(unsigned int *)data = 1 << 20;  /* 1MB */
 646                 break;
 647         case GET_TID_RANGE:
 648                 tid = data;
 649                 tid->num = t3_mc5_size(&adapter->mc5) -
 650                         adapter->params.mc5.nroutes -
 651                         adapter->params.mc5.nfilters -
 652                         adapter->params.mc5.nservers;
 653                 tid->base = 0;
 654                 break;
 655         case GET_STID_RANGE:
 656                 tid = data;
 657                 tid->num = adapter->params.mc5.nservers;
 658                 tid->base = t3_mc5_size(&adapter->mc5) - tid->num -
 659                         adapter->params.mc5.nfilters -
 660                         adapter->params.mc5.nroutes;
 661                 break;
 662         case GET_L2T_CAPACITY:
 663                 *(unsigned int *)data = 2048;
 664                 break;
 665         case GET_MTUS:
 666                 mtup = data;
 667                 mtup->size = NMTUS;
 668                 mtup->mtus = adapter->params.mtus;
 669                 break;
 670         case GET_IFF_FROM_MAC:
 671                 iffmacp = data;
 672                 iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr,
 673                                           iffmacp->vlan_tag & EVL_VLID_MASK);
 674                 break;
 675         case GET_DDP_PARAMS:
 676                 ddpp = data;
 677                 ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT);
 678                 ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT);
 679                 ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK);
 680                 break;
 681         case GET_PORTS:
 682                 ports = data;
 683                 ports->nports   = adapter->params.nports;
 684                 for_each_port(adapter, port)
 685                         ports->lldevs[port] = adapter->port[port].ifp;
 686                 break;
 687         case FAILOVER:
 688                 port = *(int *)data;
 689                 t3_port_failover(adapter, port);
 690                 failover_fixup(adapter, port);
 691                 break;
 692         case FAILOVER_DONE:
 693                 port = *(int *)data;
 694                 t3_failover_done(adapter, port);
 695                 break;
 696         case FAILOVER_CLEAR:
 697                 t3_failover_clear(adapter);
 698                 break;
 699         case GET_RX_PAGE_INFO:
 700                 rx_page_info = data;
 701                 rx_page_info->page_size = tp->rx_pg_size;
 702                 rx_page_info->num = tp->rx_num_pgs;
 703                 break;
 704         case ULP_ISCSI_GET_PARAMS:
 705         case ULP_ISCSI_SET_PARAMS:
 706                 if (!offload_running(adapter))
 707                         return (EAGAIN);
 708                 return cxgb_ulp_iscsi_ctl(adapter, req, data);
 709         case RDMA_GET_PARAMS:
 710         case RDMA_CQ_OP:
 711         case RDMA_CQ_SETUP:
 712         case RDMA_CQ_DISABLE:
 713         case RDMA_CTRL_QP_SETUP:
 714         case RDMA_GET_MEM:
 715                 if (!offload_running(adapter))
 716                         return (EAGAIN);
 717                 return cxgb_rdma_ctl(adapter, req, data);
 718         default:
 719                 return (EOPNOTSUPP);
 720         }
 721         return 0;
 722 }
 723
 724 /*
 725  * Allocate a TOM data structure,
 726  * initialize its cpl_handlers
 727  * and register it as a T3C client
 728  */
 729 static void
 730 t3c_tom_add(struct t3cdev *cdev)
 731 {
 732         int i;
 733         unsigned int wr_len;
 734         struct tom_data *t;
 735         struct toedev *tdev;
 736         struct adap_ports *port_info;
 737
 738         t = malloc(sizeof(*t), M_CXGB, M_NOWAIT|M_ZERO);
 739         if (t == NULL)
 740                 return;
 741
 742         cdev->send = t3_offload_tx;
 743         cdev->ctl = cxgb_offload_ctl;
 744
 745         if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0)
 746                 goto out_free_tom;
 747
 748         port_info = malloc(sizeof(*port_info), M_CXGB, M_NOWAIT|M_ZERO);
 749         if (!port_info)
 750                 goto out_free_tom;
 751
 752         if (cdev->ctl(cdev, GET_PORTS, port_info) < 0)
 753                 goto out_free_all;
 754
 755         t3_init_wr_tab(wr_len);
 756         t->cdev = cdev;
 757         t->client = &t3c_tom_client;
 758
 759         /* Register TCP offload device */
 760         tdev = &t->tdev;
 761         tdev->tod_ttid = cdev2type(cdev);
 762         tdev->tod_lldev = cdev->lldev;
 763
 764         if (register_toedev(tdev, "toe%d")) {
 765                 printf("unable to register offload device");
 766                 goto out_free_all;
 767         }
 768         TOM_DATA(tdev) = t;
 769
 770         for (i = 0; i < port_info->nports; i++) {
 771                 struct ifnet *ifp = port_info->lldevs[i];
 772                 TOEDEV(ifp) = tdev;
 773
 774                 CTR1(KTR_TOM, "enabling toe on %p", ifp);
 775                 ifp->if_capabilities |= IFCAP_TOE4;
 776                 ifp->if_capenable |= IFCAP_TOE4;
 777         }
 778         t->ports = port_info;
 779
 780         /* Add device to the list of offload devices */
 781         t3cdev_add(t);
 782
 783         /* Activate TCP offload device */
 784         cxgb_offload_activate(TOM_DATA(tdev)->cdev->adapter);
 785
 786         activate_offload(tdev);
 787         cxgb_register_listeners();
 788         return;
 789
 790 out_free_all:
 791         printf("out_free_all fail\n");
 792         free(port_info, M_CXGB);
 793 out_free_tom:
 794         printf("out_free_tom fail\n");
 795         free(t, M_CXGB);
 796         return;
 797 }
 798
 799
 800
 801 static int
 802 do_act_open_rpl(struct t3cdev *dev, struct mbuf *m)
 803 {
 804         struct cpl_act_open_rpl *rpl = cplhdr(m);
 805         unsigned int atid = G_TID(ntohl(rpl->atid));
 806         struct toe_tid_entry *toe_tid;
 807
 808         toe_tid = lookup_atid(&(T3C_DATA (dev))->tid_maps, atid);
 809         if (toe_tid->ctx && toe_tid->client && toe_tid->client->handlers &&
 810                 toe_tid->client->handlers[CPL_ACT_OPEN_RPL]) {
 811                 return toe_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, m,
 812                         toe_tid->ctx);
 813         } else {
 814                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
 815                         dev->name, CPL_ACT_OPEN_RPL);
 816                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
 817         }
 818 }
 819
 820 static int
 821 do_stid_rpl(struct t3cdev *dev, struct mbuf *m)
 822 {
 823         union opcode_tid *p = cplhdr(m);
 824         unsigned int stid = G_TID(ntohl(p->opcode_tid));
 825         struct toe_tid_entry *toe_tid;
 826
 827         toe_tid = lookup_stid(&(T3C_DATA (dev))->tid_maps, stid);
 828         if (toe_tid->ctx && toe_tid->client->handlers &&
 829                 toe_tid->client->handlers[p->opcode]) {
 830                 return toe_tid->client->handlers[p->opcode] (dev, m, toe_tid->ctx);
 831         } else {
 832                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
 833                         dev->name, p->opcode);
 834                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
 835         }
 836 }
 837
 838 static int
 839 do_hwtid_rpl(struct t3cdev *dev, struct mbuf *m)
 840 {
 841         union opcode_tid *p = cplhdr(m);
 842         unsigned int hwtid;
 843         struct toe_tid_entry *toe_tid;
 844
 845         DPRINTF("do_hwtid_rpl opcode=0x%x\n", p->opcode);
 846         hwtid = G_TID(ntohl(p->opcode_tid));
 847
 848         toe_tid = lookup_tid(&(T3C_DATA (dev))->tid_maps, hwtid);
 849         if (toe_tid->ctx && toe_tid->client->handlers &&
 850                 toe_tid->client->handlers[p->opcode]) {
 851                 return toe_tid->client->handlers[p->opcode]
 852                                                 (dev, m, toe_tid->ctx);
 853         } else {
 854                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
 855                         dev->name, p->opcode);
 856                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
 857         }
 858 }
 859
 860 static int
 861 do_cr(struct t3cdev *dev, struct mbuf *m)
 862 {
 863         struct cpl_pass_accept_req *req = cplhdr(m);
 864         unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
 865         struct toe_tid_entry *toe_tid;
 866
 867         toe_tid = lookup_stid(&(T3C_DATA (dev))->tid_maps, stid);
 868         if (toe_tid->ctx && toe_tid->client->handlers &&
 869                 toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) {
 870                 return toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]
 871                                                 (dev, m, toe_tid->ctx);
 872         } else {
 873                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
 874                         dev->name, CPL_PASS_ACCEPT_REQ);
 875                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
 876         }
 877 }
 878
 879 static int
 880 do_abort_req_rss(struct t3cdev *dev, struct mbuf *m)
 881 {
 882         union opcode_tid *p = cplhdr(m);
 883         unsigned int hwtid = G_TID(ntohl(p->opcode_tid));
 884         struct toe_tid_entry *toe_tid;
 885
 886         toe_tid = lookup_tid(&(T3C_DATA (dev))->tid_maps, hwtid);
 887         if (toe_tid->ctx && toe_tid->client->handlers &&
 888                 toe_tid->client->handlers[p->opcode]) {
 889                 return toe_tid->client->handlers[p->opcode]
 890                                                 (dev, m, toe_tid->ctx);
 891         } else {
 892                 struct cpl_abort_req_rss *req = cplhdr(m);
 893                 struct cpl_abort_rpl *rpl;
 894
 895                 struct mbuf *m = m_get(M_NOWAIT, MT_DATA);
 896                 if (!m) {
 897                         log(LOG_NOTICE, "do_abort_req_rss: couldn't get mbuf!\n");
 898                         goto out;
 899                 }
 900
 901                 m_set_priority(m, CPL_PRIORITY_DATA);
 902                 rpl = cplhdr(m);
 903                 rpl->wr.wr_hi =
 904                         htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
 905                 rpl->wr.wr_lo = htonl(V_WR_TID(GET_TID(req)));
 906                 OPCODE_TID(rpl) =
 907                         htonl(MK_OPCODE_TID(CPL_ABORT_RPL, GET_TID(req)));
 908                 rpl->cmd = req->status;
 909                 cxgb_ofld_send(dev, m);
 910  out:
 911                 return (CPL_RET_BUF_DONE);
 912         }
 913 }
 914
 915 static int
 916 do_act_establish(struct t3cdev *dev, struct mbuf *m)
 917 {
 918         struct cpl_act_establish *req;
 919         unsigned int atid;
 920         struct toe_tid_entry *toe_tid;
 921
 922         req = cplhdr(m);
 923         atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
 924         toe_tid = lookup_atid(&(T3C_DATA (dev))->tid_maps, atid);
 925         if (toe_tid && toe_tid->ctx && toe_tid->client->handlers &&
 926                 toe_tid->client->handlers[CPL_ACT_ESTABLISH]) {
 927
 928                 return toe_tid->client->handlers[CPL_ACT_ESTABLISH]
 929                                                 (dev, m, toe_tid->ctx);
 930         } else {
 931
 932                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
 933                         dev->name, CPL_ACT_ESTABLISH);
 934                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
 935         }
 936 }
 937
 938
 939 static int
 940 do_term(struct t3cdev *dev, struct mbuf *m)
 941 {
 942         unsigned int hwtid = ntohl(m_get_priority(m)) >> 8 & 0xfffff;
 943         unsigned int opcode = G_OPCODE(ntohl(m->m_pkthdr.csum_data));
 944         struct toe_tid_entry *toe_tid;
 945
 946         toe_tid = lookup_tid(&(T3C_DATA (dev))->tid_maps, hwtid);
 947         if (toe_tid && toe_tid->ctx && toe_tid->client->handlers &&
 948                 toe_tid->client->handlers[opcode]) {
 949                 return toe_tid->client->handlers[opcode](dev, m, toe_tid->ctx);
 950         } else {
 951                 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
 952                         dev->name, opcode);
 953                 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
 954         }
 955         return (0);
 956 }
 957
 958 /*
 959  * Process a received packet with an unknown/unexpected CPL opcode.
 960  */
 961 static int
 962 do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
 963 {
 964         log(LOG_ERR, "%s: received bad CPL command %u\n", cdev->name,
 965             0xFF & *mtod(m, unsigned int *));
 966         return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
 967 }
 968
 969 /*
 970  * Add a new handler to the CPL dispatch table.  A NULL handler may be supplied
 971  * to unregister an existing handler.
 972  */
 973 void
 974 t3tom_register_cpl_handler(unsigned int opcode, cxgb_cpl_handler_func h)
 975 {
 976         if (opcode < UCHAR_MAX)
 977                 tom_cpl_handlers[opcode] = h ? h : do_bad_cpl;
 978         else
 979                 log(LOG_ERR, "Chelsio T3 TOM: handler registration for "
 980                        "opcode %u failed\n", opcode);
 981 }
 982
 983 /*
 984  * Make a preliminary determination if a connection can be offloaded.  It's OK
 985  * to fail the offload later if we say we can offload here.  For now this
 986  * always accepts the offload request unless there are IP options.
 987  */
 988 static int
 989 can_offload(struct toedev *dev, struct socket *so)
 990 {
 991         struct tom_data *tomd = TOM_DATA(dev);
 992         struct t3cdev *cdev = T3CDEV(dev->tod_lldev);
 993         struct tid_info *t = &(T3C_DATA(cdev))->tid_maps;
 994
 995         return so_sotoinpcb(so)->inp_depend4.inp4_options == NULL &&
 996             tomd->conf.activated &&
 997             (tomd->conf.max_conn < 0 ||
 998              atomic_load_acq_int(&t->tids_in_use) + t->atids_in_use < tomd->conf.max_conn);
 999 }
1000
1001 static int
1002 tom_ctl(struct toedev *dev, unsigned int req, void *data)
1003 {
1004         struct tom_data *t = TOM_DATA(dev);
1005         struct t3cdev *cdev = t->cdev;
1006
1007         if (cdev->ctl)
1008                 return cdev->ctl(cdev, req, data);
1009
1010         return (EOPNOTSUPP);
1011 }
1012
1013 /*
1014  * Free an active-open TID.
1015  */
1016 void *
1017 cxgb_free_atid(struct t3cdev *tdev, int atid)
1018 {
1019         struct tid_info *t = &(T3C_DATA(tdev))->tid_maps;
1020         union active_open_entry *p = atid2entry(t, atid);
1021         void *ctx = p->toe_tid.ctx;
1022
1023         mtx_lock(&t->atid_lock);
1024         p->next = t->afree;
1025         t->afree = p;
1026         t->atids_in_use--;
1027         mtx_unlock(&t->atid_lock);
1028
1029         return ctx;
1030 }
1031
1032 /*
1033  * Free a server TID and return it to the free pool.
1034  */
1035 void
1036 cxgb_free_stid(struct t3cdev *tdev, int stid)
1037 {
1038         struct tid_info *t = &(T3C_DATA (tdev))->tid_maps;
1039         union listen_entry *p = stid2entry(t, stid);
1040
1041         mtx_lock(&t->stid_lock);
1042         p->next = t->sfree;
1043         t->sfree = p;
1044         t->stids_in_use--;
1045         mtx_unlock(&t->stid_lock);
1046 }
1047
1048 /*
1049  * Free a server TID and return it to the free pool.
1050  */
1051 void *
1052 cxgb_get_lctx(struct t3cdev *tdev, int stid)
1053 {
1054         struct tid_info *t = &(T3C_DATA (tdev))->tid_maps;
1055         union listen_entry *p = stid2entry(t, stid);
1056
1057         return (p->toe_tid.ctx);
1058 }
1059
1060 void
1061 cxgb_insert_tid(struct t3cdev *tdev, struct cxgb_client *client,
1062         void *ctx, unsigned int tid)
1063 {
1064         struct tid_info *t = &(T3C_DATA (tdev))->tid_maps;
1065
1066         t->tid_tab[tid].client = client;
1067         t->tid_tab[tid].ctx = ctx;
1068         atomic_add_int(&t->tids_in_use, 1);
1069 }
1070
1071 /* use ctx as a next pointer in the tid release list */
1072 void
1073 cxgb_queue_tid_release(struct t3cdev *tdev, unsigned int tid)
1074 {
1075         struct t3c_data *td = T3C_DATA (tdev);
1076         struct toe_tid_entry *p = &td->tid_maps.tid_tab[tid];
1077
1078         CTR0(KTR_TOM, "queuing tid release\n");
1079
1080         mtx_lock(&td->tid_release_lock);
1081         p->ctx = td->tid_release_list;
1082         td->tid_release_list = p;
1083
1084         if (!p->ctx)
1085                 taskqueue_enqueue(tdev->adapter->tq, &td->tid_release_task);
1086
1087         mtx_unlock(&td->tid_release_lock);
1088 }
1089
1090 /*
1091  * Remove a tid from the TID table.  A client may defer processing its last
1092  * CPL message if it is locked at the time it arrives, and while the message
1093  * sits in the client's backlog the TID may be reused for another connection.
1094  * To handle this we atomically switch the TID association if it still points
1095  * to the original client context.
1096  */
1097 void
1098 cxgb_remove_tid(struct t3cdev *tdev, void *ctx, unsigned int tid)
1099 {
1100         struct tid_info *t = &(T3C_DATA (tdev))->tid_maps;
1101
1102         if (tid >= t->ntids)
1103                 panic("tid=%d >= t->ntids=%d", tid, t->ntids);
1104
1105         if (tdev->type == T3A)
1106                 atomic_cmpset_ptr((uintptr_t *)&t->tid_tab[tid].ctx, (long)NULL, (long)ctx);
1107         else {
1108                 struct mbuf *m;
1109
1110                 m = m_get(M_NOWAIT, MT_DATA);
1111                 if (__predict_true(m != NULL)) {
1112                         mk_tid_release(m, tid);
1113                         CTR1(KTR_CXGB, "releasing tid=%u", tid);
1114
1115                         cxgb_ofld_send(tdev, m);
1116                         t->tid_tab[tid].ctx = NULL;
1117                 } else
1118                         cxgb_queue_tid_release(tdev, tid);
1119         }
1120         atomic_add_int(&t->tids_in_use, -1);
1121 }
1122
1123 int
1124 cxgb_alloc_atid(struct t3cdev *tdev, struct cxgb_client *client,
1125                      void *ctx)
1126 {
1127         int atid = -1;
1128         struct tid_info *t = &(T3C_DATA (tdev))->tid_maps;
1129
1130         mtx_lock(&t->atid_lock);
1131         if (t->afree) {
1132                 union active_open_entry *p = t->afree;
1133
1134                 atid = (p - t->atid_tab) + t->atid_base;
1135                 t->afree = p->next;
1136                 p->toe_tid.ctx = ctx;
1137                 p->toe_tid.client = client;
1138                 t->atids_in_use++;
1139         }
1140         mtx_unlock(&t->atid_lock);
1141         return atid;
1142 }
1143
1144 int
1145 cxgb_alloc_stid(struct t3cdev *tdev, struct cxgb_client *client,
1146                      void *ctx)
1147 {
1148         int stid = -1;
1149         struct tid_info *t = &(T3C_DATA (tdev))->tid_maps;
1150
1151         mtx_lock(&t->stid_lock);
1152         if (t->sfree) {
1153                 union listen_entry *p = t->sfree;
1154
1155                 stid = (p - t->stid_tab) + t->stid_base;
1156                 t->sfree = p->next;
1157                 p->toe_tid.ctx = ctx;
1158                 p->toe_tid.client = client;
1159                 t->stids_in_use++;
1160         }
1161         mtx_unlock(&t->stid_lock);
1162         return stid;
1163 }
1164
1165
1166 static int
1167 is_offloading(struct ifnet *ifp)
1168 {
1169         struct adapter *adapter;
1170         int port;
1171
1172         rw_rlock(&adapter_list_lock);
1173         TAILQ_FOREACH(adapter, &adapter_list, adapter_entry) {
1174                 for_each_port(adapter, port) {
1175                         if (ifp == adapter->port[port].ifp) {
1176                                 rw_runlock(&adapter_list_lock);
1177                                 return 1;
1178                         }
1179                 }
1180         }
1181         rw_runlock(&adapter_list_lock);
1182         return 0;
1183 }
1184
1185
1186 static void
1187 cxgb_arp_update_event(void *unused, struct rtentry *rt0,
1188     uint8_t *enaddr, struct sockaddr *sa)
1189 {
1190
1191         if (!is_offloading(rt0->rt_ifp))
1192                 return;
1193
1194         RT_ADDREF(rt0);
1195         RT_UNLOCK(rt0);
1196         cxgb_neigh_update(rt0, enaddr, sa);
1197         RT_LOCK(rt0);
1198         RT_REMREF(rt0);
1199 }
1200
1201 static void
1202 cxgb_redirect_event(void *unused, int event, struct rtentry *rt0,
1203     struct rtentry *rt1, struct sockaddr *sa)
1204 {
1205         /*
1206          * ignore events on non-offloaded interfaces
1207          */
1208         if (!is_offloading(rt0->rt_ifp))
1209                 return;
1210
1211         /*
1212          * Cannot redirect to non-offload device.
1213          */
1214         if (!is_offloading(rt1->rt_ifp)) {
1215                 log(LOG_WARNING, "%s: Redirect to non-offload"
1216                     "device ignored.\n", __FUNCTION__);
1217                 return;
1218         }
1219
1220         /*
1221          * avoid LORs by dropping the route lock but keeping a reference
1222          *
1223          */
1224         RT_ADDREF(rt0);
1225         RT_UNLOCK(rt0);
1226         RT_ADDREF(rt1);
1227         RT_UNLOCK(rt1);
1228
1229         cxgb_redirect(rt0, rt1, sa);
1230         cxgb_neigh_update(rt1, NULL, sa);
1231
1232         RT_LOCK(rt0);
1233         RT_REMREF(rt0);
1234         RT_LOCK(rt1);
1235         RT_REMREF(rt1);
1236 }
1237
1238 void
1239 cxgb_neigh_update(struct rtentry *rt, uint8_t *enaddr, struct sockaddr *sa)
1240 {
1241
1242         if (rt->rt_ifp && is_offloading(rt->rt_ifp) && (rt->rt_ifp->if_flags & IFCAP_TOE)) {
1243                 struct t3cdev *tdev = T3CDEV(rt->rt_ifp);
1244
1245                 PANIC_IF(!tdev);
1246                 t3_l2t_update(tdev, rt, enaddr, sa);
1247         }
1248 }
1249
1250 static void
1251 set_l2t_ix(struct t3cdev *tdev, u32 tid, struct l2t_entry *e)
1252 {
1253         struct mbuf *m;
1254         struct cpl_set_tcb_field *req;
1255
1256         m = m_gethdr(M_NOWAIT, MT_DATA);
1257         if (!m) {
1258                 log(LOG_ERR, "%s: cannot allocate mbuf!\n", __FUNCTION__);
1259                 return;
1260         }
1261
1262         m_set_priority(m, CPL_PRIORITY_CONTROL);
1263         req = mtod(m, struct cpl_set_tcb_field *);
1264         m->m_pkthdr.len = m->m_len = sizeof(*req);
1265
1266         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1267         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1268         req->reply = 0;
1269         req->cpu_idx = 0;
1270         req->word = htons(W_TCB_L2T_IX);
1271         req->mask = htobe64(V_TCB_L2T_IX(M_TCB_L2T_IX));
1272         req->val = htobe64(V_TCB_L2T_IX(e->idx));
1273         tdev->send(tdev, m);
1274 }
1275
1276 void
1277 cxgb_redirect(struct rtentry *old, struct rtentry *new, struct sockaddr *sa)
1278 {
1279         struct ifnet *olddev, *newdev;
1280         struct tid_info *ti;
1281         struct t3cdev *tdev;
1282         u32 tid;
1283         int update_tcb;
1284         struct l2t_entry *e;
1285         struct toe_tid_entry *te;
1286
1287         olddev = old->rt_ifp;
1288         newdev = new->rt_ifp;
1289         if (!is_offloading(olddev))
1290                 return;
1291         if (!is_offloading(newdev)) {
1292                 log(LOG_WARNING, "%s: Redirect to non-offload"
1293                     "device ignored.\n", __FUNCTION__);
1294                 return;
1295         }
1296         tdev = T3CDEV(olddev);
1297         PANIC_IF(!tdev);
1298         if (tdev != T3CDEV(newdev)) {
1299                 log(LOG_WARNING, "%s: Redirect to different "
1300                     "offload device ignored.\n", __FUNCTION__);
1301                 return;
1302         }
1303
1304         /* Add new L2T entry */
1305         e = t3_l2t_get(tdev, new, new->rt_ifp, sa);
1306         if (!e) {
1307                 log(LOG_ERR, "%s: couldn't allocate new l2t entry!\n",
1308                        __FUNCTION__);
1309                 return;
1310         }
1311
1312         /* Walk tid table and notify clients of dst change. */
1313         ti = &(T3C_DATA (tdev))->tid_maps;
1314         for (tid=0; tid < ti->ntids; tid++) {
1315                 te = lookup_tid(ti, tid);
1316                 PANIC_IF(!te);
1317                 if (te->ctx && te->client && te->client->redirect) {
1318                         update_tcb = te->client->redirect(te->ctx, old, new,
1319                                                           e);
1320                         if (update_tcb)  {
1321                                 l2t_hold(L2DATA(tdev), e);
1322                                 set_l2t_ix(tdev, tid, e);
1323                         }
1324                 }
1325         }
1326         l2t_release(L2DATA(tdev), e);
1327 }
1328
1329 /*
1330  * Initialize the CPL dispatch table.
1331  */
1332 static void
1333 init_cpl_handlers(void)
1334 {
1335         int i;
1336
1337         for (i = 0; i < 256; ++i)
1338                 tom_cpl_handlers[i] = do_bad_cpl;
1339
1340         t3_init_listen_cpl_handlers();
1341 }
1342
1343 static int
1344 t3_toe_attach(struct toedev *dev, const struct offload_id *entry)
1345 {
1346         struct tom_data *t = TOM_DATA(dev);
1347         struct t3cdev *cdev = t->cdev;
1348         struct ddp_params ddp;
1349         struct ofld_page_info rx_page_info;
1350         int err;
1351
1352         t3_init_tunables(t);
1353         mtx_init(&t->listen_lock, "tom data listeners", NULL, MTX_DEF);
1354         CTR2(KTR_TOM, "t3_toe_attach dev=%p entry=%p", dev, entry);
1355
1356         dev->tod_can_offload = can_offload;
1357         dev->tod_connect = t3_connect;
1358         dev->tod_ctl = tom_ctl;
1359 #if 0
1360         dev->tod_failover = t3_failover;
1361 #endif
1362         err = cdev->ctl(cdev, GET_DDP_PARAMS, &ddp);
1363         if (err)
1364                 return err;
1365
1366         err = cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info);
1367         if (err)
1368                 return err;
1369
1370         t->ddp_llimit = ddp.llimit;
1371         t->ddp_ulimit = ddp.ulimit;
1372         t->pdev = ddp.pdev;
1373         t->rx_page_size = rx_page_info.page_size;
1374         /* OK if this fails, we just can't do DDP */
1375         t->nppods = (ddp.ulimit + 1 - ddp.llimit) / PPOD_SIZE;
1376         t->ppod_map = malloc(t->nppods, M_DEVBUF, M_NOWAIT|M_ZERO);
1377
1378         mtx_init(&t->ppod_map_lock, "ppod map", NULL, MTX_DEF);
1379
1380
1381         t3_sysctl_register(cdev->adapter, &t->conf);
1382         return (0);
1383 }
1384
1385 static void
1386 cxgb_toe_listen_start(void *unused, struct tcpcb *tp)
1387 {
1388         struct socket *so = inp_inpcbtosocket(tp->t_inpcb);
1389         struct tom_data *p;
1390
1391         mtx_lock(&cxgb_list_lock);
1392         TAILQ_FOREACH(p, &cxgb_list, entry) {
1393                         t3_listen_start(&p->tdev, so, p->cdev);
1394         }
1395         mtx_unlock(&cxgb_list_lock);
1396 }
1397
1398 static void
1399 cxgb_toe_listen_stop(void *unused, struct tcpcb *tp)
1400 {
1401         struct socket *so = inp_inpcbtosocket(tp->t_inpcb);
1402         struct tom_data *p;
1403
1404         mtx_lock(&cxgb_list_lock);
1405         TAILQ_FOREACH(p, &cxgb_list, entry) {
1406                 if (tp->t_state == TCPS_LISTEN)
1407                         t3_listen_stop(&p->tdev, so, p->cdev);
1408         }
1409         mtx_unlock(&cxgb_list_lock);
1410 }
1411
1412 static void
1413 cxgb_toe_listen_start_handler(struct inpcb *inp, void *arg)
1414 {
1415         struct tcpcb *tp = intotcpcb(inp);
1416
1417         if (tp->t_state == TCPS_LISTEN)
1418                 cxgb_toe_listen_start(NULL, tp);
1419 }
1420
1421 static void
1422 cxgb_register_listeners(void)
1423 {
1424
1425         inp_apply_all(cxgb_toe_listen_start_handler, NULL);
1426 }
1427
1428 static int
1429 t3_tom_init(void)
1430 {
1431         init_cpl_handlers();
1432         if (t3_init_cpl_io() < 0) {
1433                 log(LOG_ERR,
1434                     "Unable to initialize cpl io ops\n");
1435                 return -1;
1436         }
1437         t3_init_socket_ops();
1438
1439          /* Register with the TOE device layer. */
1440
1441         if (register_tom(&t3_tom_info) != 0) {
1442                 log(LOG_ERR,
1443                     "Unable to register Chelsio T3 TCP offload module.\n");
1444                 return -1;
1445         }
1446
1447         rw_init(&adapter_list_lock, "ofld adap list");
1448         TAILQ_INIT(&adapter_list);
1449         EVENTHANDLER_REGISTER(route_arp_update_event, cxgb_arp_update_event,
1450             NULL, EVENTHANDLER_PRI_ANY);
1451         EVENTHANDLER_REGISTER(route_redirect_event, cxgb_redirect_event,
1452             NULL, EVENTHANDLER_PRI_ANY);
1453
1454         mtx_init(&cxgb_list_lock, "cxgb tom list", NULL, MTX_DEF);
1455         listen_tag = EVENTHANDLER_REGISTER(tcp_offload_listen_start,
1456             cxgb_toe_listen_start, NULL, EVENTHANDLER_PRI_ANY);
1457         listen_tag = EVENTHANDLER_REGISTER(tcp_offload_listen_stop,
1458             cxgb_toe_listen_stop, NULL, EVENTHANDLER_PRI_ANY);
1459         TAILQ_INIT(&cxgb_list);
1460
1461
1462
1463         t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl);
1464         t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl);
1465         t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr);
1466         t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl);
1467         t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl);
1468         t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl);
1469         t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl);
1470         t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl);
1471         t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl);
1472         t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl);
1473         t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl);
1474         t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl);
1475         t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl);
1476         t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss);
1477         t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
1478         t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term);
1479         t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl);
1480         t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl);
1481         t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl);
1482         t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl);
1483         t3_register_cpl_handler(CPL_GET_TCB_RPL, do_hwtid_rpl);
1484         t3_register_cpl_handler(CPL_SET_TCB_RPL, do_hwtid_rpl);
1485
1486         /* Register to offloading devices */
1487         cxgb_register_client(&t3c_tom_client);
1488
1489         return (0);
1490 }
1491
1492 static int
1493 t3_tom_load(module_t mod, int cmd, void *arg)
1494 {
1495         int err = 0;
1496
1497         switch (cmd) {
1498         case MOD_LOAD:
1499                 t3_tom_init();
1500                 break;
1501         case MOD_QUIESCE:
1502                 break;
1503         case MOD_UNLOAD:
1504                 printf("uhm, ... unloading isn't really supported for toe\n");
1505                 break;
1506         case MOD_SHUTDOWN:
1507                 break;
1508         default:
1509                 err = EOPNOTSUPP;
1510                 break;
1511         }
1512
1513         return (err);
1514 }
1515
1516 static moduledata_t mod_data= {
1517         "t3_tom",
1518         t3_tom_load,
1519         0
1520 };
1521 MODULE_VERSION(t3_tom, 1);
1522 MODULE_DEPEND(t3_tom, toecore, 1, 1, 1);
1523 MODULE_DEPEND(t3_tom, if_cxgb, 1, 1, 1);
1524 DECLARE_MODULE(t3_tom, mod_data, SI_SUB_EXEC, SI_ORDER_ANY);
1525