sys/dev/irdma/icrdma.c

   1 /*-
   2  * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
   3  *
   4  * Copyright (c) 2021 - 2022 Intel Corporation
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenFabrics.org BSD license below:
  11  *
  12  *   Redistribution and use in source and binary forms, with or
  13  *   without modification, are permitted provided that the following
  14  *   conditions are met:
  15  *
  16  *    - Redistributions of source code must retain the above
  17  *      copyright notice, this list of conditions and the following
  18  *      disclaimer.
  19  *
  20  *    - Redistributions in binary form must reproduce the above
  21  *      copyright notice, this list of conditions and the following
  22  *      disclaimer in the documentation and/or other materials
  23  *      provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34 /*$FreeBSD$*/
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/bus.h>
  39 #include <sys/kernel.h>
  40 #include <sys/module.h>
  41 #include <sys/sysctl.h>
  42 #include <machine/bus.h>
  43 #include <linux/device.h>
  44 #include <sys/rman.h>
  45
  46 #include "ice_rdma.h"
  47 #include "irdma_main.h"
  48 #include "icrdma_hw.h"
  49
  50 #include "irdma_if.h"
  51 #include "irdma_di_if.h"
  52
  53 /**
  54  *  Driver version
  55  */
  56 char irdma_driver_version[] = "1.1.11-k";
  57
  58 /**
  59  * irdma_init_tunable - prepare tunables
  60  * @rf: RDMA PCI function
  61  * @pf_id: id of the pf
  62  */
  63 static void
  64 irdma_init_tunable(struct irdma_pci_f *rf, uint8_t pf_id)
  65 {
  66         struct sysctl_oid_list *irdma_sysctl_oid_list;
  67         char pf_name[16];
  68
  69         snprintf(pf_name, 15, "irdma%d", pf_id);
  70         sysctl_ctx_init(&rf->tun_info.irdma_sysctl_ctx);
  71
  72         rf->tun_info.irdma_sysctl_tree = SYSCTL_ADD_NODE(&rf->tun_info.irdma_sysctl_ctx,
  73                                                          SYSCTL_STATIC_CHILDREN(_dev),
  74                                                          OID_AUTO, pf_name, CTLFLAG_RD,
  75                                                          NULL, "");
  76
  77         irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree);
  78
  79         /*
  80          * debug mask setting
  81          */
  82         SYSCTL_ADD_S32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
  83                        OID_AUTO, "debug", CTLFLAG_RWTUN, &rf->sc_dev.debug_mask,
  84                        0, "irdma debug");
  85
  86         /*
  87          * RoCEv2/iWARP setting RoCEv2 the default mode
  88          */
  89         rf->tun_info.roce_ena = 1;
  90         SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO,
  91                       "roce_enable", CTLFLAG_RDTUN, &rf->tun_info.roce_ena, 0,
  92                       "RoCEv2 mode enable");
  93
  94         rf->protocol_used = IRDMA_IWARP_PROTOCOL_ONLY;
  95         if (rf->tun_info.roce_ena == 1)
  96                 rf->protocol_used = IRDMA_ROCE_PROTOCOL_ONLY;
  97         else if (rf->tun_info.roce_ena != 0)
  98                 printf("%s:%d wrong roce_enable value (%d), using iWARP\n",
  99                        __func__, __LINE__, rf->tun_info.roce_ena);
 100         printf("%s:%d protocol: %s, roce_enable value: %d\n", __func__, __LINE__,
 101                (rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? "iWARP" : "RoCEv2",
 102                rf->tun_info.roce_ena);
 103
 104         snprintf(rf->tun_info.drv_ver, IRDMA_VER_LEN, "%s", irdma_driver_version);
 105         SYSCTL_ADD_STRING(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
 106                           OID_AUTO, "drv_ver", CTLFLAG_RDTUN, rf->tun_info.drv_ver,
 107                           IRDMA_VER_LEN, "driver version");
 108
 109         irdma_dcqcn_tunables_init(rf);
 110 }
 111
 112 /**
 113  * irdma_find_handler - obtain hdl object to identify pf
 114  * @p_dev: the peer interface structure
 115  */
 116 static struct irdma_handler *
 117 irdma_find_handler(struct ice_rdma_peer *p_dev)
 118 {
 119         struct irdma_handler *hdl;
 120         unsigned long flags;
 121
 122         spin_lock_irqsave(&irdma_handler_lock, flags);
 123         list_for_each_entry(hdl, &irdma_handlers, list) {
 124                 if (!hdl)
 125                         continue;
 126                 if (!hdl->iwdev->rf->peer_info)
 127                         continue;
 128                 if (hdl->iwdev->rf->peer_info->dev == p_dev->dev) {
 129                         spin_unlock_irqrestore(&irdma_handler_lock, flags);
 130                         return hdl;
 131                 }
 132         }
 133         spin_unlock_irqrestore(&irdma_handler_lock, flags);
 134
 135         return NULL;
 136 }
 137
 138 /**
 139  * peer_to_iwdev - return iwdev based on peer
 140  * @peer: the peer interface structure
 141  */
 142 static struct irdma_device *
 143 peer_to_iwdev(struct ice_rdma_peer *peer)
 144 {
 145         struct irdma_handler *hdl;
 146
 147         hdl = irdma_find_handler(peer);
 148         if (!hdl) {
 149                 printf("%s:%d rdma handler not found\n", __func__, __LINE__);
 150                 return NULL;
 151         }
 152
 153         return hdl->iwdev;
 154 }
 155
 156 /**
 157  * irdma_get_qos_info - save qos info from parameters to internal struct
 158  * @l2params: destination, qos, tc, mtu info structure
 159  * @qos_info: source, DCB settings structure
 160  */
 161 static void
 162 irdma_get_qos_info(struct irdma_l2params *l2params, struct ice_qos_params *qos_info)
 163 {
 164         int i;
 165
 166         l2params->num_tc = qos_info->num_tc;
 167         l2params->num_apps = qos_info->num_apps;
 168         l2params->vsi_prio_type = qos_info->vsi_priority_type;
 169         l2params->vsi_rel_bw = qos_info->vsi_relative_bw;
 170         for (i = 0; i < l2params->num_tc; i++) {
 171                 l2params->tc_info[i].egress_virt_up =
 172                     qos_info->tc_info[i].egress_virt_up;
 173                 l2params->tc_info[i].ingress_virt_up =
 174                     qos_info->tc_info[i].ingress_virt_up;
 175                 l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type;
 176                 l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw;
 177                 l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx;
 178         }
 179         for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
 180                 l2params->up2tc[i] = qos_info->up2tc[i];
 181
 182         if (qos_info->pfc_mode == IRDMA_QOS_MODE_DSCP) {
 183                 l2params->dscp_mode = true;
 184                 memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map));
 185         }
 186         printf("%s:%d: l2params settings:\n num_tc %d,\n num_apps %d,\n",
 187                __func__, __LINE__, l2params->num_tc, l2params->num_apps);
 188         printf(" vsi_prio_type %d,\n vsi_rel_bw %d,\n egress_virt_up:",
 189                l2params->vsi_prio_type, l2params->vsi_rel_bw);
 190         for (i = 0; i < l2params->num_tc; i++)
 191                 printf(" %d", l2params->tc_info[i].egress_virt_up);
 192         printf("\n ingress_virt_up:");
 193         for (i = 0; i < l2params->num_tc; i++)
 194                 printf(" %d", l2params->tc_info[i].ingress_virt_up);
 195         printf("\n prio_type:");
 196         for (i = 0; i < l2params->num_tc; i++)
 197                 printf(" %d", l2params->tc_info[i].prio_type);
 198         printf("\n rel_bw:");
 199         for (i = 0; i < l2params->num_tc; i++)
 200                 printf(" %d", l2params->tc_info[i].rel_bw);
 201         printf("\n tc_ctx:");
 202         for (i = 0; i < l2params->num_tc; i++)
 203                 printf(" %lu", l2params->tc_info[i].tc_ctx);
 204         printf("\n up2tc:");
 205         for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
 206                 printf(" %d", l2params->up2tc[i]);
 207         printf(" dscp_mode: %d,\n", l2params->dscp_mode);
 208         for (i = 0; i < IRDMA_DSCP_NUM_VAL; i++)
 209                 printf(" %d", l2params->dscp_map[i]);
 210         printf("\n");
 211
 212         dump_struct(l2params, sizeof(*l2params), "l2params");
 213 }
 214
 215 /**
 216  * irdma_log_invalid_mtu - check mtu setting validity
 217  * @mtu: mtu value
 218  * @dev: hardware control device structure
 219  */
 220 static void
 221 irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev)
 222 {
 223         if (mtu < IRDMA_MIN_MTU_IPV4)
 224                 irdma_dev_warn(to_ibdev(dev),
 225                                "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n",
 226                                mtu);
 227         else if (mtu < IRDMA_MIN_MTU_IPV6)
 228                 irdma_dev_warn(to_ibdev(dev),
 229                                "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n",
 230                                mtu);
 231 }
 232
 233 /**
 234  * irdma_event_handler - handling events from lan driver
 235  * @peer: the peer interface structure
 236  * @event: event info structure
 237  */
 238 static void
 239 irdma_event_handler(struct ice_rdma_peer *peer, struct ice_rdma_event *event)
 240 {
 241         struct irdma_device *iwdev;
 242         struct irdma_l2params l2params = {};
 243
 244         printf("%s:%d event_handler %s (%x) on pf %d (%d)\n", __func__, __LINE__,
 245                (event->type == 1) ? "LINK CHANGE" :
 246                (event->type == 2) ? "MTU CHANGE" :
 247                (event->type == 3) ? "TC CHANGE" : "UNKNOWN",
 248                event->type, peer->pf_id, if_getdunit(peer->ifp));
 249         iwdev = peer_to_iwdev(peer);
 250         if (!iwdev) {
 251                 printf("%s:%d rdma device not found\n", __func__, __LINE__);
 252                 return;
 253         }
 254
 255         switch (event->type) {
 256         case ICE_RDMA_EVENT_LINK_CHANGE:
 257                 printf("%s:%d PF: %x (%x), state: %d, speed: %lu\n", __func__, __LINE__,
 258                        peer->pf_id, if_getdunit(peer->ifp), event->linkstate,
 259                        event->baudrate);
 260                 break;
 261         case ICE_RDMA_EVENT_MTU_CHANGE:
 262                 if (iwdev->vsi.mtu != event->mtu) {
 263                         l2params.mtu = event->mtu;
 264                         l2params.mtu_changed = true;
 265                         irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev);
 266                         irdma_change_l2params(&iwdev->vsi, &l2params);
 267                 }
 268                 break;
 269         case ICE_RDMA_EVENT_TC_CHANGE:
 270                 /*
 271                  * 1. check if it is pre or post 2. check if it is currently being done
 272                  */
 273                 if (event->prep == iwdev->vsi.tc_change_pending) {
 274                         printf("%s:%d can't process %s TC change if TC change is %spending\n",
 275                                __func__, __LINE__,
 276                                event->prep ? "pre" : "post",
 277                                event->prep ? " " : "not ");
 278                         goto done;
 279                 }
 280                 if (event->prep) {
 281                         iwdev->vsi.tc_change_pending = true;
 282                         irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND);
 283                         wait_event_timeout(iwdev->suspend_wq,
 284                                            !atomic_read(&iwdev->vsi.qp_suspend_reqs),
 285                                            IRDMA_EVENT_TIMEOUT_MS * 10);
 286                         irdma_ws_reset(&iwdev->vsi);
 287                         printf("%s:%d TC change preparation done\n", __func__, __LINE__);
 288                 } else {
 289                         l2params.tc_changed = true;
 290                         irdma_get_qos_info(&l2params, &event->port_qos);
 291                         if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
 292                                 iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
 293
 294                         irdma_check_fc_for_tc_update(&iwdev->vsi, &l2params);
 295                         irdma_change_l2params(&iwdev->vsi, &l2params);
 296                         printf("%s:%d TC change done\n", __func__, __LINE__);
 297                 }
 298                 break;
 299         case ICE_RDMA_EVENT_CRIT_ERR:
 300                 printf("%s:%d event type received: %d\n", __func__, __LINE__, event->type);
 301                 break;
 302         default:
 303                 printf("%s:%d event type unsupported: %d\n", __func__, __LINE__, event->type);
 304         }
 305 done:
 306         return;
 307 }
 308
 309 /**
 310  * irdma_link_change - Callback for link state change
 311  * @peer: the peer interface structure
 312  * @linkstate: state of the link
 313  * @baudrate: speed of the link
 314  */
 315 static void
 316 irdma_link_change(struct ice_rdma_peer *peer, int linkstate, uint64_t baudrate)
 317 {
 318         printf("%s:%d PF: %x (%x), state: %d, speed: %lu\n", __func__, __LINE__,
 319                peer->pf_id, if_getdunit(peer->ifp), linkstate, baudrate);
 320 }
 321
 322 /**
 323  * irdma_finalize_task - Finish open or close phase in a separate thread
 324  * @context: instance holding peer and iwdev information
 325  *
 326  * Triggered from irdma_open or irdma_close to perform rt_init_hw or
 327  * rt_deinit_hw respectively. Does registration and unregistration of
 328  * the device.
 329  */
 330 static void
 331 irdma_finalize_task(void *context, int pending)
 332 {
 333         struct irdma_task_arg *task_arg = (struct irdma_task_arg *)context;
 334         struct irdma_device *iwdev = task_arg->iwdev;
 335         struct irdma_pci_f *rf = iwdev->rf;
 336         struct ice_rdma_peer *peer = task_arg->peer;
 337         struct irdma_l2params l2params = {{{0}}};
 338         struct ice_rdma_request req = {0};
 339         int status = 0;
 340
 341         if (iwdev->iw_status) {
 342                 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
 343                             "Starting deferred closing %d (%d)\n",
 344                             rf->peer_info->pf_id, if_getdunit(peer->ifp));
 345                 irdma_dereg_ipaddr_event_cb(rf);
 346                 irdma_ib_unregister_device(iwdev);
 347                 req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE;
 348                 req.enable_filter = false;
 349                 IRDMA_DI_REQ_HANDLER(peer, &req);
 350                 irdma_cleanup_dead_qps(&iwdev->vsi);
 351                 irdma_rt_deinit_hw(iwdev);
 352         } else {
 353                 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
 354                             "Starting deferred opening %d (%d)\n",
 355                             rf->peer_info->pf_id, if_getdunit(peer->ifp));
 356                 irdma_get_qos_info(&l2params, &peer->initial_qos_info);
 357                 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
 358                         iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
 359
 360                 l2params.mtu = peer->mtu;
 361                 status = irdma_rt_init_hw(iwdev, &l2params);
 362                 if (status) {
 363                         irdma_pr_err("RT init failed %d\n", status);
 364                         ib_dealloc_device(&iwdev->ibdev);
 365                         return;
 366                 }
 367                 status = irdma_ib_register_device(iwdev);
 368                 if (status) {
 369                         irdma_pr_err("Registration failed %d\n", status);
 370                         irdma_rt_deinit_hw(iwdev);
 371                         ib_dealloc_device(&iwdev->ibdev);
 372                 }
 373                 req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE;
 374                 req.enable_filter = true;
 375                 IRDMA_DI_REQ_HANDLER(peer, &req);
 376                 irdma_reg_ipaddr_event_cb(rf);
 377                 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
 378                             "Deferred opening finished %d (%d)\n",
 379                             rf->peer_info->pf_id, if_getdunit(peer->ifp));
 380         }
 381 }
 382
 383 /**
 384  * irdma_open - Callback for operation open for RDMA device
 385  * @peer: the new peer interface structure
 386  *
 387  * Callback implementing the RDMA_OPEN function. Called by the ice driver to
 388  * notify the RDMA client driver that a new device has been initialized.
 389  */
 390 static int
 391 irdma_open(struct ice_rdma_peer *peer)
 392 {
 393         struct ice_rdma_event event = {0};
 394
 395         event.type = ICE_RDMA_EVENT_MTU_CHANGE;
 396         event.mtu = peer->mtu;
 397
 398         irdma_event_handler(peer, &event);
 399
 400         return 0;
 401 }
 402
 403 /**
 404  * irdma_close - Callback to notify that a peer device is down
 405  * @peer: the RDMA peer device being stopped
 406  *
 407  * Callback implementing the RDMA_CLOSE function. Called by the ice driver to
 408  * notify the RDMA client driver that a peer device is being stopped.
 409  */
 410 static int
 411 irdma_close(struct ice_rdma_peer *peer)
 412 {
 413         /*
 414          * This is called when ifconfig down. Keeping it for compatibility with ice. This event might be usefull for
 415          * future.
 416          */
 417         return 0;
 418 }
 419
 420 /**
 421  * irdma_alloc_pcidev - allocate memory for pcidev and populate data
 422  * @peer: the new peer interface structure
 423  * @rf: RDMA PCI function
 424  */
 425 static int
 426 irdma_alloc_pcidev(struct ice_rdma_peer *peer, struct irdma_pci_f *rf)
 427 {
 428         rf->pcidev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL);
 429         if (!rf->pcidev) {
 430                 return -ENOMEM;
 431         }
 432         if (linux_pci_attach_device(rf->dev_ctx.dev, NULL, NULL, rf->pcidev))
 433                 return -ENOMEM;
 434
 435         return 0;
 436 }
 437
 438 /**
 439  * irdma_dealloc_pcidev - deallocate memory for pcidev
 440  * @rf: RDMA PCI function
 441  */
 442 static void
 443 irdma_dealloc_pcidev(struct irdma_pci_f *rf)
 444 {
 445         linux_pci_detach_device(rf->pcidev);
 446         kfree(rf->pcidev);
 447 }
 448
 449 /**
 450  * irdma_fill_device_info - assign initial values to rf variables
 451  * @iwdev: irdma device
 452  * @peer: the peer interface structure
 453  */
 454 static void
 455 irdma_fill_device_info(struct irdma_device *iwdev,
 456                        struct ice_rdma_peer *peer)
 457 {
 458         struct irdma_pci_f *rf = iwdev->rf;
 459
 460         rf->peer_info = peer;
 461         rf->gen_ops.register_qset = irdma_register_qset;
 462         rf->gen_ops.unregister_qset = irdma_unregister_qset;
 463
 464         rf->rdma_ver = IRDMA_GEN_2;
 465         rf->sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_2;
 466         rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT;
 467         rf->rst_to = IRDMA_RST_TIMEOUT_HZ;
 468         rf->check_fc = irdma_check_fc_for_qp;
 469         rf->gen_ops.request_reset = irdma_request_reset;
 470         irdma_set_rf_user_cfg_params(rf);
 471
 472         rf->default_vsi.vsi_idx = peer->pf_vsi_num;
 473         rf->dev_ctx.dev = peer->dev;
 474         rf->dev_ctx.mem_bus_space_tag = rman_get_bustag(peer->pci_mem);
 475         rf->dev_ctx.mem_bus_space_handle = rman_get_bushandle(peer->pci_mem);
 476         rf->dev_ctx.mem_bus_space_size = rman_get_size(peer->pci_mem);
 477
 478         rf->hw.dev_context = &rf->dev_ctx;
 479         rf->hw.hw_addr = (u8 *)rman_get_virtual(peer->pci_mem);
 480         rf->msix_count = peer->msix.count;
 481         rf->msix_info.entry = peer->msix.base;
 482         rf->msix_info.vector = peer->msix.count;
 483         printf("%s:%d msix_info: %d %d %d\n", __func__, __LINE__,
 484                rf->msix_count, rf->msix_info.entry, rf->msix_info.vector);
 485
 486         rf->iwdev = iwdev;
 487         iwdev->netdev = peer->ifp;
 488         iwdev->init_state = INITIAL_STATE;
 489         iwdev->vsi_num = peer->pf_vsi_num;
 490         iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;
 491         iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
 492         iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT;
 493         iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT;
 494         iwdev->roce_rtomin = 5;
 495
 496         if (rf->protocol_used == IRDMA_ROCE_PROTOCOL_ONLY) {
 497                 iwdev->roce_mode = true;
 498         }
 499 }
 500
 501 /**
 502  * irdma_probe - Callback to probe a new RDMA peer device
 503  * @peer: the new peer interface structure
 504  *
 505  * Callback implementing the RDMA_PROBE function. Called by the ice driver to
 506  * notify the RDMA client driver that a new device has been created
 507  */
 508 static int
 509 irdma_probe(struct ice_rdma_peer *peer)
 510 {
 511         struct irdma_device *iwdev;
 512         struct irdma_pci_f *rf;
 513         struct irdma_handler *hdl;
 514         int err = 0;
 515
 516         irdma_pr_info("probe: irdma-%s peer=%p, peer->pf_id=%d, peer->ifp=%p, peer->ifp->if_dunit=%d, peer->pci_mem->r_bustag=%p\n",
 517                       irdma_driver_version, peer, peer->pf_id, peer->ifp,
 518                       if_getdunit(peer->ifp), (void *)(uintptr_t)peer->pci_mem->r_bustag);
 519
 520         hdl = irdma_find_handler(peer);
 521         if (hdl)
 522                 return -EBUSY;
 523
 524         hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
 525         if (!hdl)
 526                 return -ENOMEM;
 527
 528         iwdev = (struct irdma_device *)ib_alloc_device(sizeof(*iwdev));
 529         if (!iwdev) {
 530                 kfree(hdl);
 531                 return -ENOMEM;
 532         }
 533
 534         iwdev->rf = kzalloc(sizeof(*rf), GFP_KERNEL);
 535         if (!iwdev->rf) {
 536                 ib_dealloc_device(&iwdev->ibdev);
 537                 kfree(hdl);
 538                 return -ENOMEM;
 539         }
 540         hdl->iwdev = iwdev;
 541         iwdev->hdl = hdl;
 542
 543         irdma_init_tunable(iwdev->rf, if_getdunit(peer->ifp));
 544         irdma_fill_device_info(iwdev, peer);
 545         rf = iwdev->rf;
 546
 547         if (irdma_alloc_pcidev(peer, rf))
 548                 goto err_pcidev;
 549
 550         irdma_add_handler(hdl);
 551
 552         if (irdma_ctrl_init_hw(rf)) {
 553                 err = -EIO;
 554                 goto err_ctrl_init;
 555         }
 556
 557         rf->dev_ctx.task_arg.peer = peer;
 558         rf->dev_ctx.task_arg.iwdev = iwdev;
 559         rf->dev_ctx.task_arg.peer = peer;
 560
 561         TASK_INIT(&hdl->deferred_task, 0, irdma_finalize_task, &rf->dev_ctx.task_arg);
 562         hdl->deferred_tq = taskqueue_create_fast("irdma_defer",
 563                                                  M_NOWAIT, taskqueue_thread_enqueue,
 564                                                  &hdl->deferred_tq);
 565         taskqueue_start_threads(&hdl->deferred_tq, 1, PI_NET, "irdma_defer_t");
 566
 567         taskqueue_enqueue(hdl->deferred_tq, &hdl->deferred_task);
 568
 569         return 0;
 570
 571 err_ctrl_init:
 572         irdma_del_handler(hdl);
 573         irdma_dealloc_pcidev(rf);
 574 err_pcidev:
 575         kfree(iwdev->rf);
 576         ib_dealloc_device(&iwdev->ibdev);
 577         kfree(hdl);
 578
 579         return err;
 580 }
 581
 582 /**
 583  * irdma_remove - Callback to remove an RDMA peer device
 584  * @peer: the new peer interface structure
 585  *
 586  * Callback implementing the RDMA_REMOVE function. Called by the ice driver to
 587  * notify the RDMA client driver that the device wille be delated
 588  */
 589 static int
 590 irdma_remove(struct ice_rdma_peer *peer)
 591 {
 592         struct irdma_handler *hdl;
 593         struct irdma_device *iwdev;
 594
 595         irdma_debug((struct irdma_sc_dev *)NULL, IRDMA_DEBUG_INIT,
 596                     "removing %s irdma%d\n", __func__, if_getdunit(peer->ifp));
 597
 598         hdl = irdma_find_handler(peer);
 599         if (!hdl)
 600                 return 0;
 601
 602         iwdev = hdl->iwdev;
 603
 604         if (iwdev->vsi.tc_change_pending) {
 605                 iwdev->vsi.tc_change_pending = false;
 606                 irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_RESUME);
 607         }
 608
 609         taskqueue_enqueue(hdl->deferred_tq, &hdl->deferred_task);
 610
 611         taskqueue_drain(hdl->deferred_tq, &hdl->deferred_task);
 612         taskqueue_free(hdl->deferred_tq);
 613         hdl->iwdev->rf->dev_ctx.task_arg.iwdev = NULL;
 614         hdl->iwdev->rf->dev_ctx.task_arg.peer = NULL;
 615
 616         sysctl_ctx_free(&iwdev->rf->tun_info.irdma_sysctl_ctx);
 617         hdl->iwdev->rf->tun_info.irdma_sysctl_tree = NULL;
 618
 619         irdma_ctrl_deinit_hw(iwdev->rf);
 620
 621         irdma_dealloc_pcidev(iwdev->rf);
 622
 623         irdma_del_handler(iwdev->hdl);
 624         kfree(iwdev->hdl);
 625         kfree(iwdev->rf);
 626         ib_dealloc_device(&iwdev->ibdev);
 627         irdma_pr_info("IRDMA hardware deinitialization complete irdma%d\n",
 628                       if_getdunit(peer->ifp));
 629
 630         return 0;
 631 }
 632
 633 /**
 634  * irdma_prep_for_unregister - ensure the driver is ready to unregister
 635  */
 636 static void
 637 irdma_prep_for_unregister(void)
 638 {
 639         struct irdma_handler *hdl;
 640         unsigned long flags;
 641         bool hdl_valid;
 642
 643         do {
 644                 hdl_valid = false;
 645                 spin_lock_irqsave(&irdma_handler_lock, flags);
 646                 list_for_each_entry(hdl, &irdma_handlers, list) {
 647                         if (!hdl)
 648                                 continue;
 649                         if (!hdl->iwdev->rf->peer_info)
 650                                 continue;
 651                         hdl_valid = true;
 652                         break;
 653                 }
 654                 spin_unlock_irqrestore(&irdma_handler_lock, flags);
 655                 if (!hdl || !hdl_valid)
 656                         break;
 657                 IRDMA_CLOSE(hdl->iwdev->rf->peer_info);
 658                 IRDMA_REMOVE(hdl->iwdev->rf->peer_info);
 659         } while (1);
 660 }
 661
 662 static kobj_method_t irdma_methods[] = {
 663         KOBJMETHOD(irdma_probe, irdma_probe),
 664             KOBJMETHOD(irdma_open, irdma_open),
 665             KOBJMETHOD(irdma_close, irdma_close),
 666             KOBJMETHOD(irdma_remove, irdma_remove),
 667             KOBJMETHOD(irdma_link_change, irdma_link_change),
 668             KOBJMETHOD(irdma_event_handler, irdma_event_handler),
 669             KOBJMETHOD_END
 670 };
 671
 672 /* declare irdma_class which extends the ice_rdma_di class */
 673 DEFINE_CLASS_1(irdma, irdma_class, irdma_methods, sizeof(struct ice_rdma_peer), ice_rdma_di_class);
 674
 675 static struct ice_rdma_info irdma_info = {
 676         .major_version = ICE_RDMA_MAJOR_VERSION,
 677         .minor_version = ICE_RDMA_MINOR_VERSION,
 678         .patch_version = ICE_RDMA_PATCH_VERSION,
 679         .rdma_class = &irdma_class,
 680 };
 681
 682 /**
 683  * irdma_module_event_handler - Module event handler callback
 684  * @mod: unused mod argument
 685  * @what: the module event to handle
 686  * @arg: unused module event argument
 687  *
 688  * Callback used by the FreeBSD module stack to notify the driver of module
 689  * events. Used to implement custom handling for certain module events such as
 690  * load and unload.
 691  */
 692 static int
 693 irdma_module_event_handler(module_t __unused mod, int what, void __unused * arg)
 694 {
 695         switch (what) {
 696         case MOD_LOAD:
 697                 printf("Loading irdma module\n");
 698                 return ice_rdma_register(&irdma_info);
 699         case MOD_UNLOAD:
 700                 printf("Unloading irdma module\n");
 701                 irdma_prep_for_unregister();
 702                 ice_rdma_unregister();
 703                 return (0);
 704         default:
 705                 return (EOPNOTSUPP);
 706         }
 707
 708         return (0);
 709 }
 710
 711 static moduledata_t irdma_moduledata = {
 712         "irdma",
 713             irdma_module_event_handler,
 714             NULL
 715 };
 716
 717 DECLARE_MODULE(irdma, irdma_moduledata, SI_SUB_LAST, SI_ORDER_ANY);
 718 MODULE_VERSION(irdma, 1);
 719 MODULE_DEPEND(irdma, ice, 1, 1, 1);
 720 MODULE_DEPEND(irdma, ibcore, 1, 1, 1);