sys/dev/irdma/icrdma.c

   1 /*-
   2  * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
   3  *
   4  * Copyright (c) 2021 - 2023 Intel Corporation
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenFabrics.org BSD license below:
  11  *
  12  *   Redistribution and use in source and binary forms, with or
  13  *   without modification, are permitted provided that the following
  14  *   conditions are met:
  15  *
  16  *    - Redistributions of source code must retain the above
  17  *      copyright notice, this list of conditions and the following
  18  *      disclaimer.
  19  *
  20  *    - Redistributions in binary form must reproduce the above
  21  *      copyright notice, this list of conditions and the following
  22  *      disclaimer in the documentation and/or other materials
  23  *      provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/bus.h>
  38 #include <sys/kernel.h>
  39 #include <sys/module.h>
  40 #include <sys/sysctl.h>
  41 #include <machine/bus.h>
  42 #include <linux/device.h>
  43 #include <sys/rman.h>
  44
  45 #include "ice_rdma.h"
  46 #include "irdma_main.h"
  47 #include "icrdma_hw.h"
  48
  49 #include "irdma_if.h"
  50 #include "irdma_di_if.h"
  51
  52 /**
  53  *  Driver version
  54  */
  55 char irdma_driver_version[] = "1.2.17-k";
  56
  57 /**
  58  * irdma_init_tunable - prepare tunables
  59  * @rf: RDMA PCI function
  60  * @pf_id: id of the pf
  61  */
  62 static void
  63 irdma_init_tunable(struct irdma_pci_f *rf, uint8_t pf_id)
  64 {
  65         struct sysctl_oid_list *irdma_oid_list;
  66         struct irdma_tunable_info *t_info = &rf->tun_info;
  67         char pf_name[16];
  68
  69         snprintf(pf_name, 15, "irdma%d", pf_id);
  70         sysctl_ctx_init(&t_info->irdma_sysctl_ctx);
  71
  72         t_info->irdma_sysctl_tree = SYSCTL_ADD_NODE(&t_info->irdma_sysctl_ctx,
  73                                                     SYSCTL_STATIC_CHILDREN(_dev),
  74                                                     OID_AUTO, pf_name,
  75                                                     CTLFLAG_RD, NULL, "");
  76
  77         irdma_oid_list = SYSCTL_CHILDREN(t_info->irdma_sysctl_tree);
  78
  79         t_info->sws_sysctl_tree = SYSCTL_ADD_NODE(&t_info->irdma_sysctl_ctx,
  80                                                   irdma_oid_list, OID_AUTO,
  81                                                   "sw_stats", CTLFLAG_RD,
  82                                                   NULL, "");
  83         /*
  84          * debug mask setting
  85          */
  86         SYSCTL_ADD_S32(&t_info->irdma_sysctl_ctx, irdma_oid_list,
  87                        OID_AUTO, "debug", CTLFLAG_RWTUN, &rf->sc_dev.debug_mask,
  88                        0, "irdma debug");
  89
  90         /*
  91          * RoCEv2/iWARP setting RoCEv2 the default mode
  92          */
  93         t_info->roce_ena = 1;
  94         SYSCTL_ADD_U8(&t_info->irdma_sysctl_ctx, irdma_oid_list, OID_AUTO,
  95                       "roce_enable", CTLFLAG_RDTUN, &t_info->roce_ena, 0,
  96                       "RoCEv2 mode enable");
  97
  98         rf->protocol_used = IRDMA_IWARP_PROTOCOL_ONLY;
  99         if (t_info->roce_ena == 1)
 100                 rf->protocol_used = IRDMA_ROCE_PROTOCOL_ONLY;
 101         else if (t_info->roce_ena != 0)
 102                 printf("%s:%d wrong roce_enable value (%d), using iWARP\n",
 103                        __func__, __LINE__, t_info->roce_ena);
 104         printf("%s:%d protocol: %s, roce_enable value: %d\n", __func__, __LINE__,
 105                (rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? "iWARP" : "RoCEv2",
 106                t_info->roce_ena);
 107
 108         snprintf(t_info->drv_ver, IRDMA_VER_LEN, "%s", irdma_driver_version);
 109         SYSCTL_ADD_STRING(&t_info->irdma_sysctl_ctx, irdma_oid_list,
 110                           OID_AUTO, "drv_ver", CTLFLAG_RDTUN, t_info->drv_ver,
 111                           IRDMA_VER_LEN, "driver version");
 112
 113         irdma_dcqcn_tunables_init(rf);
 114         irdma_sysctl_settings(rf);
 115 }
 116
 117 /**
 118  * irdma_find_handler - obtain hdl object to identify pf
 119  * @p_dev: the peer interface structure
 120  */
 121 static struct irdma_handler *
 122 irdma_find_handler(struct ice_rdma_peer *p_dev)
 123 {
 124         struct irdma_handler *hdl;
 125         unsigned long flags;
 126
 127         spin_lock_irqsave(&irdma_handler_lock, flags);
 128         list_for_each_entry(hdl, &irdma_handlers, list) {
 129                 if (!hdl->iwdev->rf->peer_info)
 130                         continue;
 131                 if (hdl->iwdev->rf->peer_info->dev == p_dev->dev) {
 132                         spin_unlock_irqrestore(&irdma_handler_lock, flags);
 133                         return hdl;
 134                 }
 135         }
 136         spin_unlock_irqrestore(&irdma_handler_lock, flags);
 137
 138         return NULL;
 139 }
 140
 141 /**
 142  * peer_to_iwdev - return iwdev based on peer
 143  * @peer: the peer interface structure
 144  */
 145 static struct irdma_device *
 146 peer_to_iwdev(struct ice_rdma_peer *peer)
 147 {
 148         struct irdma_handler *hdl;
 149
 150         hdl = irdma_find_handler(peer);
 151         if (!hdl) {
 152                 printf("%s:%d rdma handler not found\n", __func__, __LINE__);
 153                 return NULL;
 154         }
 155
 156         return hdl->iwdev;
 157 }
 158
 159 /**
 160  * irdma_get_qos_info - save qos info from parameters to internal struct
 161  * @l2params: destination, qos, tc, mtu info structure
 162  * @qos_info: source, DCB settings structure
 163  */
 164 static void
 165 irdma_get_qos_info(struct irdma_pci_f *rf, struct irdma_l2params *l2params,
 166                    struct ice_qos_params *qos_info)
 167 {
 168         int i;
 169         char txt[7][128] = {"", "", "", "", "", "", ""};
 170         u8 len;
 171
 172         l2params->num_tc = qos_info->num_tc;
 173         l2params->num_apps = qos_info->num_apps;
 174         l2params->vsi_prio_type = qos_info->vsi_priority_type;
 175         l2params->vsi_rel_bw = qos_info->vsi_relative_bw;
 176         for (i = 0; i < l2params->num_tc; i++) {
 177                 l2params->tc_info[i].egress_virt_up =
 178                     qos_info->tc_info[i].egress_virt_up;
 179                 l2params->tc_info[i].ingress_virt_up =
 180                     qos_info->tc_info[i].ingress_virt_up;
 181                 l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type;
 182                 l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw;
 183                 l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx;
 184         }
 185         for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
 186                 l2params->up2tc[i] = qos_info->up2tc[i];
 187
 188         if (qos_info->pfc_mode == IRDMA_QOS_MODE_DSCP) {
 189                 l2params->dscp_mode = true;
 190                 memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map));
 191         }
 192         if (!(rf->sc_dev.debug_mask & IRDMA_DEBUG_DCB))
 193                 return;
 194         for (i = 0; i < l2params->num_tc; i++) {
 195                 len = strlen(txt[0]);
 196                 snprintf(txt[0] + len, sizeof(txt[0]) - 5, " %d",
 197                          l2params->tc_info[i].egress_virt_up);
 198                 len = strlen(txt[1]);
 199                 snprintf(txt[1] + len, sizeof(txt[1]) - 5, " %d",
 200                          l2params->tc_info[i].ingress_virt_up);
 201                 len = strlen(txt[2]);
 202                 snprintf(txt[2] + len, sizeof(txt[2]) - 5, " %d",
 203                          l2params->tc_info[i].prio_type);
 204                 len = strlen(txt[3]);
 205                 snprintf(txt[3] + len, sizeof(txt[3]) - 5, " %d",
 206                          l2params->tc_info[i].rel_bw);
 207                 len = strlen(txt[4]);
 208                 snprintf(txt[4] + len, sizeof(txt[4]) - 5, " %lu",
 209                          l2params->tc_info[i].tc_ctx);
 210         }
 211         len = strlen(txt[5]);
 212         for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
 213                 len += snprintf(txt[5] + len, sizeof(txt[5]) - 5, " %d",
 214                                 l2params->up2tc[i]);
 215         len = strlen(txt[6]);
 216         for (i = 0; i < IRDMA_DSCP_NUM_VAL; i++)
 217                 len += snprintf(txt[6] + len, sizeof(txt[6]) - 5, " %d",
 218                                 l2params->dscp_map[i]);
 219         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "num_tc:          %d\n", l2params->num_tc);
 220         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "num_apps:        %d\n", l2params->num_apps);
 221         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "vsi_prio_type:   %d\n", l2params->vsi_prio_type);
 222         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "vsi_rel_bw:      %d\n", l2params->vsi_rel_bw);
 223         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "egress_virt_up: %s\n", txt[0]);
 224         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "ingress_virt_up:%s\n", txt[1]);
 225         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "prio_type: %s\n", txt[2]);
 226         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "rel_bw:    %s\n", txt[3]);
 227         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "tc_ctx:    %s\n", txt[4]);
 228         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "up2tc:     %s\n", txt[5]);
 229         irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "dscp_mode: %s\n", txt[6]);
 230
 231         irdma_debug_buf(&rf->sc_dev, IRDMA_DEBUG_DCB, "l2params", l2params, sizeof(*l2params));
 232 }
 233
 234 /**
 235  * irdma_log_invalid_mtu - check mtu setting validity
 236  * @mtu: mtu value
 237  * @dev: hardware control device structure
 238  */
 239 static void
 240 irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev)
 241 {
 242         if (mtu < IRDMA_MIN_MTU_IPV4)
 243                 irdma_dev_warn(to_ibdev(dev),
 244                                "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n",
 245                                mtu);
 246         else if (mtu < IRDMA_MIN_MTU_IPV6)
 247                 irdma_dev_warn(to_ibdev(dev),
 248                                "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n",
 249                                mtu);
 250 }
 251
 252 /**
 253  * irdma_get_event_name - convert type enum to string
 254  * @type: event type enum
 255  */
 256 static const char *
 257 irdma_get_event_name(enum ice_rdma_event_type type)
 258 {
 259         switch (type) {
 260         case ICE_RDMA_EVENT_LINK_CHANGE:
 261                 return "LINK CHANGE";
 262         case ICE_RDMA_EVENT_MTU_CHANGE:
 263                 return "MTU CHANGE";
 264         case ICE_RDMA_EVENT_TC_CHANGE:
 265                 return "TC CHANGE";
 266         case ICE_RDMA_EVENT_API_CHANGE:
 267                 return "API CHANGE";
 268         case ICE_RDMA_EVENT_CRIT_ERR:
 269                 return "CRITICAL ERROR";
 270         case ICE_RDMA_EVENT_RESET:
 271                 return "RESET";
 272         case ICE_RDMA_EVENT_QSET_REGISTER:
 273                 return "QSET REGISTER";
 274         case ICE_RDMA_EVENT_VSI_FILTER_UPDATE:
 275                 return "VSI FILTER UPDATE";
 276         default:
 277                 return "UNKNOWN";
 278         }
 279 }
 280
 281 /**
 282  * irdma_event_handler - handling events from lan driver
 283  * @peer: the peer interface structure
 284  * @event: event info structure
 285  */
 286 static void
 287 irdma_event_handler(struct ice_rdma_peer *peer, struct ice_rdma_event *event)
 288 {
 289         struct irdma_device *iwdev;
 290         struct irdma_l2params l2params = {};
 291
 292         printf("%s:%d event_handler %s (%x) on pf %d (%d)\n", __func__, __LINE__,
 293                irdma_get_event_name(event->type),
 294                event->type, peer->pf_id, if_getdunit(peer->ifp));
 295         iwdev = peer_to_iwdev(peer);
 296         if (!iwdev) {
 297                 printf("%s:%d rdma device not found\n", __func__, __LINE__);
 298                 return;
 299         }
 300
 301         switch (event->type) {
 302         case ICE_RDMA_EVENT_LINK_CHANGE:
 303                 printf("%s:%d PF: %x (%x), state: %d, speed: %lu\n", __func__, __LINE__,
 304                        peer->pf_id, if_getdunit(peer->ifp), event->linkstate,
 305                        event->baudrate);
 306                 break;
 307         case ICE_RDMA_EVENT_MTU_CHANGE:
 308                 if (iwdev->vsi.mtu != event->mtu) {
 309                         l2params.mtu = event->mtu;
 310                         l2params.mtu_changed = true;
 311                         irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev);
 312                         irdma_change_l2params(&iwdev->vsi, &l2params);
 313                 }
 314                 break;
 315         case ICE_RDMA_EVENT_TC_CHANGE:
 316                 /*
 317                  * 1. check if it is pre or post 2. check if it is currently being done
 318                  */
 319                 if (event->prep == iwdev->vsi.tc_change_pending) {
 320                         printf("%s:%d can't process %s TC change if TC change is %spending\n",
 321                                __func__, __LINE__,
 322                                event->prep ? "pre" : "post",
 323                                event->prep ? " " : "not ");
 324                         goto done;
 325                 }
 326                 if (!atomic_inc_not_zero(&iwdev->rf->dev_ctx.event_rfcnt)) {
 327                         printf("%s:%d (%d) EVENT_TC_CHANGE received, but not processed %d\n",
 328                                __func__, __LINE__, if_getdunit(peer->ifp),
 329                                atomic_read(&iwdev->rf->dev_ctx.event_rfcnt));
 330                         break;
 331                 }
 332                 if (event->prep) {
 333                         iwdev->vsi.tc_change_pending = true;
 334                         irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND);
 335                         wait_event_timeout(iwdev->suspend_wq,
 336                                            !atomic_read(&iwdev->vsi.qp_suspend_reqs),
 337                                            IRDMA_EVENT_TIMEOUT_MS * 10);
 338                         irdma_ws_reset(&iwdev->vsi);
 339                         printf("%s:%d TC change preparation done\n", __func__, __LINE__);
 340                 } else {
 341                         l2params.tc_changed = true;
 342                         irdma_get_qos_info(iwdev->rf, &l2params, &event->port_qos);
 343                         if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
 344                                 iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
 345
 346                         irdma_check_fc_for_tc_update(&iwdev->vsi, &l2params);
 347                         irdma_change_l2params(&iwdev->vsi, &l2params);
 348                         printf("%s:%d TC change done\n", __func__, __LINE__);
 349                 }
 350                 atomic_dec(&iwdev->rf->dev_ctx.event_rfcnt);
 351                 break;
 352         case ICE_RDMA_EVENT_CRIT_ERR:
 353                 if (event->oicr_reg & IRDMAPFINT_OICR_PE_CRITERR_M) {
 354                         u32 pe_criterr;
 355
 356 #define IRDMA_Q1_RESOURCE_ERR  0x0001024d
 357                         pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]);
 358                         if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) {
 359                                 irdma_pr_err("critical PE Error, GLPE_CRITERR=0x%08x\n",
 360                                              pe_criterr);
 361                                 iwdev->rf->reset = true;
 362                         } else {
 363                                 irdma_dev_warn(to_ibdev(&iwdev->rf->sc_dev),
 364                                                "Q1 Resource Check\n");
 365                         }
 366                 }
 367                 if (event->oicr_reg & IRDMAPFINT_OICR_HMC_ERR_M) {
 368                         irdma_pr_err("HMC Error\n");
 369                         iwdev->rf->reset = true;
 370                 }
 371                 if (iwdev->rf->reset)
 372                         iwdev->rf->gen_ops.request_reset(iwdev->rf);
 373                 break;
 374         case ICE_RDMA_EVENT_RESET:
 375                 iwdev->rf->reset = true;
 376                 break;
 377         default:
 378                 printf("%s:%d event type unsupported: %d\n", __func__, __LINE__, event->type);
 379         }
 380 done:
 381         return;
 382 }
 383
 384 /**
 385  * irdma_link_change - Callback for link state change
 386  * @peer: the peer interface structure
 387  * @linkstate: state of the link
 388  * @baudrate: speed of the link
 389  */
 390 static void
 391 irdma_link_change(struct ice_rdma_peer *peer, int linkstate, uint64_t baudrate)
 392 {
 393         printf("%s:%d PF: %x (%x), state: %d, speed: %lu\n", __func__, __LINE__,
 394                peer->pf_id, if_getdunit(peer->ifp), linkstate, baudrate);
 395 }
 396
 397 /**
 398  * irdma_finalize_task - Finish open or close phase in a separate thread
 399  * @context: instance holding peer and iwdev information
 400  *
 401  * Triggered from irdma_open or irdma_close to perform rt_init_hw or
 402  * rt_deinit_hw respectively. Does registration and unregistration of
 403  * the device.
 404  */
 405 static void
 406 irdma_finalize_task(void *context, int pending)
 407 {
 408         struct irdma_task_arg *task_arg = (struct irdma_task_arg *)context;
 409         struct irdma_device *iwdev = task_arg->iwdev;
 410         struct irdma_pci_f *rf = iwdev->rf;
 411         struct ice_rdma_peer *peer = task_arg->peer;
 412         struct irdma_l2params l2params = {{{0}}};
 413         struct ice_rdma_request req = {0};
 414         int status = 0;
 415
 416         if (iwdev->iw_status) {
 417                 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
 418                             "Starting deferred closing %d (%d)\n",
 419                             rf->peer_info->pf_id, if_getdunit(peer->ifp));
 420                 atomic_dec(&rf->dev_ctx.event_rfcnt);
 421                 wait_event_timeout(iwdev->suspend_wq,
 422                                    !atomic_read(&rf->dev_ctx.event_rfcnt),
 423                                    IRDMA_MAX_TIMEOUT);
 424                 if (atomic_read(&rf->dev_ctx.event_rfcnt) != 0) {
 425                         printf("%s:%d (%d) waiting for event_rfcnt (%d) timeout, proceed with unload\n",
 426                                __func__, __LINE__, if_getdunit(peer->ifp),
 427                                atomic_read(&rf->dev_ctx.event_rfcnt));
 428                 }
 429                 irdma_dereg_ipaddr_event_cb(rf);
 430                 irdma_ib_unregister_device(iwdev);
 431                 req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE;
 432                 req.enable_filter = false;
 433                 IRDMA_DI_REQ_HANDLER(peer, &req);
 434                 irdma_cleanup_dead_qps(&iwdev->vsi);
 435                 irdma_rt_deinit_hw(iwdev);
 436         } else {
 437                 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
 438                             "Starting deferred opening %d (%d)\n",
 439                             rf->peer_info->pf_id, if_getdunit(peer->ifp));
 440                 irdma_get_qos_info(iwdev->rf, &l2params, &peer->initial_qos_info);
 441                 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
 442                         iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
 443
 444                 l2params.mtu = peer->mtu;
 445                 status = irdma_rt_init_hw(iwdev, &l2params);
 446                 if (status) {
 447                         irdma_pr_err("RT init failed %d\n", status);
 448                         ib_dealloc_device(&iwdev->ibdev);
 449                         return;
 450                 }
 451                 status = irdma_ib_register_device(iwdev);
 452                 if (status) {
 453                         irdma_pr_err("Registration failed %d\n", status);
 454                         irdma_rt_deinit_hw(iwdev);
 455                         ib_dealloc_device(&iwdev->ibdev);
 456                 }
 457                 irdma_sw_stats_tunables_init(rf);
 458                 req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE;
 459                 req.enable_filter = true;
 460                 IRDMA_DI_REQ_HANDLER(peer, &req);
 461                 irdma_reg_ipaddr_event_cb(rf);
 462                 atomic_inc(&rf->dev_ctx.event_rfcnt);
 463                 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
 464                             "Deferred opening finished %d (%d)\n",
 465                             rf->peer_info->pf_id, if_getdunit(peer->ifp));
 466         }
 467 }
 468
 469 /**
 470  * irdma_alloc_pcidev - allocate memory for pcidev and populate data
 471  * @peer: the new peer interface structure
 472  * @rf: RDMA PCI function
 473  */
 474 static int
 475 irdma_alloc_pcidev(struct ice_rdma_peer *peer, struct irdma_pci_f *rf)
 476 {
 477         rf->pcidev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL);
 478         if (!rf->pcidev) {
 479                 return -ENOMEM;
 480         }
 481         if (linux_pci_attach_device(rf->dev_ctx.dev, NULL, NULL, rf->pcidev))
 482                 return -ENOMEM;
 483
 484         return 0;
 485 }
 486
 487 /**
 488  * irdma_dealloc_pcidev - deallocate memory for pcidev
 489  * @rf: RDMA PCI function
 490  */
 491 static void
 492 irdma_dealloc_pcidev(struct irdma_pci_f *rf)
 493 {
 494         linux_pci_detach_device(rf->pcidev);
 495         kfree(rf->pcidev);
 496 }
 497
 498 /**
 499  * irdma_fill_device_info - assign initial values to rf variables
 500  * @iwdev: irdma device
 501  * @peer: the peer interface structure
 502  */
 503 static void
 504 irdma_fill_device_info(struct irdma_device *iwdev,
 505                        struct ice_rdma_peer *peer)
 506 {
 507         struct irdma_pci_f *rf = iwdev->rf;
 508
 509         rf->peer_info = peer;
 510         rf->gen_ops.register_qset = irdma_register_qset;
 511         rf->gen_ops.unregister_qset = irdma_unregister_qset;
 512
 513         rf->rdma_ver = IRDMA_GEN_2;
 514         rf->sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_2;
 515         rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT;
 516         rf->rst_to = IRDMA_RST_TIMEOUT_HZ;
 517         rf->check_fc = irdma_check_fc_for_qp;
 518         rf->gen_ops.request_reset = irdma_request_reset;
 519         irdma_set_rf_user_cfg_params(rf);
 520
 521         rf->default_vsi.vsi_idx = peer->pf_vsi_num;
 522         rf->dev_ctx.dev = peer->dev;
 523         rf->dev_ctx.mem_bus_space_tag = rman_get_bustag(peer->pci_mem);
 524         rf->dev_ctx.mem_bus_space_handle = rman_get_bushandle(peer->pci_mem);
 525         rf->dev_ctx.mem_bus_space_size = rman_get_size(peer->pci_mem);
 526
 527         rf->hw.dev_context = &rf->dev_ctx;
 528         rf->hw.hw_addr = (u8 *)rman_get_virtual(peer->pci_mem);
 529         rf->msix_count = peer->msix.count;
 530         rf->msix_info.entry = peer->msix.base;
 531         rf->msix_info.vector = peer->msix.count;
 532         printf("%s:%d msix_info: %d %d %d\n", __func__, __LINE__,
 533                rf->msix_count, rf->msix_info.entry, rf->msix_info.vector);
 534
 535         rf->iwdev = iwdev;
 536         iwdev->netdev = peer->ifp;
 537         iwdev->init_state = INITIAL_STATE;
 538         iwdev->vsi_num = peer->pf_vsi_num;
 539         iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;
 540         iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
 541         iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT;
 542         iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT;
 543         iwdev->roce_rtomin = 5;
 544
 545         if (rf->protocol_used == IRDMA_ROCE_PROTOCOL_ONLY) {
 546                 iwdev->roce_mode = true;
 547         }
 548 }
 549
 550 /**
 551  * irdma_probe - Callback to probe a new RDMA peer device
 552  * @peer: the new peer interface structure
 553  *
 554  * Callback implementing the RDMA_PROBE function. Called by the ice driver to
 555  * notify the RDMA client driver that a new device has been created
 556  */
 557 static int
 558 irdma_probe(struct ice_rdma_peer *peer)
 559 {
 560         struct irdma_device *iwdev;
 561         struct irdma_pci_f *rf;
 562         struct irdma_handler *hdl;
 563         int err = 0;
 564
 565         irdma_pr_info("probe: irdma-%s peer=%p, peer->pf_id=%d, peer->ifp=%p, peer->ifp->if_dunit=%d, peer->pci_mem->r_bustag=%p\n",
 566                       irdma_driver_version, peer, peer->pf_id, peer->ifp,
 567                       if_getdunit(peer->ifp), (void *)(uintptr_t)peer->pci_mem->r_bustag);
 568
 569         hdl = irdma_find_handler(peer);
 570         if (hdl)
 571                 return -EBUSY;
 572
 573         hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
 574         if (!hdl)
 575                 return -ENOMEM;
 576
 577         iwdev = (struct irdma_device *)ib_alloc_device(sizeof(*iwdev));
 578         if (!iwdev) {
 579                 kfree(hdl);
 580                 return -ENOMEM;
 581         }
 582
 583         iwdev->rf = kzalloc(sizeof(*rf), GFP_KERNEL);
 584         if (!iwdev->rf) {
 585                 ib_dealloc_device(&iwdev->ibdev);
 586                 kfree(hdl);
 587                 return -ENOMEM;
 588         }
 589         hdl->iwdev = iwdev;
 590         iwdev->hdl = hdl;
 591
 592         irdma_init_tunable(iwdev->rf, if_getdunit(peer->ifp));
 593         irdma_fill_device_info(iwdev, peer);
 594         rf = iwdev->rf;
 595
 596         if (irdma_alloc_pcidev(peer, rf))
 597                 goto err_pcidev;
 598
 599         irdma_add_handler(hdl);
 600
 601         if (irdma_ctrl_init_hw(rf)) {
 602                 err = -EIO;
 603                 goto err_ctrl_init;
 604         }
 605
 606         rf->dev_ctx.task_arg.peer = peer;
 607         rf->dev_ctx.task_arg.iwdev = iwdev;
 608         rf->dev_ctx.task_arg.peer = peer;
 609
 610         TASK_INIT(&hdl->deferred_task, 0, irdma_finalize_task, &rf->dev_ctx.task_arg);
 611         hdl->deferred_tq = taskqueue_create_fast("irdma_defer",
 612                                                  M_NOWAIT, taskqueue_thread_enqueue,
 613                                                  &hdl->deferred_tq);
 614         taskqueue_start_threads(&hdl->deferred_tq, 1, PI_NET, "irdma_defer_t");
 615
 616         taskqueue_enqueue(hdl->deferred_tq, &hdl->deferred_task);
 617
 618         return 0;
 619
 620 err_ctrl_init:
 621         irdma_del_handler(hdl);
 622         irdma_dealloc_pcidev(rf);
 623 err_pcidev:
 624         kfree(iwdev->rf);
 625         ib_dealloc_device(&iwdev->ibdev);
 626         kfree(hdl);
 627
 628         return err;
 629 }
 630
 631 /**
 632  * irdma_remove - Callback to remove an RDMA peer device
 633  * @peer: the new peer interface structure
 634  *
 635  * Callback implementing the RDMA_REMOVE function. Called by the ice driver to
 636  * notify the RDMA client driver that the device wille be delated
 637  */
 638 static int
 639 irdma_remove(struct ice_rdma_peer *peer)
 640 {
 641         struct irdma_handler *hdl;
 642         struct irdma_device *iwdev;
 643
 644         irdma_debug((struct irdma_sc_dev *)NULL, IRDMA_DEBUG_INIT,
 645                     "removing %s irdma%d\n", __func__, if_getdunit(peer->ifp));
 646
 647         hdl = irdma_find_handler(peer);
 648         if (!hdl)
 649                 return 0;
 650
 651         iwdev = hdl->iwdev;
 652
 653         if (iwdev->vsi.tc_change_pending) {
 654                 iwdev->vsi.tc_change_pending = false;
 655                 irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_RESUME);
 656         }
 657
 658         taskqueue_enqueue(hdl->deferred_tq, &hdl->deferred_task);
 659
 660         taskqueue_drain(hdl->deferred_tq, &hdl->deferred_task);
 661         taskqueue_free(hdl->deferred_tq);
 662         hdl->iwdev->rf->dev_ctx.task_arg.iwdev = NULL;
 663         hdl->iwdev->rf->dev_ctx.task_arg.peer = NULL;
 664
 665         sysctl_ctx_free(&iwdev->rf->tun_info.irdma_sysctl_ctx);
 666         hdl->iwdev->rf->tun_info.irdma_sysctl_tree = NULL;
 667         hdl->iwdev->rf->tun_info.sws_sysctl_tree = NULL;
 668
 669         irdma_ctrl_deinit_hw(iwdev->rf);
 670
 671         irdma_dealloc_pcidev(iwdev->rf);
 672
 673         irdma_del_handler(iwdev->hdl);
 674         kfree(iwdev->hdl);
 675         kfree(iwdev->rf);
 676         ib_dealloc_device(&iwdev->ibdev);
 677         irdma_pr_info("IRDMA hardware deinitialization complete irdma%d\n",
 678                       if_getdunit(peer->ifp));
 679
 680         return 0;
 681 }
 682
 683 /**
 684  * irdma_open - Callback for operation open for RDMA device
 685  * @peer: the new peer interface structure
 686  *
 687  * Callback implementing the RDMA_OPEN function. Called by the ice driver to
 688  * notify the RDMA client driver that a new device has been initialized.
 689  */
 690 static int
 691 irdma_open(struct ice_rdma_peer *peer)
 692 {
 693         struct irdma_device *iwdev;
 694         struct ice_rdma_event event = {0};
 695
 696         iwdev = peer_to_iwdev(peer);
 697         if (iwdev) {
 698                 event.type = ICE_RDMA_EVENT_MTU_CHANGE;
 699                 event.mtu = peer->mtu;
 700
 701                 irdma_event_handler(peer, &event);
 702         } else {
 703                 irdma_probe(peer);
 704         }
 705
 706         return 0;
 707 }
 708
 709 /**
 710  * irdma_close - Callback to notify that a peer device is down
 711  * @peer: the RDMA peer device being stopped
 712  *
 713  * Callback implementing the RDMA_CLOSE function. Called by the ice driver to
 714  * notify the RDMA client driver that a peer device is being stopped.
 715  */
 716 static int
 717 irdma_close(struct ice_rdma_peer *peer)
 718 {
 719         /*
 720          * This is called when ifconfig down or pf-reset is about to happen.
 721          */
 722         struct irdma_device *iwdev;
 723
 724         iwdev = peer_to_iwdev(peer);
 725         if (iwdev && iwdev->rf->reset)
 726                 irdma_remove(peer);
 727
 728         return 0;
 729 }
 730
 731 /**
 732  * irdma_prep_for_unregister - ensure the driver is ready to unregister
 733  */
 734 static void
 735 irdma_prep_for_unregister(void)
 736 {
 737         struct irdma_handler *hdl;
 738         unsigned long flags;
 739         bool hdl_valid;
 740
 741         do {
 742                 hdl_valid = false;
 743                 spin_lock_irqsave(&irdma_handler_lock, flags);
 744                 list_for_each_entry(hdl, &irdma_handlers, list) {
 745                         if (!hdl->iwdev->rf->peer_info)
 746                                 continue;
 747                         hdl_valid = true;
 748                         break;
 749                 }
 750                 spin_unlock_irqrestore(&irdma_handler_lock, flags);
 751                 if (!hdl || !hdl_valid)
 752                         break;
 753                 IRDMA_CLOSE(hdl->iwdev->rf->peer_info);
 754                 IRDMA_REMOVE(hdl->iwdev->rf->peer_info);
 755         } while (1);
 756 }
 757
 758 static kobj_method_t irdma_methods[] = {
 759         KOBJMETHOD(irdma_probe, irdma_probe),
 760             KOBJMETHOD(irdma_open, irdma_open),
 761             KOBJMETHOD(irdma_close, irdma_close),
 762             KOBJMETHOD(irdma_remove, irdma_remove),
 763             KOBJMETHOD(irdma_link_change, irdma_link_change),
 764             KOBJMETHOD(irdma_event_handler, irdma_event_handler),
 765             KOBJMETHOD_END
 766 };
 767
 768 /* declare irdma_class which extends the ice_rdma_di class */
 769 DEFINE_CLASS_1(irdma, irdma_class, irdma_methods, sizeof(struct ice_rdma_peer), ice_rdma_di_class);
 770
 771 static struct ice_rdma_info irdma_info = {
 772         .major_version = ICE_RDMA_MAJOR_VERSION,
 773         .minor_version = ICE_RDMA_MINOR_VERSION,
 774         .patch_version = ICE_RDMA_PATCH_VERSION,
 775         .rdma_class = &irdma_class,
 776 };
 777
 778 /**
 779  * irdma_module_event_handler - Module event handler callback
 780  * @mod: unused mod argument
 781  * @what: the module event to handle
 782  * @arg: unused module event argument
 783  *
 784  * Callback used by the FreeBSD module stack to notify the driver of module
 785  * events. Used to implement custom handling for certain module events such as
 786  * load and unload.
 787  */
 788 static int
 789 irdma_module_event_handler(module_t __unused mod, int what, void __unused * arg)
 790 {
 791         switch (what) {
 792         case MOD_LOAD:
 793                 printf("Loading irdma module\n");
 794                 return ice_rdma_register(&irdma_info);
 795         case MOD_UNLOAD:
 796                 printf("Unloading irdma module\n");
 797                 irdma_prep_for_unregister();
 798                 ice_rdma_unregister();
 799                 return (0);
 800         default:
 801                 return (EOPNOTSUPP);
 802         }
 803
 804         return (0);
 805 }
 806
 807 static moduledata_t irdma_moduledata = {
 808         "irdma",
 809             irdma_module_event_handler,
 810             NULL
 811 };
 812
 813 DECLARE_MODULE(irdma, irdma_moduledata, SI_SUB_LAST, SI_ORDER_ANY);
 814 MODULE_VERSION(irdma, 1);
 815 MODULE_DEPEND(irdma, ice, 1, 1, 1);
 816 MODULE_DEPEND(irdma, ibcore, 1, 1, 1);