sys/dev/hyperv/netvsc/hv_net_vsc.c

   1 /*-
   2  * Copyright (c) 2009-2012,2016 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 /**
  32  * HyperV vmbus network VSC (virtual services client) module
  33  *
  34  */
  35
  36
  37 #include <sys/param.h>
  38 #include <sys/kernel.h>
  39 #include <sys/socket.h>
  40 #include <sys/lock.h>
  41 #include <net/if.h>
  42 #include <net/if_var.h>
  43 #include <net/if_arp.h>
  44 #include <machine/bus.h>
  45 #include <machine/atomic.h>
  46
  47 #include <dev/hyperv/include/hyperv.h>
  48 #include "hv_net_vsc.h"
  49 #include "hv_rndis.h"
  50 #include "hv_rndis_filter.h"
  51
  52 /* priv1 and priv2 are consumed by the main driver */
  53 #define hv_chan_rdbuf   hv_chan_priv3
  54
  55 MALLOC_DEFINE(M_NETVSC, "netvsc", "Hyper-V netvsc driver");
  56
  57 /*
  58  * Forward declarations
  59  */
  60 static void hv_nv_on_channel_callback(void *xchan);
  61 static int  hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc);
  62 static int  hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *);
  63 static int  hv_nv_destroy_send_buffer(netvsc_dev *net_dev);
  64 static int  hv_nv_destroy_rx_buffer(netvsc_dev *net_dev);
  65 static int  hv_nv_connect_to_vsp(struct hn_softc *sc);
  66 static void hv_nv_on_send_completion(netvsc_dev *net_dev,
  67     struct hv_vmbus_channel *, const struct vmbus_chanpkt_hdr *pkt);
  68 static void hv_nv_on_receive_completion(struct hv_vmbus_channel *chan,
  69     uint64_t tid, uint32_t status);
  70 static void hv_nv_on_receive(netvsc_dev *net_dev,
  71     struct hn_softc *sc, struct hv_vmbus_channel *chan,
  72     const struct vmbus_chanpkt_hdr *pkt);
  73
  74 /*
  75  *
  76  */
  77 static inline netvsc_dev *
  78 hv_nv_alloc_net_device(struct hn_softc *sc)
  79 {
  80         netvsc_dev *net_dev;
  81
  82         net_dev = malloc(sizeof(netvsc_dev), M_NETVSC, M_WAITOK | M_ZERO);
  83
  84         net_dev->sc = sc;
  85         net_dev->destroy = FALSE;
  86         sc->net_dev = net_dev;
  87
  88         return (net_dev);
  89 }
  90
  91 /*
  92  * XXX unnecessary; nuke it.
  93  */
  94 static inline netvsc_dev *
  95 hv_nv_get_outbound_net_device(struct hn_softc *sc)
  96 {
  97         return sc->net_dev;
  98 }
  99
 100 /*
 101  * XXX unnecessary; nuke it.
 102  */
 103 static inline netvsc_dev *
 104 hv_nv_get_inbound_net_device(struct hn_softc *sc)
 105 {
 106         return sc->net_dev;
 107 }
 108
 109 int
 110 hv_nv_get_next_send_section(netvsc_dev *net_dev)
 111 {
 112         unsigned long bitsmap_words = net_dev->bitsmap_words;
 113         unsigned long *bitsmap = net_dev->send_section_bitsmap;
 114         unsigned long idx;
 115         int ret = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
 116         int i;
 117
 118         for (i = 0; i < bitsmap_words; i++) {
 119                 idx = ffsl(~bitsmap[i]);
 120                 if (0 == idx)
 121                         continue;
 122
 123                 idx--;
 124                 KASSERT(i * BITS_PER_LONG + idx < net_dev->send_section_count,
 125                     ("invalid i %d and idx %lu", i, idx));
 126
 127                 if (atomic_testandset_long(&bitsmap[i], idx))
 128                         continue;
 129
 130                 ret = i * BITS_PER_LONG + idx;
 131                 break;
 132         }
 133
 134         return (ret);
 135 }
 136
 137 /*
 138  * Net VSC initialize receive buffer with net VSP
 139  *
 140  * Net VSP:  Network virtual services client, also known as the
 141  *     Hyper-V extensible switch and the synthetic data path.
 142  */
 143 static int
 144 hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *sc)
 145 {
 146         netvsc_dev *net_dev;
 147         nvsp_msg *init_pkt;
 148         int ret = 0;
 149
 150         net_dev = hv_nv_get_outbound_net_device(sc);
 151         if (!net_dev) {
 152                 return (ENODEV);
 153         }
 154
 155         net_dev->rx_buf = hyperv_dmamem_alloc(bus_get_dma_tag(sc->hn_dev),
 156             PAGE_SIZE, 0, net_dev->rx_buf_size, &net_dev->rxbuf_dma,
 157             BUS_DMA_WAITOK | BUS_DMA_ZERO);
 158         if (net_dev->rx_buf == NULL) {
 159                 device_printf(sc->hn_dev, "allocate rxbuf failed\n");
 160                 return ENOMEM;
 161         }
 162
 163         /*
 164          * Connect the RXBUF GPADL to the primary channel.
 165          *
 166          * NOTE:
 167          * Only primary channel has RXBUF connected to it.  Sub-channels
 168          * just share this RXBUF.
 169          */
 170         ret = vmbus_chan_gpadl_connect(sc->hn_prichan,
 171             net_dev->rxbuf_dma.hv_paddr, net_dev->rx_buf_size,
 172             &net_dev->rx_buf_gpadl_handle);
 173         if (ret != 0) {
 174                 device_printf(sc->hn_dev, "rxbuf gpadl connect failed: %d\n",
 175                     ret);
 176                 goto cleanup;
 177         }
 178
 179         /* sema_wait(&ext->channel_init_sema); KYS CHECK */
 180
 181         /* Notify the NetVsp of the gpadl handle */
 182         init_pkt = &net_dev->channel_init_packet;
 183
 184         memset(init_pkt, 0, sizeof(nvsp_msg));
 185
 186         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_rx_buf;
 187         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 188             net_dev->rx_buf_gpadl_handle;
 189         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 190             NETVSC_RECEIVE_BUFFER_ID;
 191
 192         /* Send the gpadl notification request */
 193
 194         ret = vmbus_chan_send(sc->hn_prichan,
 195             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 196             init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
 197         if (ret != 0) {
 198                 goto cleanup;
 199         }
 200
 201         sema_wait(&net_dev->channel_init_sema);
 202
 203         /* Check the response */
 204         if (init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.status
 205             != nvsp_status_success) {
 206                 ret = EINVAL;
 207                 goto cleanup;
 208         }
 209
 210         net_dev->rx_section_count =
 211             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.num_sections;
 212
 213         net_dev->rx_sections = malloc(net_dev->rx_section_count *
 214             sizeof(nvsp_1_rx_buf_section), M_NETVSC, M_WAITOK);
 215         memcpy(net_dev->rx_sections,
 216             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.sections,
 217             net_dev->rx_section_count * sizeof(nvsp_1_rx_buf_section));
 218
 219
 220         /*
 221          * For first release, there should only be 1 section that represents
 222          * the entire receive buffer
 223          */
 224         if (net_dev->rx_section_count != 1
 225             || net_dev->rx_sections->offset != 0) {
 226                 ret = EINVAL;
 227                 goto cleanup;
 228         }
 229
 230         goto exit;
 231
 232 cleanup:
 233         hv_nv_destroy_rx_buffer(net_dev);
 234
 235 exit:
 236         return (ret);
 237 }
 238
 239 /*
 240  * Net VSC initialize send buffer with net VSP
 241  */
 242 static int
 243 hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc)
 244 {
 245         netvsc_dev *net_dev;
 246         nvsp_msg *init_pkt;
 247         int ret = 0;
 248
 249         net_dev = hv_nv_get_outbound_net_device(sc);
 250         if (!net_dev) {
 251                 return (ENODEV);
 252         }
 253
 254         net_dev->send_buf = hyperv_dmamem_alloc(bus_get_dma_tag(sc->hn_dev),
 255             PAGE_SIZE, 0, net_dev->send_buf_size, &net_dev->txbuf_dma,
 256             BUS_DMA_WAITOK | BUS_DMA_ZERO);
 257         if (net_dev->send_buf == NULL) {
 258                 device_printf(sc->hn_dev, "allocate chimney txbuf failed\n");
 259                 return ENOMEM;
 260         }
 261
 262         /*
 263          * Connect chimney sending buffer GPADL to the primary channel.
 264          *
 265          * NOTE:
 266          * Only primary channel has chimney sending buffer connected to it.
 267          * Sub-channels just share this chimney sending buffer.
 268          */
 269         ret = vmbus_chan_gpadl_connect(sc->hn_prichan,
 270             net_dev->txbuf_dma.hv_paddr, net_dev->send_buf_size,
 271             &net_dev->send_buf_gpadl_handle);
 272         if (ret != 0) {
 273                 device_printf(sc->hn_dev, "chimney sending buffer gpadl "
 274                     "connect failed: %d\n", ret);
 275                 goto cleanup;
 276         }
 277
 278         /* Notify the NetVsp of the gpadl handle */
 279
 280         init_pkt = &net_dev->channel_init_packet;
 281
 282         memset(init_pkt, 0, sizeof(nvsp_msg));
 283
 284         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_send_buf;
 285         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 286             net_dev->send_buf_gpadl_handle;
 287         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 288             NETVSC_SEND_BUFFER_ID;
 289
 290         /* Send the gpadl notification request */
 291
 292         ret = vmbus_chan_send(sc->hn_prichan,
 293             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 294             init_pkt, sizeof(nvsp_msg), (uint64_t)init_pkt);
 295         if (ret != 0) {
 296                 goto cleanup;
 297         }
 298
 299         sema_wait(&net_dev->channel_init_sema);
 300
 301         /* Check the response */
 302         if (init_pkt->msgs.vers_1_msgs.send_send_buf_complete.status
 303             != nvsp_status_success) {
 304                 ret = EINVAL;
 305                 goto cleanup;
 306         }
 307
 308         net_dev->send_section_size =
 309             init_pkt->msgs.vers_1_msgs.send_send_buf_complete.section_size;
 310         net_dev->send_section_count =
 311             net_dev->send_buf_size / net_dev->send_section_size;
 312         net_dev->bitsmap_words = howmany(net_dev->send_section_count,
 313             BITS_PER_LONG);
 314         net_dev->send_section_bitsmap =
 315             malloc(net_dev->bitsmap_words * sizeof(long), M_NETVSC,
 316             M_WAITOK | M_ZERO);
 317
 318         goto exit;
 319
 320 cleanup:
 321         hv_nv_destroy_send_buffer(net_dev);
 322
 323 exit:
 324         return (ret);
 325 }
 326
 327 /*
 328  * Net VSC destroy receive buffer
 329  */
 330 static int
 331 hv_nv_destroy_rx_buffer(netvsc_dev *net_dev)
 332 {
 333         nvsp_msg *revoke_pkt;
 334         int ret = 0;
 335
 336         /*
 337          * If we got a section count, it means we received a
 338          * send_rx_buf_complete msg
 339          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 340          * we need to send a revoke msg here
 341          */
 342         if (net_dev->rx_section_count) {
 343                 /* Send the revoke receive buffer */
 344                 revoke_pkt = &net_dev->revoke_packet;
 345                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 346
 347                 revoke_pkt->hdr.msg_type = nvsp_msg_1_type_revoke_rx_buf;
 348                 revoke_pkt->msgs.vers_1_msgs.revoke_rx_buf.id =
 349                     NETVSC_RECEIVE_BUFFER_ID;
 350
 351                 ret = vmbus_chan_send(net_dev->sc->hn_prichan,
 352                     VMBUS_CHANPKT_TYPE_INBAND, 0, revoke_pkt, sizeof(nvsp_msg),
 353                     (uint64_t)(uintptr_t)revoke_pkt);
 354
 355                 /*
 356                  * If we failed here, we might as well return and have a leak
 357                  * rather than continue and a bugchk
 358                  */
 359                 if (ret != 0) {
 360                         return (ret);
 361                 }
 362         }
 363
 364         /* Tear down the gpadl on the vsp end */
 365         if (net_dev->rx_buf_gpadl_handle) {
 366                 ret = vmbus_chan_gpadl_disconnect(net_dev->sc->hn_prichan,
 367                     net_dev->rx_buf_gpadl_handle);
 368                 /*
 369                  * If we failed here, we might as well return and have a leak
 370                  * rather than continue and a bugchk
 371                  */
 372                 if (ret != 0) {
 373                         return (ret);
 374                 }
 375                 net_dev->rx_buf_gpadl_handle = 0;
 376         }
 377
 378         if (net_dev->rx_buf) {
 379                 /* Free up the receive buffer */
 380                 hyperv_dmamem_free(&net_dev->rxbuf_dma, net_dev->rx_buf);
 381                 net_dev->rx_buf = NULL;
 382         }
 383
 384         if (net_dev->rx_sections) {
 385                 free(net_dev->rx_sections, M_NETVSC);
 386                 net_dev->rx_sections = NULL;
 387                 net_dev->rx_section_count = 0;
 388         }
 389
 390         return (ret);
 391 }
 392
 393 /*
 394  * Net VSC destroy send buffer
 395  */
 396 static int
 397 hv_nv_destroy_send_buffer(netvsc_dev *net_dev)
 398 {
 399         nvsp_msg *revoke_pkt;
 400         int ret = 0;
 401
 402         /*
 403          * If we got a section count, it means we received a
 404          * send_rx_buf_complete msg
 405          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 406          * we need to send a revoke msg here
 407          */
 408         if (net_dev->send_section_size) {
 409                 /* Send the revoke send buffer */
 410                 revoke_pkt = &net_dev->revoke_packet;
 411                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 412
 413                 revoke_pkt->hdr.msg_type =
 414                     nvsp_msg_1_type_revoke_send_buf;
 415                 revoke_pkt->msgs.vers_1_msgs.revoke_send_buf.id =
 416                     NETVSC_SEND_BUFFER_ID;
 417
 418                 ret = vmbus_chan_send(net_dev->sc->hn_prichan,
 419                     VMBUS_CHANPKT_TYPE_INBAND, 0,
 420                     revoke_pkt, sizeof(nvsp_msg),
 421                     (uint64_t)(uintptr_t)revoke_pkt);
 422                 /*
 423                  * If we failed here, we might as well return and have a leak
 424                  * rather than continue and a bugchk
 425                  */
 426                 if (ret != 0) {
 427                         return (ret);
 428                 }
 429         }
 430
 431         /* Tear down the gpadl on the vsp end */
 432         if (net_dev->send_buf_gpadl_handle) {
 433                 ret = vmbus_chan_gpadl_disconnect(net_dev->sc->hn_prichan,
 434                     net_dev->send_buf_gpadl_handle);
 435
 436                 /*
 437                  * If we failed here, we might as well return and have a leak
 438                  * rather than continue and a bugchk
 439                  */
 440                 if (ret != 0) {
 441                         return (ret);
 442                 }
 443                 net_dev->send_buf_gpadl_handle = 0;
 444         }
 445
 446         if (net_dev->send_buf) {
 447                 /* Free up the receive buffer */
 448                 hyperv_dmamem_free(&net_dev->txbuf_dma, net_dev->send_buf);
 449                 net_dev->send_buf = NULL;
 450         }
 451
 452         if (net_dev->send_section_bitsmap) {
 453                 free(net_dev->send_section_bitsmap, M_NETVSC);
 454         }
 455
 456         return (ret);
 457 }
 458
 459
 460 /*
 461  * Attempt to negotiate the caller-specified NVSP version
 462  *
 463  * For NVSP v2, Server 2008 R2 does not set
 464  * init_pkt->msgs.init_msgs.init_compl.negotiated_prot_vers
 465  * to the negotiated version, so we cannot rely on that.
 466  */
 467 static int
 468 hv_nv_negotiate_nvsp_protocol(struct hn_softc *sc, netvsc_dev *net_dev,
 469     uint32_t nvsp_ver)
 470 {
 471         nvsp_msg *init_pkt;
 472         int ret;
 473
 474         init_pkt = &net_dev->channel_init_packet;
 475         memset(init_pkt, 0, sizeof(nvsp_msg));
 476         init_pkt->hdr.msg_type = nvsp_msg_type_init;
 477
 478         /*
 479          * Specify parameter as the only acceptable protocol version
 480          */
 481         init_pkt->msgs.init_msgs.init.p1.protocol_version = nvsp_ver;
 482         init_pkt->msgs.init_msgs.init.protocol_version_2 = nvsp_ver;
 483
 484         /* Send the init request */
 485         ret = vmbus_chan_send(sc->hn_prichan,
 486             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 487             init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
 488         if (ret != 0)
 489                 return (-1);
 490
 491         sema_wait(&net_dev->channel_init_sema);
 492
 493         if (init_pkt->msgs.init_msgs.init_compl.status != nvsp_status_success)
 494                 return (EINVAL);
 495
 496         return (0);
 497 }
 498
 499 /*
 500  * Send NDIS version 2 config packet containing MTU.
 501  *
 502  * Not valid for NDIS version 1.
 503  */
 504 static int
 505 hv_nv_send_ndis_config(struct hn_softc *sc, uint32_t mtu)
 506 {
 507         netvsc_dev *net_dev;
 508         nvsp_msg *init_pkt;
 509         int ret;
 510
 511         net_dev = hv_nv_get_outbound_net_device(sc);
 512         if (!net_dev)
 513                 return (-ENODEV);
 514
 515         /*
 516          * Set up configuration packet, write MTU
 517          * Indicate we are capable of handling VLAN tags
 518          */
 519         init_pkt = &net_dev->channel_init_packet;
 520         memset(init_pkt, 0, sizeof(nvsp_msg));
 521         init_pkt->hdr.msg_type = nvsp_msg_2_type_send_ndis_config;
 522         init_pkt->msgs.vers_2_msgs.send_ndis_config.mtu = mtu;
 523         init_pkt->
 524                 msgs.vers_2_msgs.send_ndis_config.capabilities.u1.u2.ieee8021q
 525                 = 1;
 526
 527         /* Send the configuration packet */
 528         ret = vmbus_chan_send(sc->hn_prichan, VMBUS_CHANPKT_TYPE_INBAND, 0,
 529             init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
 530         if (ret != 0)
 531                 return (-EINVAL);
 532
 533         return (0);
 534 }
 535
 536 /*
 537  * Net VSC connect to VSP
 538  */
 539 static int
 540 hv_nv_connect_to_vsp(struct hn_softc *sc)
 541 {
 542         netvsc_dev *net_dev;
 543         nvsp_msg *init_pkt;
 544         uint32_t ndis_version;
 545         uint32_t protocol_list[] = { NVSP_PROTOCOL_VERSION_1,
 546             NVSP_PROTOCOL_VERSION_2,
 547             NVSP_PROTOCOL_VERSION_4,
 548             NVSP_PROTOCOL_VERSION_5 };
 549         int i;
 550         int protocol_number = nitems(protocol_list);
 551         int ret = 0;
 552         device_t dev = sc->hn_dev;
 553         struct ifnet *ifp = sc->hn_ifp;
 554
 555         net_dev = hv_nv_get_outbound_net_device(sc);
 556
 557         /*
 558          * Negotiate the NVSP version.  Try the latest NVSP first.
 559          */
 560         for (i = protocol_number - 1; i >= 0; i--) {
 561                 if (hv_nv_negotiate_nvsp_protocol(sc, net_dev,
 562                     protocol_list[i]) == 0) {
 563                         net_dev->nvsp_version = protocol_list[i];
 564                         if (bootverbose)
 565                                 device_printf(dev, "Netvsc: got version 0x%x\n",
 566                                     net_dev->nvsp_version);
 567                         break;
 568                 }
 569         }
 570
 571         if (i < 0) {
 572                 if (bootverbose)
 573                         device_printf(dev, "failed to negotiate a valid "
 574                             "protocol.\n");
 575                 return (EPROTO);
 576         }
 577
 578         /*
 579          * Set the MTU if supported by this NVSP protocol version
 580          * This needs to be right after the NVSP init message per Haiyang
 581          */
 582         if (net_dev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
 583                 ret = hv_nv_send_ndis_config(sc, ifp->if_mtu);
 584
 585         /*
 586          * Send the NDIS version
 587          */
 588         init_pkt = &net_dev->channel_init_packet;
 589
 590         memset(init_pkt, 0, sizeof(nvsp_msg));
 591
 592         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_4) {
 593                 ndis_version = NDIS_VERSION_6_1;
 594         } else {
 595                 ndis_version = NDIS_VERSION_6_30;
 596         }
 597
 598         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_ndis_vers;
 599         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_major_vers =
 600             (ndis_version & 0xFFFF0000) >> 16;
 601         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_minor_vers =
 602             ndis_version & 0xFFFF;
 603
 604         /* Send the init request */
 605
 606         ret = vmbus_chan_send(sc->hn_prichan, VMBUS_CHANPKT_TYPE_INBAND, 0,
 607             init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
 608         if (ret != 0) {
 609                 goto cleanup;
 610         }
 611         /*
 612          * TODO:  BUGBUG - We have to wait for the above msg since the netvsp
 613          * uses KMCL which acknowledges packet (completion packet)
 614          * since our Vmbus always set the VMBUS_CHANPKT_FLAG_RC flag
 615          */
 616         /* sema_wait(&NetVscChannel->channel_init_sema); */
 617
 618         /* Post the big receive buffer to NetVSP */
 619         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
 620                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
 621         else
 622                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
 623         net_dev->send_buf_size = NETVSC_SEND_BUFFER_SIZE;
 624
 625         ret = hv_nv_init_rx_buffer_with_net_vsp(sc);
 626         if (ret == 0)
 627                 ret = hv_nv_init_send_buffer_with_net_vsp(sc);
 628
 629 cleanup:
 630         return (ret);
 631 }
 632
 633 /*
 634  * Net VSC disconnect from VSP
 635  */
 636 static void
 637 hv_nv_disconnect_from_vsp(netvsc_dev *net_dev)
 638 {
 639         hv_nv_destroy_rx_buffer(net_dev);
 640         hv_nv_destroy_send_buffer(net_dev);
 641 }
 642
 643 void
 644 hv_nv_subchan_attach(struct hv_vmbus_channel *chan)
 645 {
 646
 647         chan->hv_chan_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK);
 648         vmbus_chan_open(chan, NETVSC_DEVICE_RING_BUFFER_SIZE,
 649             NETVSC_DEVICE_RING_BUFFER_SIZE, NULL, 0,
 650             hv_nv_on_channel_callback, chan);
 651 }
 652
 653 /*
 654  * Net VSC on device add
 655  *
 656  * Callback when the device belonging to this driver is added
 657  */
 658 netvsc_dev *
 659 hv_nv_on_device_add(struct hn_softc *sc, void *additional_info)
 660 {
 661         struct hv_vmbus_channel *chan = sc->hn_prichan;
 662         netvsc_dev *net_dev;
 663         int ret = 0;
 664
 665         net_dev = hv_nv_alloc_net_device(sc);
 666         if (net_dev == NULL)
 667                 return NULL;
 668
 669         /* Initialize the NetVSC channel extension */
 670
 671         sema_init(&net_dev->channel_init_sema, 0, "netdev_sema");
 672
 673         chan->hv_chan_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK);
 674
 675         /*
 676          * Open the channel
 677          */
 678         ret = vmbus_chan_open(chan,
 679             NETVSC_DEVICE_RING_BUFFER_SIZE, NETVSC_DEVICE_RING_BUFFER_SIZE,
 680             NULL, 0, hv_nv_on_channel_callback, chan);
 681         if (ret != 0) {
 682                 free(chan->hv_chan_rdbuf, M_NETVSC);
 683                 goto cleanup;
 684         }
 685
 686         /*
 687          * Connect with the NetVsp
 688          */
 689         ret = hv_nv_connect_to_vsp(sc);
 690         if (ret != 0)
 691                 goto close;
 692
 693         return (net_dev);
 694
 695 close:
 696         /* Now, we can close the channel safely */
 697         free(chan->hv_chan_rdbuf, M_NETVSC);
 698         vmbus_chan_close(chan);
 699
 700 cleanup:
 701         /*
 702          * Free the packet buffers on the netvsc device packet queue.
 703          * Release other resources.
 704          */
 705         sema_destroy(&net_dev->channel_init_sema);
 706         free(net_dev, M_NETVSC);
 707
 708         return (NULL);
 709 }
 710
 711 /*
 712  * Net VSC on device remove
 713  */
 714 int
 715 hv_nv_on_device_remove(struct hn_softc *sc, boolean_t destroy_channel)
 716 {
 717         netvsc_dev *net_dev = sc->net_dev;;
 718
 719         /* Stop outbound traffic ie sends and receives completions */
 720         net_dev->destroy = TRUE;
 721
 722         hv_nv_disconnect_from_vsp(net_dev);
 723
 724         /* At this point, no one should be accessing net_dev except in here */
 725
 726         /* Now, we can close the channel safely */
 727
 728         free(sc->hn_prichan->hv_chan_rdbuf, M_NETVSC);
 729         vmbus_chan_close(sc->hn_prichan);
 730
 731         sema_destroy(&net_dev->channel_init_sema);
 732         free(net_dev, M_NETVSC);
 733
 734         return (0);
 735 }
 736
 737 /*
 738  * Net VSC on send completion
 739  */
 740 static void
 741 hv_nv_on_send_completion(netvsc_dev *net_dev, struct hv_vmbus_channel *chan,
 742     const struct vmbus_chanpkt_hdr *pkt)
 743 {
 744         const nvsp_msg *nvsp_msg_pkt;
 745         netvsc_packet *net_vsc_pkt;
 746
 747         nvsp_msg_pkt = VMBUS_CHANPKT_CONST_DATA(pkt);
 748
 749         if (nvsp_msg_pkt->hdr.msg_type == nvsp_msg_type_init_complete
 750                 || nvsp_msg_pkt->hdr.msg_type
 751                         == nvsp_msg_1_type_send_rx_buf_complete
 752                 || nvsp_msg_pkt->hdr.msg_type
 753                         == nvsp_msg_1_type_send_send_buf_complete
 754                 || nvsp_msg_pkt->hdr.msg_type
 755                         == nvsp_msg5_type_subchannel) {
 756                 /* Copy the response back */
 757                 memcpy(&net_dev->channel_init_packet, nvsp_msg_pkt,
 758                     sizeof(nvsp_msg));
 759                 sema_post(&net_dev->channel_init_sema);
 760         } else if (nvsp_msg_pkt->hdr.msg_type ==
 761                     nvsp_msg_1_type_send_rndis_pkt_complete) {
 762                 /* Get the send context */
 763                 net_vsc_pkt =
 764                     (netvsc_packet *)(unsigned long)pkt->cph_xactid;
 765                 if (NULL != net_vsc_pkt) {
 766                         if (net_vsc_pkt->send_buf_section_idx !=
 767                             NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) {
 768                                 u_long mask;
 769                                 int idx;
 770
 771                                 idx = net_vsc_pkt->send_buf_section_idx /
 772                                     BITS_PER_LONG;
 773                                 KASSERT(idx < net_dev->bitsmap_words,
 774                                     ("invalid section index %u",
 775                                      net_vsc_pkt->send_buf_section_idx));
 776                                 mask = 1UL <<
 777                                     (net_vsc_pkt->send_buf_section_idx %
 778                                      BITS_PER_LONG);
 779
 780                                 KASSERT(net_dev->send_section_bitsmap[idx] &
 781                                     mask,
 782                                     ("index bitmap 0x%lx, section index %u, "
 783                                      "bitmap idx %d, bitmask 0x%lx",
 784                                      net_dev->send_section_bitsmap[idx],
 785                                      net_vsc_pkt->send_buf_section_idx,
 786                                      idx, mask));
 787                                 atomic_clear_long(
 788                                     &net_dev->send_section_bitsmap[idx], mask);
 789                         }
 790
 791                         /* Notify the layer above us */
 792                         net_vsc_pkt->compl.send.on_send_completion(chan,
 793                             net_vsc_pkt->compl.send.send_completion_context);
 794
 795                 }
 796         }
 797 }
 798
 799 /*
 800  * Net VSC on send
 801  * Sends a packet on the specified Hyper-V device.
 802  * Returns 0 on success, non-zero on failure.
 803  */
 804 int
 805 hv_nv_on_send(struct hv_vmbus_channel *chan, netvsc_packet *pkt)
 806 {
 807         nvsp_msg send_msg;
 808         int ret;
 809
 810         send_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt;
 811         if (pkt->is_data_pkt) {
 812                 /* 0 is RMC_DATA */
 813                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 0;
 814         } else {
 815                 /* 1 is RMC_CONTROL */
 816                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 1;
 817         }
 818
 819         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_idx =
 820             pkt->send_buf_section_idx;
 821         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_size =
 822             pkt->send_buf_section_size;
 823
 824         if (pkt->gpa_cnt) {
 825                 ret = vmbus_chan_send_sglist(chan, pkt->gpa, pkt->gpa_cnt,
 826                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt);
 827         } else {
 828                 ret = vmbus_chan_send(chan,
 829                     VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 830                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt);
 831         }
 832
 833         return (ret);
 834 }
 835
 836 /*
 837  * Net VSC on receive
 838  *
 839  * In the FreeBSD Hyper-V virtual world, this function deals exclusively
 840  * with virtual addresses.
 841  */
 842 static void
 843 hv_nv_on_receive(netvsc_dev *net_dev, struct hn_softc *sc,
 844     struct hv_vmbus_channel *chan, const struct vmbus_chanpkt_hdr *pkthdr)
 845 {
 846         const struct vmbus_chanpkt_rxbuf *pkt;
 847         const nvsp_msg *nvsp_msg_pkt;
 848         netvsc_packet vsc_pkt;
 849         netvsc_packet *net_vsc_pkt = &vsc_pkt;
 850         device_t dev = sc->hn_dev;
 851         int count = 0;
 852         int i = 0;
 853         int status = nvsp_status_success;
 854
 855         nvsp_msg_pkt = VMBUS_CHANPKT_CONST_DATA(pkthdr);
 856
 857         /* Make sure this is a valid nvsp packet */
 858         if (nvsp_msg_pkt->hdr.msg_type != nvsp_msg_1_type_send_rndis_pkt) {
 859                 device_printf(dev, "packet hdr type %u is invalid!\n",
 860                     nvsp_msg_pkt->hdr.msg_type);
 861                 return;
 862         }
 863
 864         pkt = (const struct vmbus_chanpkt_rxbuf *)pkthdr;
 865
 866         if (pkt->cp_rxbuf_id != NETVSC_RECEIVE_BUFFER_ID) {
 867                 device_printf(dev, "rxbuf_id %d is invalid!\n",
 868                     pkt->cp_rxbuf_id);
 869                 return;
 870         }
 871
 872         count = pkt->cp_rxbuf_cnt;
 873
 874         /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */
 875         for (i = 0; i < count; i++) {
 876                 net_vsc_pkt->status = nvsp_status_success;
 877                 net_vsc_pkt->data = ((uint8_t *)net_dev->rx_buf +
 878                     pkt->cp_rxbuf[i].rb_ofs);
 879                 net_vsc_pkt->tot_data_buf_len = pkt->cp_rxbuf[i].rb_len;
 880
 881                 hv_rf_on_receive(net_dev, chan, net_vsc_pkt);
 882                 if (net_vsc_pkt->status != nvsp_status_success) {
 883                         status = nvsp_status_failure;
 884                 }
 885         }
 886
 887         /*
 888          * Moved completion call back here so that all received
 889          * messages (not just data messages) will trigger a response
 890          * message back to the host.
 891          */
 892         hv_nv_on_receive_completion(chan, pkt->cp_hdr.cph_xactid, status);
 893 }
 894
 895 /*
 896  * Net VSC on receive completion
 897  *
 898  * Send a receive completion packet to RNDIS device (ie NetVsp)
 899  */
 900 static void
 901 hv_nv_on_receive_completion(struct hv_vmbus_channel *chan, uint64_t tid,
 902     uint32_t status)
 903 {
 904         nvsp_msg rx_comp_msg;
 905         int retries = 0;
 906         int ret = 0;
 907
 908         rx_comp_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt_complete;
 909
 910         /* Pass in the status */
 911         rx_comp_msg.msgs.vers_1_msgs.send_rndis_pkt_complete.status =
 912             status;
 913
 914 retry_send_cmplt:
 915         /* Send the completion */
 916         ret = vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP, 0,
 917             &rx_comp_msg, sizeof(nvsp_msg), tid);
 918         if (ret == 0) {
 919                 /* success */
 920                 /* no-op */
 921         } else if (ret == EAGAIN) {
 922                 /* no more room... wait a bit and attempt to retry 3 times */
 923                 retries++;
 924
 925                 if (retries < 4) {
 926                         DELAY(100);
 927                         goto retry_send_cmplt;
 928                 }
 929         }
 930 }
 931
 932 /*
 933  * Net VSC receiving vRSS send table from VSP
 934  */
 935 static void
 936 hv_nv_send_table(struct hn_softc *sc, const struct vmbus_chanpkt_hdr *pkt)
 937 {
 938         netvsc_dev *net_dev;
 939         const nvsp_msg *nvsp_msg_pkt;
 940         int i;
 941         uint32_t count;
 942         const uint32_t *table;
 943
 944         net_dev = hv_nv_get_inbound_net_device(sc);
 945         if (!net_dev)
 946                 return;
 947
 948         nvsp_msg_pkt = VMBUS_CHANPKT_CONST_DATA(pkt);
 949
 950         if (nvsp_msg_pkt->hdr.msg_type !=
 951             nvsp_msg5_type_send_indirection_table) {
 952                 printf("Netvsc: !Warning! receive msg type not "
 953                         "send_indirection_table. type = %d\n",
 954                         nvsp_msg_pkt->hdr.msg_type);
 955                 return;
 956         }
 957
 958         count = nvsp_msg_pkt->msgs.vers_5_msgs.send_table.count;
 959         if (count != VRSS_SEND_TABLE_SIZE) {
 960                 printf("Netvsc: Received wrong send table size: %u\n", count);
 961                 return;
 962         }
 963
 964         table = (const uint32_t *)
 965             ((const uint8_t *)&nvsp_msg_pkt->msgs.vers_5_msgs.send_table +
 966              nvsp_msg_pkt->msgs.vers_5_msgs.send_table.offset);
 967
 968         for (i = 0; i < count; i++)
 969                 net_dev->vrss_send_table[i] = table[i];
 970 }
 971
 972 /*
 973  * Net VSC on channel callback
 974  */
 975 static void
 976 hv_nv_on_channel_callback(void *xchan)
 977 {
 978         struct hv_vmbus_channel *chan = xchan;
 979         device_t dev = chan->ch_dev;
 980         struct hn_softc *sc = device_get_softc(dev);
 981         netvsc_dev *net_dev;
 982         void *buffer;
 983         int bufferlen = NETVSC_PACKET_SIZE;
 984
 985         net_dev = hv_nv_get_inbound_net_device(sc);
 986         if (net_dev == NULL)
 987                 return;
 988
 989         buffer = chan->hv_chan_rdbuf;
 990         do {
 991                 struct vmbus_chanpkt_hdr *pkt = buffer;
 992                 uint32_t bytes_rxed;
 993                 int ret;
 994
 995                 bytes_rxed = bufferlen;
 996                 ret = vmbus_chan_recv_pkt(chan, pkt, &bytes_rxed);
 997                 if (ret == 0) {
 998                         if (bytes_rxed > 0) {
 999                                 switch (pkt->cph_type) {
1000                                 case VMBUS_CHANPKT_TYPE_COMP:
1001                                         hv_nv_on_send_completion(net_dev, chan,
1002                                             pkt);
1003                                         break;
1004                                 case VMBUS_CHANPKT_TYPE_RXBUF:
1005                                         hv_nv_on_receive(net_dev, sc, chan, pkt);
1006                                         break;
1007                                 case VMBUS_CHANPKT_TYPE_INBAND:
1008                                         hv_nv_send_table(sc, pkt);
1009                                         break;
1010                                 default:
1011                                         device_printf(dev,
1012                                             "unknown chan pkt %u\n",
1013                                             pkt->cph_type);
1014                                         break;
1015                                 }
1016                         }
1017                 } else if (ret == ENOBUFS) {
1018                         /* Handle large packet */
1019                         if (bufferlen > NETVSC_PACKET_SIZE) {
1020                                 free(buffer, M_NETVSC);
1021                                 buffer = NULL;
1022                         }
1023
1024                         /* alloc new buffer */
1025                         buffer = malloc(bytes_rxed, M_NETVSC, M_NOWAIT);
1026                         if (buffer == NULL) {
1027                                 device_printf(dev,
1028                                     "hv_cb malloc buffer failed, len=%u\n",
1029                                     bytes_rxed);
1030                                 bufferlen = 0;
1031                                 break;
1032                         }
1033                         bufferlen = bytes_rxed;
1034                 } else {
1035                         /* No more packets */
1036                         break;
1037                 }
1038         } while (1);
1039
1040         if (bufferlen > NETVSC_PACKET_SIZE)
1041                 free(buffer, M_NETVSC);
1042
1043         hv_rf_channel_rollup(chan);
1044 }