sys/dev/hyperv/netvsc/hv_net_vsc.c

   1 /*-
   2  * Copyright (c) 2009-2012 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 /**
  32  * HyperV vmbus network VSC (virtual services client) module
  33  *
  34  */
  35
  36
  37 #include <sys/param.h>
  38 #include <sys/kernel.h>
  39 #include <sys/socket.h>
  40 #include <sys/lock.h>
  41 #include <net/if.h>
  42 #include <net/if_arp.h>
  43 #include <machine/bus.h>
  44 #include <machine/atomic.h>
  45
  46 #include <dev/hyperv/include/hyperv.h>
  47 #include "hv_net_vsc.h"
  48 #include "hv_rndis.h"
  49 #include "hv_rndis_filter.h"
  50
  51 /* priv1 and priv2 are consumed by the main driver */
  52 #define hv_chan_rdbuf   hv_chan_priv3
  53
  54 MALLOC_DEFINE(M_NETVSC, "netvsc", "Hyper-V netvsc driver");
  55
  56 /*
  57  * Forward declarations
  58  */
  59 static void hv_nv_on_channel_callback(void *xchan);
  60 static int  hv_nv_init_send_buffer_with_net_vsp(struct hv_device *device);
  61 static int  hv_nv_init_rx_buffer_with_net_vsp(struct hv_device *device);
  62 static int  hv_nv_destroy_send_buffer(netvsc_dev *net_dev);
  63 static int  hv_nv_destroy_rx_buffer(netvsc_dev *net_dev);
  64 static int  hv_nv_connect_to_vsp(struct hv_device *device);
  65 static void hv_nv_on_send_completion(netvsc_dev *net_dev,
  66     struct hv_device *device, struct hv_vmbus_channel *, hv_vm_packet_descriptor *pkt);
  67 static void hv_nv_on_receive_completion(struct hv_vmbus_channel *chan,
  68     uint64_t tid, uint32_t status);
  69 static void hv_nv_on_receive(netvsc_dev *net_dev,
  70     struct hv_device *device, struct hv_vmbus_channel *chan,
  71     hv_vm_packet_descriptor *pkt);
  72
  73 /*
  74  *
  75  */
  76 static inline netvsc_dev *
  77 hv_nv_alloc_net_device(struct hv_device *device)
  78 {
  79         netvsc_dev *net_dev;
  80         hn_softc_t *sc = device_get_softc(device->device);
  81
  82         net_dev = malloc(sizeof(netvsc_dev), M_NETVSC, M_WAITOK | M_ZERO);
  83
  84         net_dev->dev = device;
  85         net_dev->destroy = FALSE;
  86         sc->net_dev = net_dev;
  87
  88         return (net_dev);
  89 }
  90
  91 /*
  92  *
  93  */
  94 static inline netvsc_dev *
  95 hv_nv_get_outbound_net_device(struct hv_device *device)
  96 {
  97         hn_softc_t *sc = device_get_softc(device->device);
  98         netvsc_dev *net_dev = sc->net_dev;;
  99
 100         if ((net_dev != NULL) && net_dev->destroy) {
 101                 return (NULL);
 102         }
 103
 104         return (net_dev);
 105 }
 106
 107 /*
 108  *
 109  */
 110 static inline netvsc_dev *
 111 hv_nv_get_inbound_net_device(struct hv_device *device)
 112 {
 113         hn_softc_t *sc = device_get_softc(device->device);
 114         netvsc_dev *net_dev = sc->net_dev;;
 115
 116         if (net_dev == NULL) {
 117                 return (net_dev);
 118         }
 119         /*
 120          * When the device is being destroyed; we only
 121          * permit incoming packets if and only if there
 122          * are outstanding sends.
 123          */
 124         if (net_dev->destroy) {
 125                 return (NULL);
 126         }
 127
 128         return (net_dev);
 129 }
 130
 131 int
 132 hv_nv_get_next_send_section(netvsc_dev *net_dev)
 133 {
 134         unsigned long bitsmap_words = net_dev->bitsmap_words;
 135         unsigned long *bitsmap = net_dev->send_section_bitsmap;
 136         unsigned long idx;
 137         int ret = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
 138         int i;
 139
 140         for (i = 0; i < bitsmap_words; i++) {
 141                 idx = ffsl(~bitsmap[i]);
 142                 if (0 == idx)
 143                         continue;
 144
 145                 idx--;
 146                 KASSERT(i * BITS_PER_LONG + idx < net_dev->send_section_count,
 147                     ("invalid i %d and idx %lu", i, idx));
 148
 149                 if (atomic_testandset_long(&bitsmap[i], idx))
 150                         continue;
 151
 152                 ret = i * BITS_PER_LONG + idx;
 153                 break;
 154         }
 155
 156         return (ret);
 157 }
 158
 159 /*
 160  * Net VSC initialize receive buffer with net VSP
 161  *
 162  * Net VSP:  Network virtual services client, also known as the
 163  *     Hyper-V extensible switch and the synthetic data path.
 164  */
 165 static int
 166 hv_nv_init_rx_buffer_with_net_vsp(struct hv_device *device)
 167 {
 168         netvsc_dev *net_dev;
 169         nvsp_msg *init_pkt;
 170         int ret = 0;
 171
 172         net_dev = hv_nv_get_outbound_net_device(device);
 173         if (!net_dev) {
 174                 return (ENODEV);
 175         }
 176
 177         net_dev->rx_buf = contigmalloc(net_dev->rx_buf_size, M_NETVSC,
 178             M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 179
 180         /*
 181          * Establish the GPADL handle for this buffer on this channel.
 182          * Note:  This call uses the vmbus connection rather than the
 183          * channel to establish the gpadl handle.
 184          * GPADL:  Guest physical address descriptor list.
 185          */
 186         ret = hv_vmbus_channel_establish_gpadl(
 187                 device->channel, net_dev->rx_buf,
 188                 net_dev->rx_buf_size, &net_dev->rx_buf_gpadl_handle);
 189         if (ret != 0) {
 190                 goto cleanup;
 191         }
 192
 193         /* sema_wait(&ext->channel_init_sema); KYS CHECK */
 194
 195         /* Notify the NetVsp of the gpadl handle */
 196         init_pkt = &net_dev->channel_init_packet;
 197
 198         memset(init_pkt, 0, sizeof(nvsp_msg));
 199
 200         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_rx_buf;
 201         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 202             net_dev->rx_buf_gpadl_handle;
 203         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 204             NETVSC_RECEIVE_BUFFER_ID;
 205
 206         /* Send the gpadl notification request */
 207
 208         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 209             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 210             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 211             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 212         if (ret != 0) {
 213                 goto cleanup;
 214         }
 215
 216         sema_wait(&net_dev->channel_init_sema);
 217
 218         /* Check the response */
 219         if (init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.status
 220             != nvsp_status_success) {
 221                 ret = EINVAL;
 222                 goto cleanup;
 223         }
 224
 225         net_dev->rx_section_count =
 226             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.num_sections;
 227
 228         net_dev->rx_sections = malloc(net_dev->rx_section_count *
 229             sizeof(nvsp_1_rx_buf_section), M_NETVSC, M_WAITOK);
 230         memcpy(net_dev->rx_sections,
 231             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.sections,
 232             net_dev->rx_section_count * sizeof(nvsp_1_rx_buf_section));
 233
 234
 235         /*
 236          * For first release, there should only be 1 section that represents
 237          * the entire receive buffer
 238          */
 239         if (net_dev->rx_section_count != 1
 240             || net_dev->rx_sections->offset != 0) {
 241                 ret = EINVAL;
 242                 goto cleanup;
 243         }
 244
 245         goto exit;
 246
 247 cleanup:
 248         hv_nv_destroy_rx_buffer(net_dev);
 249
 250 exit:
 251         return (ret);
 252 }
 253
 254 /*
 255  * Net VSC initialize send buffer with net VSP
 256  */
 257 static int
 258 hv_nv_init_send_buffer_with_net_vsp(struct hv_device *device)
 259 {
 260         netvsc_dev *net_dev;
 261         nvsp_msg *init_pkt;
 262         int ret = 0;
 263
 264         net_dev = hv_nv_get_outbound_net_device(device);
 265         if (!net_dev) {
 266                 return (ENODEV);
 267         }
 268
 269         net_dev->send_buf  = contigmalloc(net_dev->send_buf_size, M_NETVSC,
 270             M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 271         if (net_dev->send_buf == NULL) {
 272                 ret = ENOMEM;
 273                 goto cleanup;
 274         }
 275
 276         /*
 277          * Establish the gpadl handle for this buffer on this channel.
 278          * Note:  This call uses the vmbus connection rather than the
 279          * channel to establish the gpadl handle.
 280          */
 281         ret = hv_vmbus_channel_establish_gpadl(device->channel,
 282             net_dev->send_buf, net_dev->send_buf_size,
 283             &net_dev->send_buf_gpadl_handle);
 284         if (ret != 0) {
 285                 goto cleanup;
 286         }
 287
 288         /* Notify the NetVsp of the gpadl handle */
 289
 290         init_pkt = &net_dev->channel_init_packet;
 291
 292         memset(init_pkt, 0, sizeof(nvsp_msg));
 293
 294         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_send_buf;
 295         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 296             net_dev->send_buf_gpadl_handle;
 297         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 298             NETVSC_SEND_BUFFER_ID;
 299
 300         /* Send the gpadl notification request */
 301
 302         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 303             sizeof(nvsp_msg), (uint64_t)init_pkt,
 304             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 305             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 306         if (ret != 0) {
 307                 goto cleanup;
 308         }
 309
 310         sema_wait(&net_dev->channel_init_sema);
 311
 312         /* Check the response */
 313         if (init_pkt->msgs.vers_1_msgs.send_send_buf_complete.status
 314             != nvsp_status_success) {
 315                 ret = EINVAL;
 316                 goto cleanup;
 317         }
 318
 319         net_dev->send_section_size =
 320             init_pkt->msgs.vers_1_msgs.send_send_buf_complete.section_size;
 321         net_dev->send_section_count =
 322             net_dev->send_buf_size / net_dev->send_section_size;
 323         net_dev->bitsmap_words = howmany(net_dev->send_section_count,
 324             BITS_PER_LONG);
 325         net_dev->send_section_bitsmap =
 326             malloc(net_dev->bitsmap_words * sizeof(long), M_NETVSC,
 327             M_WAITOK | M_ZERO);
 328
 329         goto exit;
 330
 331 cleanup:
 332         hv_nv_destroy_send_buffer(net_dev);
 333
 334 exit:
 335         return (ret);
 336 }
 337
 338 /*
 339  * Net VSC destroy receive buffer
 340  */
 341 static int
 342 hv_nv_destroy_rx_buffer(netvsc_dev *net_dev)
 343 {
 344         nvsp_msg *revoke_pkt;
 345         int ret = 0;
 346
 347         /*
 348          * If we got a section count, it means we received a
 349          * send_rx_buf_complete msg
 350          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 351          * we need to send a revoke msg here
 352          */
 353         if (net_dev->rx_section_count) {
 354                 /* Send the revoke receive buffer */
 355                 revoke_pkt = &net_dev->revoke_packet;
 356                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 357
 358                 revoke_pkt->hdr.msg_type = nvsp_msg_1_type_revoke_rx_buf;
 359                 revoke_pkt->msgs.vers_1_msgs.revoke_rx_buf.id =
 360                     NETVSC_RECEIVE_BUFFER_ID;
 361
 362                 ret = hv_vmbus_channel_send_packet(net_dev->dev->channel,
 363                     revoke_pkt, sizeof(nvsp_msg),
 364                     (uint64_t)(uintptr_t)revoke_pkt,
 365                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 366
 367                 /*
 368                  * If we failed here, we might as well return and have a leak
 369                  * rather than continue and a bugchk
 370                  */
 371                 if (ret != 0) {
 372                         return (ret);
 373                 }
 374         }
 375
 376         /* Tear down the gpadl on the vsp end */
 377         if (net_dev->rx_buf_gpadl_handle) {
 378                 ret = hv_vmbus_channel_teardown_gpdal(net_dev->dev->channel,
 379                     net_dev->rx_buf_gpadl_handle);
 380                 /*
 381                  * If we failed here, we might as well return and have a leak
 382                  * rather than continue and a bugchk
 383                  */
 384                 if (ret != 0) {
 385                         return (ret);
 386                 }
 387                 net_dev->rx_buf_gpadl_handle = 0;
 388         }
 389
 390         if (net_dev->rx_buf) {
 391                 /* Free up the receive buffer */
 392                 contigfree(net_dev->rx_buf, net_dev->rx_buf_size, M_NETVSC);
 393                 net_dev->rx_buf = NULL;
 394         }
 395
 396         if (net_dev->rx_sections) {
 397                 free(net_dev->rx_sections, M_NETVSC);
 398                 net_dev->rx_sections = NULL;
 399                 net_dev->rx_section_count = 0;
 400         }
 401
 402         return (ret);
 403 }
 404
 405 /*
 406  * Net VSC destroy send buffer
 407  */
 408 static int
 409 hv_nv_destroy_send_buffer(netvsc_dev *net_dev)
 410 {
 411         nvsp_msg *revoke_pkt;
 412         int ret = 0;
 413
 414         /*
 415          * If we got a section count, it means we received a
 416          * send_rx_buf_complete msg
 417          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 418          * we need to send a revoke msg here
 419          */
 420         if (net_dev->send_section_size) {
 421                 /* Send the revoke send buffer */
 422                 revoke_pkt = &net_dev->revoke_packet;
 423                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 424
 425                 revoke_pkt->hdr.msg_type =
 426                     nvsp_msg_1_type_revoke_send_buf;
 427                 revoke_pkt->msgs.vers_1_msgs.revoke_send_buf.id =
 428                     NETVSC_SEND_BUFFER_ID;
 429
 430                 ret = hv_vmbus_channel_send_packet(net_dev->dev->channel,
 431                     revoke_pkt, sizeof(nvsp_msg),
 432                     (uint64_t)(uintptr_t)revoke_pkt,
 433                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 434                 /*
 435                  * If we failed here, we might as well return and have a leak
 436                  * rather than continue and a bugchk
 437                  */
 438                 if (ret != 0) {
 439                         return (ret);
 440                 }
 441         }
 442
 443         /* Tear down the gpadl on the vsp end */
 444         if (net_dev->send_buf_gpadl_handle) {
 445                 ret = hv_vmbus_channel_teardown_gpdal(net_dev->dev->channel,
 446                     net_dev->send_buf_gpadl_handle);
 447
 448                 /*
 449                  * If we failed here, we might as well return and have a leak
 450                  * rather than continue and a bugchk
 451                  */
 452                 if (ret != 0) {
 453                         return (ret);
 454                 }
 455                 net_dev->send_buf_gpadl_handle = 0;
 456         }
 457
 458         if (net_dev->send_buf) {
 459                 /* Free up the receive buffer */
 460                 contigfree(net_dev->send_buf, net_dev->send_buf_size, M_NETVSC);
 461                 net_dev->send_buf = NULL;
 462         }
 463
 464         if (net_dev->send_section_bitsmap) {
 465                 free(net_dev->send_section_bitsmap, M_NETVSC);
 466         }
 467
 468         return (ret);
 469 }
 470
 471
 472 /*
 473  * Attempt to negotiate the caller-specified NVSP version
 474  *
 475  * For NVSP v2, Server 2008 R2 does not set
 476  * init_pkt->msgs.init_msgs.init_compl.negotiated_prot_vers
 477  * to the negotiated version, so we cannot rely on that.
 478  */
 479 static int
 480 hv_nv_negotiate_nvsp_protocol(struct hv_device *device, netvsc_dev *net_dev,
 481     uint32_t nvsp_ver)
 482 {
 483         nvsp_msg *init_pkt;
 484         int ret;
 485
 486         init_pkt = &net_dev->channel_init_packet;
 487         memset(init_pkt, 0, sizeof(nvsp_msg));
 488         init_pkt->hdr.msg_type = nvsp_msg_type_init;
 489
 490         /*
 491          * Specify parameter as the only acceptable protocol version
 492          */
 493         init_pkt->msgs.init_msgs.init.p1.protocol_version = nvsp_ver;
 494         init_pkt->msgs.init_msgs.init.protocol_version_2 = nvsp_ver;
 495
 496         /* Send the init request */
 497         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 498             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 499             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 500             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 501         if (ret != 0)
 502                 return (-1);
 503
 504         sema_wait(&net_dev->channel_init_sema);
 505
 506         if (init_pkt->msgs.init_msgs.init_compl.status != nvsp_status_success)
 507                 return (EINVAL);
 508
 509         return (0);
 510 }
 511
 512 /*
 513  * Send NDIS version 2 config packet containing MTU.
 514  *
 515  * Not valid for NDIS version 1.
 516  */
 517 static int
 518 hv_nv_send_ndis_config(struct hv_device *device, uint32_t mtu)
 519 {
 520         netvsc_dev *net_dev;
 521         nvsp_msg *init_pkt;
 522         int ret;
 523
 524         net_dev = hv_nv_get_outbound_net_device(device);
 525         if (!net_dev)
 526                 return (-ENODEV);
 527
 528         /*
 529          * Set up configuration packet, write MTU
 530          * Indicate we are capable of handling VLAN tags
 531          */
 532         init_pkt = &net_dev->channel_init_packet;
 533         memset(init_pkt, 0, sizeof(nvsp_msg));
 534         init_pkt->hdr.msg_type = nvsp_msg_2_type_send_ndis_config;
 535         init_pkt->msgs.vers_2_msgs.send_ndis_config.mtu = mtu;
 536         init_pkt->
 537                 msgs.vers_2_msgs.send_ndis_config.capabilities.u1.u2.ieee8021q
 538                 = 1;
 539
 540         /* Send the configuration packet */
 541         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 542             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 543             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 544         if (ret != 0)
 545                 return (-EINVAL);
 546
 547         return (0);
 548 }
 549
 550 /*
 551  * Net VSC connect to VSP
 552  */
 553 static int
 554 hv_nv_connect_to_vsp(struct hv_device *device)
 555 {
 556         netvsc_dev *net_dev;
 557         nvsp_msg *init_pkt;
 558         uint32_t ndis_version;
 559         uint32_t protocol_list[] = { NVSP_PROTOCOL_VERSION_1,
 560             NVSP_PROTOCOL_VERSION_2,
 561             NVSP_PROTOCOL_VERSION_4,
 562             NVSP_PROTOCOL_VERSION_5 };
 563         int i;
 564         int protocol_number = nitems(protocol_list);
 565         int ret = 0;
 566         device_t dev = device->device;
 567         hn_softc_t *sc = device_get_softc(dev);
 568         struct ifnet *ifp = sc->arpcom.ac_ifp;
 569
 570         net_dev = hv_nv_get_outbound_net_device(device);
 571         if (!net_dev) {
 572                 return (ENODEV);
 573         }
 574
 575         /*
 576          * Negotiate the NVSP version.  Try the latest NVSP first.
 577          */
 578         for (i = protocol_number - 1; i >= 0; i--) {
 579                 if (hv_nv_negotiate_nvsp_protocol(device, net_dev,
 580                     protocol_list[i]) == 0) {
 581                         net_dev->nvsp_version = protocol_list[i];
 582                         if (bootverbose)
 583                                 device_printf(dev, "Netvsc: got version 0x%x\n",
 584                                     net_dev->nvsp_version);
 585                         break;
 586                 }
 587         }
 588
 589         if (i < 0) {
 590                 if (bootverbose)
 591                         device_printf(dev, "failed to negotiate a valid "
 592                             "protocol.\n");
 593                 return (EPROTO);
 594         }
 595
 596         /*
 597          * Set the MTU if supported by this NVSP protocol version
 598          * This needs to be right after the NVSP init message per Haiyang
 599          */
 600         if (net_dev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
 601                 ret = hv_nv_send_ndis_config(device, ifp->if_mtu);
 602
 603         /*
 604          * Send the NDIS version
 605          */
 606         init_pkt = &net_dev->channel_init_packet;
 607
 608         memset(init_pkt, 0, sizeof(nvsp_msg));
 609
 610         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_4) {
 611                 ndis_version = NDIS_VERSION_6_1;
 612         } else {
 613                 ndis_version = NDIS_VERSION_6_30;
 614         }
 615
 616         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_ndis_vers;
 617         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_major_vers =
 618             (ndis_version & 0xFFFF0000) >> 16;
 619         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_minor_vers =
 620             ndis_version & 0xFFFF;
 621
 622         /* Send the init request */
 623
 624         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 625             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 626             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 627         if (ret != 0) {
 628                 goto cleanup;
 629         }
 630         /*
 631          * TODO:  BUGBUG - We have to wait for the above msg since the netvsp
 632          * uses KMCL which acknowledges packet (completion packet)
 633          * since our Vmbus always set the
 634          * HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED flag
 635          */
 636         /* sema_wait(&NetVscChannel->channel_init_sema); */
 637
 638         /* Post the big receive buffer to NetVSP */
 639         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
 640                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
 641         else
 642                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
 643         net_dev->send_buf_size = NETVSC_SEND_BUFFER_SIZE;
 644
 645         ret = hv_nv_init_rx_buffer_with_net_vsp(device);
 646         if (ret == 0)
 647                 ret = hv_nv_init_send_buffer_with_net_vsp(device);
 648
 649 cleanup:
 650         return (ret);
 651 }
 652
 653 /*
 654  * Net VSC disconnect from VSP
 655  */
 656 static void
 657 hv_nv_disconnect_from_vsp(netvsc_dev *net_dev)
 658 {
 659         hv_nv_destroy_rx_buffer(net_dev);
 660         hv_nv_destroy_send_buffer(net_dev);
 661 }
 662
 663 /*
 664  * Callback handler for subchannel offer
 665  * @@param context new subchannel
 666  */
 667 static void
 668 hv_nv_subchan_callback(void *xchan)
 669 {
 670         struct hv_vmbus_channel *chan = xchan;
 671         netvsc_dev *net_dev;
 672         uint16_t chn_index = chan->offer_msg.offer.sub_channel_index;
 673         struct hv_device *device = chan->device;
 674         hn_softc_t *sc = device_get_softc(device->device);
 675         int ret;
 676
 677         net_dev = sc->net_dev;
 678
 679         if (chn_index >= net_dev->num_channel) {
 680                 /* Would this ever happen? */
 681                 return;
 682         }
 683         netvsc_subchan_callback(sc, chan);
 684
 685         chan->hv_chan_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK);
 686         ret = hv_vmbus_channel_open(chan, NETVSC_DEVICE_RING_BUFFER_SIZE,
 687             NETVSC_DEVICE_RING_BUFFER_SIZE, NULL, 0,
 688             hv_nv_on_channel_callback, chan);
 689 }
 690
 691 /*
 692  * Net VSC on device add
 693  *
 694  * Callback when the device belonging to this driver is added
 695  */
 696 netvsc_dev *
 697 hv_nv_on_device_add(struct hv_device *device, void *additional_info)
 698 {
 699         struct hv_vmbus_channel *chan = device->channel;
 700         netvsc_dev *net_dev;
 701         int ret = 0;
 702
 703         net_dev = hv_nv_alloc_net_device(device);
 704         if (net_dev == NULL)
 705                 return NULL;
 706
 707         /* Initialize the NetVSC channel extension */
 708
 709         sema_init(&net_dev->channel_init_sema, 0, "netdev_sema");
 710
 711         chan->hv_chan_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK);
 712
 713         /*
 714          * Open the channel
 715          */
 716         ret = hv_vmbus_channel_open(chan,
 717             NETVSC_DEVICE_RING_BUFFER_SIZE, NETVSC_DEVICE_RING_BUFFER_SIZE,
 718             NULL, 0, hv_nv_on_channel_callback, chan);
 719         if (ret != 0) {
 720                 free(chan->hv_chan_rdbuf, M_NETVSC);
 721                 goto cleanup;
 722         }
 723         chan->sc_creation_callback = hv_nv_subchan_callback;
 724
 725         /*
 726          * Connect with the NetVsp
 727          */
 728         ret = hv_nv_connect_to_vsp(device);
 729         if (ret != 0)
 730                 goto close;
 731
 732         return (net_dev);
 733
 734 close:
 735         /* Now, we can close the channel safely */
 736         free(chan->hv_chan_rdbuf, M_NETVSC);
 737         hv_vmbus_channel_close(chan);
 738
 739 cleanup:
 740         /*
 741          * Free the packet buffers on the netvsc device packet queue.
 742          * Release other resources.
 743          */
 744         sema_destroy(&net_dev->channel_init_sema);
 745         free(net_dev, M_NETVSC);
 746
 747         return (NULL);
 748 }
 749
 750 /*
 751  * Net VSC on device remove
 752  */
 753 int
 754 hv_nv_on_device_remove(struct hv_device *device, boolean_t destroy_channel)
 755 {
 756         hn_softc_t *sc = device_get_softc(device->device);
 757         netvsc_dev *net_dev = sc->net_dev;;
 758
 759         /* Stop outbound traffic ie sends and receives completions */
 760         net_dev->destroy = TRUE;
 761
 762         hv_nv_disconnect_from_vsp(net_dev);
 763
 764         /* At this point, no one should be accessing net_dev except in here */
 765
 766         /* Now, we can close the channel safely */
 767
 768         if (!destroy_channel) {
 769                 device->channel->state =
 770                     HV_CHANNEL_CLOSING_NONDESTRUCTIVE_STATE;
 771         }
 772
 773         free(device->channel->hv_chan_rdbuf, M_NETVSC);
 774         hv_vmbus_channel_close(device->channel);
 775
 776         sema_destroy(&net_dev->channel_init_sema);
 777         free(net_dev, M_NETVSC);
 778
 779         return (0);
 780 }
 781
 782 /*
 783  * Net VSC on send completion
 784  */
 785 static void
 786 hv_nv_on_send_completion(netvsc_dev *net_dev,
 787     struct hv_device *device, struct hv_vmbus_channel *chan,
 788     hv_vm_packet_descriptor *pkt)
 789 {
 790         nvsp_msg *nvsp_msg_pkt;
 791         netvsc_packet *net_vsc_pkt;
 792
 793         nvsp_msg_pkt =
 794             (nvsp_msg *)((unsigned long)pkt + (pkt->data_offset8 << 3));
 795
 796         if (nvsp_msg_pkt->hdr.msg_type == nvsp_msg_type_init_complete
 797                 || nvsp_msg_pkt->hdr.msg_type
 798                         == nvsp_msg_1_type_send_rx_buf_complete
 799                 || nvsp_msg_pkt->hdr.msg_type
 800                         == nvsp_msg_1_type_send_send_buf_complete
 801                 || nvsp_msg_pkt->hdr.msg_type
 802                         == nvsp_msg5_type_subchannel) {
 803                 /* Copy the response back */
 804                 memcpy(&net_dev->channel_init_packet, nvsp_msg_pkt,
 805                     sizeof(nvsp_msg));
 806                 sema_post(&net_dev->channel_init_sema);
 807         } else if (nvsp_msg_pkt->hdr.msg_type ==
 808                     nvsp_msg_1_type_send_rndis_pkt_complete) {
 809                 /* Get the send context */
 810                 net_vsc_pkt =
 811                     (netvsc_packet *)(unsigned long)pkt->transaction_id;
 812                 if (NULL != net_vsc_pkt) {
 813                         if (net_vsc_pkt->send_buf_section_idx !=
 814                             NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) {
 815                                 u_long mask;
 816                                 int idx;
 817
 818                                 idx = net_vsc_pkt->send_buf_section_idx /
 819                                     BITS_PER_LONG;
 820                                 KASSERT(idx < net_dev->bitsmap_words,
 821                                     ("invalid section index %u",
 822                                      net_vsc_pkt->send_buf_section_idx));
 823                                 mask = 1UL <<
 824                                     (net_vsc_pkt->send_buf_section_idx %
 825                                      BITS_PER_LONG);
 826
 827                                 KASSERT(net_dev->send_section_bitsmap[idx] &
 828                                     mask,
 829                                     ("index bitmap 0x%lx, section index %u, "
 830                                      "bitmap idx %d, bitmask 0x%lx",
 831                                      net_dev->send_section_bitsmap[idx],
 832                                      net_vsc_pkt->send_buf_section_idx,
 833                                      idx, mask));
 834                                 atomic_clear_long(
 835                                     &net_dev->send_section_bitsmap[idx], mask);
 836                         }
 837
 838                         /* Notify the layer above us */
 839                         net_vsc_pkt->compl.send.on_send_completion(chan,
 840                             net_vsc_pkt->compl.send.send_completion_context);
 841
 842                 }
 843         }
 844 }
 845
 846 /*
 847  * Net VSC on send
 848  * Sends a packet on the specified Hyper-V device.
 849  * Returns 0 on success, non-zero on failure.
 850  */
 851 int
 852 hv_nv_on_send(struct hv_vmbus_channel *chan, netvsc_packet *pkt)
 853 {
 854         nvsp_msg send_msg;
 855         int ret;
 856
 857         send_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt;
 858         if (pkt->is_data_pkt) {
 859                 /* 0 is RMC_DATA */
 860                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 0;
 861         } else {
 862                 /* 1 is RMC_CONTROL */
 863                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 1;
 864         }
 865
 866         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_idx =
 867             pkt->send_buf_section_idx;
 868         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_size =
 869             pkt->send_buf_section_size;
 870
 871         if (pkt->page_buf_count) {
 872                 ret = hv_vmbus_channel_send_packet_pagebuffer(chan,
 873                     pkt->page_buffers, pkt->page_buf_count,
 874                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt);
 875         } else {
 876                 ret = hv_vmbus_channel_send_packet(chan,
 877                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt,
 878                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 879                     HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 880         }
 881
 882         return (ret);
 883 }
 884
 885 /*
 886  * Net VSC on receive
 887  *
 888  * In the FreeBSD Hyper-V virtual world, this function deals exclusively
 889  * with virtual addresses.
 890  */
 891 static void
 892 hv_nv_on_receive(netvsc_dev *net_dev, struct hv_device *device,
 893     struct hv_vmbus_channel *chan, hv_vm_packet_descriptor *pkt)
 894 {
 895         hv_vm_transfer_page_packet_header *vm_xfer_page_pkt;
 896         nvsp_msg *nvsp_msg_pkt;
 897         netvsc_packet vsc_pkt;
 898         netvsc_packet *net_vsc_pkt = &vsc_pkt;
 899         device_t dev = device->device;
 900         int count = 0;
 901         int i = 0;
 902         int status = nvsp_status_success;
 903
 904         /*
 905          * All inbound packets other than send completion should be
 906          * xfer page packet.
 907          */
 908         if (pkt->type != HV_VMBUS_PACKET_TYPE_DATA_USING_TRANSFER_PAGES) {
 909                 device_printf(dev, "packet type %d is invalid!\n", pkt->type);
 910                 return;
 911         }
 912
 913         nvsp_msg_pkt = (nvsp_msg *)((unsigned long)pkt
 914                 + (pkt->data_offset8 << 3));
 915
 916         /* Make sure this is a valid nvsp packet */
 917         if (nvsp_msg_pkt->hdr.msg_type != nvsp_msg_1_type_send_rndis_pkt) {
 918                 device_printf(dev, "packet hdr type %d is invalid!\n",
 919                     pkt->type);
 920                 return;
 921         }
 922
 923         vm_xfer_page_pkt = (hv_vm_transfer_page_packet_header *)pkt;
 924
 925         if (vm_xfer_page_pkt->transfer_page_set_id !=
 926             NETVSC_RECEIVE_BUFFER_ID) {
 927                 device_printf(dev, "transfer_page_set_id %d is invalid!\n",
 928                     vm_xfer_page_pkt->transfer_page_set_id);
 929                 return;
 930         }
 931
 932         count = vm_xfer_page_pkt->range_count;
 933         net_vsc_pkt->device = device;
 934
 935         /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */
 936         for (i = 0; i < count; i++) {
 937                 net_vsc_pkt->status = nvsp_status_success;
 938                 net_vsc_pkt->data = (void *)((unsigned long)net_dev->rx_buf +
 939                     vm_xfer_page_pkt->ranges[i].byte_offset);
 940                 net_vsc_pkt->tot_data_buf_len =
 941                     vm_xfer_page_pkt->ranges[i].byte_count;
 942
 943                 hv_rf_on_receive(net_dev, device, chan, net_vsc_pkt);
 944                 if (net_vsc_pkt->status != nvsp_status_success) {
 945                         status = nvsp_status_failure;
 946                 }
 947         }
 948
 949         /*
 950          * Moved completion call back here so that all received
 951          * messages (not just data messages) will trigger a response
 952          * message back to the host.
 953          */
 954         hv_nv_on_receive_completion(chan, vm_xfer_page_pkt->d.transaction_id,
 955             status);
 956 }
 957
 958 /*
 959  * Net VSC on receive completion
 960  *
 961  * Send a receive completion packet to RNDIS device (ie NetVsp)
 962  */
 963 static void
 964 hv_nv_on_receive_completion(struct hv_vmbus_channel *chan, uint64_t tid,
 965     uint32_t status)
 966 {
 967         nvsp_msg rx_comp_msg;
 968         int retries = 0;
 969         int ret = 0;
 970
 971         rx_comp_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt_complete;
 972
 973         /* Pass in the status */
 974         rx_comp_msg.msgs.vers_1_msgs.send_rndis_pkt_complete.status =
 975             status;
 976
 977 retry_send_cmplt:
 978         /* Send the completion */
 979         ret = hv_vmbus_channel_send_packet(chan, &rx_comp_msg,
 980             sizeof(nvsp_msg), tid, HV_VMBUS_PACKET_TYPE_COMPLETION, 0);
 981         if (ret == 0) {
 982                 /* success */
 983                 /* no-op */
 984         } else if (ret == EAGAIN) {
 985                 /* no more room... wait a bit and attempt to retry 3 times */
 986                 retries++;
 987
 988                 if (retries < 4) {
 989                         DELAY(100);
 990                         goto retry_send_cmplt;
 991                 }
 992         }
 993 }
 994
 995 /*
 996  * Net VSC receiving vRSS send table from VSP
 997  */
 998 static void
 999 hv_nv_send_table(struct hv_device *device, hv_vm_packet_descriptor *pkt)
1000 {
1001         netvsc_dev *net_dev;
1002         nvsp_msg *nvsp_msg_pkt;
1003         int i;
1004         uint32_t count, *table;
1005
1006         net_dev = hv_nv_get_inbound_net_device(device);
1007         if (!net_dev)
1008                 return;
1009
1010         nvsp_msg_pkt =
1011             (nvsp_msg *)((unsigned long)pkt + (pkt->data_offset8 << 3));
1012
1013         if (nvsp_msg_pkt->hdr.msg_type !=
1014             nvsp_msg5_type_send_indirection_table) {
1015                 printf("Netvsc: !Warning! receive msg type not "
1016                         "send_indirection_table. type = %d\n",
1017                         nvsp_msg_pkt->hdr.msg_type);
1018                 return;
1019         }
1020
1021         count = nvsp_msg_pkt->msgs.vers_5_msgs.send_table.count;
1022         if (count != VRSS_SEND_TABLE_SIZE) {
1023                 printf("Netvsc: Received wrong send table size: %u\n", count);
1024                 return;
1025         }
1026
1027         table = (uint32_t *)
1028             ((unsigned long)&nvsp_msg_pkt->msgs.vers_5_msgs.send_table +
1029              nvsp_msg_pkt->msgs.vers_5_msgs.send_table.offset);
1030
1031         for (i = 0; i < count; i++)
1032                 net_dev->vrss_send_table[i] = table[i];
1033 }
1034
1035 /*
1036  * Net VSC on channel callback
1037  */
1038 static void
1039 hv_nv_on_channel_callback(void *xchan)
1040 {
1041         struct hv_vmbus_channel *chan = xchan;
1042         struct hv_device *device = chan->device;
1043         netvsc_dev *net_dev;
1044         device_t dev = device->device;
1045         uint32_t bytes_rxed;
1046         uint64_t request_id;
1047         hv_vm_packet_descriptor *desc;
1048         uint8_t *buffer;
1049         int bufferlen = NETVSC_PACKET_SIZE;
1050         int ret = 0;
1051
1052         net_dev = hv_nv_get_inbound_net_device(device);
1053         if (net_dev == NULL)
1054                 return;
1055
1056         buffer = chan->hv_chan_rdbuf;
1057
1058         do {
1059                 ret = hv_vmbus_channel_recv_packet_raw(chan,
1060                     buffer, bufferlen, &bytes_rxed, &request_id);
1061                 if (ret == 0) {
1062                         if (bytes_rxed > 0) {
1063                                 desc = (hv_vm_packet_descriptor *)buffer;
1064                                 switch (desc->type) {
1065                                 case HV_VMBUS_PACKET_TYPE_COMPLETION:
1066                                         hv_nv_on_send_completion(net_dev, device,
1067                                             chan, desc);
1068                                         break;
1069                                 case HV_VMBUS_PACKET_TYPE_DATA_USING_TRANSFER_PAGES:
1070                                         hv_nv_on_receive(net_dev, device, chan, desc);
1071                                         break;
1072                                 case HV_VMBUS_PACKET_TYPE_DATA_IN_BAND:
1073                                         hv_nv_send_table(device, desc);
1074                                         break;
1075                                 default:
1076                                         device_printf(dev,
1077                                             "hv_cb recv unknow type %d "
1078                                             " packet\n", desc->type);
1079                                         break;
1080                                 }
1081                         } else {
1082                                 break;
1083                         }
1084                 } else if (ret == ENOBUFS) {
1085                         /* Handle large packet */
1086                         if (bufferlen > NETVSC_PACKET_SIZE) {
1087                                 free(buffer, M_NETVSC);
1088                                 buffer = NULL;
1089                         }
1090
1091                         /* alloc new buffer */
1092                         buffer = malloc(bytes_rxed, M_NETVSC, M_NOWAIT);
1093                         if (buffer == NULL) {
1094                                 device_printf(dev,
1095                                     "hv_cb malloc buffer failed, len=%u\n",
1096                                     bytes_rxed);
1097                                 bufferlen = 0;
1098                                 break;
1099                         }
1100                         bufferlen = bytes_rxed;
1101                 }
1102         } while (1);
1103
1104         if (bufferlen > NETVSC_PACKET_SIZE)
1105                 free(buffer, M_NETVSC);
1106
1107         hv_rf_channel_rollup(chan);
1108 }