sys/dev/hyperv/netvsc/hv_net_vsc.c

   1 /*-
   2  * Copyright (c) 2009-2012 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 /**
  32  * HyperV vmbus network VSC (virtual services client) module
  33  *
  34  */
  35
  36
  37 #include <sys/param.h>
  38 #include <sys/kernel.h>
  39 #include <sys/socket.h>
  40 #include <sys/lock.h>
  41 #include <net/if.h>
  42 #include <net/if_var.h>
  43 #include <net/if_arp.h>
  44 #include <machine/bus.h>
  45 #include <machine/atomic.h>
  46
  47 #include <dev/hyperv/include/hyperv.h>
  48 #include "hv_net_vsc.h"
  49 #include "hv_rndis.h"
  50 #include "hv_rndis_filter.h"
  51
  52 /* priv1 and priv2 are consumed by the main driver */
  53 #define hv_chan_rdbuf   hv_chan_priv3
  54
  55 MALLOC_DEFINE(M_NETVSC, "netvsc", "Hyper-V netvsc driver");
  56
  57 /*
  58  * Forward declarations
  59  */
  60 static void hv_nv_on_channel_callback(void *xchan);
  61 static int  hv_nv_init_send_buffer_with_net_vsp(struct hv_device *device);
  62 static int  hv_nv_init_rx_buffer_with_net_vsp(struct hv_device *device);
  63 static int  hv_nv_destroy_send_buffer(netvsc_dev *net_dev);
  64 static int  hv_nv_destroy_rx_buffer(netvsc_dev *net_dev);
  65 static int  hv_nv_connect_to_vsp(struct hv_device *device);
  66 static void hv_nv_on_send_completion(netvsc_dev *net_dev,
  67     struct hv_device *device, struct hv_vmbus_channel *, hv_vm_packet_descriptor *pkt);
  68 static void hv_nv_on_receive_completion(struct hv_vmbus_channel *chan,
  69     uint64_t tid, uint32_t status);
  70 static void hv_nv_on_receive(netvsc_dev *net_dev,
  71     struct hv_device *device, struct hv_vmbus_channel *chan,
  72     hv_vm_packet_descriptor *pkt);
  73
  74 /*
  75  *
  76  */
  77 static inline netvsc_dev *
  78 hv_nv_alloc_net_device(struct hv_device *device)
  79 {
  80         netvsc_dev *net_dev;
  81         hn_softc_t *sc = device_get_softc(device->device);
  82
  83         net_dev = malloc(sizeof(netvsc_dev), M_NETVSC, M_WAITOK | M_ZERO);
  84
  85         net_dev->dev = device;
  86         net_dev->destroy = FALSE;
  87         sc->net_dev = net_dev;
  88
  89         return (net_dev);
  90 }
  91
  92 /*
  93  *
  94  */
  95 static inline netvsc_dev *
  96 hv_nv_get_outbound_net_device(struct hv_device *device)
  97 {
  98         hn_softc_t *sc = device_get_softc(device->device);
  99         netvsc_dev *net_dev = sc->net_dev;;
 100
 101         if ((net_dev != NULL) && net_dev->destroy) {
 102                 return (NULL);
 103         }
 104
 105         return (net_dev);
 106 }
 107
 108 /*
 109  *
 110  */
 111 static inline netvsc_dev *
 112 hv_nv_get_inbound_net_device(struct hv_device *device)
 113 {
 114         hn_softc_t *sc = device_get_softc(device->device);
 115         netvsc_dev *net_dev = sc->net_dev;;
 116
 117         if (net_dev == NULL) {
 118                 return (net_dev);
 119         }
 120         /*
 121          * When the device is being destroyed; we only
 122          * permit incoming packets if and only if there
 123          * are outstanding sends.
 124          */
 125         if (net_dev->destroy) {
 126                 return (NULL);
 127         }
 128
 129         return (net_dev);
 130 }
 131
 132 int
 133 hv_nv_get_next_send_section(netvsc_dev *net_dev)
 134 {
 135         unsigned long bitsmap_words = net_dev->bitsmap_words;
 136         unsigned long *bitsmap = net_dev->send_section_bitsmap;
 137         unsigned long idx;
 138         int ret = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
 139         int i;
 140
 141         for (i = 0; i < bitsmap_words; i++) {
 142                 idx = ffsl(~bitsmap[i]);
 143                 if (0 == idx)
 144                         continue;
 145
 146                 idx--;
 147                 KASSERT(i * BITS_PER_LONG + idx < net_dev->send_section_count,
 148                     ("invalid i %d and idx %lu", i, idx));
 149
 150                 if (atomic_testandset_long(&bitsmap[i], idx))
 151                         continue;
 152
 153                 ret = i * BITS_PER_LONG + idx;
 154                 break;
 155         }
 156
 157         return (ret);
 158 }
 159
 160 /*
 161  * Net VSC initialize receive buffer with net VSP
 162  *
 163  * Net VSP:  Network virtual services client, also known as the
 164  *     Hyper-V extensible switch and the synthetic data path.
 165  */
 166 static int
 167 hv_nv_init_rx_buffer_with_net_vsp(struct hv_device *device)
 168 {
 169         netvsc_dev *net_dev;
 170         nvsp_msg *init_pkt;
 171         int ret = 0;
 172
 173         net_dev = hv_nv_get_outbound_net_device(device);
 174         if (!net_dev) {
 175                 return (ENODEV);
 176         }
 177
 178         net_dev->rx_buf = contigmalloc(net_dev->rx_buf_size, M_NETVSC,
 179             M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 180
 181         /*
 182          * Establish the GPADL handle for this buffer on this channel.
 183          * Note:  This call uses the vmbus connection rather than the
 184          * channel to establish the gpadl handle.
 185          * GPADL:  Guest physical address descriptor list.
 186          */
 187         ret = hv_vmbus_channel_establish_gpadl(
 188                 device->channel, net_dev->rx_buf,
 189                 net_dev->rx_buf_size, &net_dev->rx_buf_gpadl_handle);
 190         if (ret != 0) {
 191                 goto cleanup;
 192         }
 193
 194         /* sema_wait(&ext->channel_init_sema); KYS CHECK */
 195
 196         /* Notify the NetVsp of the gpadl handle */
 197         init_pkt = &net_dev->channel_init_packet;
 198
 199         memset(init_pkt, 0, sizeof(nvsp_msg));
 200
 201         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_rx_buf;
 202         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 203             net_dev->rx_buf_gpadl_handle;
 204         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 205             NETVSC_RECEIVE_BUFFER_ID;
 206
 207         /* Send the gpadl notification request */
 208
 209         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 210             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 211             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 212             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 213         if (ret != 0) {
 214                 goto cleanup;
 215         }
 216
 217         sema_wait(&net_dev->channel_init_sema);
 218
 219         /* Check the response */
 220         if (init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.status
 221             != nvsp_status_success) {
 222                 ret = EINVAL;
 223                 goto cleanup;
 224         }
 225
 226         net_dev->rx_section_count =
 227             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.num_sections;
 228
 229         net_dev->rx_sections = malloc(net_dev->rx_section_count *
 230             sizeof(nvsp_1_rx_buf_section), M_NETVSC, M_WAITOK);
 231         memcpy(net_dev->rx_sections,
 232             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.sections,
 233             net_dev->rx_section_count * sizeof(nvsp_1_rx_buf_section));
 234
 235
 236         /*
 237          * For first release, there should only be 1 section that represents
 238          * the entire receive buffer
 239          */
 240         if (net_dev->rx_section_count != 1
 241             || net_dev->rx_sections->offset != 0) {
 242                 ret = EINVAL;
 243                 goto cleanup;
 244         }
 245
 246         goto exit;
 247
 248 cleanup:
 249         hv_nv_destroy_rx_buffer(net_dev);
 250
 251 exit:
 252         return (ret);
 253 }
 254
 255 /*
 256  * Net VSC initialize send buffer with net VSP
 257  */
 258 static int
 259 hv_nv_init_send_buffer_with_net_vsp(struct hv_device *device)
 260 {
 261         netvsc_dev *net_dev;
 262         nvsp_msg *init_pkt;
 263         int ret = 0;
 264
 265         net_dev = hv_nv_get_outbound_net_device(device);
 266         if (!net_dev) {
 267                 return (ENODEV);
 268         }
 269
 270         net_dev->send_buf  = contigmalloc(net_dev->send_buf_size, M_NETVSC,
 271             M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 272         if (net_dev->send_buf == NULL) {
 273                 ret = ENOMEM;
 274                 goto cleanup;
 275         }
 276
 277         /*
 278          * Establish the gpadl handle for this buffer on this channel.
 279          * Note:  This call uses the vmbus connection rather than the
 280          * channel to establish the gpadl handle.
 281          */
 282         ret = hv_vmbus_channel_establish_gpadl(device->channel,
 283             net_dev->send_buf, net_dev->send_buf_size,
 284             &net_dev->send_buf_gpadl_handle);
 285         if (ret != 0) {
 286                 goto cleanup;
 287         }
 288
 289         /* Notify the NetVsp of the gpadl handle */
 290
 291         init_pkt = &net_dev->channel_init_packet;
 292
 293         memset(init_pkt, 0, sizeof(nvsp_msg));
 294
 295         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_send_buf;
 296         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 297             net_dev->send_buf_gpadl_handle;
 298         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 299             NETVSC_SEND_BUFFER_ID;
 300
 301         /* Send the gpadl notification request */
 302
 303         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 304             sizeof(nvsp_msg), (uint64_t)init_pkt,
 305             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 306             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 307         if (ret != 0) {
 308                 goto cleanup;
 309         }
 310
 311         sema_wait(&net_dev->channel_init_sema);
 312
 313         /* Check the response */
 314         if (init_pkt->msgs.vers_1_msgs.send_send_buf_complete.status
 315             != nvsp_status_success) {
 316                 ret = EINVAL;
 317                 goto cleanup;
 318         }
 319
 320         net_dev->send_section_size =
 321             init_pkt->msgs.vers_1_msgs.send_send_buf_complete.section_size;
 322         net_dev->send_section_count =
 323             net_dev->send_buf_size / net_dev->send_section_size;
 324         net_dev->bitsmap_words = howmany(net_dev->send_section_count,
 325             BITS_PER_LONG);
 326         net_dev->send_section_bitsmap =
 327             malloc(net_dev->bitsmap_words * sizeof(long), M_NETVSC,
 328             M_WAITOK | M_ZERO);
 329
 330         goto exit;
 331
 332 cleanup:
 333         hv_nv_destroy_send_buffer(net_dev);
 334
 335 exit:
 336         return (ret);
 337 }
 338
 339 /*
 340  * Net VSC destroy receive buffer
 341  */
 342 static int
 343 hv_nv_destroy_rx_buffer(netvsc_dev *net_dev)
 344 {
 345         nvsp_msg *revoke_pkt;
 346         int ret = 0;
 347
 348         /*
 349          * If we got a section count, it means we received a
 350          * send_rx_buf_complete msg
 351          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 352          * we need to send a revoke msg here
 353          */
 354         if (net_dev->rx_section_count) {
 355                 /* Send the revoke receive buffer */
 356                 revoke_pkt = &net_dev->revoke_packet;
 357                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 358
 359                 revoke_pkt->hdr.msg_type = nvsp_msg_1_type_revoke_rx_buf;
 360                 revoke_pkt->msgs.vers_1_msgs.revoke_rx_buf.id =
 361                     NETVSC_RECEIVE_BUFFER_ID;
 362
 363                 ret = hv_vmbus_channel_send_packet(net_dev->dev->channel,
 364                     revoke_pkt, sizeof(nvsp_msg),
 365                     (uint64_t)(uintptr_t)revoke_pkt,
 366                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 367
 368                 /*
 369                  * If we failed here, we might as well return and have a leak
 370                  * rather than continue and a bugchk
 371                  */
 372                 if (ret != 0) {
 373                         return (ret);
 374                 }
 375         }
 376
 377         /* Tear down the gpadl on the vsp end */
 378         if (net_dev->rx_buf_gpadl_handle) {
 379                 ret = hv_vmbus_channel_teardown_gpdal(net_dev->dev->channel,
 380                     net_dev->rx_buf_gpadl_handle);
 381                 /*
 382                  * If we failed here, we might as well return and have a leak
 383                  * rather than continue and a bugchk
 384                  */
 385                 if (ret != 0) {
 386                         return (ret);
 387                 }
 388                 net_dev->rx_buf_gpadl_handle = 0;
 389         }
 390
 391         if (net_dev->rx_buf) {
 392                 /* Free up the receive buffer */
 393                 contigfree(net_dev->rx_buf, net_dev->rx_buf_size, M_NETVSC);
 394                 net_dev->rx_buf = NULL;
 395         }
 396
 397         if (net_dev->rx_sections) {
 398                 free(net_dev->rx_sections, M_NETVSC);
 399                 net_dev->rx_sections = NULL;
 400                 net_dev->rx_section_count = 0;
 401         }
 402
 403         return (ret);
 404 }
 405
 406 /*
 407  * Net VSC destroy send buffer
 408  */
 409 static int
 410 hv_nv_destroy_send_buffer(netvsc_dev *net_dev)
 411 {
 412         nvsp_msg *revoke_pkt;
 413         int ret = 0;
 414
 415         /*
 416          * If we got a section count, it means we received a
 417          * send_rx_buf_complete msg
 418          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 419          * we need to send a revoke msg here
 420          */
 421         if (net_dev->send_section_size) {
 422                 /* Send the revoke send buffer */
 423                 revoke_pkt = &net_dev->revoke_packet;
 424                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 425
 426                 revoke_pkt->hdr.msg_type =
 427                     nvsp_msg_1_type_revoke_send_buf;
 428                 revoke_pkt->msgs.vers_1_msgs.revoke_send_buf.id =
 429                     NETVSC_SEND_BUFFER_ID;
 430
 431                 ret = hv_vmbus_channel_send_packet(net_dev->dev->channel,
 432                     revoke_pkt, sizeof(nvsp_msg),
 433                     (uint64_t)(uintptr_t)revoke_pkt,
 434                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 435                 /*
 436                  * If we failed here, we might as well return and have a leak
 437                  * rather than continue and a bugchk
 438                  */
 439                 if (ret != 0) {
 440                         return (ret);
 441                 }
 442         }
 443
 444         /* Tear down the gpadl on the vsp end */
 445         if (net_dev->send_buf_gpadl_handle) {
 446                 ret = hv_vmbus_channel_teardown_gpdal(net_dev->dev->channel,
 447                     net_dev->send_buf_gpadl_handle);
 448
 449                 /*
 450                  * If we failed here, we might as well return and have a leak
 451                  * rather than continue and a bugchk
 452                  */
 453                 if (ret != 0) {
 454                         return (ret);
 455                 }
 456                 net_dev->send_buf_gpadl_handle = 0;
 457         }
 458
 459         if (net_dev->send_buf) {
 460                 /* Free up the receive buffer */
 461                 contigfree(net_dev->send_buf, net_dev->send_buf_size, M_NETVSC);
 462                 net_dev->send_buf = NULL;
 463         }
 464
 465         if (net_dev->send_section_bitsmap) {
 466                 free(net_dev->send_section_bitsmap, M_NETVSC);
 467         }
 468
 469         return (ret);
 470 }
 471
 472
 473 /*
 474  * Attempt to negotiate the caller-specified NVSP version
 475  *
 476  * For NVSP v2, Server 2008 R2 does not set
 477  * init_pkt->msgs.init_msgs.init_compl.negotiated_prot_vers
 478  * to the negotiated version, so we cannot rely on that.
 479  */
 480 static int
 481 hv_nv_negotiate_nvsp_protocol(struct hv_device *device, netvsc_dev *net_dev,
 482     uint32_t nvsp_ver)
 483 {
 484         nvsp_msg *init_pkt;
 485         int ret;
 486
 487         init_pkt = &net_dev->channel_init_packet;
 488         memset(init_pkt, 0, sizeof(nvsp_msg));
 489         init_pkt->hdr.msg_type = nvsp_msg_type_init;
 490
 491         /*
 492          * Specify parameter as the only acceptable protocol version
 493          */
 494         init_pkt->msgs.init_msgs.init.p1.protocol_version = nvsp_ver;
 495         init_pkt->msgs.init_msgs.init.protocol_version_2 = nvsp_ver;
 496
 497         /* Send the init request */
 498         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 499             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 500             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 501             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 502         if (ret != 0)
 503                 return (-1);
 504
 505         sema_wait(&net_dev->channel_init_sema);
 506
 507         if (init_pkt->msgs.init_msgs.init_compl.status != nvsp_status_success)
 508                 return (EINVAL);
 509
 510         return (0);
 511 }
 512
 513 /*
 514  * Send NDIS version 2 config packet containing MTU.
 515  *
 516  * Not valid for NDIS version 1.
 517  */
 518 static int
 519 hv_nv_send_ndis_config(struct hv_device *device, uint32_t mtu)
 520 {
 521         netvsc_dev *net_dev;
 522         nvsp_msg *init_pkt;
 523         int ret;
 524
 525         net_dev = hv_nv_get_outbound_net_device(device);
 526         if (!net_dev)
 527                 return (-ENODEV);
 528
 529         /*
 530          * Set up configuration packet, write MTU
 531          * Indicate we are capable of handling VLAN tags
 532          */
 533         init_pkt = &net_dev->channel_init_packet;
 534         memset(init_pkt, 0, sizeof(nvsp_msg));
 535         init_pkt->hdr.msg_type = nvsp_msg_2_type_send_ndis_config;
 536         init_pkt->msgs.vers_2_msgs.send_ndis_config.mtu = mtu;
 537         init_pkt->
 538                 msgs.vers_2_msgs.send_ndis_config.capabilities.u1.u2.ieee8021q
 539                 = 1;
 540
 541         /* Send the configuration packet */
 542         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 543             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 544             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 545         if (ret != 0)
 546                 return (-EINVAL);
 547
 548         return (0);
 549 }
 550
 551 /*
 552  * Net VSC connect to VSP
 553  */
 554 static int
 555 hv_nv_connect_to_vsp(struct hv_device *device)
 556 {
 557         netvsc_dev *net_dev;
 558         nvsp_msg *init_pkt;
 559         uint32_t ndis_version;
 560         uint32_t protocol_list[] = { NVSP_PROTOCOL_VERSION_1,
 561             NVSP_PROTOCOL_VERSION_2,
 562             NVSP_PROTOCOL_VERSION_4,
 563             NVSP_PROTOCOL_VERSION_5 };
 564         int i;
 565         int protocol_number = nitems(protocol_list);
 566         int ret = 0;
 567         device_t dev = device->device;
 568         hn_softc_t *sc = device_get_softc(dev);
 569         struct ifnet *ifp = sc->hn_ifp;
 570
 571         net_dev = hv_nv_get_outbound_net_device(device);
 572         if (!net_dev) {
 573                 return (ENODEV);
 574         }
 575
 576         /*
 577          * Negotiate the NVSP version.  Try the latest NVSP first.
 578          */
 579         for (i = protocol_number - 1; i >= 0; i--) {
 580                 if (hv_nv_negotiate_nvsp_protocol(device, net_dev,
 581                     protocol_list[i]) == 0) {
 582                         net_dev->nvsp_version = protocol_list[i];
 583                         if (bootverbose)
 584                                 device_printf(dev, "Netvsc: got version 0x%x\n",
 585                                     net_dev->nvsp_version);
 586                         break;
 587                 }
 588         }
 589
 590         if (i < 0) {
 591                 if (bootverbose)
 592                         device_printf(dev, "failed to negotiate a valid "
 593                             "protocol.\n");
 594                 return (EPROTO);
 595         }
 596
 597         /*
 598          * Set the MTU if supported by this NVSP protocol version
 599          * This needs to be right after the NVSP init message per Haiyang
 600          */
 601         if (net_dev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
 602                 ret = hv_nv_send_ndis_config(device, ifp->if_mtu);
 603
 604         /*
 605          * Send the NDIS version
 606          */
 607         init_pkt = &net_dev->channel_init_packet;
 608
 609         memset(init_pkt, 0, sizeof(nvsp_msg));
 610
 611         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_4) {
 612                 ndis_version = NDIS_VERSION_6_1;
 613         } else {
 614                 ndis_version = NDIS_VERSION_6_30;
 615         }
 616
 617         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_ndis_vers;
 618         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_major_vers =
 619             (ndis_version & 0xFFFF0000) >> 16;
 620         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_minor_vers =
 621             ndis_version & 0xFFFF;
 622
 623         /* Send the init request */
 624
 625         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 626             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 627             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 628         if (ret != 0) {
 629                 goto cleanup;
 630         }
 631         /*
 632          * TODO:  BUGBUG - We have to wait for the above msg since the netvsp
 633          * uses KMCL which acknowledges packet (completion packet)
 634          * since our Vmbus always set the
 635          * HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED flag
 636          */
 637         /* sema_wait(&NetVscChannel->channel_init_sema); */
 638
 639         /* Post the big receive buffer to NetVSP */
 640         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
 641                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
 642         else
 643                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
 644         net_dev->send_buf_size = NETVSC_SEND_BUFFER_SIZE;
 645
 646         ret = hv_nv_init_rx_buffer_with_net_vsp(device);
 647         if (ret == 0)
 648                 ret = hv_nv_init_send_buffer_with_net_vsp(device);
 649
 650 cleanup:
 651         return (ret);
 652 }
 653
 654 /*
 655  * Net VSC disconnect from VSP
 656  */
 657 static void
 658 hv_nv_disconnect_from_vsp(netvsc_dev *net_dev)
 659 {
 660         hv_nv_destroy_rx_buffer(net_dev);
 661         hv_nv_destroy_send_buffer(net_dev);
 662 }
 663
 664 /*
 665  * Callback handler for subchannel offer
 666  * @@param context new subchannel
 667  */
 668 static void
 669 hv_nv_subchan_callback(void *xchan)
 670 {
 671         struct hv_vmbus_channel *chan = xchan;
 672         netvsc_dev *net_dev;
 673         uint16_t chn_index = chan->offer_msg.offer.sub_channel_index;
 674         struct hv_device *device = chan->device;
 675         hn_softc_t *sc = device_get_softc(device->device);
 676         int ret;
 677
 678         net_dev = sc->net_dev;
 679
 680         if (chn_index >= net_dev->num_channel) {
 681                 /* Would this ever happen? */
 682                 return;
 683         }
 684         netvsc_subchan_callback(sc, chan);
 685
 686         chan->hv_chan_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK);
 687         ret = hv_vmbus_channel_open(chan, NETVSC_DEVICE_RING_BUFFER_SIZE,
 688             NETVSC_DEVICE_RING_BUFFER_SIZE, NULL, 0,
 689             hv_nv_on_channel_callback, chan);
 690 }
 691
 692 /*
 693  * Net VSC on device add
 694  *
 695  * Callback when the device belonging to this driver is added
 696  */
 697 netvsc_dev *
 698 hv_nv_on_device_add(struct hv_device *device, void *additional_info)
 699 {
 700         struct hv_vmbus_channel *chan = device->channel;
 701         netvsc_dev *net_dev;
 702         int ret = 0;
 703
 704         net_dev = hv_nv_alloc_net_device(device);
 705         if (net_dev == NULL)
 706                 return NULL;
 707
 708         /* Initialize the NetVSC channel extension */
 709
 710         sema_init(&net_dev->channel_init_sema, 0, "netdev_sema");
 711
 712         chan->hv_chan_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK);
 713
 714         /*
 715          * Open the channel
 716          */
 717         ret = hv_vmbus_channel_open(chan,
 718             NETVSC_DEVICE_RING_BUFFER_SIZE, NETVSC_DEVICE_RING_BUFFER_SIZE,
 719             NULL, 0, hv_nv_on_channel_callback, chan);
 720         if (ret != 0) {
 721                 free(chan->hv_chan_rdbuf, M_NETVSC);
 722                 goto cleanup;
 723         }
 724         chan->sc_creation_callback = hv_nv_subchan_callback;
 725
 726         /*
 727          * Connect with the NetVsp
 728          */
 729         ret = hv_nv_connect_to_vsp(device);
 730         if (ret != 0)
 731                 goto close;
 732
 733         return (net_dev);
 734
 735 close:
 736         /* Now, we can close the channel safely */
 737         free(chan->hv_chan_rdbuf, M_NETVSC);
 738         hv_vmbus_channel_close(chan);
 739
 740 cleanup:
 741         /*
 742          * Free the packet buffers on the netvsc device packet queue.
 743          * Release other resources.
 744          */
 745         sema_destroy(&net_dev->channel_init_sema);
 746         free(net_dev, M_NETVSC);
 747
 748         return (NULL);
 749 }
 750
 751 /*
 752  * Net VSC on device remove
 753  */
 754 int
 755 hv_nv_on_device_remove(struct hv_device *device, boolean_t destroy_channel)
 756 {
 757         hn_softc_t *sc = device_get_softc(device->device);
 758         netvsc_dev *net_dev = sc->net_dev;;
 759
 760         /* Stop outbound traffic ie sends and receives completions */
 761         net_dev->destroy = TRUE;
 762
 763         hv_nv_disconnect_from_vsp(net_dev);
 764
 765         /* At this point, no one should be accessing net_dev except in here */
 766
 767         /* Now, we can close the channel safely */
 768
 769         if (!destroy_channel) {
 770                 device->channel->state =
 771                     HV_CHANNEL_CLOSING_NONDESTRUCTIVE_STATE;
 772         }
 773
 774         free(device->channel->hv_chan_rdbuf, M_NETVSC);
 775         hv_vmbus_channel_close(device->channel);
 776
 777         sema_destroy(&net_dev->channel_init_sema);
 778         free(net_dev, M_NETVSC);
 779
 780         return (0);
 781 }
 782
 783 /*
 784  * Net VSC on send completion
 785  */
 786 static void
 787 hv_nv_on_send_completion(netvsc_dev *net_dev,
 788     struct hv_device *device, struct hv_vmbus_channel *chan,
 789     hv_vm_packet_descriptor *pkt)
 790 {
 791         nvsp_msg *nvsp_msg_pkt;
 792         netvsc_packet *net_vsc_pkt;
 793
 794         nvsp_msg_pkt =
 795             (nvsp_msg *)((unsigned long)pkt + (pkt->data_offset8 << 3));
 796
 797         if (nvsp_msg_pkt->hdr.msg_type == nvsp_msg_type_init_complete
 798                 || nvsp_msg_pkt->hdr.msg_type
 799                         == nvsp_msg_1_type_send_rx_buf_complete
 800                 || nvsp_msg_pkt->hdr.msg_type
 801                         == nvsp_msg_1_type_send_send_buf_complete
 802                 || nvsp_msg_pkt->hdr.msg_type
 803                         == nvsp_msg5_type_subchannel) {
 804                 /* Copy the response back */
 805                 memcpy(&net_dev->channel_init_packet, nvsp_msg_pkt,
 806                     sizeof(nvsp_msg));
 807                 sema_post(&net_dev->channel_init_sema);
 808         } else if (nvsp_msg_pkt->hdr.msg_type ==
 809                     nvsp_msg_1_type_send_rndis_pkt_complete) {
 810                 /* Get the send context */
 811                 net_vsc_pkt =
 812                     (netvsc_packet *)(unsigned long)pkt->transaction_id;
 813                 if (NULL != net_vsc_pkt) {
 814                         if (net_vsc_pkt->send_buf_section_idx !=
 815                             NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) {
 816                                 u_long mask;
 817                                 int idx;
 818
 819                                 idx = net_vsc_pkt->send_buf_section_idx /
 820                                     BITS_PER_LONG;
 821                                 KASSERT(idx < net_dev->bitsmap_words,
 822                                     ("invalid section index %u",
 823                                      net_vsc_pkt->send_buf_section_idx));
 824                                 mask = 1UL <<
 825                                     (net_vsc_pkt->send_buf_section_idx %
 826                                      BITS_PER_LONG);
 827
 828                                 KASSERT(net_dev->send_section_bitsmap[idx] &
 829                                     mask,
 830                                     ("index bitmap 0x%lx, section index %u, "
 831                                      "bitmap idx %d, bitmask 0x%lx",
 832                                      net_dev->send_section_bitsmap[idx],
 833                                      net_vsc_pkt->send_buf_section_idx,
 834                                      idx, mask));
 835                                 atomic_clear_long(
 836                                     &net_dev->send_section_bitsmap[idx], mask);
 837                         }
 838
 839                         /* Notify the layer above us */
 840                         net_vsc_pkt->compl.send.on_send_completion(chan,
 841                             net_vsc_pkt->compl.send.send_completion_context);
 842
 843                 }
 844         }
 845 }
 846
 847 /*
 848  * Net VSC on send
 849  * Sends a packet on the specified Hyper-V device.
 850  * Returns 0 on success, non-zero on failure.
 851  */
 852 int
 853 hv_nv_on_send(struct hv_vmbus_channel *chan, netvsc_packet *pkt)
 854 {
 855         nvsp_msg send_msg;
 856         int ret;
 857
 858         send_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt;
 859         if (pkt->is_data_pkt) {
 860                 /* 0 is RMC_DATA */
 861                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 0;
 862         } else {
 863                 /* 1 is RMC_CONTROL */
 864                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 1;
 865         }
 866
 867         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_idx =
 868             pkt->send_buf_section_idx;
 869         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_size =
 870             pkt->send_buf_section_size;
 871
 872         if (pkt->page_buf_count) {
 873                 ret = hv_vmbus_channel_send_packet_pagebuffer(chan,
 874                     pkt->page_buffers, pkt->page_buf_count,
 875                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt);
 876         } else {
 877                 ret = hv_vmbus_channel_send_packet(chan,
 878                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt,
 879                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 880                     HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 881         }
 882
 883         return (ret);
 884 }
 885
 886 /*
 887  * Net VSC on receive
 888  *
 889  * In the FreeBSD Hyper-V virtual world, this function deals exclusively
 890  * with virtual addresses.
 891  */
 892 static void
 893 hv_nv_on_receive(netvsc_dev *net_dev, struct hv_device *device,
 894     struct hv_vmbus_channel *chan, hv_vm_packet_descriptor *pkt)
 895 {
 896         hv_vm_transfer_page_packet_header *vm_xfer_page_pkt;
 897         nvsp_msg *nvsp_msg_pkt;
 898         netvsc_packet vsc_pkt;
 899         netvsc_packet *net_vsc_pkt = &vsc_pkt;
 900         device_t dev = device->device;
 901         int count = 0;
 902         int i = 0;
 903         int status = nvsp_status_success;
 904
 905         /*
 906          * All inbound packets other than send completion should be
 907          * xfer page packet.
 908          */
 909         if (pkt->type != HV_VMBUS_PACKET_TYPE_DATA_USING_TRANSFER_PAGES) {
 910                 device_printf(dev, "packet type %d is invalid!\n", pkt->type);
 911                 return;
 912         }
 913
 914         nvsp_msg_pkt = (nvsp_msg *)((unsigned long)pkt
 915                 + (pkt->data_offset8 << 3));
 916
 917         /* Make sure this is a valid nvsp packet */
 918         if (nvsp_msg_pkt->hdr.msg_type != nvsp_msg_1_type_send_rndis_pkt) {
 919                 device_printf(dev, "packet hdr type %d is invalid!\n",
 920                     pkt->type);
 921                 return;
 922         }
 923
 924         vm_xfer_page_pkt = (hv_vm_transfer_page_packet_header *)pkt;
 925
 926         if (vm_xfer_page_pkt->transfer_page_set_id !=
 927             NETVSC_RECEIVE_BUFFER_ID) {
 928                 device_printf(dev, "transfer_page_set_id %d is invalid!\n",
 929                     vm_xfer_page_pkt->transfer_page_set_id);
 930                 return;
 931         }
 932
 933         count = vm_xfer_page_pkt->range_count;
 934         net_vsc_pkt->device = device;
 935
 936         /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */
 937         for (i = 0; i < count; i++) {
 938                 net_vsc_pkt->status = nvsp_status_success;
 939                 net_vsc_pkt->data = (void *)((unsigned long)net_dev->rx_buf +
 940                     vm_xfer_page_pkt->ranges[i].byte_offset);
 941                 net_vsc_pkt->tot_data_buf_len =
 942                     vm_xfer_page_pkt->ranges[i].byte_count;
 943
 944                 hv_rf_on_receive(net_dev, device, chan, net_vsc_pkt);
 945                 if (net_vsc_pkt->status != nvsp_status_success) {
 946                         status = nvsp_status_failure;
 947                 }
 948         }
 949
 950         /*
 951          * Moved completion call back here so that all received
 952          * messages (not just data messages) will trigger a response
 953          * message back to the host.
 954          */
 955         hv_nv_on_receive_completion(chan, vm_xfer_page_pkt->d.transaction_id,
 956             status);
 957 }
 958
 959 /*
 960  * Net VSC on receive completion
 961  *
 962  * Send a receive completion packet to RNDIS device (ie NetVsp)
 963  */
 964 static void
 965 hv_nv_on_receive_completion(struct hv_vmbus_channel *chan, uint64_t tid,
 966     uint32_t status)
 967 {
 968         nvsp_msg rx_comp_msg;
 969         int retries = 0;
 970         int ret = 0;
 971
 972         rx_comp_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt_complete;
 973
 974         /* Pass in the status */
 975         rx_comp_msg.msgs.vers_1_msgs.send_rndis_pkt_complete.status =
 976             status;
 977
 978 retry_send_cmplt:
 979         /* Send the completion */
 980         ret = hv_vmbus_channel_send_packet(chan, &rx_comp_msg,
 981             sizeof(nvsp_msg), tid, HV_VMBUS_PACKET_TYPE_COMPLETION, 0);
 982         if (ret == 0) {
 983                 /* success */
 984                 /* no-op */
 985         } else if (ret == EAGAIN) {
 986                 /* no more room... wait a bit and attempt to retry 3 times */
 987                 retries++;
 988
 989                 if (retries < 4) {
 990                         DELAY(100);
 991                         goto retry_send_cmplt;
 992                 }
 993         }
 994 }
 995
 996 /*
 997  * Net VSC receiving vRSS send table from VSP
 998  */
 999 static void
1000 hv_nv_send_table(struct hv_device *device, hv_vm_packet_descriptor *pkt)
1001 {
1002         netvsc_dev *net_dev;
1003         nvsp_msg *nvsp_msg_pkt;
1004         int i;
1005         uint32_t count, *table;
1006
1007         net_dev = hv_nv_get_inbound_net_device(device);
1008         if (!net_dev)
1009                 return;
1010
1011         nvsp_msg_pkt =
1012             (nvsp_msg *)((unsigned long)pkt + (pkt->data_offset8 << 3));
1013
1014         if (nvsp_msg_pkt->hdr.msg_type !=
1015             nvsp_msg5_type_send_indirection_table) {
1016                 printf("Netvsc: !Warning! receive msg type not "
1017                         "send_indirection_table. type = %d\n",
1018                         nvsp_msg_pkt->hdr.msg_type);
1019                 return;
1020         }
1021
1022         count = nvsp_msg_pkt->msgs.vers_5_msgs.send_table.count;
1023         if (count != VRSS_SEND_TABLE_SIZE) {
1024                 printf("Netvsc: Received wrong send table size: %u\n", count);
1025                 return;
1026         }
1027
1028         table = (uint32_t *)
1029             ((unsigned long)&nvsp_msg_pkt->msgs.vers_5_msgs.send_table +
1030              nvsp_msg_pkt->msgs.vers_5_msgs.send_table.offset);
1031
1032         for (i = 0; i < count; i++)
1033                 net_dev->vrss_send_table[i] = table[i];
1034 }
1035
1036 /*
1037  * Net VSC on channel callback
1038  */
1039 static void
1040 hv_nv_on_channel_callback(void *xchan)
1041 {
1042         struct hv_vmbus_channel *chan = xchan;
1043         struct hv_device *device = chan->device;
1044         netvsc_dev *net_dev;
1045         device_t dev = device->device;
1046         uint32_t bytes_rxed;
1047         uint64_t request_id;
1048         hv_vm_packet_descriptor *desc;
1049         uint8_t *buffer;
1050         int bufferlen = NETVSC_PACKET_SIZE;
1051         int ret = 0;
1052
1053         net_dev = hv_nv_get_inbound_net_device(device);
1054         if (net_dev == NULL)
1055                 return;
1056
1057         buffer = chan->hv_chan_rdbuf;
1058
1059         do {
1060                 ret = hv_vmbus_channel_recv_packet_raw(chan,
1061                     buffer, bufferlen, &bytes_rxed, &request_id);
1062                 if (ret == 0) {
1063                         if (bytes_rxed > 0) {
1064                                 desc = (hv_vm_packet_descriptor *)buffer;
1065                                 switch (desc->type) {
1066                                 case HV_VMBUS_PACKET_TYPE_COMPLETION:
1067                                         hv_nv_on_send_completion(net_dev, device,
1068                                             chan, desc);
1069                                         break;
1070                                 case HV_VMBUS_PACKET_TYPE_DATA_USING_TRANSFER_PAGES:
1071                                         hv_nv_on_receive(net_dev, device, chan, desc);
1072                                         break;
1073                                 case HV_VMBUS_PACKET_TYPE_DATA_IN_BAND:
1074                                         hv_nv_send_table(device, desc);
1075                                         break;
1076                                 default:
1077                                         device_printf(dev,
1078                                             "hv_cb recv unknow type %d "
1079                                             " packet\n", desc->type);
1080                                         break;
1081                                 }
1082                         } else {
1083                                 break;
1084                         }
1085                 } else if (ret == ENOBUFS) {
1086                         /* Handle large packet */
1087                         if (bufferlen > NETVSC_PACKET_SIZE) {
1088                                 free(buffer, M_NETVSC);
1089                                 buffer = NULL;
1090                         }
1091
1092                         /* alloc new buffer */
1093                         buffer = malloc(bytes_rxed, M_NETVSC, M_NOWAIT);
1094                         if (buffer == NULL) {
1095                                 device_printf(dev,
1096                                     "hv_cb malloc buffer failed, len=%u\n",
1097                                     bytes_rxed);
1098                                 bufferlen = 0;
1099                                 break;
1100                         }
1101                         bufferlen = bytes_rxed;
1102                 }
1103         } while (1);
1104
1105         if (bufferlen > NETVSC_PACKET_SIZE)
1106                 free(buffer, M_NETVSC);
1107
1108         hv_rf_channel_rollup(chan);
1109 }