sys/dev/hyperv/netvsc/hv_net_vsc.c

   1 /*-
   2  * Copyright (c) 2009-2012 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 /**
  32  * HyperV vmbus network VSC (virtual services client) module
  33  *
  34  */
  35
  36
  37 #include <sys/param.h>
  38 #include <sys/kernel.h>
  39 #include <sys/socket.h>
  40 #include <sys/lock.h>
  41 #include <net/if.h>
  42 #include <net/if_arp.h>
  43 #include <machine/bus.h>
  44 #include <machine/atomic.h>
  45
  46 #include <dev/hyperv/include/hyperv.h>
  47 #include "hv_net_vsc.h"
  48 #include "hv_rndis.h"
  49 #include "hv_rndis_filter.h"
  50
  51 MALLOC_DEFINE(M_NETVSC, "netvsc", "Hyper-V netvsc driver");
  52
  53 /*
  54  * Forward declarations
  55  */
  56 static void hv_nv_on_channel_callback(void *context);
  57 static int  hv_nv_init_send_buffer_with_net_vsp(struct hv_device *device);
  58 static int  hv_nv_init_rx_buffer_with_net_vsp(struct hv_device *device);
  59 static int  hv_nv_destroy_send_buffer(netvsc_dev *net_dev);
  60 static int  hv_nv_destroy_rx_buffer(netvsc_dev *net_dev);
  61 static int  hv_nv_connect_to_vsp(struct hv_device *device);
  62 static void hv_nv_on_send_completion(netvsc_dev *net_dev,
  63     struct hv_device *device, hv_vm_packet_descriptor *pkt);
  64 static void hv_nv_on_receive(netvsc_dev *net_dev,
  65     struct hv_device *device, hv_vm_packet_descriptor *pkt);
  66
  67 /*
  68  *
  69  */
  70 static inline netvsc_dev *
  71 hv_nv_alloc_net_device(struct hv_device *device)
  72 {
  73         netvsc_dev *net_dev;
  74         hn_softc_t *sc = device_get_softc(device->device);
  75
  76         net_dev = malloc(sizeof(netvsc_dev), M_NETVSC, M_WAITOK | M_ZERO);
  77
  78         net_dev->dev = device;
  79         net_dev->destroy = FALSE;
  80         sc->net_dev = net_dev;
  81
  82         return (net_dev);
  83 }
  84
  85 /*
  86  *
  87  */
  88 static inline netvsc_dev *
  89 hv_nv_get_outbound_net_device(struct hv_device *device)
  90 {
  91         hn_softc_t *sc = device_get_softc(device->device);
  92         netvsc_dev *net_dev = sc->net_dev;;
  93
  94         if ((net_dev != NULL) && net_dev->destroy) {
  95                 return (NULL);
  96         }
  97
  98         return (net_dev);
  99 }
 100
 101 /*
 102  *
 103  */
 104 static inline netvsc_dev *
 105 hv_nv_get_inbound_net_device(struct hv_device *device)
 106 {
 107         hn_softc_t *sc = device_get_softc(device->device);
 108         netvsc_dev *net_dev = sc->net_dev;;
 109
 110         if (net_dev == NULL) {
 111                 return (net_dev);
 112         }
 113         /*
 114          * When the device is being destroyed; we only
 115          * permit incoming packets if and only if there
 116          * are outstanding sends.
 117          */
 118         if (net_dev->destroy && net_dev->num_outstanding_sends == 0) {
 119                 return (NULL);
 120         }
 121
 122         return (net_dev);
 123 }
 124
 125 int
 126 hv_nv_get_next_send_section(netvsc_dev *net_dev)
 127 {
 128         unsigned long bitsmap_words = net_dev->bitsmap_words;
 129         unsigned long *bitsmap = net_dev->send_section_bitsmap;
 130         unsigned long idx;
 131         int ret = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
 132         int i;
 133
 134         for (i = 0; i < bitsmap_words; i++) {
 135                 idx = ffsl(~bitsmap[i]);
 136                 if (0 == idx)
 137                         continue;
 138
 139                 idx--;
 140                 KASSERT(i * BITS_PER_LONG + idx < net_dev->send_section_count,
 141                     ("invalid i %d and idx %lu", i, idx));
 142
 143                 if (atomic_testandset_long(&bitsmap[i], idx))
 144                         continue;
 145
 146                 ret = i * BITS_PER_LONG + idx;
 147                 break;
 148         }
 149
 150         return (ret);
 151 }
 152
 153 /*
 154  * Net VSC initialize receive buffer with net VSP
 155  *
 156  * Net VSP:  Network virtual services client, also known as the
 157  *     Hyper-V extensible switch and the synthetic data path.
 158  */
 159 static int
 160 hv_nv_init_rx_buffer_with_net_vsp(struct hv_device *device)
 161 {
 162         netvsc_dev *net_dev;
 163         nvsp_msg *init_pkt;
 164         int ret = 0;
 165
 166         net_dev = hv_nv_get_outbound_net_device(device);
 167         if (!net_dev) {
 168                 return (ENODEV);
 169         }
 170
 171         net_dev->rx_buf = contigmalloc(net_dev->rx_buf_size, M_NETVSC,
 172             M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 173
 174         /*
 175          * Establish the GPADL handle for this buffer on this channel.
 176          * Note:  This call uses the vmbus connection rather than the
 177          * channel to establish the gpadl handle.
 178          * GPADL:  Guest physical address descriptor list.
 179          */
 180         ret = hv_vmbus_channel_establish_gpadl(
 181                 device->channel, net_dev->rx_buf,
 182                 net_dev->rx_buf_size, &net_dev->rx_buf_gpadl_handle);
 183         if (ret != 0) {
 184                 goto cleanup;
 185         }
 186
 187         /* sema_wait(&ext->channel_init_sema); KYS CHECK */
 188
 189         /* Notify the NetVsp of the gpadl handle */
 190         init_pkt = &net_dev->channel_init_packet;
 191
 192         memset(init_pkt, 0, sizeof(nvsp_msg));
 193
 194         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_rx_buf;
 195         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 196             net_dev->rx_buf_gpadl_handle;
 197         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 198             NETVSC_RECEIVE_BUFFER_ID;
 199
 200         /* Send the gpadl notification request */
 201
 202         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 203             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 204             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 205             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 206         if (ret != 0) {
 207                 goto cleanup;
 208         }
 209
 210         sema_wait(&net_dev->channel_init_sema);
 211
 212         /* Check the response */
 213         if (init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.status
 214             != nvsp_status_success) {
 215                 ret = EINVAL;
 216                 goto cleanup;
 217         }
 218
 219         net_dev->rx_section_count =
 220             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.num_sections;
 221
 222         net_dev->rx_sections = malloc(net_dev->rx_section_count *
 223             sizeof(nvsp_1_rx_buf_section), M_NETVSC, M_WAITOK);
 224         memcpy(net_dev->rx_sections,
 225             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.sections,
 226             net_dev->rx_section_count * sizeof(nvsp_1_rx_buf_section));
 227
 228
 229         /*
 230          * For first release, there should only be 1 section that represents
 231          * the entire receive buffer
 232          */
 233         if (net_dev->rx_section_count != 1
 234             || net_dev->rx_sections->offset != 0) {
 235                 ret = EINVAL;
 236                 goto cleanup;
 237         }
 238
 239         goto exit;
 240
 241 cleanup:
 242         hv_nv_destroy_rx_buffer(net_dev);
 243
 244 exit:
 245         return (ret);
 246 }
 247
 248 /*
 249  * Net VSC initialize send buffer with net VSP
 250  */
 251 static int
 252 hv_nv_init_send_buffer_with_net_vsp(struct hv_device *device)
 253 {
 254         netvsc_dev *net_dev;
 255         nvsp_msg *init_pkt;
 256         int ret = 0;
 257
 258         net_dev = hv_nv_get_outbound_net_device(device);
 259         if (!net_dev) {
 260                 return (ENODEV);
 261         }
 262
 263         net_dev->send_buf  = contigmalloc(net_dev->send_buf_size, M_NETVSC,
 264             M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 265         if (net_dev->send_buf == NULL) {
 266                 ret = ENOMEM;
 267                 goto cleanup;
 268         }
 269
 270         /*
 271          * Establish the gpadl handle for this buffer on this channel.
 272          * Note:  This call uses the vmbus connection rather than the
 273          * channel to establish the gpadl handle.
 274          */
 275         ret = hv_vmbus_channel_establish_gpadl(device->channel,
 276             net_dev->send_buf, net_dev->send_buf_size,
 277             &net_dev->send_buf_gpadl_handle);
 278         if (ret != 0) {
 279                 goto cleanup;
 280         }
 281
 282         /* Notify the NetVsp of the gpadl handle */
 283
 284         init_pkt = &net_dev->channel_init_packet;
 285
 286         memset(init_pkt, 0, sizeof(nvsp_msg));
 287
 288         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_send_buf;
 289         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 290             net_dev->send_buf_gpadl_handle;
 291         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 292             NETVSC_SEND_BUFFER_ID;
 293
 294         /* Send the gpadl notification request */
 295
 296         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 297             sizeof(nvsp_msg), (uint64_t)init_pkt,
 298             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 299             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 300         if (ret != 0) {
 301                 goto cleanup;
 302         }
 303
 304         sema_wait(&net_dev->channel_init_sema);
 305
 306         /* Check the response */
 307         if (init_pkt->msgs.vers_1_msgs.send_send_buf_complete.status
 308             != nvsp_status_success) {
 309                 ret = EINVAL;
 310                 goto cleanup;
 311         }
 312
 313         net_dev->send_section_size =
 314             init_pkt->msgs.vers_1_msgs.send_send_buf_complete.section_size;
 315         net_dev->send_section_count =
 316             net_dev->send_buf_size / net_dev->send_section_size;
 317         net_dev->bitsmap_words = howmany(net_dev->send_section_count,
 318             BITS_PER_LONG);
 319         net_dev->send_section_bitsmap =
 320             malloc(net_dev->bitsmap_words * sizeof(long), M_NETVSC,
 321             M_WAITOK | M_ZERO);
 322
 323         goto exit;
 324
 325 cleanup:
 326         hv_nv_destroy_send_buffer(net_dev);
 327
 328 exit:
 329         return (ret);
 330 }
 331
 332 /*
 333  * Net VSC destroy receive buffer
 334  */
 335 static int
 336 hv_nv_destroy_rx_buffer(netvsc_dev *net_dev)
 337 {
 338         nvsp_msg *revoke_pkt;
 339         int ret = 0;
 340
 341         /*
 342          * If we got a section count, it means we received a
 343          * send_rx_buf_complete msg
 344          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 345          * we need to send a revoke msg here
 346          */
 347         if (net_dev->rx_section_count) {
 348                 /* Send the revoke receive buffer */
 349                 revoke_pkt = &net_dev->revoke_packet;
 350                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 351
 352                 revoke_pkt->hdr.msg_type = nvsp_msg_1_type_revoke_rx_buf;
 353                 revoke_pkt->msgs.vers_1_msgs.revoke_rx_buf.id =
 354                     NETVSC_RECEIVE_BUFFER_ID;
 355
 356                 ret = hv_vmbus_channel_send_packet(net_dev->dev->channel,
 357                     revoke_pkt, sizeof(nvsp_msg),
 358                     (uint64_t)(uintptr_t)revoke_pkt,
 359                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 360
 361                 /*
 362                  * If we failed here, we might as well return and have a leak
 363                  * rather than continue and a bugchk
 364                  */
 365                 if (ret != 0) {
 366                         return (ret);
 367                 }
 368         }
 369
 370         /* Tear down the gpadl on the vsp end */
 371         if (net_dev->rx_buf_gpadl_handle) {
 372                 ret = hv_vmbus_channel_teardown_gpdal(net_dev->dev->channel,
 373                     net_dev->rx_buf_gpadl_handle);
 374                 /*
 375                  * If we failed here, we might as well return and have a leak
 376                  * rather than continue and a bugchk
 377                  */
 378                 if (ret != 0) {
 379                         return (ret);
 380                 }
 381                 net_dev->rx_buf_gpadl_handle = 0;
 382         }
 383
 384         if (net_dev->rx_buf) {
 385                 /* Free up the receive buffer */
 386                 contigfree(net_dev->rx_buf, net_dev->rx_buf_size, M_NETVSC);
 387                 net_dev->rx_buf = NULL;
 388         }
 389
 390         if (net_dev->rx_sections) {
 391                 free(net_dev->rx_sections, M_NETVSC);
 392                 net_dev->rx_sections = NULL;
 393                 net_dev->rx_section_count = 0;
 394         }
 395
 396         return (ret);
 397 }
 398
 399 /*
 400  * Net VSC destroy send buffer
 401  */
 402 static int
 403 hv_nv_destroy_send_buffer(netvsc_dev *net_dev)
 404 {
 405         nvsp_msg *revoke_pkt;
 406         int ret = 0;
 407
 408         /*
 409          * If we got a section count, it means we received a
 410          * send_rx_buf_complete msg
 411          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 412          * we need to send a revoke msg here
 413          */
 414         if (net_dev->send_section_size) {
 415                 /* Send the revoke send buffer */
 416                 revoke_pkt = &net_dev->revoke_packet;
 417                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 418
 419                 revoke_pkt->hdr.msg_type =
 420                     nvsp_msg_1_type_revoke_send_buf;
 421                 revoke_pkt->msgs.vers_1_msgs.revoke_send_buf.id =
 422                     NETVSC_SEND_BUFFER_ID;
 423
 424                 ret = hv_vmbus_channel_send_packet(net_dev->dev->channel,
 425                     revoke_pkt, sizeof(nvsp_msg),
 426                     (uint64_t)(uintptr_t)revoke_pkt,
 427                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 428                 /*
 429                  * If we failed here, we might as well return and have a leak
 430                  * rather than continue and a bugchk
 431                  */
 432                 if (ret != 0) {
 433                         return (ret);
 434                 }
 435         }
 436
 437         /* Tear down the gpadl on the vsp end */
 438         if (net_dev->send_buf_gpadl_handle) {
 439                 ret = hv_vmbus_channel_teardown_gpdal(net_dev->dev->channel,
 440                     net_dev->send_buf_gpadl_handle);
 441
 442                 /*
 443                  * If we failed here, we might as well return and have a leak
 444                  * rather than continue and a bugchk
 445                  */
 446                 if (ret != 0) {
 447                         return (ret);
 448                 }
 449                 net_dev->send_buf_gpadl_handle = 0;
 450         }
 451
 452         if (net_dev->send_buf) {
 453                 /* Free up the receive buffer */
 454                 contigfree(net_dev->send_buf, net_dev->send_buf_size, M_NETVSC);
 455                 net_dev->send_buf = NULL;
 456         }
 457
 458         if (net_dev->send_section_bitsmap) {
 459                 free(net_dev->send_section_bitsmap, M_NETVSC);
 460         }
 461
 462         return (ret);
 463 }
 464
 465
 466 /*
 467  * Attempt to negotiate the caller-specified NVSP version
 468  *
 469  * For NVSP v2, Server 2008 R2 does not set
 470  * init_pkt->msgs.init_msgs.init_compl.negotiated_prot_vers
 471  * to the negotiated version, so we cannot rely on that.
 472  */
 473 static int
 474 hv_nv_negotiate_nvsp_protocol(struct hv_device *device, netvsc_dev *net_dev,
 475     uint32_t nvsp_ver)
 476 {
 477         nvsp_msg *init_pkt;
 478         int ret;
 479
 480         init_pkt = &net_dev->channel_init_packet;
 481         memset(init_pkt, 0, sizeof(nvsp_msg));
 482         init_pkt->hdr.msg_type = nvsp_msg_type_init;
 483
 484         /*
 485          * Specify parameter as the only acceptable protocol version
 486          */
 487         init_pkt->msgs.init_msgs.init.p1.protocol_version = nvsp_ver;
 488         init_pkt->msgs.init_msgs.init.protocol_version_2 = nvsp_ver;
 489
 490         /* Send the init request */
 491         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 492             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 493             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 494             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 495         if (ret != 0)
 496                 return (-1);
 497
 498         sema_wait(&net_dev->channel_init_sema);
 499
 500         if (init_pkt->msgs.init_msgs.init_compl.status != nvsp_status_success)
 501                 return (EINVAL);
 502
 503         return (0);
 504 }
 505
 506 /*
 507  * Send NDIS version 2 config packet containing MTU.
 508  *
 509  * Not valid for NDIS version 1.
 510  */
 511 static int
 512 hv_nv_send_ndis_config(struct hv_device *device, uint32_t mtu)
 513 {
 514         netvsc_dev *net_dev;
 515         nvsp_msg *init_pkt;
 516         int ret;
 517
 518         net_dev = hv_nv_get_outbound_net_device(device);
 519         if (!net_dev)
 520                 return (-ENODEV);
 521
 522         /*
 523          * Set up configuration packet, write MTU
 524          * Indicate we are capable of handling VLAN tags
 525          */
 526         init_pkt = &net_dev->channel_init_packet;
 527         memset(init_pkt, 0, sizeof(nvsp_msg));
 528         init_pkt->hdr.msg_type = nvsp_msg_2_type_send_ndis_config;
 529         init_pkt->msgs.vers_2_msgs.send_ndis_config.mtu = mtu;
 530         init_pkt->
 531                 msgs.vers_2_msgs.send_ndis_config.capabilities.u1.u2.ieee8021q
 532                 = 1;
 533
 534         /* Send the configuration packet */
 535         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 536             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 537             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 538         if (ret != 0)
 539                 return (-EINVAL);
 540
 541         return (0);
 542 }
 543
 544 /*
 545  * Net VSC connect to VSP
 546  */
 547 static int
 548 hv_nv_connect_to_vsp(struct hv_device *device)
 549 {
 550         netvsc_dev *net_dev;
 551         nvsp_msg *init_pkt;
 552         uint32_t ndis_version;
 553         uint32_t protocol_list[] = { NVSP_PROTOCOL_VERSION_1,
 554             NVSP_PROTOCOL_VERSION_2,
 555             NVSP_PROTOCOL_VERSION_4,
 556             NVSP_PROTOCOL_VERSION_5 };
 557         int i;
 558         int protocol_number = nitems(protocol_list);
 559         int ret = 0;
 560         device_t dev = device->device;
 561         hn_softc_t *sc = device_get_softc(dev);
 562         struct ifnet *ifp = sc->arpcom.ac_ifp;
 563
 564         net_dev = hv_nv_get_outbound_net_device(device);
 565         if (!net_dev) {
 566                 return (ENODEV);
 567         }
 568
 569         /*
 570          * Negotiate the NVSP version.  Try the latest NVSP first.
 571          */
 572         for (i = protocol_number - 1; i >= 0; i--) {
 573                 if (hv_nv_negotiate_nvsp_protocol(device, net_dev,
 574                     protocol_list[i]) == 0) {
 575                         net_dev->nvsp_version = protocol_list[i];
 576                         if (bootverbose)
 577                                 device_printf(dev, "Netvsc: got version 0x%x\n",
 578                                     net_dev->nvsp_version);
 579                         break;
 580                 }
 581         }
 582
 583         if (i < 0) {
 584                 if (bootverbose)
 585                         device_printf(dev, "failed to negotiate a valid "
 586                             "protocol.\n");
 587                 return (EPROTO);
 588         }
 589
 590         /*
 591          * Set the MTU if supported by this NVSP protocol version
 592          * This needs to be right after the NVSP init message per Haiyang
 593          */
 594         if (net_dev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
 595                 ret = hv_nv_send_ndis_config(device, ifp->if_mtu);
 596
 597         /*
 598          * Send the NDIS version
 599          */
 600         init_pkt = &net_dev->channel_init_packet;
 601
 602         memset(init_pkt, 0, sizeof(nvsp_msg));
 603
 604         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_4) {
 605                 ndis_version = NDIS_VERSION_6_1;
 606         } else {
 607                 ndis_version = NDIS_VERSION_6_30;
 608         }
 609
 610         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_ndis_vers;
 611         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_major_vers =
 612             (ndis_version & 0xFFFF0000) >> 16;
 613         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_minor_vers =
 614             ndis_version & 0xFFFF;
 615
 616         /* Send the init request */
 617
 618         ret = hv_vmbus_channel_send_packet(device->channel, init_pkt,
 619             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 620             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND, 0);
 621         if (ret != 0) {
 622                 goto cleanup;
 623         }
 624         /*
 625          * TODO:  BUGBUG - We have to wait for the above msg since the netvsp
 626          * uses KMCL which acknowledges packet (completion packet)
 627          * since our Vmbus always set the
 628          * HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED flag
 629          */
 630         /* sema_wait(&NetVscChannel->channel_init_sema); */
 631
 632         /* Post the big receive buffer to NetVSP */
 633         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
 634                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
 635         else
 636                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
 637         net_dev->send_buf_size = NETVSC_SEND_BUFFER_SIZE;
 638
 639         ret = hv_nv_init_rx_buffer_with_net_vsp(device);
 640         if (ret == 0)
 641                 ret = hv_nv_init_send_buffer_with_net_vsp(device);
 642
 643 cleanup:
 644         return (ret);
 645 }
 646
 647 /*
 648  * Net VSC disconnect from VSP
 649  */
 650 static void
 651 hv_nv_disconnect_from_vsp(netvsc_dev *net_dev)
 652 {
 653         hv_nv_destroy_rx_buffer(net_dev);
 654         hv_nv_destroy_send_buffer(net_dev);
 655 }
 656
 657 /*
 658  * Net VSC on device add
 659  *
 660  * Callback when the device belonging to this driver is added
 661  */
 662 netvsc_dev *
 663 hv_nv_on_device_add(struct hv_device *device, void *additional_info)
 664 {
 665         netvsc_dev *net_dev;
 666         int ret = 0;
 667
 668         net_dev = hv_nv_alloc_net_device(device);
 669         if (!net_dev)
 670                 goto cleanup;
 671
 672         /* Initialize the NetVSC channel extension */
 673
 674         sema_init(&net_dev->channel_init_sema, 0, "netdev_sema");
 675
 676         /*
 677          * Open the channel
 678          */
 679         ret = hv_vmbus_channel_open(device->channel,
 680             NETVSC_DEVICE_RING_BUFFER_SIZE, NETVSC_DEVICE_RING_BUFFER_SIZE,
 681             NULL, 0, hv_nv_on_channel_callback, device);
 682         if (ret != 0)
 683                 goto cleanup;
 684
 685         /*
 686          * Connect with the NetVsp
 687          */
 688         ret = hv_nv_connect_to_vsp(device);
 689         if (ret != 0)
 690                 goto close;
 691
 692         return (net_dev);
 693
 694 close:
 695         /* Now, we can close the channel safely */
 696
 697         hv_vmbus_channel_close(device->channel);
 698
 699 cleanup:
 700         /*
 701          * Free the packet buffers on the netvsc device packet queue.
 702          * Release other resources.
 703          */
 704         if (net_dev) {
 705                 sema_destroy(&net_dev->channel_init_sema);
 706                 free(net_dev, M_NETVSC);
 707         }
 708
 709         return (NULL);
 710 }
 711
 712 /*
 713  * Net VSC on device remove
 714  */
 715 int
 716 hv_nv_on_device_remove(struct hv_device *device, boolean_t destroy_channel)
 717 {
 718         hn_softc_t *sc = device_get_softc(device->device);
 719         netvsc_dev *net_dev = sc->net_dev;;
 720
 721         /* Stop outbound traffic ie sends and receives completions */
 722         mtx_lock(&device->channel->inbound_lock);
 723         net_dev->destroy = TRUE;
 724         mtx_unlock(&device->channel->inbound_lock);
 725
 726         /* Wait for all send completions */
 727         while (net_dev->num_outstanding_sends) {
 728                 DELAY(100);
 729         }
 730
 731         hv_nv_disconnect_from_vsp(net_dev);
 732
 733         /* At this point, no one should be accessing net_dev except in here */
 734
 735         /* Now, we can close the channel safely */
 736
 737         if (!destroy_channel) {
 738                 device->channel->state =
 739                     HV_CHANNEL_CLOSING_NONDESTRUCTIVE_STATE;
 740         }
 741
 742         hv_vmbus_channel_close(device->channel);
 743
 744         sema_destroy(&net_dev->channel_init_sema);
 745         free(net_dev, M_NETVSC);
 746
 747         return (0);
 748 }
 749
 750 /*
 751  * Net VSC on send completion
 752  */
 753 static void
 754 hv_nv_on_send_completion(netvsc_dev *net_dev,
 755     struct hv_device *device, hv_vm_packet_descriptor *pkt)
 756 {
 757         nvsp_msg *nvsp_msg_pkt;
 758         netvsc_packet *net_vsc_pkt;
 759
 760         nvsp_msg_pkt =
 761             (nvsp_msg *)((unsigned long)pkt + (pkt->data_offset8 << 3));
 762
 763         if (nvsp_msg_pkt->hdr.msg_type == nvsp_msg_type_init_complete
 764                 || nvsp_msg_pkt->hdr.msg_type
 765                         == nvsp_msg_1_type_send_rx_buf_complete
 766                 || nvsp_msg_pkt->hdr.msg_type
 767                         == nvsp_msg_1_type_send_send_buf_complete) {
 768                 /* Copy the response back */
 769                 memcpy(&net_dev->channel_init_packet, nvsp_msg_pkt,
 770                     sizeof(nvsp_msg));
 771                 sema_post(&net_dev->channel_init_sema);
 772         } else if (nvsp_msg_pkt->hdr.msg_type ==
 773                     nvsp_msg_1_type_send_rndis_pkt_complete) {
 774                 /* Get the send context */
 775                 net_vsc_pkt =
 776                     (netvsc_packet *)(unsigned long)pkt->transaction_id;
 777                 if (NULL != net_vsc_pkt) {
 778                         if (net_vsc_pkt->send_buf_section_idx !=
 779                             NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) {
 780                                 u_long mask;
 781                                 int idx;
 782
 783                                 idx = net_vsc_pkt->send_buf_section_idx /
 784                                     BITS_PER_LONG;
 785                                 KASSERT(idx < net_dev->bitsmap_words,
 786                                     ("invalid section index %u",
 787                                      net_vsc_pkt->send_buf_section_idx));
 788                                 mask = 1UL <<
 789                                     (net_vsc_pkt->send_buf_section_idx %
 790                                      BITS_PER_LONG);
 791
 792                                 KASSERT(net_dev->send_section_bitsmap[idx] &
 793                                     mask,
 794                                     ("index bitmap 0x%lx, section index %u, "
 795                                      "bitmap idx %d, bitmask 0x%lx",
 796                                      net_dev->send_section_bitsmap[idx],
 797                                      net_vsc_pkt->send_buf_section_idx,
 798                                      idx, mask));
 799                                 atomic_clear_long(
 800                                     &net_dev->send_section_bitsmap[idx], mask);
 801                         }
 802
 803                         /* Notify the layer above us */
 804                         net_vsc_pkt->compl.send.on_send_completion(
 805                             net_vsc_pkt->compl.send.send_completion_context);
 806
 807                 }
 808
 809                 atomic_subtract_int(&net_dev->num_outstanding_sends, 1);
 810         }
 811 }
 812
 813 /*
 814  * Net VSC on send
 815  * Sends a packet on the specified Hyper-V device.
 816  * Returns 0 on success, non-zero on failure.
 817  */
 818 int
 819 hv_nv_on_send(struct hv_device *device, netvsc_packet *pkt)
 820 {
 821         netvsc_dev *net_dev;
 822         nvsp_msg send_msg;
 823         int ret;
 824
 825         net_dev = hv_nv_get_outbound_net_device(device);
 826         if (!net_dev)
 827                 return (ENODEV);
 828
 829         send_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt;
 830         if (pkt->is_data_pkt) {
 831                 /* 0 is RMC_DATA */
 832                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 0;
 833         } else {
 834                 /* 1 is RMC_CONTROL */
 835                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 1;
 836         }
 837
 838         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_idx =
 839             pkt->send_buf_section_idx;
 840         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_size =
 841             pkt->send_buf_section_size;
 842
 843         if (pkt->page_buf_count) {
 844                 ret = hv_vmbus_channel_send_packet_pagebuffer(device->channel,
 845                     pkt->page_buffers, pkt->page_buf_count,
 846                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt);
 847         } else {
 848                 ret = hv_vmbus_channel_send_packet(device->channel,
 849                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt,
 850                     HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 851                     HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 852         }
 853
 854         /* Record outstanding send only if send_packet() succeeded */
 855         if (ret == 0)
 856                 atomic_add_int(&net_dev->num_outstanding_sends, 1);
 857
 858         return (ret);
 859 }
 860
 861 /*
 862  * Net VSC on receive
 863  *
 864  * In the FreeBSD Hyper-V virtual world, this function deals exclusively
 865  * with virtual addresses.
 866  */
 867 static void
 868 hv_nv_on_receive(netvsc_dev *net_dev, struct hv_device *device,
 869     hv_vm_packet_descriptor *pkt)
 870 {
 871         hv_vm_transfer_page_packet_header *vm_xfer_page_pkt;
 872         nvsp_msg *nvsp_msg_pkt;
 873         netvsc_packet vsc_pkt;
 874         netvsc_packet *net_vsc_pkt = &vsc_pkt;
 875         device_t dev = device->device;
 876         int count = 0;
 877         int i = 0;
 878         int status = nvsp_status_success;
 879
 880         /*
 881          * All inbound packets other than send completion should be
 882          * xfer page packet.
 883          */
 884         if (pkt->type != HV_VMBUS_PACKET_TYPE_DATA_USING_TRANSFER_PAGES) {
 885                 device_printf(dev, "packet type %d is invalid!\n", pkt->type);
 886                 return;
 887         }
 888
 889         nvsp_msg_pkt = (nvsp_msg *)((unsigned long)pkt
 890                 + (pkt->data_offset8 << 3));
 891
 892         /* Make sure this is a valid nvsp packet */
 893         if (nvsp_msg_pkt->hdr.msg_type != nvsp_msg_1_type_send_rndis_pkt) {
 894                 device_printf(dev, "packet hdr type %d is invalid!\n",
 895                     pkt->type);
 896                 return;
 897         }
 898
 899         vm_xfer_page_pkt = (hv_vm_transfer_page_packet_header *)pkt;
 900
 901         if (vm_xfer_page_pkt->transfer_page_set_id !=
 902             NETVSC_RECEIVE_BUFFER_ID) {
 903                 device_printf(dev, "transfer_page_set_id %d is invalid!\n",
 904                     vm_xfer_page_pkt->transfer_page_set_id);
 905                 return;
 906         }
 907
 908         count = vm_xfer_page_pkt->range_count;
 909         net_vsc_pkt->device = device;
 910
 911         /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */
 912         for (i = 0; i < count; i++) {
 913                 net_vsc_pkt->status = nvsp_status_success;
 914                 net_vsc_pkt->data = (void *)((unsigned long)net_dev->rx_buf +
 915                     vm_xfer_page_pkt->ranges[i].byte_offset);
 916                 net_vsc_pkt->tot_data_buf_len =
 917                     vm_xfer_page_pkt->ranges[i].byte_count;
 918
 919                 hv_rf_on_receive(net_dev, device, net_vsc_pkt);
 920                 if (net_vsc_pkt->status != nvsp_status_success) {
 921                         status = nvsp_status_failure;
 922                 }
 923         }
 924
 925         /*
 926          * Moved completion call back here so that all received
 927          * messages (not just data messages) will trigger a response
 928          * message back to the host.
 929          */
 930         hv_nv_on_receive_completion(device, vm_xfer_page_pkt->d.transaction_id,
 931             status);
 932         hv_rf_receive_rollup(net_dev);
 933 }
 934
 935 /*
 936  * Net VSC on receive completion
 937  *
 938  * Send a receive completion packet to RNDIS device (ie NetVsp)
 939  */
 940 void
 941 hv_nv_on_receive_completion(struct hv_device *device, uint64_t tid,
 942     uint32_t status)
 943 {
 944         nvsp_msg rx_comp_msg;
 945         int retries = 0;
 946         int ret = 0;
 947
 948         rx_comp_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt_complete;
 949
 950         /* Pass in the status */
 951         rx_comp_msg.msgs.vers_1_msgs.send_rndis_pkt_complete.status =
 952             status;
 953
 954 retry_send_cmplt:
 955         /* Send the completion */
 956         ret = hv_vmbus_channel_send_packet(device->channel, &rx_comp_msg,
 957             sizeof(nvsp_msg), tid, HV_VMBUS_PACKET_TYPE_COMPLETION, 0);
 958         if (ret == 0) {
 959                 /* success */
 960                 /* no-op */
 961         } else if (ret == EAGAIN) {
 962                 /* no more room... wait a bit and attempt to retry 3 times */
 963                 retries++;
 964
 965                 if (retries < 4) {
 966                         DELAY(100);
 967                         goto retry_send_cmplt;
 968                 }
 969         }
 970 }
 971
 972 /*
 973  * Net VSC on channel callback
 974  */
 975 static void
 976 hv_nv_on_channel_callback(void *context)
 977 {
 978         struct hv_device *device = (struct hv_device *)context;
 979         netvsc_dev *net_dev;
 980         device_t dev = device->device;
 981         uint32_t bytes_rxed;
 982         uint64_t request_id;
 983         hv_vm_packet_descriptor *desc;
 984         uint8_t *buffer;
 985         int bufferlen = NETVSC_PACKET_SIZE;
 986         int ret = 0;
 987
 988         net_dev = hv_nv_get_inbound_net_device(device);
 989         if (net_dev == NULL)
 990                 return;
 991
 992         buffer = net_dev->callback_buf;
 993
 994         do {
 995                 ret = hv_vmbus_channel_recv_packet_raw(device->channel,
 996                     buffer, bufferlen, &bytes_rxed, &request_id);
 997                 if (ret == 0) {
 998                         if (bytes_rxed > 0) {
 999                                 desc = (hv_vm_packet_descriptor *)buffer;
1000                                 switch (desc->type) {
1001                                 case HV_VMBUS_PACKET_TYPE_COMPLETION:
1002                                         hv_nv_on_send_completion(net_dev, device, desc);
1003                                         break;
1004                                 case HV_VMBUS_PACKET_TYPE_DATA_USING_TRANSFER_PAGES:
1005                                         hv_nv_on_receive(net_dev, device, desc);
1006                                         break;
1007                                 default:
1008                                         device_printf(dev,
1009                                             "hv_cb recv unknow type %d "
1010                                             " packet\n", desc->type);
1011                                         break;
1012                                 }
1013                         } else {
1014                                 break;
1015                         }
1016                 } else if (ret == ENOBUFS) {
1017                         /* Handle large packet */
1018                         if (bufferlen > NETVSC_PACKET_SIZE) {
1019                                 free(buffer, M_NETVSC);
1020                                 buffer = NULL;
1021                         }
1022
1023                         /* alloc new buffer */
1024                         buffer = malloc(bytes_rxed, M_NETVSC, M_NOWAIT);
1025                         if (buffer == NULL) {
1026                                 device_printf(dev,
1027                                     "hv_cb malloc buffer failed, len=%u\n",
1028                                     bytes_rxed);
1029                                 bufferlen = 0;
1030                                 break;
1031                         }
1032                         bufferlen = bytes_rxed;
1033                 }
1034         } while (1);
1035
1036         if (bufferlen > NETVSC_PACKET_SIZE)
1037                 free(buffer, M_NETVSC);
1038
1039         hv_rf_channel_rollup(net_dev);
1040 }