sys/dev/hyperv/netvsc/hv_net_vsc.c

   1 /*-
   2  * Copyright (c) 2009-2012,2016 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 /**
  32  * HyperV vmbus network VSC (virtual services client) module
  33  *
  34  */
  35
  36
  37 #include <sys/param.h>
  38 #include <sys/kernel.h>
  39 #include <sys/socket.h>
  40 #include <sys/lock.h>
  41 #include <net/if.h>
  42 #include <net/if_var.h>
  43 #include <net/if_arp.h>
  44 #include <machine/bus.h>
  45 #include <machine/atomic.h>
  46
  47 #include <dev/hyperv/include/hyperv.h>
  48 #include "hv_net_vsc.h"
  49 #include "hv_rndis.h"
  50 #include "hv_rndis_filter.h"
  51
  52 /* priv1 and priv2 are consumed by the main driver */
  53 #define hv_chan_rdbuf   hv_chan_priv3
  54
  55 MALLOC_DEFINE(M_NETVSC, "netvsc", "Hyper-V netvsc driver");
  56
  57 /*
  58  * Forward declarations
  59  */
  60 static void hv_nv_on_channel_callback(void *xchan);
  61 static int  hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc);
  62 static int  hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *);
  63 static int  hv_nv_destroy_send_buffer(netvsc_dev *net_dev);
  64 static int  hv_nv_destroy_rx_buffer(netvsc_dev *net_dev);
  65 static int  hv_nv_connect_to_vsp(struct hn_softc *sc);
  66 static void hv_nv_on_send_completion(netvsc_dev *net_dev,
  67     struct hv_vmbus_channel *, hv_vm_packet_descriptor *pkt);
  68 static void hv_nv_on_receive_completion(struct hv_vmbus_channel *chan,
  69     uint64_t tid, uint32_t status);
  70 static void hv_nv_on_receive(netvsc_dev *net_dev,
  71     struct hn_softc *sc, struct hv_vmbus_channel *chan,
  72     hv_vm_packet_descriptor *pkt);
  73
  74 /*
  75  *
  76  */
  77 static inline netvsc_dev *
  78 hv_nv_alloc_net_device(struct hn_softc *sc)
  79 {
  80         netvsc_dev *net_dev;
  81
  82         net_dev = malloc(sizeof(netvsc_dev), M_NETVSC, M_WAITOK | M_ZERO);
  83
  84         net_dev->sc = sc;
  85         net_dev->destroy = FALSE;
  86         sc->net_dev = net_dev;
  87
  88         return (net_dev);
  89 }
  90
  91 /*
  92  * XXX unnecessary; nuke it.
  93  */
  94 static inline netvsc_dev *
  95 hv_nv_get_outbound_net_device(struct hn_softc *sc)
  96 {
  97         return sc->net_dev;
  98 }
  99
 100 /*
 101  * XXX unnecessary; nuke it.
 102  */
 103 static inline netvsc_dev *
 104 hv_nv_get_inbound_net_device(struct hn_softc *sc)
 105 {
 106         return sc->net_dev;
 107 }
 108
 109 int
 110 hv_nv_get_next_send_section(netvsc_dev *net_dev)
 111 {
 112         unsigned long bitsmap_words = net_dev->bitsmap_words;
 113         unsigned long *bitsmap = net_dev->send_section_bitsmap;
 114         unsigned long idx;
 115         int ret = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
 116         int i;
 117
 118         for (i = 0; i < bitsmap_words; i++) {
 119                 idx = ffsl(~bitsmap[i]);
 120                 if (0 == idx)
 121                         continue;
 122
 123                 idx--;
 124                 KASSERT(i * BITS_PER_LONG + idx < net_dev->send_section_count,
 125                     ("invalid i %d and idx %lu", i, idx));
 126
 127                 if (atomic_testandset_long(&bitsmap[i], idx))
 128                         continue;
 129
 130                 ret = i * BITS_PER_LONG + idx;
 131                 break;
 132         }
 133
 134         return (ret);
 135 }
 136
 137 /*
 138  * Net VSC initialize receive buffer with net VSP
 139  *
 140  * Net VSP:  Network virtual services client, also known as the
 141  *     Hyper-V extensible switch and the synthetic data path.
 142  */
 143 static int
 144 hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *sc)
 145 {
 146         netvsc_dev *net_dev;
 147         nvsp_msg *init_pkt;
 148         int ret = 0;
 149
 150         net_dev = hv_nv_get_outbound_net_device(sc);
 151         if (!net_dev) {
 152                 return (ENODEV);
 153         }
 154
 155         net_dev->rx_buf = contigmalloc(net_dev->rx_buf_size, M_NETVSC,
 156             M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 157
 158         /*
 159          * Establish the GPADL handle for this buffer on this channel.
 160          * Note:  This call uses the vmbus connection rather than the
 161          * channel to establish the gpadl handle.
 162          * GPADL:  Guest physical address descriptor list.
 163          */
 164         ret = hv_vmbus_channel_establish_gpadl(
 165                 sc->hn_prichan, net_dev->rx_buf,
 166                 net_dev->rx_buf_size, &net_dev->rx_buf_gpadl_handle);
 167         if (ret != 0) {
 168                 goto cleanup;
 169         }
 170
 171         /* sema_wait(&ext->channel_init_sema); KYS CHECK */
 172
 173         /* Notify the NetVsp of the gpadl handle */
 174         init_pkt = &net_dev->channel_init_packet;
 175
 176         memset(init_pkt, 0, sizeof(nvsp_msg));
 177
 178         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_rx_buf;
 179         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 180             net_dev->rx_buf_gpadl_handle;
 181         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 182             NETVSC_RECEIVE_BUFFER_ID;
 183
 184         /* Send the gpadl notification request */
 185
 186         ret = hv_vmbus_channel_send_packet(sc->hn_prichan, init_pkt,
 187             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 188             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC);
 189         if (ret != 0) {
 190                 goto cleanup;
 191         }
 192
 193         sema_wait(&net_dev->channel_init_sema);
 194
 195         /* Check the response */
 196         if (init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.status
 197             != nvsp_status_success) {
 198                 ret = EINVAL;
 199                 goto cleanup;
 200         }
 201
 202         net_dev->rx_section_count =
 203             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.num_sections;
 204
 205         net_dev->rx_sections = malloc(net_dev->rx_section_count *
 206             sizeof(nvsp_1_rx_buf_section), M_NETVSC, M_WAITOK);
 207         memcpy(net_dev->rx_sections,
 208             init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.sections,
 209             net_dev->rx_section_count * sizeof(nvsp_1_rx_buf_section));
 210
 211
 212         /*
 213          * For first release, there should only be 1 section that represents
 214          * the entire receive buffer
 215          */
 216         if (net_dev->rx_section_count != 1
 217             || net_dev->rx_sections->offset != 0) {
 218                 ret = EINVAL;
 219                 goto cleanup;
 220         }
 221
 222         goto exit;
 223
 224 cleanup:
 225         hv_nv_destroy_rx_buffer(net_dev);
 226
 227 exit:
 228         return (ret);
 229 }
 230
 231 /*
 232  * Net VSC initialize send buffer with net VSP
 233  */
 234 static int
 235 hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc)
 236 {
 237         netvsc_dev *net_dev;
 238         nvsp_msg *init_pkt;
 239         int ret = 0;
 240
 241         net_dev = hv_nv_get_outbound_net_device(sc);
 242         if (!net_dev) {
 243                 return (ENODEV);
 244         }
 245
 246         net_dev->send_buf  = contigmalloc(net_dev->send_buf_size, M_NETVSC,
 247             M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 248         if (net_dev->send_buf == NULL) {
 249                 ret = ENOMEM;
 250                 goto cleanup;
 251         }
 252
 253         /*
 254          * Establish the gpadl handle for this buffer on this channel.
 255          * Note:  This call uses the vmbus connection rather than the
 256          * channel to establish the gpadl handle.
 257          */
 258         ret = hv_vmbus_channel_establish_gpadl(sc->hn_prichan,
 259             net_dev->send_buf, net_dev->send_buf_size,
 260             &net_dev->send_buf_gpadl_handle);
 261         if (ret != 0) {
 262                 goto cleanup;
 263         }
 264
 265         /* Notify the NetVsp of the gpadl handle */
 266
 267         init_pkt = &net_dev->channel_init_packet;
 268
 269         memset(init_pkt, 0, sizeof(nvsp_msg));
 270
 271         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_send_buf;
 272         init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle =
 273             net_dev->send_buf_gpadl_handle;
 274         init_pkt->msgs.vers_1_msgs.send_rx_buf.id =
 275             NETVSC_SEND_BUFFER_ID;
 276
 277         /* Send the gpadl notification request */
 278
 279         ret = hv_vmbus_channel_send_packet(sc->hn_prichan, init_pkt,
 280             sizeof(nvsp_msg), (uint64_t)init_pkt,
 281             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC);
 282         if (ret != 0) {
 283                 goto cleanup;
 284         }
 285
 286         sema_wait(&net_dev->channel_init_sema);
 287
 288         /* Check the response */
 289         if (init_pkt->msgs.vers_1_msgs.send_send_buf_complete.status
 290             != nvsp_status_success) {
 291                 ret = EINVAL;
 292                 goto cleanup;
 293         }
 294
 295         net_dev->send_section_size =
 296             init_pkt->msgs.vers_1_msgs.send_send_buf_complete.section_size;
 297         net_dev->send_section_count =
 298             net_dev->send_buf_size / net_dev->send_section_size;
 299         net_dev->bitsmap_words = howmany(net_dev->send_section_count,
 300             BITS_PER_LONG);
 301         net_dev->send_section_bitsmap =
 302             malloc(net_dev->bitsmap_words * sizeof(long), M_NETVSC,
 303             M_WAITOK | M_ZERO);
 304
 305         goto exit;
 306
 307 cleanup:
 308         hv_nv_destroy_send_buffer(net_dev);
 309
 310 exit:
 311         return (ret);
 312 }
 313
 314 /*
 315  * Net VSC destroy receive buffer
 316  */
 317 static int
 318 hv_nv_destroy_rx_buffer(netvsc_dev *net_dev)
 319 {
 320         nvsp_msg *revoke_pkt;
 321         int ret = 0;
 322
 323         /*
 324          * If we got a section count, it means we received a
 325          * send_rx_buf_complete msg
 326          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 327          * we need to send a revoke msg here
 328          */
 329         if (net_dev->rx_section_count) {
 330                 /* Send the revoke receive buffer */
 331                 revoke_pkt = &net_dev->revoke_packet;
 332                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 333
 334                 revoke_pkt->hdr.msg_type = nvsp_msg_1_type_revoke_rx_buf;
 335                 revoke_pkt->msgs.vers_1_msgs.revoke_rx_buf.id =
 336                     NETVSC_RECEIVE_BUFFER_ID;
 337
 338                 ret = hv_vmbus_channel_send_packet(net_dev->sc->hn_prichan,
 339                     revoke_pkt, sizeof(nvsp_msg),
 340                     (uint64_t)(uintptr_t)revoke_pkt,
 341                     VMBUS_CHANPKT_TYPE_INBAND, 0);
 342
 343                 /*
 344                  * If we failed here, we might as well return and have a leak
 345                  * rather than continue and a bugchk
 346                  */
 347                 if (ret != 0) {
 348                         return (ret);
 349                 }
 350         }
 351
 352         /* Tear down the gpadl on the vsp end */
 353         if (net_dev->rx_buf_gpadl_handle) {
 354                 ret = hv_vmbus_channel_teardown_gpdal(net_dev->sc->hn_prichan,
 355                     net_dev->rx_buf_gpadl_handle);
 356                 /*
 357                  * If we failed here, we might as well return and have a leak
 358                  * rather than continue and a bugchk
 359                  */
 360                 if (ret != 0) {
 361                         return (ret);
 362                 }
 363                 net_dev->rx_buf_gpadl_handle = 0;
 364         }
 365
 366         if (net_dev->rx_buf) {
 367                 /* Free up the receive buffer */
 368                 contigfree(net_dev->rx_buf, net_dev->rx_buf_size, M_NETVSC);
 369                 net_dev->rx_buf = NULL;
 370         }
 371
 372         if (net_dev->rx_sections) {
 373                 free(net_dev->rx_sections, M_NETVSC);
 374                 net_dev->rx_sections = NULL;
 375                 net_dev->rx_section_count = 0;
 376         }
 377
 378         return (ret);
 379 }
 380
 381 /*
 382  * Net VSC destroy send buffer
 383  */
 384 static int
 385 hv_nv_destroy_send_buffer(netvsc_dev *net_dev)
 386 {
 387         nvsp_msg *revoke_pkt;
 388         int ret = 0;
 389
 390         /*
 391          * If we got a section count, it means we received a
 392          * send_rx_buf_complete msg
 393          * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore,
 394          * we need to send a revoke msg here
 395          */
 396         if (net_dev->send_section_size) {
 397                 /* Send the revoke send buffer */
 398                 revoke_pkt = &net_dev->revoke_packet;
 399                 memset(revoke_pkt, 0, sizeof(nvsp_msg));
 400
 401                 revoke_pkt->hdr.msg_type =
 402                     nvsp_msg_1_type_revoke_send_buf;
 403                 revoke_pkt->msgs.vers_1_msgs.revoke_send_buf.id =
 404                     NETVSC_SEND_BUFFER_ID;
 405
 406                 ret = hv_vmbus_channel_send_packet(net_dev->sc->hn_prichan,
 407                     revoke_pkt, sizeof(nvsp_msg),
 408                     (uint64_t)(uintptr_t)revoke_pkt,
 409                     VMBUS_CHANPKT_TYPE_INBAND, 0);
 410                 /*
 411                  * If we failed here, we might as well return and have a leak
 412                  * rather than continue and a bugchk
 413                  */
 414                 if (ret != 0) {
 415                         return (ret);
 416                 }
 417         }
 418
 419         /* Tear down the gpadl on the vsp end */
 420         if (net_dev->send_buf_gpadl_handle) {
 421                 ret = hv_vmbus_channel_teardown_gpdal(net_dev->sc->hn_prichan,
 422                     net_dev->send_buf_gpadl_handle);
 423
 424                 /*
 425                  * If we failed here, we might as well return and have a leak
 426                  * rather than continue and a bugchk
 427                  */
 428                 if (ret != 0) {
 429                         return (ret);
 430                 }
 431                 net_dev->send_buf_gpadl_handle = 0;
 432         }
 433
 434         if (net_dev->send_buf) {
 435                 /* Free up the receive buffer */
 436                 contigfree(net_dev->send_buf, net_dev->send_buf_size, M_NETVSC);
 437                 net_dev->send_buf = NULL;
 438         }
 439
 440         if (net_dev->send_section_bitsmap) {
 441                 free(net_dev->send_section_bitsmap, M_NETVSC);
 442         }
 443
 444         return (ret);
 445 }
 446
 447
 448 /*
 449  * Attempt to negotiate the caller-specified NVSP version
 450  *
 451  * For NVSP v2, Server 2008 R2 does not set
 452  * init_pkt->msgs.init_msgs.init_compl.negotiated_prot_vers
 453  * to the negotiated version, so we cannot rely on that.
 454  */
 455 static int
 456 hv_nv_negotiate_nvsp_protocol(struct hn_softc *sc, netvsc_dev *net_dev,
 457     uint32_t nvsp_ver)
 458 {
 459         nvsp_msg *init_pkt;
 460         int ret;
 461
 462         init_pkt = &net_dev->channel_init_packet;
 463         memset(init_pkt, 0, sizeof(nvsp_msg));
 464         init_pkt->hdr.msg_type = nvsp_msg_type_init;
 465
 466         /*
 467          * Specify parameter as the only acceptable protocol version
 468          */
 469         init_pkt->msgs.init_msgs.init.p1.protocol_version = nvsp_ver;
 470         init_pkt->msgs.init_msgs.init.protocol_version_2 = nvsp_ver;
 471
 472         /* Send the init request */
 473         ret = hv_vmbus_channel_send_packet(sc->hn_prichan, init_pkt,
 474             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 475             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC);
 476         if (ret != 0)
 477                 return (-1);
 478
 479         sema_wait(&net_dev->channel_init_sema);
 480
 481         if (init_pkt->msgs.init_msgs.init_compl.status != nvsp_status_success)
 482                 return (EINVAL);
 483
 484         return (0);
 485 }
 486
 487 /*
 488  * Send NDIS version 2 config packet containing MTU.
 489  *
 490  * Not valid for NDIS version 1.
 491  */
 492 static int
 493 hv_nv_send_ndis_config(struct hn_softc *sc, uint32_t mtu)
 494 {
 495         netvsc_dev *net_dev;
 496         nvsp_msg *init_pkt;
 497         int ret;
 498
 499         net_dev = hv_nv_get_outbound_net_device(sc);
 500         if (!net_dev)
 501                 return (-ENODEV);
 502
 503         /*
 504          * Set up configuration packet, write MTU
 505          * Indicate we are capable of handling VLAN tags
 506          */
 507         init_pkt = &net_dev->channel_init_packet;
 508         memset(init_pkt, 0, sizeof(nvsp_msg));
 509         init_pkt->hdr.msg_type = nvsp_msg_2_type_send_ndis_config;
 510         init_pkt->msgs.vers_2_msgs.send_ndis_config.mtu = mtu;
 511         init_pkt->
 512                 msgs.vers_2_msgs.send_ndis_config.capabilities.u1.u2.ieee8021q
 513                 = 1;
 514
 515         /* Send the configuration packet */
 516         ret = hv_vmbus_channel_send_packet(sc->hn_prichan, init_pkt,
 517             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 518             VMBUS_CHANPKT_TYPE_INBAND, 0);
 519         if (ret != 0)
 520                 return (-EINVAL);
 521
 522         return (0);
 523 }
 524
 525 /*
 526  * Net VSC connect to VSP
 527  */
 528 static int
 529 hv_nv_connect_to_vsp(struct hn_softc *sc)
 530 {
 531         netvsc_dev *net_dev;
 532         nvsp_msg *init_pkt;
 533         uint32_t ndis_version;
 534         uint32_t protocol_list[] = { NVSP_PROTOCOL_VERSION_1,
 535             NVSP_PROTOCOL_VERSION_2,
 536             NVSP_PROTOCOL_VERSION_4,
 537             NVSP_PROTOCOL_VERSION_5 };
 538         int i;
 539         int protocol_number = nitems(protocol_list);
 540         int ret = 0;
 541         device_t dev = sc->hn_dev;
 542         struct ifnet *ifp = sc->hn_ifp;
 543
 544         net_dev = hv_nv_get_outbound_net_device(sc);
 545
 546         /*
 547          * Negotiate the NVSP version.  Try the latest NVSP first.
 548          */
 549         for (i = protocol_number - 1; i >= 0; i--) {
 550                 if (hv_nv_negotiate_nvsp_protocol(sc, net_dev,
 551                     protocol_list[i]) == 0) {
 552                         net_dev->nvsp_version = protocol_list[i];
 553                         if (bootverbose)
 554                                 device_printf(dev, "Netvsc: got version 0x%x\n",
 555                                     net_dev->nvsp_version);
 556                         break;
 557                 }
 558         }
 559
 560         if (i < 0) {
 561                 if (bootverbose)
 562                         device_printf(dev, "failed to negotiate a valid "
 563                             "protocol.\n");
 564                 return (EPROTO);
 565         }
 566
 567         /*
 568          * Set the MTU if supported by this NVSP protocol version
 569          * This needs to be right after the NVSP init message per Haiyang
 570          */
 571         if (net_dev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
 572                 ret = hv_nv_send_ndis_config(sc, ifp->if_mtu);
 573
 574         /*
 575          * Send the NDIS version
 576          */
 577         init_pkt = &net_dev->channel_init_packet;
 578
 579         memset(init_pkt, 0, sizeof(nvsp_msg));
 580
 581         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_4) {
 582                 ndis_version = NDIS_VERSION_6_1;
 583         } else {
 584                 ndis_version = NDIS_VERSION_6_30;
 585         }
 586
 587         init_pkt->hdr.msg_type = nvsp_msg_1_type_send_ndis_vers;
 588         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_major_vers =
 589             (ndis_version & 0xFFFF0000) >> 16;
 590         init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_minor_vers =
 591             ndis_version & 0xFFFF;
 592
 593         /* Send the init request */
 594
 595         ret = hv_vmbus_channel_send_packet(sc->hn_prichan, init_pkt,
 596             sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt,
 597             VMBUS_CHANPKT_TYPE_INBAND, 0);
 598         if (ret != 0) {
 599                 goto cleanup;
 600         }
 601         /*
 602          * TODO:  BUGBUG - We have to wait for the above msg since the netvsp
 603          * uses KMCL which acknowledges packet (completion packet)
 604          * since our Vmbus always set the VMBUS_CHANPKT_FLAG_RC flag
 605          */
 606         /* sema_wait(&NetVscChannel->channel_init_sema); */
 607
 608         /* Post the big receive buffer to NetVSP */
 609         if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
 610                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
 611         else
 612                 net_dev->rx_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
 613         net_dev->send_buf_size = NETVSC_SEND_BUFFER_SIZE;
 614
 615         ret = hv_nv_init_rx_buffer_with_net_vsp(sc);
 616         if (ret == 0)
 617                 ret = hv_nv_init_send_buffer_with_net_vsp(sc);
 618
 619 cleanup:
 620         return (ret);
 621 }
 622
 623 /*
 624  * Net VSC disconnect from VSP
 625  */
 626 static void
 627 hv_nv_disconnect_from_vsp(netvsc_dev *net_dev)
 628 {
 629         hv_nv_destroy_rx_buffer(net_dev);
 630         hv_nv_destroy_send_buffer(net_dev);
 631 }
 632
 633 void
 634 hv_nv_subchan_attach(struct hv_vmbus_channel *chan)
 635 {
 636
 637         chan->hv_chan_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK);
 638         hv_vmbus_channel_open(chan, NETVSC_DEVICE_RING_BUFFER_SIZE,
 639             NETVSC_DEVICE_RING_BUFFER_SIZE, NULL, 0,
 640             hv_nv_on_channel_callback, chan);
 641 }
 642
 643 /*
 644  * Net VSC on device add
 645  *
 646  * Callback when the device belonging to this driver is added
 647  */
 648 netvsc_dev *
 649 hv_nv_on_device_add(struct hn_softc *sc, void *additional_info)
 650 {
 651         struct hv_vmbus_channel *chan = sc->hn_prichan;
 652         netvsc_dev *net_dev;
 653         int ret = 0;
 654
 655         net_dev = hv_nv_alloc_net_device(sc);
 656         if (net_dev == NULL)
 657                 return NULL;
 658
 659         /* Initialize the NetVSC channel extension */
 660
 661         sema_init(&net_dev->channel_init_sema, 0, "netdev_sema");
 662
 663         chan->hv_chan_rdbuf = malloc(NETVSC_PACKET_SIZE, M_NETVSC, M_WAITOK);
 664
 665         /*
 666          * Open the channel
 667          */
 668         ret = hv_vmbus_channel_open(chan,
 669             NETVSC_DEVICE_RING_BUFFER_SIZE, NETVSC_DEVICE_RING_BUFFER_SIZE,
 670             NULL, 0, hv_nv_on_channel_callback, chan);
 671         if (ret != 0) {
 672                 free(chan->hv_chan_rdbuf, M_NETVSC);
 673                 goto cleanup;
 674         }
 675
 676         /*
 677          * Connect with the NetVsp
 678          */
 679         ret = hv_nv_connect_to_vsp(sc);
 680         if (ret != 0)
 681                 goto close;
 682
 683         return (net_dev);
 684
 685 close:
 686         /* Now, we can close the channel safely */
 687         free(chan->hv_chan_rdbuf, M_NETVSC);
 688         hv_vmbus_channel_close(chan);
 689
 690 cleanup:
 691         /*
 692          * Free the packet buffers on the netvsc device packet queue.
 693          * Release other resources.
 694          */
 695         sema_destroy(&net_dev->channel_init_sema);
 696         free(net_dev, M_NETVSC);
 697
 698         return (NULL);
 699 }
 700
 701 /*
 702  * Net VSC on device remove
 703  */
 704 int
 705 hv_nv_on_device_remove(struct hn_softc *sc, boolean_t destroy_channel)
 706 {
 707         netvsc_dev *net_dev = sc->net_dev;;
 708
 709         /* Stop outbound traffic ie sends and receives completions */
 710         net_dev->destroy = TRUE;
 711
 712         hv_nv_disconnect_from_vsp(net_dev);
 713
 714         /* At this point, no one should be accessing net_dev except in here */
 715
 716         /* Now, we can close the channel safely */
 717
 718         free(sc->hn_prichan->hv_chan_rdbuf, M_NETVSC);
 719         hv_vmbus_channel_close(sc->hn_prichan);
 720
 721         sema_destroy(&net_dev->channel_init_sema);
 722         free(net_dev, M_NETVSC);
 723
 724         return (0);
 725 }
 726
 727 /*
 728  * Net VSC on send completion
 729  */
 730 static void
 731 hv_nv_on_send_completion(netvsc_dev *net_dev, struct hv_vmbus_channel *chan,
 732     hv_vm_packet_descriptor *pkt)
 733 {
 734         nvsp_msg *nvsp_msg_pkt;
 735         netvsc_packet *net_vsc_pkt;
 736
 737         nvsp_msg_pkt =
 738             (nvsp_msg *)((unsigned long)pkt + (pkt->data_offset8 << 3));
 739
 740         if (nvsp_msg_pkt->hdr.msg_type == nvsp_msg_type_init_complete
 741                 || nvsp_msg_pkt->hdr.msg_type
 742                         == nvsp_msg_1_type_send_rx_buf_complete
 743                 || nvsp_msg_pkt->hdr.msg_type
 744                         == nvsp_msg_1_type_send_send_buf_complete
 745                 || nvsp_msg_pkt->hdr.msg_type
 746                         == nvsp_msg5_type_subchannel) {
 747                 /* Copy the response back */
 748                 memcpy(&net_dev->channel_init_packet, nvsp_msg_pkt,
 749                     sizeof(nvsp_msg));
 750                 sema_post(&net_dev->channel_init_sema);
 751         } else if (nvsp_msg_pkt->hdr.msg_type ==
 752                     nvsp_msg_1_type_send_rndis_pkt_complete) {
 753                 /* Get the send context */
 754                 net_vsc_pkt =
 755                     (netvsc_packet *)(unsigned long)pkt->transaction_id;
 756                 if (NULL != net_vsc_pkt) {
 757                         if (net_vsc_pkt->send_buf_section_idx !=
 758                             NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) {
 759                                 u_long mask;
 760                                 int idx;
 761
 762                                 idx = net_vsc_pkt->send_buf_section_idx /
 763                                     BITS_PER_LONG;
 764                                 KASSERT(idx < net_dev->bitsmap_words,
 765                                     ("invalid section index %u",
 766                                      net_vsc_pkt->send_buf_section_idx));
 767                                 mask = 1UL <<
 768                                     (net_vsc_pkt->send_buf_section_idx %
 769                                      BITS_PER_LONG);
 770
 771                                 KASSERT(net_dev->send_section_bitsmap[idx] &
 772                                     mask,
 773                                     ("index bitmap 0x%lx, section index %u, "
 774                                      "bitmap idx %d, bitmask 0x%lx",
 775                                      net_dev->send_section_bitsmap[idx],
 776                                      net_vsc_pkt->send_buf_section_idx,
 777                                      idx, mask));
 778                                 atomic_clear_long(
 779                                     &net_dev->send_section_bitsmap[idx], mask);
 780                         }
 781
 782                         /* Notify the layer above us */
 783                         net_vsc_pkt->compl.send.on_send_completion(chan,
 784                             net_vsc_pkt->compl.send.send_completion_context);
 785
 786                 }
 787         }
 788 }
 789
 790 /*
 791  * Net VSC on send
 792  * Sends a packet on the specified Hyper-V device.
 793  * Returns 0 on success, non-zero on failure.
 794  */
 795 int
 796 hv_nv_on_send(struct hv_vmbus_channel *chan, netvsc_packet *pkt)
 797 {
 798         nvsp_msg send_msg;
 799         int ret;
 800
 801         send_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt;
 802         if (pkt->is_data_pkt) {
 803                 /* 0 is RMC_DATA */
 804                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 0;
 805         } else {
 806                 /* 1 is RMC_CONTROL */
 807                 send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 1;
 808         }
 809
 810         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_idx =
 811             pkt->send_buf_section_idx;
 812         send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_size =
 813             pkt->send_buf_section_size;
 814
 815         if (pkt->gpa_cnt) {
 816                 ret = vmbus_chan_send_sglist(chan, pkt->gpa, pkt->gpa_cnt,
 817                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt);
 818         } else {
 819                 ret = hv_vmbus_channel_send_packet(chan,
 820                     &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt,
 821                     VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC);
 822         }
 823
 824         return (ret);
 825 }
 826
 827 /*
 828  * Net VSC on receive
 829  *
 830  * In the FreeBSD Hyper-V virtual world, this function deals exclusively
 831  * with virtual addresses.
 832  */
 833 static void
 834 hv_nv_on_receive(netvsc_dev *net_dev, struct hn_softc *sc,
 835     struct hv_vmbus_channel *chan, hv_vm_packet_descriptor *pkt)
 836 {
 837         hv_vm_transfer_page_packet_header *vm_xfer_page_pkt;
 838         nvsp_msg *nvsp_msg_pkt;
 839         netvsc_packet vsc_pkt;
 840         netvsc_packet *net_vsc_pkt = &vsc_pkt;
 841         device_t dev = sc->hn_dev;
 842         int count = 0;
 843         int i = 0;
 844         int status = nvsp_status_success;
 845
 846         /*
 847          * All inbound packets other than send completion should be
 848          * xfer page packet.
 849          */
 850         if (pkt->type != VMBUS_CHANPKT_TYPE_RXBUF) {
 851                 device_printf(dev, "packet type %d is invalid!\n", pkt->type);
 852                 return;
 853         }
 854
 855         nvsp_msg_pkt = (nvsp_msg *)((unsigned long)pkt
 856                 + (pkt->data_offset8 << 3));
 857
 858         /* Make sure this is a valid nvsp packet */
 859         if (nvsp_msg_pkt->hdr.msg_type != nvsp_msg_1_type_send_rndis_pkt) {
 860                 device_printf(dev, "packet hdr type %d is invalid!\n",
 861                     pkt->type);
 862                 return;
 863         }
 864
 865         vm_xfer_page_pkt = (hv_vm_transfer_page_packet_header *)pkt;
 866
 867         if (vm_xfer_page_pkt->transfer_page_set_id !=
 868             NETVSC_RECEIVE_BUFFER_ID) {
 869                 device_printf(dev, "transfer_page_set_id %d is invalid!\n",
 870                     vm_xfer_page_pkt->transfer_page_set_id);
 871                 return;
 872         }
 873
 874         count = vm_xfer_page_pkt->range_count;
 875
 876         /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */
 877         for (i = 0; i < count; i++) {
 878                 net_vsc_pkt->status = nvsp_status_success;
 879                 net_vsc_pkt->data = (void *)((unsigned long)net_dev->rx_buf +
 880                     vm_xfer_page_pkt->ranges[i].byte_offset);
 881                 net_vsc_pkt->tot_data_buf_len =
 882                     vm_xfer_page_pkt->ranges[i].byte_count;
 883
 884                 hv_rf_on_receive(net_dev, chan, net_vsc_pkt);
 885                 if (net_vsc_pkt->status != nvsp_status_success) {
 886                         status = nvsp_status_failure;
 887                 }
 888         }
 889
 890         /*
 891          * Moved completion call back here so that all received
 892          * messages (not just data messages) will trigger a response
 893          * message back to the host.
 894          */
 895         hv_nv_on_receive_completion(chan, vm_xfer_page_pkt->d.transaction_id,
 896             status);
 897 }
 898
 899 /*
 900  * Net VSC on receive completion
 901  *
 902  * Send a receive completion packet to RNDIS device (ie NetVsp)
 903  */
 904 static void
 905 hv_nv_on_receive_completion(struct hv_vmbus_channel *chan, uint64_t tid,
 906     uint32_t status)
 907 {
 908         nvsp_msg rx_comp_msg;
 909         int retries = 0;
 910         int ret = 0;
 911
 912         rx_comp_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt_complete;
 913
 914         /* Pass in the status */
 915         rx_comp_msg.msgs.vers_1_msgs.send_rndis_pkt_complete.status =
 916             status;
 917
 918 retry_send_cmplt:
 919         /* Send the completion */
 920         ret = hv_vmbus_channel_send_packet(chan, &rx_comp_msg,
 921             sizeof(nvsp_msg), tid, VMBUS_CHANPKT_TYPE_COMP, 0);
 922         if (ret == 0) {
 923                 /* success */
 924                 /* no-op */
 925         } else if (ret == EAGAIN) {
 926                 /* no more room... wait a bit and attempt to retry 3 times */
 927                 retries++;
 928
 929                 if (retries < 4) {
 930                         DELAY(100);
 931                         goto retry_send_cmplt;
 932                 }
 933         }
 934 }
 935
 936 /*
 937  * Net VSC receiving vRSS send table from VSP
 938  */
 939 static void
 940 hv_nv_send_table(struct hn_softc *sc, hv_vm_packet_descriptor *pkt)
 941 {
 942         netvsc_dev *net_dev;
 943         nvsp_msg *nvsp_msg_pkt;
 944         int i;
 945         uint32_t count, *table;
 946
 947         net_dev = hv_nv_get_inbound_net_device(sc);
 948         if (!net_dev)
 949                 return;
 950
 951         nvsp_msg_pkt =
 952             (nvsp_msg *)((unsigned long)pkt + (pkt->data_offset8 << 3));
 953
 954         if (nvsp_msg_pkt->hdr.msg_type !=
 955             nvsp_msg5_type_send_indirection_table) {
 956                 printf("Netvsc: !Warning! receive msg type not "
 957                         "send_indirection_table. type = %d\n",
 958                         nvsp_msg_pkt->hdr.msg_type);
 959                 return;
 960         }
 961
 962         count = nvsp_msg_pkt->msgs.vers_5_msgs.send_table.count;
 963         if (count != VRSS_SEND_TABLE_SIZE) {
 964                 printf("Netvsc: Received wrong send table size: %u\n", count);
 965                 return;
 966         }
 967
 968         table = (uint32_t *)
 969             ((unsigned long)&nvsp_msg_pkt->msgs.vers_5_msgs.send_table +
 970              nvsp_msg_pkt->msgs.vers_5_msgs.send_table.offset);
 971
 972         for (i = 0; i < count; i++)
 973                 net_dev->vrss_send_table[i] = table[i];
 974 }
 975
 976 /*
 977  * Net VSC on channel callback
 978  */
 979 static void
 980 hv_nv_on_channel_callback(void *xchan)
 981 {
 982         struct hv_vmbus_channel *chan = xchan;
 983         device_t dev = chan->ch_dev;
 984         struct hn_softc *sc = device_get_softc(dev);
 985         netvsc_dev *net_dev;
 986         uint32_t bytes_rxed;
 987         uint64_t request_id;
 988         hv_vm_packet_descriptor *desc;
 989         uint8_t *buffer;
 990         int bufferlen = NETVSC_PACKET_SIZE;
 991         int ret = 0;
 992
 993         net_dev = hv_nv_get_inbound_net_device(sc);
 994         if (net_dev == NULL)
 995                 return;
 996
 997         buffer = chan->hv_chan_rdbuf;
 998
 999         do {
1000                 ret = hv_vmbus_channel_recv_packet_raw(chan,
1001                     buffer, bufferlen, &bytes_rxed, &request_id);
1002                 if (ret == 0) {
1003                         if (bytes_rxed > 0) {
1004                                 desc = (hv_vm_packet_descriptor *)buffer;
1005                                 switch (desc->type) {
1006                                 case VMBUS_CHANPKT_TYPE_COMP:
1007                                         hv_nv_on_send_completion(net_dev, chan,
1008                                             desc);
1009                                         break;
1010                                 case VMBUS_CHANPKT_TYPE_RXBUF:
1011                                         hv_nv_on_receive(net_dev, sc, chan, desc);
1012                                         break;
1013                                 case VMBUS_CHANPKT_TYPE_INBAND:
1014                                         hv_nv_send_table(sc, desc);
1015                                         break;
1016                                 default:
1017                                         device_printf(dev,
1018                                             "hv_cb recv unknow type %d "
1019                                             " packet\n", desc->type);
1020                                         break;
1021                                 }
1022                         } else {
1023                                 break;
1024                         }
1025                 } else if (ret == ENOBUFS) {
1026                         /* Handle large packet */
1027                         if (bufferlen > NETVSC_PACKET_SIZE) {
1028                                 free(buffer, M_NETVSC);
1029                                 buffer = NULL;
1030                         }
1031
1032                         /* alloc new buffer */
1033                         buffer = malloc(bytes_rxed, M_NETVSC, M_NOWAIT);
1034                         if (buffer == NULL) {
1035                                 device_printf(dev,
1036                                     "hv_cb malloc buffer failed, len=%u\n",
1037                                     bytes_rxed);
1038                                 bufferlen = 0;
1039                                 break;
1040                         }
1041                         bufferlen = bytes_rxed;
1042                 }
1043         } while (1);
1044
1045         if (bufferlen > NETVSC_PACKET_SIZE)
1046                 free(buffer, M_NETVSC);
1047
1048         hv_rf_channel_rollup(chan);
1049 }