sys/dev/hyperv/vmbus/hv_channel.c

   1 /*-
   2  * Copyright (c) 2009-2012 Microsoft Corp.
   3  * Copyright (c) 2012 NetApp Inc.
   4  * Copyright (c) 2012 Citrix Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/kernel.h>
  34 #include <sys/malloc.h>
  35 #include <sys/systm.h>
  36 #include <sys/mbuf.h>
  37 #include <sys/lock.h>
  38 #include <sys/mutex.h>
  39 #include <machine/bus.h>
  40 #include <vm/vm.h>
  41 #include <vm/vm_param.h>
  42 #include <vm/pmap.h>
  43
  44 #include "hv_vmbus_priv.h"
  45
  46 static int      vmbus_channel_create_gpadl_header(
  47                         /* must be phys and virt contiguous*/
  48                         void*                           contig_buffer,
  49                         /* page-size multiple */
  50                         uint32_t                        size,
  51                         hv_vmbus_channel_msg_info**     msg_info,
  52                         uint32_t*                       message_count);
  53
  54 static void     vmbus_channel_set_event(hv_vmbus_channel* channel);
  55 static void     VmbusProcessChannelEvent(void* channel, int pending);
  56
  57 /**
  58  *  @brief Trigger an event notification on the specified channel
  59  */
  60 static void
  61 vmbus_channel_set_event(hv_vmbus_channel *channel)
  62 {
  63         hv_vmbus_monitor_page *monitor_page;
  64
  65         if (channel->offer_msg.monitor_allocated) {
  66                 /* Each uint32_t represents 32 channels */
  67                 synch_set_bit((channel->offer_msg.child_rel_id & 31),
  68                         ((uint32_t *)hv_vmbus_g_connection.send_interrupt_page
  69                                 + ((channel->offer_msg.child_rel_id >> 5))));
  70
  71                 monitor_page = (hv_vmbus_monitor_page *)
  72                         hv_vmbus_g_connection.monitor_pages;
  73
  74                 monitor_page++; /* Get the child to parent monitor page */
  75
  76                 synch_set_bit(channel->monitor_bit,
  77                         (uint32_t *)&monitor_page->
  78                                 trigger_group[channel->monitor_group].u.pending);
  79         } else {
  80                 hv_vmbus_set_event(channel);
  81         }
  82
  83 }
  84
  85 /**
  86  * @brief Open the specified channel
  87  */
  88 int
  89 hv_vmbus_channel_open(
  90         hv_vmbus_channel*               new_channel,
  91         uint32_t                        send_ring_buffer_size,
  92         uint32_t                        recv_ring_buffer_size,
  93         void*                           user_data,
  94         uint32_t                        user_data_len,
  95         hv_vmbus_pfn_channel_callback   pfn_on_channel_callback,
  96         void*                           context)
  97 {
  98
  99         int ret = 0;
 100         void *in, *out;
 101         hv_vmbus_channel_open_channel*  open_msg;
 102         hv_vmbus_channel_msg_info*      open_info;
 103
 104         mtx_lock(&new_channel->sc_lock);
 105         if (new_channel->state == HV_CHANNEL_OPEN_STATE) {
 106             new_channel->state = HV_CHANNEL_OPENING_STATE;
 107         } else {
 108             mtx_unlock(&new_channel->sc_lock);
 109             if(bootverbose)
 110                 printf("VMBUS: Trying to open channel <%p> which in "
 111                     "%d state.\n", new_channel, new_channel->state);
 112             return (EINVAL);
 113         }
 114         mtx_unlock(&new_channel->sc_lock);
 115
 116         new_channel->on_channel_callback = pfn_on_channel_callback;
 117         new_channel->channel_callback_context = context;
 118
 119         new_channel->rxq = hv_vmbus_g_context.hv_event_queue[new_channel->target_cpu];
 120         TASK_INIT(&new_channel->channel_task, 0, VmbusProcessChannelEvent, new_channel);
 121
 122         /* Allocate the ring buffer */
 123         out = contigmalloc((send_ring_buffer_size + recv_ring_buffer_size),
 124             M_DEVBUF, M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 125         KASSERT(out != NULL,
 126             ("Error VMBUS: contigmalloc failed to allocate Ring Buffer!"));
 127         if (out == NULL)
 128                 return (ENOMEM);
 129
 130         in = ((uint8_t *) out + send_ring_buffer_size);
 131
 132         new_channel->ring_buffer_pages = out;
 133         new_channel->ring_buffer_page_count = (send_ring_buffer_size +
 134             recv_ring_buffer_size) >> PAGE_SHIFT;
 135         new_channel->ring_buffer_size = send_ring_buffer_size +
 136             recv_ring_buffer_size;
 137
 138         hv_vmbus_ring_buffer_init(
 139                 &new_channel->outbound,
 140                 out,
 141                 send_ring_buffer_size);
 142
 143         hv_vmbus_ring_buffer_init(
 144                 &new_channel->inbound,
 145                 in,
 146                 recv_ring_buffer_size);
 147
 148         /**
 149          * Establish the gpadl for the ring buffer
 150          */
 151         new_channel->ring_buffer_gpadl_handle = 0;
 152
 153         ret = hv_vmbus_channel_establish_gpadl(new_channel,
 154                 new_channel->outbound.ring_buffer,
 155                 send_ring_buffer_size + recv_ring_buffer_size,
 156                 &new_channel->ring_buffer_gpadl_handle);
 157
 158         /**
 159          * Create and init the channel open message
 160          */
 161         open_info = (hv_vmbus_channel_msg_info*) malloc(
 162                 sizeof(hv_vmbus_channel_msg_info) +
 163                         sizeof(hv_vmbus_channel_open_channel),
 164                 M_DEVBUF,
 165                 M_NOWAIT);
 166         KASSERT(open_info != NULL,
 167             ("Error VMBUS: malloc failed to allocate Open Channel message!"));
 168
 169         if (open_info == NULL)
 170                 return (ENOMEM);
 171
 172         sema_init(&open_info->wait_sema, 0, "Open Info Sema");
 173
 174         open_msg = (hv_vmbus_channel_open_channel*) open_info->msg;
 175         open_msg->header.message_type = HV_CHANNEL_MESSAGE_OPEN_CHANNEL;
 176         open_msg->open_id = new_channel->offer_msg.child_rel_id;
 177         open_msg->child_rel_id = new_channel->offer_msg.child_rel_id;
 178         open_msg->ring_buffer_gpadl_handle =
 179                 new_channel->ring_buffer_gpadl_handle;
 180         open_msg->downstream_ring_buffer_page_offset = send_ring_buffer_size
 181                 >> PAGE_SHIFT;
 182         open_msg->target_vcpu = new_channel->target_vcpu;
 183
 184         if (user_data_len)
 185                 memcpy(open_msg->user_data, user_data, user_data_len);
 186
 187         mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 188         TAILQ_INSERT_TAIL(
 189                 &hv_vmbus_g_connection.channel_msg_anchor,
 190                 open_info,
 191                 msg_list_entry);
 192         mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 193
 194         ret = hv_vmbus_post_message(
 195                 open_msg, sizeof(hv_vmbus_channel_open_channel));
 196
 197         if (ret != 0)
 198             goto cleanup;
 199
 200         ret = sema_timedwait(&open_info->wait_sema, 5 * hz); /* KYS 5 seconds */
 201
 202         if (ret) {
 203             if(bootverbose)
 204                 printf("VMBUS: channel <%p> open timeout.\n", new_channel);
 205             goto cleanup;
 206         }
 207
 208         if (open_info->response.open_result.status == 0) {
 209             new_channel->state = HV_CHANNEL_OPENED_STATE;
 210             if(bootverbose)
 211                 printf("VMBUS: channel <%p> open success.\n", new_channel);
 212         } else {
 213             if(bootverbose)
 214                 printf("Error VMBUS: channel <%p> open failed - %d!\n",
 215                         new_channel, open_info->response.open_result.status);
 216         }
 217
 218         cleanup:
 219         mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 220         TAILQ_REMOVE(
 221                 &hv_vmbus_g_connection.channel_msg_anchor,
 222                 open_info,
 223                 msg_list_entry);
 224         mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 225         sema_destroy(&open_info->wait_sema);
 226         free(open_info, M_DEVBUF);
 227
 228         return (ret);
 229 }
 230
 231 /**
 232  * @brief Create a gpadl for the specified buffer
 233  */
 234 static int
 235 vmbus_channel_create_gpadl_header(
 236         void*                           contig_buffer,
 237         uint32_t                        size,   /* page-size multiple */
 238         hv_vmbus_channel_msg_info**     msg_info,
 239         uint32_t*                       message_count)
 240 {
 241         int                             i;
 242         int                             page_count;
 243         unsigned long long              pfn;
 244         uint32_t                        msg_size;
 245         hv_vmbus_channel_gpadl_header*  gpa_header;
 246         hv_vmbus_channel_gpadl_body*    gpadl_body;
 247         hv_vmbus_channel_msg_info*      msg_header;
 248         hv_vmbus_channel_msg_info*      msg_body;
 249
 250         int pfnSum, pfnCount, pfnLeft, pfnCurr, pfnSize;
 251
 252         page_count = size >> PAGE_SHIFT;
 253         pfn = hv_get_phys_addr(contig_buffer) >> PAGE_SHIFT;
 254
 255         /*do we need a gpadl body msg */
 256         pfnSize = HV_MAX_SIZE_CHANNEL_MESSAGE
 257             - sizeof(hv_vmbus_channel_gpadl_header)
 258             - sizeof(hv_gpa_range);
 259         pfnCount = pfnSize / sizeof(uint64_t);
 260
 261         if (page_count > pfnCount) { /* if(we need a gpadl body)        */
 262             /* fill in the header               */
 263             msg_size = sizeof(hv_vmbus_channel_msg_info)
 264                 + sizeof(hv_vmbus_channel_gpadl_header)
 265                 + sizeof(hv_gpa_range)
 266                 + pfnCount * sizeof(uint64_t);
 267             msg_header = malloc(msg_size, M_DEVBUF, M_NOWAIT | M_ZERO);
 268             KASSERT(
 269                 msg_header != NULL,
 270                 ("Error VMBUS: malloc failed to allocate Gpadl Message!"));
 271             if (msg_header == NULL)
 272                 return (ENOMEM);
 273
 274             TAILQ_INIT(&msg_header->sub_msg_list_anchor);
 275             msg_header->message_size = msg_size;
 276
 277             gpa_header = (hv_vmbus_channel_gpadl_header*) msg_header->msg;
 278             gpa_header->range_count = 1;
 279             gpa_header->range_buf_len = sizeof(hv_gpa_range)
 280                 + page_count * sizeof(uint64_t);
 281             gpa_header->range[0].byte_offset = 0;
 282             gpa_header->range[0].byte_count = size;
 283             for (i = 0; i < pfnCount; i++) {
 284                 gpa_header->range[0].pfn_array[i] = pfn + i;
 285             }
 286             *msg_info = msg_header;
 287             *message_count = 1;
 288
 289             pfnSum = pfnCount;
 290             pfnLeft = page_count - pfnCount;
 291
 292             /*
 293              *  figure out how many pfns we can fit
 294              */
 295             pfnSize = HV_MAX_SIZE_CHANNEL_MESSAGE
 296                 - sizeof(hv_vmbus_channel_gpadl_body);
 297             pfnCount = pfnSize / sizeof(uint64_t);
 298
 299             /*
 300              * fill in the body
 301              */
 302             while (pfnLeft) {
 303                 if (pfnLeft > pfnCount) {
 304                     pfnCurr = pfnCount;
 305                 } else {
 306                     pfnCurr = pfnLeft;
 307                 }
 308
 309                 msg_size = sizeof(hv_vmbus_channel_msg_info) +
 310                     sizeof(hv_vmbus_channel_gpadl_body) +
 311                     pfnCurr * sizeof(uint64_t);
 312                 msg_body = malloc(msg_size, M_DEVBUF, M_NOWAIT | M_ZERO);
 313                 KASSERT(
 314                     msg_body != NULL,
 315                     ("Error VMBUS: malloc failed to allocate Gpadl msg_body!"));
 316                 if (msg_body == NULL)
 317                     return (ENOMEM);
 318
 319                 msg_body->message_size = msg_size;
 320                 (*message_count)++;
 321                 gpadl_body =
 322                     (hv_vmbus_channel_gpadl_body*) msg_body->msg;
 323                 /*
 324                  * gpadl_body->gpadl = kbuffer;
 325                  */
 326                 for (i = 0; i < pfnCurr; i++) {
 327                     gpadl_body->pfn[i] = pfn + pfnSum + i;
 328                 }
 329
 330                 TAILQ_INSERT_TAIL(
 331                     &msg_header->sub_msg_list_anchor,
 332                     msg_body,
 333                     msg_list_entry);
 334                 pfnSum += pfnCurr;
 335                 pfnLeft -= pfnCurr;
 336             }
 337         } else { /* else everything fits in a header */
 338
 339             msg_size = sizeof(hv_vmbus_channel_msg_info) +
 340                 sizeof(hv_vmbus_channel_gpadl_header) +
 341                 sizeof(hv_gpa_range) +
 342                 page_count * sizeof(uint64_t);
 343             msg_header = malloc(msg_size, M_DEVBUF, M_NOWAIT | M_ZERO);
 344             KASSERT(
 345                 msg_header != NULL,
 346                 ("Error VMBUS: malloc failed to allocate Gpadl Message!"));
 347             if (msg_header == NULL)
 348                 return (ENOMEM);
 349
 350             msg_header->message_size = msg_size;
 351
 352             gpa_header = (hv_vmbus_channel_gpadl_header*) msg_header->msg;
 353             gpa_header->range_count = 1;
 354             gpa_header->range_buf_len = sizeof(hv_gpa_range) +
 355                 page_count * sizeof(uint64_t);
 356             gpa_header->range[0].byte_offset = 0;
 357             gpa_header->range[0].byte_count = size;
 358             for (i = 0; i < page_count; i++) {
 359                 gpa_header->range[0].pfn_array[i] = pfn + i;
 360             }
 361
 362             *msg_info = msg_header;
 363             *message_count = 1;
 364         }
 365
 366         return (0);
 367 }
 368
 369 /**
 370  * @brief Establish a GPADL for the specified buffer
 371  */
 372 int
 373 hv_vmbus_channel_establish_gpadl(
 374         hv_vmbus_channel*       channel,
 375         void*                   contig_buffer,
 376         uint32_t                size, /* page-size multiple */
 377         uint32_t*               gpadl_handle)
 378
 379 {
 380         int ret = 0;
 381         hv_vmbus_channel_gpadl_header*  gpadl_msg;
 382         hv_vmbus_channel_gpadl_body*    gpadl_body;
 383         hv_vmbus_channel_msg_info*      msg_info;
 384         hv_vmbus_channel_msg_info*      sub_msg_info;
 385         uint32_t                        msg_count;
 386         hv_vmbus_channel_msg_info*      curr;
 387         uint32_t                        next_gpadl_handle;
 388
 389         next_gpadl_handle = hv_vmbus_g_connection.next_gpadl_handle;
 390         atomic_add_int((int*) &hv_vmbus_g_connection.next_gpadl_handle, 1);
 391
 392         ret = vmbus_channel_create_gpadl_header(
 393                 contig_buffer, size, &msg_info, &msg_count);
 394
 395         if(ret != 0) { /* if(allocation failed) return immediately */
 396             /* reverse atomic_add_int above */
 397             atomic_subtract_int((int*)
 398                     &hv_vmbus_g_connection.next_gpadl_handle, 1);
 399             return ret;
 400         }
 401
 402         sema_init(&msg_info->wait_sema, 0, "Open Info Sema");
 403         gpadl_msg = (hv_vmbus_channel_gpadl_header*) msg_info->msg;
 404         gpadl_msg->header.message_type = HV_CHANNEL_MESSAGEL_GPADL_HEADER;
 405         gpadl_msg->child_rel_id = channel->offer_msg.child_rel_id;
 406         gpadl_msg->gpadl = next_gpadl_handle;
 407
 408         mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 409         TAILQ_INSERT_TAIL(
 410                 &hv_vmbus_g_connection.channel_msg_anchor,
 411                 msg_info,
 412                 msg_list_entry);
 413
 414         mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 415
 416         ret = hv_vmbus_post_message(
 417                 gpadl_msg,
 418                 msg_info->message_size -
 419                     (uint32_t) sizeof(hv_vmbus_channel_msg_info));
 420
 421         if (ret != 0)
 422             goto cleanup;
 423
 424         if (msg_count > 1) {
 425             TAILQ_FOREACH(curr,
 426                     &msg_info->sub_msg_list_anchor, msg_list_entry) {
 427                 sub_msg_info = curr;
 428                 gpadl_body =
 429                     (hv_vmbus_channel_gpadl_body*) sub_msg_info->msg;
 430
 431                 gpadl_body->header.message_type =
 432                     HV_CHANNEL_MESSAGE_GPADL_BODY;
 433                 gpadl_body->gpadl = next_gpadl_handle;
 434
 435                 ret = hv_vmbus_post_message(
 436                         gpadl_body,
 437                         sub_msg_info->message_size
 438                             - (uint32_t) sizeof(hv_vmbus_channel_msg_info));
 439                  /* if (the post message failed) give up and clean up */
 440                 if(ret != 0)
 441                     goto cleanup;
 442             }
 443         }
 444
 445         ret = sema_timedwait(&msg_info->wait_sema, 5 * hz); /* KYS 5 seconds*/
 446         if (ret != 0)
 447             goto cleanup;
 448
 449         *gpadl_handle = gpadl_msg->gpadl;
 450
 451 cleanup:
 452
 453         mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 454         TAILQ_REMOVE(&hv_vmbus_g_connection.channel_msg_anchor,
 455                 msg_info, msg_list_entry);
 456         mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 457
 458         sema_destroy(&msg_info->wait_sema);
 459         free(msg_info, M_DEVBUF);
 460
 461         return (ret);
 462 }
 463
 464 /**
 465  * @brief Teardown the specified GPADL handle
 466  */
 467 int
 468 hv_vmbus_channel_teardown_gpdal(
 469         hv_vmbus_channel*       channel,
 470         uint32_t                gpadl_handle)
 471 {
 472         int                                     ret = 0;
 473         hv_vmbus_channel_gpadl_teardown*        msg;
 474         hv_vmbus_channel_msg_info*              info;
 475
 476         info = (hv_vmbus_channel_msg_info *)
 477                 malloc( sizeof(hv_vmbus_channel_msg_info) +
 478                         sizeof(hv_vmbus_channel_gpadl_teardown),
 479                                 M_DEVBUF, M_NOWAIT);
 480         KASSERT(info != NULL,
 481             ("Error VMBUS: malloc failed to allocate Gpadl Teardown Msg!"));
 482         if (info == NULL) {
 483             ret = ENOMEM;
 484             goto cleanup;
 485         }
 486
 487         sema_init(&info->wait_sema, 0, "Open Info Sema");
 488
 489         msg = (hv_vmbus_channel_gpadl_teardown*) info->msg;
 490
 491         msg->header.message_type = HV_CHANNEL_MESSAGE_GPADL_TEARDOWN;
 492         msg->child_rel_id = channel->offer_msg.child_rel_id;
 493         msg->gpadl = gpadl_handle;
 494
 495         mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 496         TAILQ_INSERT_TAIL(&hv_vmbus_g_connection.channel_msg_anchor,
 497                         info, msg_list_entry);
 498         mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 499
 500         ret = hv_vmbus_post_message(msg,
 501                         sizeof(hv_vmbus_channel_gpadl_teardown));
 502         if (ret != 0)
 503             goto cleanup;
 504
 505         ret = sema_timedwait(&info->wait_sema, 5 * hz); /* KYS 5 seconds */
 506
 507 cleanup:
 508         /*
 509          * Received a torndown response
 510          */
 511         mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 512         TAILQ_REMOVE(&hv_vmbus_g_connection.channel_msg_anchor,
 513                         info, msg_list_entry);
 514         mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
 515         sema_destroy(&info->wait_sema);
 516         free(info, M_DEVBUF);
 517
 518         return (ret);
 519 }
 520
 521 static void
 522 hv_vmbus_channel_close_internal(hv_vmbus_channel *channel)
 523 {
 524         int ret = 0;
 525         struct taskqueue *rxq = channel->rxq;
 526         hv_vmbus_channel_close_channel* msg;
 527         hv_vmbus_channel_msg_info* info;
 528
 529         channel->state = HV_CHANNEL_OPEN_STATE;
 530         channel->sc_creation_callback = NULL;
 531
 532         /*
 533          * set rxq to NULL to avoid more requests be scheduled
 534          */
 535         channel->rxq = NULL;
 536         taskqueue_drain(rxq, &channel->channel_task);
 537         /*
 538          * Grab the lock to prevent race condition when a packet received
 539          * and unloading driver is in the process.
 540          */
 541         mtx_lock(&channel->inbound_lock);
 542         channel->on_channel_callback = NULL;
 543         mtx_unlock(&channel->inbound_lock);
 544
 545         /**
 546          * Send a closing message
 547          */
 548         info = (hv_vmbus_channel_msg_info *)
 549                 malloc( sizeof(hv_vmbus_channel_msg_info) +
 550                         sizeof(hv_vmbus_channel_close_channel),
 551                                 M_DEVBUF, M_NOWAIT);
 552         KASSERT(info != NULL, ("VMBUS: malloc failed hv_vmbus_channel_close!"));
 553         if(info == NULL)
 554             return;
 555
 556         msg = (hv_vmbus_channel_close_channel*) info->msg;
 557         msg->header.message_type = HV_CHANNEL_MESSAGE_CLOSE_CHANNEL;
 558         msg->child_rel_id = channel->offer_msg.child_rel_id;
 559
 560         ret = hv_vmbus_post_message(
 561                 msg, sizeof(hv_vmbus_channel_close_channel));
 562
 563         /* Tear down the gpadl for the channel's ring buffer */
 564         if (channel->ring_buffer_gpadl_handle) {
 565                 hv_vmbus_channel_teardown_gpdal(channel,
 566                         channel->ring_buffer_gpadl_handle);
 567         }
 568
 569         /* TODO: Send a msg to release the childRelId */
 570
 571         /* cleanup the ring buffers for this channel */
 572         hv_ring_buffer_cleanup(&channel->outbound);
 573         hv_ring_buffer_cleanup(&channel->inbound);
 574
 575         contigfree(channel->ring_buffer_pages, channel->ring_buffer_size,
 576             M_DEVBUF);
 577
 578         free(info, M_DEVBUF);
 579 }
 580
 581 /**
 582  * @brief Close the specified channel
 583  */
 584 void
 585 hv_vmbus_channel_close(hv_vmbus_channel *channel)
 586 {
 587         hv_vmbus_channel*       sub_channel;
 588
 589         if (channel->primary_channel != NULL) {
 590                 /*
 591                  * We only close multi-channels when the primary is
 592                  * closed.
 593                  */
 594                 return;
 595         }
 596
 597         /*
 598          * Close all multi-channels first.
 599          */
 600         TAILQ_FOREACH(sub_channel, &channel->sc_list_anchor,
 601             sc_list_entry) {
 602                 if (sub_channel->state != HV_CHANNEL_OPENED_STATE)
 603                         continue;
 604                 hv_vmbus_channel_close_internal(sub_channel);
 605         }
 606         /*
 607          * Then close the primary channel.
 608          */
 609         hv_vmbus_channel_close_internal(channel);
 610 }
 611
 612 /**
 613  * @brief Send the specified buffer on the given channel
 614  */
 615 int
 616 hv_vmbus_channel_send_packet(
 617         hv_vmbus_channel*       channel,
 618         void*                   buffer,
 619         uint32_t                buffer_len,
 620         uint64_t                request_id,
 621         hv_vmbus_packet_type    type,
 622         uint32_t                flags)
 623 {
 624         int                     ret = 0;
 625         hv_vm_packet_descriptor desc;
 626         uint32_t                packet_len;
 627         uint64_t                aligned_data;
 628         uint32_t                packet_len_aligned;
 629         boolean_t               need_sig;
 630         hv_vmbus_sg_buffer_list buffer_list[3];
 631
 632         packet_len = sizeof(hv_vm_packet_descriptor) + buffer_len;
 633         packet_len_aligned = HV_ALIGN_UP(packet_len, sizeof(uint64_t));
 634         aligned_data = 0;
 635
 636         /* Setup the descriptor */
 637         desc.type = type;   /* HV_VMBUS_PACKET_TYPE_DATA_IN_BAND;             */
 638         desc.flags = flags; /* HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED */
 639                             /* in 8-bytes granularity */
 640         desc.data_offset8 = sizeof(hv_vm_packet_descriptor) >> 3;
 641         desc.length8 = (uint16_t) (packet_len_aligned >> 3);
 642         desc.transaction_id = request_id;
 643
 644         buffer_list[0].data = &desc;
 645         buffer_list[0].length = sizeof(hv_vm_packet_descriptor);
 646
 647         buffer_list[1].data = buffer;
 648         buffer_list[1].length = buffer_len;
 649
 650         buffer_list[2].data = &aligned_data;
 651         buffer_list[2].length = packet_len_aligned - packet_len;
 652
 653         ret = hv_ring_buffer_write(&channel->outbound, buffer_list, 3,
 654             &need_sig);
 655
 656         /* TODO: We should determine if this is optional */
 657         if (ret == 0 && need_sig) {
 658                 vmbus_channel_set_event(channel);
 659         }
 660
 661         return (ret);
 662 }
 663
 664 /**
 665  * @brief Send a range of single-page buffer packets using
 666  * a GPADL Direct packet type
 667  */
 668 int
 669 hv_vmbus_channel_send_packet_pagebuffer(
 670         hv_vmbus_channel*       channel,
 671         hv_vmbus_page_buffer    page_buffers[],
 672         uint32_t                page_count,
 673         void*                   buffer,
 674         uint32_t                buffer_len,
 675         uint64_t                request_id)
 676 {
 677
 678         int                                     ret = 0;
 679         boolean_t                               need_sig;
 680         uint32_t                                packet_len;
 681         uint32_t                                page_buflen;
 682         uint32_t                                packetLen_aligned;
 683         hv_vmbus_sg_buffer_list                 buffer_list[4];
 684         hv_vmbus_channel_packet_page_buffer     desc;
 685         uint32_t                                descSize;
 686         uint64_t                                alignedData = 0;
 687
 688         if (page_count > HV_MAX_PAGE_BUFFER_COUNT)
 689                 return (EINVAL);
 690
 691         /*
 692          * Adjust the size down since hv_vmbus_channel_packet_page_buffer
 693          *  is the largest size we support
 694          */
 695         descSize = __offsetof(hv_vmbus_channel_packet_page_buffer, range);
 696         page_buflen = sizeof(hv_vmbus_page_buffer) * page_count;
 697         packet_len = descSize + page_buflen + buffer_len;
 698         packetLen_aligned = HV_ALIGN_UP(packet_len, sizeof(uint64_t));
 699
 700         /* Setup the descriptor */
 701         desc.type = HV_VMBUS_PACKET_TYPE_DATA_USING_GPA_DIRECT;
 702         desc.flags = HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
 703         /* in 8-bytes granularity */
 704         desc.data_offset8 = (descSize + page_buflen) >> 3;
 705         desc.length8 = (uint16_t) (packetLen_aligned >> 3);
 706         desc.transaction_id = request_id;
 707         desc.range_count = page_count;
 708
 709         buffer_list[0].data = &desc;
 710         buffer_list[0].length = descSize;
 711
 712         buffer_list[1].data = page_buffers;
 713         buffer_list[1].length = page_buflen;
 714
 715         buffer_list[2].data = buffer;
 716         buffer_list[2].length = buffer_len;
 717
 718         buffer_list[3].data = &alignedData;
 719         buffer_list[3].length = packetLen_aligned - packet_len;
 720
 721         ret = hv_ring_buffer_write(&channel->outbound, buffer_list, 4,
 722             &need_sig);
 723
 724         /* TODO: We should determine if this is optional */
 725         if (ret == 0 && need_sig) {
 726                 vmbus_channel_set_event(channel);
 727         }
 728
 729         return (ret);
 730 }
 731
 732 /**
 733  * @brief Send a multi-page buffer packet using a GPADL Direct packet type
 734  */
 735 int
 736 hv_vmbus_channel_send_packet_multipagebuffer(
 737         hv_vmbus_channel*               channel,
 738         hv_vmbus_multipage_buffer*      multi_page_buffer,
 739         void*                           buffer,
 740         uint32_t                        buffer_len,
 741         uint64_t                        request_id)
 742 {
 743
 744         int                     ret = 0;
 745         uint32_t                desc_size;
 746         boolean_t               need_sig;
 747         uint32_t                packet_len;
 748         uint32_t                packet_len_aligned;
 749         uint32_t                pfn_count;
 750         uint64_t                aligned_data = 0;
 751         hv_vmbus_sg_buffer_list buffer_list[3];
 752         hv_vmbus_channel_packet_multipage_buffer desc;
 753
 754         pfn_count =
 755             HV_NUM_PAGES_SPANNED(
 756                     multi_page_buffer->offset,
 757                     multi_page_buffer->length);
 758
 759         if ((pfn_count == 0) || (pfn_count > HV_MAX_MULTIPAGE_BUFFER_COUNT))
 760             return (EINVAL);
 761         /*
 762          * Adjust the size down since hv_vmbus_channel_packet_multipage_buffer
 763          * is the largest size we support
 764          */
 765         desc_size =
 766             sizeof(hv_vmbus_channel_packet_multipage_buffer) -
 767                     ((HV_MAX_MULTIPAGE_BUFFER_COUNT - pfn_count) *
 768                         sizeof(uint64_t));
 769         packet_len = desc_size + buffer_len;
 770         packet_len_aligned = HV_ALIGN_UP(packet_len, sizeof(uint64_t));
 771
 772         /*
 773          * Setup the descriptor
 774          */
 775         desc.type = HV_VMBUS_PACKET_TYPE_DATA_USING_GPA_DIRECT;
 776         desc.flags = HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
 777         desc.data_offset8 = desc_size >> 3; /* in 8-bytes granularity */
 778         desc.length8 = (uint16_t) (packet_len_aligned >> 3);
 779         desc.transaction_id = request_id;
 780         desc.range_count = 1;
 781
 782         desc.range.length = multi_page_buffer->length;
 783         desc.range.offset = multi_page_buffer->offset;
 784
 785         memcpy(desc.range.pfn_array, multi_page_buffer->pfn_array,
 786                 pfn_count * sizeof(uint64_t));
 787
 788         buffer_list[0].data = &desc;
 789         buffer_list[0].length = desc_size;
 790
 791         buffer_list[1].data = buffer;
 792         buffer_list[1].length = buffer_len;
 793
 794         buffer_list[2].data = &aligned_data;
 795         buffer_list[2].length = packet_len_aligned - packet_len;
 796
 797         ret = hv_ring_buffer_write(&channel->outbound, buffer_list, 3,
 798             &need_sig);
 799
 800         /* TODO: We should determine if this is optional */
 801         if (ret == 0 && need_sig) {
 802             vmbus_channel_set_event(channel);
 803         }
 804
 805         return (ret);
 806 }
 807
 808 /**
 809  * @brief Retrieve the user packet on the specified channel
 810  */
 811 int
 812 hv_vmbus_channel_recv_packet(
 813         hv_vmbus_channel*       channel,
 814         void*                   Buffer,
 815         uint32_t                buffer_len,
 816         uint32_t*               buffer_actual_len,
 817         uint64_t*               request_id)
 818 {
 819         int                     ret;
 820         uint32_t                user_len;
 821         uint32_t                packet_len;
 822         hv_vm_packet_descriptor desc;
 823
 824         *buffer_actual_len = 0;
 825         *request_id = 0;
 826
 827         ret = hv_ring_buffer_peek(&channel->inbound, &desc,
 828                 sizeof(hv_vm_packet_descriptor));
 829         if (ret != 0)
 830                 return (0);
 831
 832         packet_len = desc.length8 << 3;
 833         user_len = packet_len - (desc.data_offset8 << 3);
 834
 835         *buffer_actual_len = user_len;
 836
 837         if (user_len > buffer_len)
 838                 return (EINVAL);
 839
 840         *request_id = desc.transaction_id;
 841
 842         /* Copy over the packet to the user buffer */
 843         ret = hv_ring_buffer_read(&channel->inbound, Buffer, user_len,
 844                 (desc.data_offset8 << 3));
 845
 846         return (0);
 847 }
 848
 849 /**
 850  * @brief Retrieve the raw packet on the specified channel
 851  */
 852 int
 853 hv_vmbus_channel_recv_packet_raw(
 854         hv_vmbus_channel*       channel,
 855         void*                   buffer,
 856         uint32_t                buffer_len,
 857         uint32_t*               buffer_actual_len,
 858         uint64_t*               request_id)
 859 {
 860         int             ret;
 861         uint32_t        packetLen;
 862         uint32_t        userLen;
 863         hv_vm_packet_descriptor desc;
 864
 865         *buffer_actual_len = 0;
 866         *request_id = 0;
 867
 868         ret = hv_ring_buffer_peek(
 869                 &channel->inbound, &desc,
 870                 sizeof(hv_vm_packet_descriptor));
 871
 872         if (ret != 0)
 873             return (0);
 874
 875         packetLen = desc.length8 << 3;
 876         userLen = packetLen - (desc.data_offset8 << 3);
 877
 878         *buffer_actual_len = packetLen;
 879
 880         if (packetLen > buffer_len)
 881             return (ENOBUFS);
 882
 883         *request_id = desc.transaction_id;
 884
 885         /* Copy over the entire packet to the user buffer */
 886         ret = hv_ring_buffer_read(&channel->inbound, buffer, packetLen, 0);
 887
 888         return (0);
 889 }
 890
 891
 892 /**
 893  * Process a channel event notification
 894  */
 895 static void
 896 VmbusProcessChannelEvent(void* context, int pending)
 897 {
 898         void* arg;
 899         uint32_t bytes_to_read;
 900         hv_vmbus_channel* channel = (hv_vmbus_channel*)context;
 901         boolean_t is_batched_reading;
 902
 903         /**
 904          * Find the channel based on this relid and invokes
 905          * the channel callback to process the event
 906          */
 907
 908         if (channel == NULL) {
 909                 return;
 910         }
 911         /**
 912          * To deal with the race condition where we might
 913          * receive a packet while the relevant driver is
 914          * being unloaded, dispatch the callback while
 915          * holding the channel lock. The unloading driver
 916          * will acquire the same channel lock to set the
 917          * callback to NULL. This closes the window.
 918          */
 919
 920         /*
 921          * Disable the lock due to newly added WITNESS check in r277723.
 922          * Will seek other way to avoid race condition.
 923          * -- whu
 924          */
 925         // mtx_lock(&channel->inbound_lock);
 926         if (channel->on_channel_callback != NULL) {
 927                 arg = channel->channel_callback_context;
 928                 is_batched_reading = channel->batched_reading;
 929                 /*
 930                  * Optimize host to guest signaling by ensuring:
 931                  * 1. While reading the channel, we disable interrupts from
 932                  *    host.
 933                  * 2. Ensure that we process all posted messages from the host
 934                  *    before returning from this callback.
 935                  * 3. Once we return, enable signaling from the host. Once this
 936                  *    state is set we check to see if additional packets are
 937                  *    available to read. In this case we repeat the process.
 938                  */
 939                 do {
 940                         if (is_batched_reading)
 941                                 hv_ring_buffer_read_begin(&channel->inbound);
 942
 943                         channel->on_channel_callback(arg);
 944
 945                         if (is_batched_reading)
 946                                 bytes_to_read =
 947                                     hv_ring_buffer_read_end(&channel->inbound);
 948                         else
 949                                 bytes_to_read = 0;
 950                 } while (is_batched_reading && (bytes_to_read != 0));
 951         }
 952         // mtx_unlock(&channel->inbound_lock);
 953 }