sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c

   1 /*-
   2  * Copyright (c) 2009-2012 Microsoft Corp.
   3  * Copyright (c) 2012 NetApp Inc.
   4  * Copyright (c) 2012 Citrix Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /**
  30  * StorVSC driver for Hyper-V.  This driver presents a SCSI HBA interface
  31  * to the Comman Access Method (CAM) layer.  CAM control blocks (CCBs) are
  32  * converted into VSCSI protocol messages which are delivered to the parent
  33  * partition StorVSP driver over the Hyper-V VMBUS.
  34  */
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/proc.h>
  40 #include <sys/condvar.h>
  41 #include <sys/time.h>
  42 #include <sys/systm.h>
  43 #include <sys/sockio.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/malloc.h>
  46 #include <sys/module.h>
  47 #include <sys/kernel.h>
  48 #include <sys/queue.h>
  49 #include <sys/lock.h>
  50 #include <sys/sx.h>
  51 #include <sys/taskqueue.h>
  52 #include <sys/bus.h>
  53 #include <sys/mutex.h>
  54 #include <sys/callout.h>
  55 #include <vm/vm.h>
  56 #include <vm/pmap.h>
  57 #include <vm/uma.h>
  58 #include <sys/lock.h>
  59 #include <sys/sema.h>
  60 #include <sys/sglist.h>
  61 #include <machine/bus.h>
  62 #include <sys/bus_dma.h>
  63
  64 #include <cam/cam.h>
  65 #include <cam/cam_ccb.h>
  66 #include <cam/cam_periph.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_xpt_sim.h>
  69 #include <cam/cam_xpt_internal.h>
  70 #include <cam/cam_debug.h>
  71 #include <cam/scsi/scsi_all.h>
  72 #include <cam/scsi/scsi_message.h>
  73
  74 #include <dev/hyperv/include/hyperv.h>
  75 #include "hv_vstorage.h"
  76
  77 #define STORVSC_RINGBUFFER_SIZE         (20*PAGE_SIZE)
  78 #define STORVSC_MAX_LUNS_PER_TARGET     (64)
  79 #define STORVSC_MAX_IO_REQUESTS         (STORVSC_MAX_LUNS_PER_TARGET * 2)
  80 #define BLKVSC_MAX_IDE_DISKS_PER_TARGET (1)
  81 #define BLKVSC_MAX_IO_REQUESTS          STORVSC_MAX_IO_REQUESTS
  82 #define STORVSC_MAX_TARGETS             (2)
  83
  84 #define VSTOR_PKT_SIZE  (sizeof(struct vstor_packet) - vmscsi_size_delta)
  85
  86 #define HV_ALIGN(x, a) roundup2(x, a)
  87
  88 struct storvsc_softc;
  89
  90 struct hv_sgl_node {
  91         LIST_ENTRY(hv_sgl_node) link;
  92         struct sglist *sgl_data;
  93 };
  94
  95 struct hv_sgl_page_pool{
  96         LIST_HEAD(, hv_sgl_node) in_use_sgl_list;
  97         LIST_HEAD(, hv_sgl_node) free_sgl_list;
  98         boolean_t                is_init;
  99 } g_hv_sgl_page_pool;
 100
 101 #define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * HV_MAX_MULTIPAGE_BUFFER_COUNT
 102
 103 enum storvsc_request_type {
 104         WRITE_TYPE,
 105         READ_TYPE,
 106         UNKNOWN_TYPE
 107 };
 108
 109 struct hv_storvsc_request {
 110         LIST_ENTRY(hv_storvsc_request) link;
 111         struct vstor_packet     vstor_packet;
 112         hv_vmbus_multipage_buffer data_buf;
 113         void *sense_data;
 114         uint8_t sense_info_len;
 115         uint8_t retries;
 116         union ccb *ccb;
 117         struct storvsc_softc *softc;
 118         struct callout callout;
 119         struct sema synch_sema; /*Synchronize the request/response if needed */
 120         struct sglist *bounce_sgl;
 121         unsigned int bounce_sgl_count;
 122         uint64_t not_aligned_seg_bits;
 123 };
 124
 125 struct storvsc_softc {
 126         struct hv_device                *hs_dev;
 127         LIST_HEAD(, hv_storvsc_request) hs_free_list;
 128         struct mtx                      hs_lock;
 129         struct storvsc_driver_props     *hs_drv_props;
 130         int                             hs_unit;
 131         uint32_t                        hs_frozen;
 132         struct cam_sim                  *hs_sim;
 133         struct cam_path                 *hs_path;
 134         uint32_t                        hs_num_out_reqs;
 135         boolean_t                       hs_destroy;
 136         boolean_t                       hs_drain_notify;
 137         boolean_t                       hs_open_multi_channel;
 138         struct sema                     hs_drain_sema;
 139         struct hv_storvsc_request       hs_init_req;
 140         struct hv_storvsc_request       hs_reset_req;
 141 };
 142
 143
 144 /**
 145  * HyperV storvsc timeout testing cases:
 146  * a. IO returned after first timeout;
 147  * b. IO returned after second timeout and queue freeze;
 148  * c. IO returned while timer handler is running
 149  * The first can be tested by "sg_senddiag -vv /dev/daX",
 150  * and the second and third can be done by
 151  * "sg_wr_mode -v -p 08 -c 0,1a -m 0,ff /dev/daX".
 152  */
 153 #define HVS_TIMEOUT_TEST 0
 154
 155 /*
 156  * Bus/adapter reset functionality on the Hyper-V host is
 157  * buggy and it will be disabled until
 158  * it can be further tested.
 159  */
 160 #define HVS_HOST_RESET 0
 161
 162 struct storvsc_driver_props {
 163         char            *drv_name;
 164         char            *drv_desc;
 165         uint8_t         drv_max_luns_per_target;
 166         uint8_t         drv_max_ios_per_target;
 167         uint32_t        drv_ringbuffer_size;
 168 };
 169
 170 enum hv_storage_type {
 171         DRIVER_BLKVSC,
 172         DRIVER_STORVSC,
 173         DRIVER_UNKNOWN
 174 };
 175
 176 #define HS_MAX_ADAPTERS 10
 177
 178 #define HV_STORAGE_SUPPORTS_MULTI_CHANNEL 0x1
 179
 180 /* {ba6163d9-04a1-4d29-b605-72e2ffb1dc7f} */
 181 static const hv_guid gStorVscDeviceType={
 182         .data = {0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d,
 183                  0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f}
 184 };
 185
 186 /* {32412632-86cb-44a2-9b5c-50d1417354f5} */
 187 static const hv_guid gBlkVscDeviceType={
 188         .data = {0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44,
 189                  0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5}
 190 };
 191
 192 static struct storvsc_driver_props g_drv_props_table[] = {
 193         {"blkvsc", "Hyper-V IDE Storage Interface",
 194          BLKVSC_MAX_IDE_DISKS_PER_TARGET, BLKVSC_MAX_IO_REQUESTS,
 195          STORVSC_RINGBUFFER_SIZE},
 196         {"storvsc", "Hyper-V SCSI Storage Interface",
 197          STORVSC_MAX_LUNS_PER_TARGET, STORVSC_MAX_IO_REQUESTS,
 198          STORVSC_RINGBUFFER_SIZE}
 199 };
 200
 201 /*
 202  * Sense buffer size changed in win8; have a run-time
 203  * variable to track the size we should use.
 204  */
 205 static int sense_buffer_size = PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE;
 206
 207 /*
 208  * The size of the vmscsi_request has changed in win8. The
 209  * additional size is for the newly added elements in the
 210  * structure. These elements are valid only when we are talking
 211  * to a win8 host.
 212  * Track the correct size we need to apply.
 213  */
 214 static int vmscsi_size_delta;
 215 /*
 216  * The storage protocol version is determined during the
 217  * initial exchange with the host.  It will indicate which
 218  * storage functionality is available in the host.
 219 */
 220 static int vmstor_proto_version;
 221
 222 struct vmstor_proto {
 223         int proto_version;
 224         int sense_buffer_size;
 225         int vmscsi_size_delta;
 226 };
 227
 228 static const struct vmstor_proto vmstor_proto_list[] = {
 229         {
 230                 VMSTOR_PROTOCOL_VERSION_WIN10,
 231                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 232                 0
 233         },
 234         {
 235                 VMSTOR_PROTOCOL_VERSION_WIN8_1,
 236                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 237                 0
 238         },
 239         {
 240                 VMSTOR_PROTOCOL_VERSION_WIN8,
 241                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 242                 0
 243         },
 244         {
 245                 VMSTOR_PROTOCOL_VERSION_WIN7,
 246                 PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE,
 247                 sizeof(struct vmscsi_win8_extension),
 248         },
 249         {
 250                 VMSTOR_PROTOCOL_VERSION_WIN6,
 251                 PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE,
 252                 sizeof(struct vmscsi_win8_extension),
 253         }
 254 };
 255
 256 /* static functions */
 257 static int storvsc_probe(device_t dev);
 258 static int storvsc_attach(device_t dev);
 259 static int storvsc_detach(device_t dev);
 260 static void storvsc_poll(struct cam_sim * sim);
 261 static void storvsc_action(struct cam_sim * sim, union ccb * ccb);
 262 static int create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp);
 263 static void storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp);
 264 static enum hv_storage_type storvsc_get_storage_type(device_t dev);
 265 static void hv_storvsc_rescan_target(struct storvsc_softc *sc);
 266 static void hv_storvsc_on_channel_callback(void *context);
 267 static void hv_storvsc_on_iocompletion( struct storvsc_softc *sc,
 268                                         struct vstor_packet *vstor_packet,
 269                                         struct hv_storvsc_request *request);
 270 static int hv_storvsc_connect_vsp(struct hv_device *device);
 271 static void storvsc_io_done(struct hv_storvsc_request *reqp);
 272 static void storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
 273                                 bus_dma_segment_t *orig_sgl,
 274                                 unsigned int orig_sgl_count,
 275                                 uint64_t seg_bits);
 276 void storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
 277                                 unsigned int dest_sgl_count,
 278                                 struct sglist* src_sgl,
 279                                 uint64_t seg_bits);
 280
 281 static device_method_t storvsc_methods[] = {
 282         /* Device interface */
 283         DEVMETHOD(device_probe,         storvsc_probe),
 284         DEVMETHOD(device_attach,        storvsc_attach),
 285         DEVMETHOD(device_detach,        storvsc_detach),
 286         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 287         DEVMETHOD_END
 288 };
 289
 290 static driver_t storvsc_driver = {
 291         "storvsc", storvsc_methods, sizeof(struct storvsc_softc),
 292 };
 293
 294 static devclass_t storvsc_devclass;
 295 DRIVER_MODULE(storvsc, vmbus, storvsc_driver, storvsc_devclass, 0, 0);
 296 MODULE_VERSION(storvsc, 1);
 297 MODULE_DEPEND(storvsc, vmbus, 1, 1, 1);
 298
 299
 300 /**
 301  * The host is capable of sending messages to us that are
 302  * completely unsolicited. So, we need to address the race
 303  * condition where we may be in the process of unloading the
 304  * driver when the host may send us an unsolicited message.
 305  * We address this issue by implementing a sequentially
 306  * consistent protocol:
 307  *
 308  * 1. Channel callback is invoked while holding the the channel lock
 309  *    and an unloading driver will reset the channel callback under
 310  *    the protection of this channel lock.
 311  *
 312  * 2. To ensure bounded wait time for unloading a driver, we don't
 313  *    permit outgoing traffic once the device is marked as being
 314  *    destroyed.
 315  *
 316  * 3. Once the device is marked as being destroyed, we only
 317  *    permit incoming traffic to properly account for
 318  *    packets already sent out.
 319  */
 320 static inline struct storvsc_softc *
 321 get_stor_device(struct hv_device *device,
 322                                 boolean_t outbound)
 323 {
 324         struct storvsc_softc *sc;
 325
 326         sc = device_get_softc(device->device);
 327         if (sc == NULL) {
 328                 return NULL;
 329         }
 330
 331         if (outbound) {
 332                 /*
 333                  * Here we permit outgoing I/O only
 334                  * if the device is not being destroyed.
 335                  */
 336
 337                 if (sc->hs_destroy) {
 338                         sc = NULL;
 339                 }
 340         } else {
 341                 /*
 342                  * inbound case; if being destroyed
 343                  * only permit to account for
 344                  * messages already sent out.
 345                  */
 346                 if (sc->hs_destroy && (sc->hs_num_out_reqs == 0)) {
 347                         sc = NULL;
 348                 }
 349         }
 350         return sc;
 351 }
 352
 353 /**
 354  * @brief Callback handler, will be invoked when receive mutil-channel offer
 355  *
 356  * @param context  new multi-channel
 357  */
 358 static void
 359 storvsc_handle_sc_creation(void *context)
 360 {
 361         hv_vmbus_channel *new_channel;
 362         struct hv_device *device;
 363         struct storvsc_softc *sc;
 364         struct vmstor_chan_props props;
 365         int ret = 0;
 366
 367         new_channel = (hv_vmbus_channel *)context;
 368         device = new_channel->primary_channel->device;
 369         sc = get_stor_device(device, TRUE);
 370         if (sc == NULL)
 371                 return;
 372
 373         if (FALSE == sc->hs_open_multi_channel)
 374                 return;
 375
 376         memset(&props, 0, sizeof(props));
 377
 378         ret = hv_vmbus_channel_open(new_channel,
 379             sc->hs_drv_props->drv_ringbuffer_size,
 380             sc->hs_drv_props->drv_ringbuffer_size,
 381             (void *)&props,
 382             sizeof(struct vmstor_chan_props),
 383             hv_storvsc_on_channel_callback,
 384             new_channel);
 385
 386         return;
 387 }
 388
 389 /**
 390  * @brief Send multi-channel creation request to host
 391  *
 392  * @param device  a Hyper-V device pointer
 393  * @param max_chans  the max channels supported by vmbus
 394  */
 395 static void
 396 storvsc_send_multichannel_request(struct hv_device *dev, int max_chans)
 397 {
 398         struct storvsc_softc *sc;
 399         struct hv_storvsc_request *request;
 400         struct vstor_packet *vstor_packet;
 401         int request_channels_cnt = 0;
 402         int ret;
 403
 404         /* get multichannels count that need to create */
 405         request_channels_cnt = MIN(max_chans, mp_ncpus);
 406
 407         sc = get_stor_device(dev, TRUE);
 408         if (sc == NULL) {
 409                 printf("Storvsc_error: get sc failed while send mutilchannel "
 410                     "request\n");
 411                 return;
 412         }
 413
 414         request = &sc->hs_init_req;
 415
 416         /* Establish a handler for multi-channel */
 417         dev->channel->sc_creation_callback = storvsc_handle_sc_creation;
 418
 419         /* request the host to create multi-channel */
 420         memset(request, 0, sizeof(struct hv_storvsc_request));
 421
 422         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 423
 424         vstor_packet = &request->vstor_packet;
 425
 426         vstor_packet->operation = VSTOR_OPERATION_CREATE_MULTI_CHANNELS;
 427         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 428         vstor_packet->u.multi_channels_cnt = request_channels_cnt;
 429
 430         ret = hv_vmbus_channel_send_packet(
 431             dev->channel,
 432             vstor_packet,
 433             VSTOR_PKT_SIZE,
 434             (uint64_t)(uintptr_t)request,
 435             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 436             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 437
 438         /* wait for 5 seconds */
 439         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 440         if (ret != 0) {
 441                 printf("Storvsc_error: create multi-channel timeout, %d\n",
 442                     ret);
 443                 return;
 444         }
 445
 446         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 447             vstor_packet->status != 0) {
 448                 printf("Storvsc_error: create multi-channel invalid operation "
 449                     "(%d) or statue (%u)\n",
 450                     vstor_packet->operation, vstor_packet->status);
 451                 return;
 452         }
 453
 454         sc->hs_open_multi_channel = TRUE;
 455
 456         if (bootverbose)
 457                 printf("Storvsc create multi-channel success!\n");
 458 }
 459
 460 /**
 461  * @brief initialize channel connection to parent partition
 462  *
 463  * @param dev  a Hyper-V device pointer
 464  * @returns  0 on success, non-zero error on failure
 465  */
 466 static int
 467 hv_storvsc_channel_init(struct hv_device *dev)
 468 {
 469         int ret = 0, i;
 470         struct hv_storvsc_request *request;
 471         struct vstor_packet *vstor_packet;
 472         struct storvsc_softc *sc;
 473         uint16_t max_chans = 0;
 474         boolean_t support_multichannel = FALSE;
 475
 476         max_chans = 0;
 477         support_multichannel = FALSE;
 478
 479         sc = get_stor_device(dev, TRUE);
 480         if (sc == NULL)
 481                 return (ENODEV);
 482
 483         request = &sc->hs_init_req;
 484         memset(request, 0, sizeof(struct hv_storvsc_request));
 485         vstor_packet = &request->vstor_packet;
 486         request->softc = sc;
 487
 488         /**
 489          * Initiate the vsc/vsp initialization protocol on the open channel
 490          */
 491         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 492
 493         vstor_packet->operation = VSTOR_OPERATION_BEGININITIALIZATION;
 494         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 495
 496
 497         ret = hv_vmbus_channel_send_packet(
 498                         dev->channel,
 499                         vstor_packet,
 500                         VSTOR_PKT_SIZE,
 501                         (uint64_t)(uintptr_t)request,
 502                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 503                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 504
 505         if (ret != 0)
 506                 goto cleanup;
 507
 508         /* wait 5 seconds */
 509         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 510         if (ret != 0)
 511                 goto cleanup;
 512
 513         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 514                 vstor_packet->status != 0) {
 515                 goto cleanup;
 516         }
 517
 518         for (i = 0; i < nitems(vmstor_proto_list); i++) {
 519                 /* reuse the packet for version range supported */
 520
 521                 memset(vstor_packet, 0, sizeof(struct vstor_packet));
 522                 vstor_packet->operation = VSTOR_OPERATION_QUERYPROTOCOLVERSION;
 523                 vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 524
 525                 vstor_packet->u.version.major_minor =
 526                         vmstor_proto_list[i].proto_version;
 527
 528                 /* revision is only significant for Windows guests */
 529                 vstor_packet->u.version.revision = 0;
 530
 531                 ret = hv_vmbus_channel_send_packet(
 532                         dev->channel,
 533                         vstor_packet,
 534                         VSTOR_PKT_SIZE,
 535                         (uint64_t)(uintptr_t)request,
 536                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 537                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 538
 539                 if (ret != 0)
 540                         goto cleanup;
 541
 542                 /* wait 5 seconds */
 543                 ret = sema_timedwait(&request->synch_sema, 5 * hz);
 544
 545                 if (ret)
 546                         goto cleanup;
 547
 548                 if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO) {
 549                         ret = EINVAL;
 550                         goto cleanup;
 551                 }
 552                 if (vstor_packet->status == 0) {
 553                         vmstor_proto_version =
 554                                 vmstor_proto_list[i].proto_version;
 555                         sense_buffer_size =
 556                                 vmstor_proto_list[i].sense_buffer_size;
 557                         vmscsi_size_delta =
 558                                 vmstor_proto_list[i].vmscsi_size_delta;
 559                         break;
 560                 }
 561         }
 562
 563         if (vstor_packet->status != 0) {
 564                 ret = EINVAL;
 565                 goto cleanup;
 566         }
 567         /**
 568          * Query channel properties
 569          */
 570         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 571         vstor_packet->operation = VSTOR_OPERATION_QUERYPROPERTIES;
 572         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 573
 574         ret = hv_vmbus_channel_send_packet(
 575                                 dev->channel,
 576                                 vstor_packet,
 577                                 VSTOR_PKT_SIZE,
 578                                 (uint64_t)(uintptr_t)request,
 579                                 HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 580                                 HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 581
 582         if ( ret != 0)
 583                 goto cleanup;
 584
 585         /* wait 5 seconds */
 586         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 587
 588         if (ret != 0)
 589                 goto cleanup;
 590
 591         /* TODO: Check returned version */
 592         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 593             vstor_packet->status != 0) {
 594                 goto cleanup;
 595         }
 596
 597         /* multi-channels feature is supported by WIN8 and above version */
 598         max_chans = vstor_packet->u.chan_props.max_channel_cnt;
 599         if ((hv_vmbus_protocal_version != HV_VMBUS_VERSION_WIN7) &&
 600             (hv_vmbus_protocal_version != HV_VMBUS_VERSION_WS2008) &&
 601             (vstor_packet->u.chan_props.flags &
 602              HV_STORAGE_SUPPORTS_MULTI_CHANNEL)) {
 603                 support_multichannel = TRUE;
 604         }
 605
 606         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 607         vstor_packet->operation = VSTOR_OPERATION_ENDINITIALIZATION;
 608         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 609
 610         ret = hv_vmbus_channel_send_packet(
 611                         dev->channel,
 612                         vstor_packet,
 613                         VSTOR_PKT_SIZE,
 614                         (uint64_t)(uintptr_t)request,
 615                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 616                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 617
 618         if (ret != 0) {
 619                 goto cleanup;
 620         }
 621
 622         /* wait 5 seconds */
 623         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 624
 625         if (ret != 0)
 626                 goto cleanup;
 627
 628         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 629             vstor_packet->status != 0)
 630                 goto cleanup;
 631
 632         /*
 633          * If multi-channel is supported, send multichannel create
 634          * request to host.
 635          */
 636         if (support_multichannel)
 637                 storvsc_send_multichannel_request(dev, max_chans);
 638
 639 cleanup:
 640         sema_destroy(&request->synch_sema);
 641         return (ret);
 642 }
 643
 644 /**
 645  * @brief Open channel connection to paraent partition StorVSP driver
 646  *
 647  * Open and initialize channel connection to parent partition StorVSP driver.
 648  *
 649  * @param pointer to a Hyper-V device
 650  * @returns 0 on success, non-zero error on failure
 651  */
 652 static int
 653 hv_storvsc_connect_vsp(struct hv_device *dev)
 654 {
 655         int ret = 0;
 656         struct vmstor_chan_props props;
 657         struct storvsc_softc *sc;
 658
 659         sc = device_get_softc(dev->device);
 660
 661         memset(&props, 0, sizeof(struct vmstor_chan_props));
 662
 663         /*
 664          * Open the channel
 665          */
 666
 667         ret = hv_vmbus_channel_open(
 668                 dev->channel,
 669                 sc->hs_drv_props->drv_ringbuffer_size,
 670                 sc->hs_drv_props->drv_ringbuffer_size,
 671                 (void *)&props,
 672                 sizeof(struct vmstor_chan_props),
 673                 hv_storvsc_on_channel_callback,
 674                 dev->channel);
 675
 676         if (ret != 0) {
 677                 return ret;
 678         }
 679
 680         ret = hv_storvsc_channel_init(dev);
 681
 682         return (ret);
 683 }
 684
 685 #if HVS_HOST_RESET
 686 static int
 687 hv_storvsc_host_reset(struct hv_device *dev)
 688 {
 689         int ret = 0;
 690         struct storvsc_softc *sc;
 691
 692         struct hv_storvsc_request *request;
 693         struct vstor_packet *vstor_packet;
 694
 695         sc = get_stor_device(dev, TRUE);
 696         if (sc == NULL) {
 697                 return ENODEV;
 698         }
 699
 700         request = &sc->hs_reset_req;
 701         request->softc = sc;
 702         vstor_packet = &request->vstor_packet;
 703
 704         sema_init(&request->synch_sema, 0, "stor synch sema");
 705
 706         vstor_packet->operation = VSTOR_OPERATION_RESETBUS;
 707         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 708
 709         ret = hv_vmbus_channel_send_packet(dev->channel,
 710                         vstor_packet,
 711                         VSTOR_PKT_SIZE,
 712                         (uint64_t)(uintptr_t)&sc->hs_reset_req,
 713                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 714                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 715
 716         if (ret != 0) {
 717                 goto cleanup;
 718         }
 719
 720         ret = sema_timedwait(&request->synch_sema, 5 * hz); /* KYS 5 seconds */
 721
 722         if (ret) {
 723                 goto cleanup;
 724         }
 725
 726
 727         /*
 728          * At this point, all outstanding requests in the adapter
 729          * should have been flushed out and return to us
 730          */
 731
 732 cleanup:
 733         sema_destroy(&request->synch_sema);
 734         return (ret);
 735 }
 736 #endif /* HVS_HOST_RESET */
 737
 738 /**
 739  * @brief Function to initiate an I/O request
 740  *
 741  * @param device Hyper-V device pointer
 742  * @param request pointer to a request structure
 743  * @returns 0 on success, non-zero error on failure
 744  */
 745 static int
 746 hv_storvsc_io_request(struct hv_device *device,
 747                                           struct hv_storvsc_request *request)
 748 {
 749         struct storvsc_softc *sc;
 750         struct vstor_packet *vstor_packet = &request->vstor_packet;
 751         struct hv_vmbus_channel* outgoing_channel = NULL;
 752         int ret = 0;
 753
 754         sc = get_stor_device(device, TRUE);
 755
 756         if (sc == NULL) {
 757                 return ENODEV;
 758         }
 759
 760         vstor_packet->flags |= REQUEST_COMPLETION_FLAG;
 761
 762         vstor_packet->u.vm_srb.length = VSTOR_PKT_SIZE;
 763
 764         vstor_packet->u.vm_srb.sense_info_len = sense_buffer_size;
 765
 766         vstor_packet->u.vm_srb.transfer_len = request->data_buf.length;
 767
 768         vstor_packet->operation = VSTOR_OPERATION_EXECUTESRB;
 769
 770         outgoing_channel = vmbus_select_outgoing_channel(device->channel);
 771
 772         mtx_unlock(&request->softc->hs_lock);
 773         if (request->data_buf.length) {
 774                 ret = hv_vmbus_channel_send_packet_multipagebuffer(
 775                                 outgoing_channel,
 776                                 &request->data_buf,
 777                                 vstor_packet,
 778                                 VSTOR_PKT_SIZE,
 779                                 (uint64_t)(uintptr_t)request);
 780
 781         } else {
 782                 ret = hv_vmbus_channel_send_packet(
 783                         outgoing_channel,
 784                         vstor_packet,
 785                         VSTOR_PKT_SIZE,
 786                         (uint64_t)(uintptr_t)request,
 787                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 788                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 789         }
 790         mtx_lock(&request->softc->hs_lock);
 791
 792         if (ret != 0) {
 793                 printf("Unable to send packet %p ret %d", vstor_packet, ret);
 794         } else {
 795                 atomic_add_int(&sc->hs_num_out_reqs, 1);
 796         }
 797
 798         return (ret);
 799 }
 800
 801
 802 /**
 803  * Process IO_COMPLETION_OPERATION and ready
 804  * the result to be completed for upper layer
 805  * processing by the CAM layer.
 806  */
 807 static void
 808 hv_storvsc_on_iocompletion(struct storvsc_softc *sc,
 809                            struct vstor_packet *vstor_packet,
 810                            struct hv_storvsc_request *request)
 811 {
 812         struct vmscsi_req *vm_srb;
 813
 814         vm_srb = &vstor_packet->u.vm_srb;
 815
 816         if (((vm_srb->scsi_status & 0xFF) == SCSI_STATUS_CHECK_COND) &&
 817                         (vm_srb->srb_status & SRB_STATUS_AUTOSENSE_VALID)) {
 818                 /* Autosense data available */
 819
 820                 KASSERT(vm_srb->sense_info_len <= request->sense_info_len,
 821                                 ("vm_srb->sense_info_len <= "
 822                                  "request->sense_info_len"));
 823
 824                 memcpy(request->sense_data, vm_srb->u.sense_data,
 825                         vm_srb->sense_info_len);
 826
 827                 request->sense_info_len = vm_srb->sense_info_len;
 828         }
 829
 830         /* Complete request by passing to the CAM layer */
 831         storvsc_io_done(request);
 832         atomic_subtract_int(&sc->hs_num_out_reqs, 1);
 833         if (sc->hs_drain_notify && (sc->hs_num_out_reqs == 0)) {
 834                 sema_post(&sc->hs_drain_sema);
 835         }
 836 }
 837
 838 static void
 839 hv_storvsc_rescan_target(struct storvsc_softc *sc)
 840 {
 841         path_id_t pathid;
 842         target_id_t targetid;
 843         union ccb *ccb;
 844
 845         pathid = cam_sim_path(sc->hs_sim);
 846         targetid = CAM_TARGET_WILDCARD;
 847
 848         /*
 849          * Allocate a CCB and schedule a rescan.
 850          */
 851         ccb = xpt_alloc_ccb_nowait();
 852         if (ccb == NULL) {
 853                 printf("unable to alloc CCB for rescan\n");
 854                 return;
 855         }
 856
 857         if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid,
 858             CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 859                 printf("unable to create path for rescan, pathid: %d,"
 860                     "targetid: %d\n", pathid, targetid);
 861                 xpt_free_ccb(ccb);
 862                 return;
 863         }
 864
 865         if (targetid == CAM_TARGET_WILDCARD)
 866                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 867         else
 868                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 869
 870         xpt_rescan(ccb);
 871 }
 872
 873 static void
 874 hv_storvsc_on_channel_callback(void *context)
 875 {
 876         int ret = 0;
 877         hv_vmbus_channel *channel = (hv_vmbus_channel *)context;
 878         struct hv_device *device = NULL;
 879         struct storvsc_softc *sc;
 880         uint32_t bytes_recvd;
 881         uint64_t request_id;
 882         uint8_t packet[roundup2(sizeof(struct vstor_packet), 8)];
 883         struct hv_storvsc_request *request;
 884         struct vstor_packet *vstor_packet;
 885
 886         if (channel->primary_channel != NULL){
 887                 device = channel->primary_channel->device;
 888         } else {
 889                 device = channel->device;
 890         }
 891
 892         KASSERT(device, ("device is NULL"));
 893
 894         sc = get_stor_device(device, FALSE);
 895         if (sc == NULL) {
 896                 printf("Storvsc_error: get stor device failed.\n");
 897                 return;
 898         }
 899
 900         ret = hv_vmbus_channel_recv_packet(
 901                         channel,
 902                         packet,
 903                         roundup2(VSTOR_PKT_SIZE, 8),
 904                         &bytes_recvd,
 905                         &request_id);
 906
 907         while ((ret == 0) && (bytes_recvd > 0)) {
 908                 request = (struct hv_storvsc_request *)(uintptr_t)request_id;
 909
 910                 if ((request == &sc->hs_init_req) ||
 911                         (request == &sc->hs_reset_req)) {
 912                         memcpy(&request->vstor_packet, packet,
 913                                    sizeof(struct vstor_packet));
 914                         sema_post(&request->synch_sema);
 915                 } else {
 916                         vstor_packet = (struct vstor_packet *)packet;
 917                         switch(vstor_packet->operation) {
 918                         case VSTOR_OPERATION_COMPLETEIO:
 919                                 if (request == NULL)
 920                                         panic("VMBUS: storvsc received a "
 921                                             "packet with NULL request id in "
 922                                             "COMPLETEIO operation.");
 923
 924                                 hv_storvsc_on_iocompletion(sc,
 925                                                         vstor_packet, request);
 926                                 break;
 927                         case VSTOR_OPERATION_REMOVEDEVICE:
 928                                 printf("VMBUS: storvsc operation %d not "
 929                                     "implemented.\n", vstor_packet->operation);
 930                                 /* TODO: implement */
 931                                 break;
 932                         case VSTOR_OPERATION_ENUMERATE_BUS:
 933                                 hv_storvsc_rescan_target(sc);
 934                                 break;
 935                         default:
 936                                 break;
 937                         }
 938                 }
 939                 ret = hv_vmbus_channel_recv_packet(
 940                                 channel,
 941                                 packet,
 942                                 roundup2(VSTOR_PKT_SIZE, 8),
 943                                 &bytes_recvd,
 944                                 &request_id);
 945         }
 946 }
 947
 948 /**
 949  * @brief StorVSC probe function
 950  *
 951  * Device probe function.  Returns 0 if the input device is a StorVSC
 952  * device.  Otherwise, a ENXIO is returned.  If the input device is
 953  * for BlkVSC (paravirtual IDE) device and this support is disabled in
 954  * favor of the emulated ATA/IDE device, return ENXIO.
 955  *
 956  * @param a device
 957  * @returns 0 on success, ENXIO if not a matcing StorVSC device
 958  */
 959 static int
 960 storvsc_probe(device_t dev)
 961 {
 962         int ata_disk_enable = 0;
 963         int ret = ENXIO;
 964
 965         switch (storvsc_get_storage_type(dev)) {
 966         case DRIVER_BLKVSC:
 967                 if(bootverbose)
 968                         device_printf(dev, "DRIVER_BLKVSC-Emulated ATA/IDE probe\n");
 969                 if (!getenv_int("hw.ata.disk_enable", &ata_disk_enable)) {
 970                         if(bootverbose)
 971                                 device_printf(dev,
 972                                         "Enlightened ATA/IDE detected\n");
 973                         ret = BUS_PROBE_DEFAULT;
 974                 } else if(bootverbose)
 975                         device_printf(dev, "Emulated ATA/IDE set (hw.ata.disk_enable set)\n");
 976                 break;
 977         case DRIVER_STORVSC:
 978                 if(bootverbose)
 979                         device_printf(dev, "Enlightened SCSI device detected\n");
 980                 ret = BUS_PROBE_DEFAULT;
 981                 break;
 982         default:
 983                 ret = ENXIO;
 984         }
 985         return (ret);
 986 }
 987
 988 /**
 989  * @brief StorVSC attach function
 990  *
 991  * Function responsible for allocating per-device structures,
 992  * setting up CAM interfaces and scanning for available LUNs to
 993  * be used for SCSI device peripherals.
 994  *
 995  * @param a device
 996  * @returns 0 on success or an error on failure
 997  */
 998 static int
 999 storvsc_attach(device_t dev)
1000 {
1001         struct hv_device *hv_dev = vmbus_get_devctx(dev);
1002         enum hv_storage_type stor_type;
1003         struct storvsc_softc *sc;
1004         struct cam_devq *devq;
1005         int ret, i, j;
1006         struct hv_storvsc_request *reqp;
1007         struct root_hold_token *root_mount_token = NULL;
1008         struct hv_sgl_node *sgl_node = NULL;
1009         void *tmp_buff = NULL;
1010
1011         /*
1012          * We need to serialize storvsc attach calls.
1013          */
1014         root_mount_token = root_mount_hold("storvsc");
1015
1016         sc = device_get_softc(dev);
1017         if (sc == NULL) {
1018                 ret = ENOMEM;
1019                 goto cleanup;
1020         }
1021
1022         stor_type = storvsc_get_storage_type(dev);
1023
1024         if (stor_type == DRIVER_UNKNOWN) {
1025                 ret = ENODEV;
1026                 goto cleanup;
1027         }
1028
1029         bzero(sc, sizeof(struct storvsc_softc));
1030
1031         /* fill in driver specific properties */
1032         sc->hs_drv_props = &g_drv_props_table[stor_type];
1033
1034         /* fill in device specific properties */
1035         sc->hs_unit     = device_get_unit(dev);
1036         sc->hs_dev      = hv_dev;
1037         device_set_desc(dev, g_drv_props_table[stor_type].drv_desc);
1038
1039         LIST_INIT(&sc->hs_free_list);
1040         mtx_init(&sc->hs_lock, "hvslck", NULL, MTX_DEF);
1041
1042         for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; ++i) {
1043                 reqp = malloc(sizeof(struct hv_storvsc_request),
1044                                  M_DEVBUF, M_WAITOK|M_ZERO);
1045                 reqp->softc = sc;
1046
1047                 LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
1048         }
1049
1050         /* create sg-list page pool */
1051         if (FALSE == g_hv_sgl_page_pool.is_init) {
1052                 g_hv_sgl_page_pool.is_init = TRUE;
1053                 LIST_INIT(&g_hv_sgl_page_pool.in_use_sgl_list);
1054                 LIST_INIT(&g_hv_sgl_page_pool.free_sgl_list);
1055
1056                 /*
1057                  * Pre-create SG list, each SG list with
1058                  * HV_MAX_MULTIPAGE_BUFFER_COUNT segments, each
1059                  * segment has one page buffer
1060                  */
1061                 for (i = 0; i < STORVSC_MAX_IO_REQUESTS; i++) {
1062                         sgl_node = malloc(sizeof(struct hv_sgl_node),
1063                             M_DEVBUF, M_WAITOK|M_ZERO);
1064
1065                         sgl_node->sgl_data =
1066                             sglist_alloc(HV_MAX_MULTIPAGE_BUFFER_COUNT,
1067                             M_WAITOK|M_ZERO);
1068
1069                         for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
1070                                 tmp_buff = malloc(PAGE_SIZE,
1071                                     M_DEVBUF, M_WAITOK|M_ZERO);
1072
1073                                 sgl_node->sgl_data->sg_segs[j].ss_paddr =
1074                                     (vm_paddr_t)tmp_buff;
1075                         }
1076
1077                         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list,
1078                             sgl_node, link);
1079                 }
1080         }
1081
1082         sc->hs_destroy = FALSE;
1083         sc->hs_drain_notify = FALSE;
1084         sc->hs_open_multi_channel = FALSE;
1085         sema_init(&sc->hs_drain_sema, 0, "Store Drain Sema");
1086
1087         ret = hv_storvsc_connect_vsp(hv_dev);
1088         if (ret != 0) {
1089                 goto cleanup;
1090         }
1091
1092         /*
1093          * Create the device queue.
1094          * Hyper-V maps each target to one SCSI HBA
1095          */
1096         devq = cam_simq_alloc(sc->hs_drv_props->drv_max_ios_per_target);
1097         if (devq == NULL) {
1098                 device_printf(dev, "Failed to alloc device queue\n");
1099                 ret = ENOMEM;
1100                 goto cleanup;
1101         }
1102
1103         sc->hs_sim = cam_sim_alloc(storvsc_action,
1104                                 storvsc_poll,
1105                                 sc->hs_drv_props->drv_name,
1106                                 sc,
1107                                 sc->hs_unit,
1108                                 &sc->hs_lock, 1,
1109                                 sc->hs_drv_props->drv_max_ios_per_target,
1110                                 devq);
1111
1112         if (sc->hs_sim == NULL) {
1113                 device_printf(dev, "Failed to alloc sim\n");
1114                 cam_simq_free(devq);
1115                 ret = ENOMEM;
1116                 goto cleanup;
1117         }
1118
1119         mtx_lock(&sc->hs_lock);
1120         /* bus_id is set to 0, need to get it from VMBUS channel query? */
1121         if (xpt_bus_register(sc->hs_sim, dev, 0) != CAM_SUCCESS) {
1122                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1123                 mtx_unlock(&sc->hs_lock);
1124                 device_printf(dev, "Unable to register SCSI bus\n");
1125                 ret = ENXIO;
1126                 goto cleanup;
1127         }
1128
1129         if (xpt_create_path(&sc->hs_path, /*periph*/NULL,
1130                  cam_sim_path(sc->hs_sim),
1131                 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1132                 xpt_bus_deregister(cam_sim_path(sc->hs_sim));
1133                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1134                 mtx_unlock(&sc->hs_lock);
1135                 device_printf(dev, "Unable to create path\n");
1136                 ret = ENXIO;
1137                 goto cleanup;
1138         }
1139
1140         mtx_unlock(&sc->hs_lock);
1141
1142         root_mount_rel(root_mount_token);
1143         return (0);
1144
1145
1146 cleanup:
1147         root_mount_rel(root_mount_token);
1148         while (!LIST_EMPTY(&sc->hs_free_list)) {
1149                 reqp = LIST_FIRST(&sc->hs_free_list);
1150                 LIST_REMOVE(reqp, link);
1151                 free(reqp, M_DEVBUF);
1152         }
1153
1154         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1155                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1156                 LIST_REMOVE(sgl_node, link);
1157                 for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
1158                         if (NULL !=
1159                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1160                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1161                         }
1162                 }
1163                 sglist_free(sgl_node->sgl_data);
1164                 free(sgl_node, M_DEVBUF);
1165         }
1166
1167         return (ret);
1168 }
1169
1170 /**
1171  * @brief StorVSC device detach function
1172  *
1173  * This function is responsible for safely detaching a
1174  * StorVSC device.  This includes waiting for inbound responses
1175  * to complete and freeing associated per-device structures.
1176  *
1177  * @param dev a device
1178  * returns 0 on success
1179  */
1180 static int
1181 storvsc_detach(device_t dev)
1182 {
1183         struct storvsc_softc *sc = device_get_softc(dev);
1184         struct hv_storvsc_request *reqp = NULL;
1185         struct hv_device *hv_device = vmbus_get_devctx(dev);
1186         struct hv_sgl_node *sgl_node = NULL;
1187         int j = 0;
1188
1189         mtx_lock(&hv_device->channel->inbound_lock);
1190         sc->hs_destroy = TRUE;
1191         mtx_unlock(&hv_device->channel->inbound_lock);
1192
1193         /*
1194          * At this point, all outbound traffic should be disabled. We
1195          * only allow inbound traffic (responses) to proceed so that
1196          * outstanding requests can be completed.
1197          */
1198
1199         sc->hs_drain_notify = TRUE;
1200         sema_wait(&sc->hs_drain_sema);
1201         sc->hs_drain_notify = FALSE;
1202
1203         /*
1204          * Since we have already drained, we don't need to busy wait.
1205          * The call to close the channel will reset the callback
1206          * under the protection of the incoming channel lock.
1207          */
1208
1209         hv_vmbus_channel_close(hv_device->channel);
1210
1211         mtx_lock(&sc->hs_lock);
1212         while (!LIST_EMPTY(&sc->hs_free_list)) {
1213                 reqp = LIST_FIRST(&sc->hs_free_list);
1214                 LIST_REMOVE(reqp, link);
1215
1216                 free(reqp, M_DEVBUF);
1217         }
1218         mtx_unlock(&sc->hs_lock);
1219
1220         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1221                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1222                 LIST_REMOVE(sgl_node, link);
1223                 for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++){
1224                         if (NULL !=
1225                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1226                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1227                         }
1228                 }
1229                 sglist_free(sgl_node->sgl_data);
1230                 free(sgl_node, M_DEVBUF);
1231         }
1232
1233         return (0);
1234 }
1235
1236 #if HVS_TIMEOUT_TEST
1237 /**
1238  * @brief unit test for timed out operations
1239  *
1240  * This function provides unit testing capability to simulate
1241  * timed out operations.  Recompilation with HV_TIMEOUT_TEST=1
1242  * is required.
1243  *
1244  * @param reqp pointer to a request structure
1245  * @param opcode SCSI operation being performed
1246  * @param wait if 1, wait for I/O to complete
1247  */
1248 static void
1249 storvsc_timeout_test(struct hv_storvsc_request *reqp,
1250                 uint8_t opcode, int wait)
1251 {
1252         int ret;
1253         union ccb *ccb = reqp->ccb;
1254         struct storvsc_softc *sc = reqp->softc;
1255
1256         if (reqp->vstor_packet.vm_srb.cdb[0] != opcode) {
1257                 return;
1258         }
1259
1260         if (wait) {
1261                 mtx_lock(&reqp->event.mtx);
1262         }
1263         ret = hv_storvsc_io_request(sc->hs_dev, reqp);
1264         if (ret != 0) {
1265                 if (wait) {
1266                         mtx_unlock(&reqp->event.mtx);
1267                 }
1268                 printf("%s: io_request failed with %d.\n",
1269                                 __func__, ret);
1270                 ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1271                 mtx_lock(&sc->hs_lock);
1272                 storvsc_free_request(sc, reqp);
1273                 xpt_done(ccb);
1274                 mtx_unlock(&sc->hs_lock);
1275                 return;
1276         }
1277
1278         if (wait) {
1279                 xpt_print(ccb->ccb_h.path,
1280                                 "%u: %s: waiting for IO return.\n",
1281                                 ticks, __func__);
1282                 ret = cv_timedwait(&reqp->event.cv, &reqp->event.mtx, 60*hz);
1283                 mtx_unlock(&reqp->event.mtx);
1284                 xpt_print(ccb->ccb_h.path, "%u: %s: %s.\n",
1285                                 ticks, __func__, (ret == 0)?
1286                                 "IO return detected" :
1287                                 "IO return not detected");
1288                 /*
1289                  * Now both the timer handler and io done are running
1290                  * simultaneously. We want to confirm the io done always
1291                  * finishes after the timer handler exits. So reqp used by
1292                  * timer handler is not freed or stale. Do busy loop for
1293                  * another 1/10 second to make sure io done does
1294                  * wait for the timer handler to complete.
1295                  */
1296                 DELAY(100*1000);
1297                 mtx_lock(&sc->hs_lock);
1298                 xpt_print(ccb->ccb_h.path,
1299                                 "%u: %s: finishing, queue frozen %d, "
1300                                 "ccb status 0x%x scsi_status 0x%x.\n",
1301                                 ticks, __func__, sc->hs_frozen,
1302                                 ccb->ccb_h.status,
1303                                 ccb->csio.scsi_status);
1304                 mtx_unlock(&sc->hs_lock);
1305         }
1306 }
1307 #endif /* HVS_TIMEOUT_TEST */
1308
1309 #ifdef notyet
1310 /**
1311  * @brief timeout handler for requests
1312  *
1313  * This function is called as a result of a callout expiring.
1314  *
1315  * @param arg pointer to a request
1316  */
1317 static void
1318 storvsc_timeout(void *arg)
1319 {
1320         struct hv_storvsc_request *reqp = arg;
1321         struct storvsc_softc *sc = reqp->softc;
1322         union ccb *ccb = reqp->ccb;
1323
1324         if (reqp->retries == 0) {
1325                 mtx_lock(&sc->hs_lock);
1326                 xpt_print(ccb->ccb_h.path,
1327                     "%u: IO timed out (req=0x%p), wait for another %u secs.\n",
1328                     ticks, reqp, ccb->ccb_h.timeout / 1000);
1329                 cam_error_print(ccb, CAM_ESF_ALL, CAM_EPF_ALL);
1330                 mtx_unlock(&sc->hs_lock);
1331
1332                 reqp->retries++;
1333                 callout_reset_sbt(&reqp->callout, SBT_1MS * ccb->ccb_h.timeout,
1334                     0, storvsc_timeout, reqp, 0);
1335 #if HVS_TIMEOUT_TEST
1336                 storvsc_timeout_test(reqp, SEND_DIAGNOSTIC, 0);
1337 #endif
1338                 return;
1339         }
1340
1341         mtx_lock(&sc->hs_lock);
1342         xpt_print(ccb->ccb_h.path,
1343                 "%u: IO (reqp = 0x%p) did not return for %u seconds, %s.\n",
1344                 ticks, reqp, ccb->ccb_h.timeout * (reqp->retries+1) / 1000,
1345                 (sc->hs_frozen == 0)?
1346                 "freezing the queue" : "the queue is already frozen");
1347         if (sc->hs_frozen == 0) {
1348                 sc->hs_frozen = 1;
1349                 xpt_freeze_simq(xpt_path_sim(ccb->ccb_h.path), 1);
1350         }
1351         mtx_unlock(&sc->hs_lock);
1352
1353 #if HVS_TIMEOUT_TEST
1354         storvsc_timeout_test(reqp, MODE_SELECT_10, 1);
1355 #endif
1356 }
1357 #endif
1358
1359 /**
1360  * @brief StorVSC device poll function
1361  *
1362  * This function is responsible for servicing requests when
1363  * interrupts are disabled (i.e when we are dumping core.)
1364  *
1365  * @param sim a pointer to a CAM SCSI interface module
1366  */
1367 static void
1368 storvsc_poll(struct cam_sim *sim)
1369 {
1370         struct storvsc_softc *sc = cam_sim_softc(sim);
1371
1372         mtx_assert(&sc->hs_lock, MA_OWNED);
1373         mtx_unlock(&sc->hs_lock);
1374         hv_storvsc_on_channel_callback(sc->hs_dev->channel);
1375         mtx_lock(&sc->hs_lock);
1376 }
1377
1378 /**
1379  * @brief StorVSC device action function
1380  *
1381  * This function is responsible for handling SCSI operations which
1382  * are passed from the CAM layer.  The requests are in the form of
1383  * CAM control blocks which indicate the action being performed.
1384  * Not all actions require converting the request to a VSCSI protocol
1385  * message - these actions can be responded to by this driver.
1386  * Requests which are destined for a backend storage device are converted
1387  * to a VSCSI protocol message and sent on the channel connection associated
1388  * with this device.
1389  *
1390  * @param sim pointer to a CAM SCSI interface module
1391  * @param ccb pointer to a CAM control block
1392  */
1393 static void
1394 storvsc_action(struct cam_sim *sim, union ccb *ccb)
1395 {
1396         struct storvsc_softc *sc = cam_sim_softc(sim);
1397         int res;
1398
1399         mtx_assert(&sc->hs_lock, MA_OWNED);
1400         switch (ccb->ccb_h.func_code) {
1401         case XPT_PATH_INQ: {
1402                 struct ccb_pathinq *cpi = &ccb->cpi;
1403
1404                 cpi->version_num = 1;
1405                 cpi->hba_inquiry = PI_TAG_ABLE|PI_SDTR_ABLE;
1406                 cpi->target_sprt = 0;
1407                 cpi->hba_misc = PIM_NOBUSRESET;
1408                 cpi->hba_eng_cnt = 0;
1409                 cpi->max_target = STORVSC_MAX_TARGETS;
1410                 cpi->max_lun = sc->hs_drv_props->drv_max_luns_per_target;
1411                 cpi->initiator_id = cpi->max_target;
1412                 cpi->bus_id = cam_sim_bus(sim);
1413                 cpi->base_transfer_speed = 300000;
1414                 cpi->transport = XPORT_SAS;
1415                 cpi->transport_version = 0;
1416                 cpi->protocol = PROTO_SCSI;
1417                 cpi->protocol_version = SCSI_REV_SPC2;
1418                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1419                 strncpy(cpi->hba_vid, sc->hs_drv_props->drv_name, HBA_IDLEN);
1420                 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1421                 cpi->unit_number = cam_sim_unit(sim);
1422
1423                 ccb->ccb_h.status = CAM_REQ_CMP;
1424                 xpt_done(ccb);
1425                 return;
1426         }
1427         case XPT_GET_TRAN_SETTINGS: {
1428                 struct  ccb_trans_settings *cts = &ccb->cts;
1429
1430                 cts->transport = XPORT_SAS;
1431                 cts->transport_version = 0;
1432                 cts->protocol = PROTO_SCSI;
1433                 cts->protocol_version = SCSI_REV_SPC2;
1434
1435                 /* enable tag queuing and disconnected mode */
1436                 cts->proto_specific.valid = CTS_SCSI_VALID_TQ;
1437                 cts->proto_specific.scsi.valid = CTS_SCSI_VALID_TQ;
1438                 cts->proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB;
1439                 cts->xport_specific.valid = CTS_SPI_VALID_DISC;
1440                 cts->xport_specific.spi.flags = CTS_SPI_FLAGS_DISC_ENB;
1441
1442                 ccb->ccb_h.status = CAM_REQ_CMP;
1443                 xpt_done(ccb);
1444                 return;
1445         }
1446         case XPT_SET_TRAN_SETTINGS:     {
1447                 ccb->ccb_h.status = CAM_REQ_CMP;
1448                 xpt_done(ccb);
1449                 return;
1450         }
1451         case XPT_CALC_GEOMETRY:{
1452                 cam_calc_geometry(&ccb->ccg, 1);
1453                 xpt_done(ccb);
1454                 return;
1455         }
1456         case  XPT_RESET_BUS:
1457         case  XPT_RESET_DEV:{
1458 #if HVS_HOST_RESET
1459                 if ((res = hv_storvsc_host_reset(sc->hs_dev)) != 0) {
1460                         xpt_print(ccb->ccb_h.path,
1461                                 "hv_storvsc_host_reset failed with %d\n", res);
1462                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1463                         xpt_done(ccb);
1464                         return;
1465                 }
1466                 ccb->ccb_h.status = CAM_REQ_CMP;
1467                 xpt_done(ccb);
1468                 return;
1469 #else
1470                 xpt_print(ccb->ccb_h.path,
1471                                   "%s reset not supported.\n",
1472                                   (ccb->ccb_h.func_code == XPT_RESET_BUS)?
1473                                   "bus" : "dev");
1474                 ccb->ccb_h.status = CAM_REQ_INVALID;
1475                 xpt_done(ccb);
1476                 return;
1477 #endif  /* HVS_HOST_RESET */
1478         }
1479         case XPT_SCSI_IO:
1480         case XPT_IMMED_NOTIFY: {
1481                 struct hv_storvsc_request *reqp = NULL;
1482
1483                 if (ccb->csio.cdb_len == 0) {
1484                         panic("cdl_len is 0\n");
1485                 }
1486
1487                 if (LIST_EMPTY(&sc->hs_free_list)) {
1488                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
1489                         if (sc->hs_frozen == 0) {
1490                                 sc->hs_frozen = 1;
1491                                 xpt_freeze_simq(sim, /* count*/1);
1492                         }
1493                         xpt_done(ccb);
1494                         return;
1495                 }
1496
1497                 reqp = LIST_FIRST(&sc->hs_free_list);
1498                 LIST_REMOVE(reqp, link);
1499
1500                 bzero(reqp, sizeof(struct hv_storvsc_request));
1501                 reqp->softc = sc;
1502
1503                 ccb->ccb_h.status |= CAM_SIM_QUEUED;
1504                 if ((res = create_storvsc_request(ccb, reqp)) != 0) {
1505                         ccb->ccb_h.status = CAM_REQ_INVALID;
1506                         xpt_done(ccb);
1507                         return;
1508                 }
1509
1510 #ifdef notyet
1511                 if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
1512                         callout_init(&reqp->callout, CALLOUT_MPSAFE);
1513                         callout_reset_sbt(&reqp->callout,
1514                             SBT_1MS * ccb->ccb_h.timeout, 0,
1515                             storvsc_timeout, reqp, 0);
1516 #if HVS_TIMEOUT_TEST
1517                         cv_init(&reqp->event.cv, "storvsc timeout cv");
1518                         mtx_init(&reqp->event.mtx, "storvsc timeout mutex",
1519                                         NULL, MTX_DEF);
1520                         switch (reqp->vstor_packet.vm_srb.cdb[0]) {
1521                                 case MODE_SELECT_10:
1522                                 case SEND_DIAGNOSTIC:
1523                                         /* To have timer send the request. */
1524                                         return;
1525                                 default:
1526                                         break;
1527                         }
1528 #endif /* HVS_TIMEOUT_TEST */
1529                 }
1530 #endif
1531
1532                 if ((res = hv_storvsc_io_request(sc->hs_dev, reqp)) != 0) {
1533                         xpt_print(ccb->ccb_h.path,
1534                                 "hv_storvsc_io_request failed with %d\n", res);
1535                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1536                         storvsc_free_request(sc, reqp);
1537                         xpt_done(ccb);
1538                         return;
1539                 }
1540                 return;
1541         }
1542
1543         default:
1544                 ccb->ccb_h.status = CAM_REQ_INVALID;
1545                 xpt_done(ccb);
1546                 return;
1547         }
1548 }
1549
1550 /**
1551  * @brief destroy bounce buffer
1552  *
1553  * This function is responsible for destroy a Scatter/Gather list
1554  * that create by storvsc_create_bounce_buffer()
1555  *
1556  * @param sgl- the Scatter/Gather need be destroy
1557  * @param sg_count- page count of the SG list.
1558  *
1559  */
1560 static void
1561 storvsc_destroy_bounce_buffer(struct sglist *sgl)
1562 {
1563         struct hv_sgl_node *sgl_node = NULL;
1564
1565         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.in_use_sgl_list);
1566         LIST_REMOVE(sgl_node, link);
1567         if (NULL == sgl_node) {
1568                 printf("storvsc error: not enough in use sgl\n");
1569                 return;
1570         }
1571         sgl_node->sgl_data = sgl;
1572         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list, sgl_node, link);
1573 }
1574
1575 /**
1576  * @brief create bounce buffer
1577  *
1578  * This function is responsible for create a Scatter/Gather list,
1579  * which hold several pages that can be aligned with page size.
1580  *
1581  * @param seg_count- SG-list segments count
1582  * @param write - if WRITE_TYPE, set SG list page used size to 0,
1583  * otherwise set used size to page size.
1584  *
1585  * return NULL if create failed
1586  */
1587 static struct sglist *
1588 storvsc_create_bounce_buffer(uint16_t seg_count, int write)
1589 {
1590         int i = 0;
1591         struct sglist *bounce_sgl = NULL;
1592         unsigned int buf_len = ((write == WRITE_TYPE) ? 0 : PAGE_SIZE);
1593         struct hv_sgl_node *sgl_node = NULL;
1594
1595         /* get struct sglist from free_sgl_list */
1596         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1597         LIST_REMOVE(sgl_node, link);
1598         if (NULL == sgl_node) {
1599                 printf("storvsc error: not enough free sgl\n");
1600                 return NULL;
1601         }
1602         bounce_sgl = sgl_node->sgl_data;
1603         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.in_use_sgl_list, sgl_node, link);
1604
1605         bounce_sgl->sg_maxseg = seg_count;
1606
1607         if (write == WRITE_TYPE)
1608                 bounce_sgl->sg_nseg = 0;
1609         else
1610                 bounce_sgl->sg_nseg = seg_count;
1611
1612         for (i = 0; i < seg_count; i++)
1613                 bounce_sgl->sg_segs[i].ss_len = buf_len;
1614
1615         return bounce_sgl;
1616 }
1617
1618 /**
1619  * @brief copy data from SG list to bounce buffer
1620  *
1621  * This function is responsible for copy data from one SG list's segments
1622  * to another SG list which used as bounce buffer.
1623  *
1624  * @param bounce_sgl - the destination SG list
1625  * @param orig_sgl - the segment of the source SG list.
1626  * @param orig_sgl_count - the count of segments.
1627  * @param orig_sgl_count - indicate which segment need bounce buffer,
1628  *  set 1 means need.
1629  *
1630  */
1631 static void
1632 storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
1633                                bus_dma_segment_t *orig_sgl,
1634                                unsigned int orig_sgl_count,
1635                                uint64_t seg_bits)
1636 {
1637         int src_sgl_idx = 0;
1638
1639         for (src_sgl_idx = 0; src_sgl_idx < orig_sgl_count; src_sgl_idx++) {
1640                 if (seg_bits & (1 << src_sgl_idx)) {
1641                         memcpy((void*)bounce_sgl->sg_segs[src_sgl_idx].ss_paddr,
1642                             (void*)orig_sgl[src_sgl_idx].ds_addr,
1643                             orig_sgl[src_sgl_idx].ds_len);
1644
1645                         bounce_sgl->sg_segs[src_sgl_idx].ss_len =
1646                             orig_sgl[src_sgl_idx].ds_len;
1647                 }
1648         }
1649 }
1650
1651 /**
1652  * @brief copy data from SG list which used as bounce to another SG list
1653  *
1654  * This function is responsible for copy data from one SG list with bounce
1655  * buffer to another SG list's segments.
1656  *
1657  * @param dest_sgl - the destination SG list's segments
1658  * @param dest_sgl_count - the count of destination SG list's segment.
1659  * @param src_sgl - the source SG list.
1660  * @param seg_bits - indicate which segment used bounce buffer of src SG-list.
1661  *
1662  */
1663 void
1664 storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
1665                                     unsigned int dest_sgl_count,
1666                                     struct sglist* src_sgl,
1667                                     uint64_t seg_bits)
1668 {
1669         int sgl_idx = 0;
1670
1671         for (sgl_idx = 0; sgl_idx < dest_sgl_count; sgl_idx++) {
1672                 if (seg_bits & (1 << sgl_idx)) {
1673                         memcpy((void*)(dest_sgl[sgl_idx].ds_addr),
1674                             (void*)(src_sgl->sg_segs[sgl_idx].ss_paddr),
1675                             src_sgl->sg_segs[sgl_idx].ss_len);
1676                 }
1677         }
1678 }
1679
1680 /**
1681  * @brief check SG list with bounce buffer or not
1682  *
1683  * This function is responsible for check if need bounce buffer for SG list.
1684  *
1685  * @param sgl - the SG list's segments
1686  * @param sg_count - the count of SG list's segment.
1687  * @param bits - segmengs number that need bounce buffer
1688  *
1689  * return -1 if SG list needless bounce buffer
1690  */
1691 static int
1692 storvsc_check_bounce_buffer_sgl(bus_dma_segment_t *sgl,
1693                                 unsigned int sg_count,
1694                                 uint64_t *bits)
1695 {
1696         int i = 0;
1697         int offset = 0;
1698         uint64_t phys_addr = 0;
1699         uint64_t tmp_bits = 0;
1700         boolean_t found_hole = FALSE;
1701         boolean_t pre_aligned = TRUE;
1702
1703         if (sg_count < 2){
1704                 return -1;
1705         }
1706
1707         *bits = 0;
1708
1709         phys_addr = vtophys(sgl[0].ds_addr);
1710         offset =  phys_addr - trunc_page(phys_addr);
1711
1712         if (offset != 0) {
1713                 pre_aligned = FALSE;
1714                 tmp_bits |= 1;
1715         }
1716
1717         for (i = 1; i < sg_count; i++) {
1718                 phys_addr = vtophys(sgl[i].ds_addr);
1719                 offset =  phys_addr - trunc_page(phys_addr);
1720
1721                 if (offset == 0) {
1722                         if (FALSE == pre_aligned){
1723                                 /*
1724                                  * This segment is aligned, if the previous
1725                                  * one is not aligned, find a hole
1726                                  */
1727                                 found_hole = TRUE;
1728                         }
1729                         pre_aligned = TRUE;
1730                 } else {
1731                         tmp_bits |= 1 << i;
1732                         if (!pre_aligned) {
1733                                 if (phys_addr != vtophys(sgl[i-1].ds_addr +
1734                                     sgl[i-1].ds_len)) {
1735                                         /*
1736                                          * Check whether connect to previous
1737                                          * segment,if not, find the hole
1738                                          */
1739                                         found_hole = TRUE;
1740                                 }
1741                         } else {
1742                                 found_hole = TRUE;
1743                         }
1744                         pre_aligned = FALSE;
1745                 }
1746         }
1747
1748         if (!found_hole) {
1749                 return (-1);
1750         } else {
1751                 *bits = tmp_bits;
1752                 return 0;
1753         }
1754 }
1755
1756 /**
1757  * @brief Fill in a request structure based on a CAM control block
1758  *
1759  * Fills in a request structure based on the contents of a CAM control
1760  * block.  The request structure holds the payload information for
1761  * VSCSI protocol request.
1762  *
1763  * @param ccb pointer to a CAM contorl block
1764  * @param reqp pointer to a request structure
1765  */
1766 static int
1767 create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
1768 {
1769         struct ccb_scsiio *csio = &ccb->csio;
1770         uint64_t phys_addr;
1771         uint32_t bytes_to_copy = 0;
1772         uint32_t pfn_num = 0;
1773         uint32_t pfn;
1774         uint64_t not_aligned_seg_bits = 0;
1775
1776         /* refer to struct vmscsi_req for meanings of these two fields */
1777         reqp->vstor_packet.u.vm_srb.port =
1778                 cam_sim_unit(xpt_path_sim(ccb->ccb_h.path));
1779         reqp->vstor_packet.u.vm_srb.path_id =
1780                 cam_sim_bus(xpt_path_sim(ccb->ccb_h.path));
1781
1782         reqp->vstor_packet.u.vm_srb.target_id = ccb->ccb_h.target_id;
1783         reqp->vstor_packet.u.vm_srb.lun = ccb->ccb_h.target_lun;
1784
1785         reqp->vstor_packet.u.vm_srb.cdb_len = csio->cdb_len;
1786         if(ccb->ccb_h.flags & CAM_CDB_POINTER) {
1787                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_ptr,
1788                         csio->cdb_len);
1789         } else {
1790                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_bytes,
1791                         csio->cdb_len);
1792         }
1793
1794         switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
1795         case CAM_DIR_OUT:
1796                 reqp->vstor_packet.u.vm_srb.data_in = WRITE_TYPE;
1797                 break;
1798         case CAM_DIR_IN:
1799                 reqp->vstor_packet.u.vm_srb.data_in = READ_TYPE;
1800                 break;
1801         case CAM_DIR_NONE:
1802                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1803                 break;
1804         default:
1805                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1806                 break;
1807         }
1808
1809         reqp->sense_data     = &csio->sense_data;
1810         reqp->sense_info_len = csio->sense_len;
1811
1812         reqp->ccb = ccb;
1813
1814         if (0 == csio->dxfer_len) {
1815                 return (0);
1816         }
1817
1818         reqp->data_buf.length = csio->dxfer_len;
1819
1820         switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
1821         case CAM_DATA_VADDR:
1822         {
1823                 bytes_to_copy = csio->dxfer_len;
1824                 phys_addr = vtophys(csio->data_ptr);
1825                 reqp->data_buf.offset = phys_addr & PAGE_MASK;
1826
1827                 while (bytes_to_copy != 0) {
1828                         int bytes, page_offset;
1829                         phys_addr =
1830                             vtophys(&csio->data_ptr[reqp->data_buf.length -
1831                             bytes_to_copy]);
1832                         pfn = phys_addr >> PAGE_SHIFT;
1833                         reqp->data_buf.pfn_array[pfn_num] = pfn;
1834                         page_offset = phys_addr & PAGE_MASK;
1835
1836                         bytes = min(PAGE_SIZE - page_offset, bytes_to_copy);
1837
1838                         bytes_to_copy -= bytes;
1839                         pfn_num++;
1840                 }
1841                 break;
1842         }
1843
1844         case CAM_DATA_SG:
1845         {
1846                 int i = 0;
1847                 int offset = 0;
1848                 int ret;
1849
1850                 bus_dma_segment_t *storvsc_sglist =
1851                     (bus_dma_segment_t *)ccb->csio.data_ptr;
1852                 u_int16_t storvsc_sg_count = ccb->csio.sglist_cnt;
1853
1854                 printf("Storvsc: get SG I/O operation, %d\n",
1855                     reqp->vstor_packet.u.vm_srb.data_in);
1856
1857                 if (storvsc_sg_count > HV_MAX_MULTIPAGE_BUFFER_COUNT){
1858                         printf("Storvsc: %d segments is too much, "
1859                             "only support %d segments\n",
1860                             storvsc_sg_count, HV_MAX_MULTIPAGE_BUFFER_COUNT);
1861                         return (EINVAL);
1862                 }
1863
1864                 /*
1865                  * We create our own bounce buffer function currently. Idealy
1866                  * we should use BUS_DMA(9) framework. But with current BUS_DMA
1867                  * code there is no callback API to check the page alignment of
1868                  * middle segments before busdma can decide if a bounce buffer
1869                  * is needed for particular segment. There is callback,
1870                  * "bus_dma_filter_t *filter", but the parrameters are not
1871                  * sufficient for storvsc driver.
1872                  * TODO:
1873                  *      Add page alignment check in BUS_DMA(9) callback. Once
1874                  *      this is complete, switch the following code to use
1875                  *      BUS_DMA(9) for storvsc bounce buffer support.
1876                  */
1877                 /* check if we need to create bounce buffer */
1878                 ret = storvsc_check_bounce_buffer_sgl(storvsc_sglist,
1879                     storvsc_sg_count, &not_aligned_seg_bits);
1880                 if (ret != -1) {
1881                         reqp->bounce_sgl =
1882                             storvsc_create_bounce_buffer(storvsc_sg_count,
1883                             reqp->vstor_packet.u.vm_srb.data_in);
1884                         if (NULL == reqp->bounce_sgl) {
1885                                 printf("Storvsc_error: "
1886                                     "create bounce buffer failed.\n");
1887                                 return (ENOMEM);
1888                         }
1889
1890                         reqp->bounce_sgl_count = storvsc_sg_count;
1891                         reqp->not_aligned_seg_bits = not_aligned_seg_bits;
1892
1893                         /*
1894                          * if it is write, we need copy the original data
1895                          *to bounce buffer
1896                          */
1897                         if (WRITE_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
1898                                 storvsc_copy_sgl_to_bounce_buf(
1899                                     reqp->bounce_sgl,
1900                                     storvsc_sglist,
1901                                     storvsc_sg_count,
1902                                     reqp->not_aligned_seg_bits);
1903                         }
1904
1905                         /* transfer virtual address to physical frame number */
1906                         if (reqp->not_aligned_seg_bits & 0x1){
1907                                 phys_addr =
1908                                     vtophys(reqp->bounce_sgl->sg_segs[0].ss_paddr);
1909                         }else{
1910                                 phys_addr =
1911                                         vtophys(storvsc_sglist[0].ds_addr);
1912                         }
1913                         reqp->data_buf.offset = phys_addr & PAGE_MASK;
1914
1915                         pfn = phys_addr >> PAGE_SHIFT;
1916                         reqp->data_buf.pfn_array[0] = pfn;
1917
1918                         for (i = 1; i < storvsc_sg_count; i++) {
1919                                 if (reqp->not_aligned_seg_bits & (1 << i)) {
1920                                         phys_addr =
1921                                             vtophys(reqp->bounce_sgl->sg_segs[i].ss_paddr);
1922                                 } else {
1923                                         phys_addr =
1924                                             vtophys(storvsc_sglist[i].ds_addr);
1925                                 }
1926
1927                                 pfn = phys_addr >> PAGE_SHIFT;
1928                                 reqp->data_buf.pfn_array[i] = pfn;
1929                         }
1930                 } else {
1931                         phys_addr = vtophys(storvsc_sglist[0].ds_addr);
1932
1933                         reqp->data_buf.offset = phys_addr & PAGE_MASK;
1934
1935                         for (i = 0; i < storvsc_sg_count; i++) {
1936                                 phys_addr = vtophys(storvsc_sglist[i].ds_addr);
1937                                 pfn = phys_addr >> PAGE_SHIFT;
1938                                 reqp->data_buf.pfn_array[i] = pfn;
1939                         }
1940
1941                         /* check the last segment cross boundary or not */
1942                         offset = phys_addr & PAGE_MASK;
1943                         if (offset) {
1944                                 phys_addr =
1945                                     vtophys(storvsc_sglist[i-1].ds_addr +
1946                                     PAGE_SIZE - offset);
1947                                 pfn = phys_addr >> PAGE_SHIFT;
1948                                 reqp->data_buf.pfn_array[i] = pfn;
1949                         }
1950
1951                         reqp->bounce_sgl_count = 0;
1952                 }
1953                 break;
1954         }
1955         default:
1956                 printf("Unknow flags: %d\n", ccb->ccb_h.flags);
1957                 return(EINVAL);
1958         }
1959
1960         return(0);
1961 }
1962
1963 /*
1964  * Modified based on scsi_print_inquiry which is responsible to
1965  * print the detail information for scsi_inquiry_data.
1966  *
1967  * Return 1 if it is valid, 0 otherwise.
1968  */
1969 static inline int
1970 is_inquiry_valid(const struct scsi_inquiry_data *inq_data)
1971 {
1972         uint8_t type;
1973         char vendor[16], product[48], revision[16];
1974
1975         /*
1976          * Check device type and qualifier
1977          */
1978         if (!(SID_QUAL_IS_VENDOR_UNIQUE(inq_data) ||
1979             SID_QUAL(inq_data) == SID_QUAL_LU_CONNECTED))
1980                 return (0);
1981
1982         type = SID_TYPE(inq_data);
1983         switch (type) {
1984         case T_DIRECT:
1985         case T_SEQUENTIAL:
1986         case T_PRINTER:
1987         case T_PROCESSOR:
1988         case T_WORM:
1989         case T_CDROM:
1990         case T_SCANNER:
1991         case T_OPTICAL:
1992         case T_CHANGER:
1993         case T_COMM:
1994         case T_STORARRAY:
1995         case T_ENCLOSURE:
1996         case T_RBC:
1997         case T_OCRW:
1998         case T_OSD:
1999         case T_ADC:
2000                 break;
2001         case T_NODEVICE:
2002         default:
2003                 return (0);
2004         }
2005
2006         /*
2007          * Check vendor, product, and revision
2008          */
2009         cam_strvis(vendor, inq_data->vendor, sizeof(inq_data->vendor),
2010             sizeof(vendor));
2011         cam_strvis(product, inq_data->product, sizeof(inq_data->product),
2012             sizeof(product));
2013         cam_strvis(revision, inq_data->revision, sizeof(inq_data->revision),
2014             sizeof(revision));
2015         if (strlen(vendor) == 0  ||
2016             strlen(product) == 0 ||
2017             strlen(revision) == 0)
2018                 return (0);
2019
2020         return (1);
2021 }
2022
2023 /**
2024  * @brief completion function before returning to CAM
2025  *
2026  * I/O process has been completed and the result needs
2027  * to be passed to the CAM layer.
2028  * Free resources related to this request.
2029  *
2030  * @param reqp pointer to a request structure
2031  */
2032 static void
2033 storvsc_io_done(struct hv_storvsc_request *reqp)
2034 {
2035         union ccb *ccb = reqp->ccb;
2036         struct ccb_scsiio *csio = &ccb->csio;
2037         struct storvsc_softc *sc = reqp->softc;
2038         struct vmscsi_req *vm_srb = &reqp->vstor_packet.u.vm_srb;
2039         bus_dma_segment_t *ori_sglist = NULL;
2040         int ori_sg_count = 0;
2041
2042         /* destroy bounce buffer if it is used */
2043         if (reqp->bounce_sgl_count) {
2044                 ori_sglist = (bus_dma_segment_t *)ccb->csio.data_ptr;
2045                 ori_sg_count = ccb->csio.sglist_cnt;
2046
2047                 /*
2048                  * If it is READ operation, we should copy back the data
2049                  * to original SG list.
2050                  */
2051                 if (READ_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
2052                         storvsc_copy_from_bounce_buf_to_sgl(ori_sglist,
2053                             ori_sg_count,
2054                             reqp->bounce_sgl,
2055                             reqp->not_aligned_seg_bits);
2056                 }
2057
2058                 storvsc_destroy_bounce_buffer(reqp->bounce_sgl);
2059                 reqp->bounce_sgl_count = 0;
2060         }
2061
2062         if (reqp->retries > 0) {
2063                 mtx_lock(&sc->hs_lock);
2064 #if HVS_TIMEOUT_TEST
2065                 xpt_print(ccb->ccb_h.path,
2066                         "%u: IO returned after timeout, "
2067                         "waking up timer handler if any.\n", ticks);
2068                 mtx_lock(&reqp->event.mtx);
2069                 cv_signal(&reqp->event.cv);
2070                 mtx_unlock(&reqp->event.mtx);
2071 #endif
2072                 reqp->retries = 0;
2073                 xpt_print(ccb->ccb_h.path,
2074                         "%u: IO returned after timeout, "
2075                         "stopping timer if any.\n", ticks);
2076                 mtx_unlock(&sc->hs_lock);
2077         }
2078
2079 #ifdef notyet
2080         /*
2081          * callout_drain() will wait for the timer handler to finish
2082          * if it is running. So we don't need any lock to synchronize
2083          * between this routine and the timer handler.
2084          * Note that we need to make sure reqp is not freed when timer
2085          * handler is using or will use it.
2086          */
2087         if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
2088                 callout_drain(&reqp->callout);
2089         }
2090 #endif
2091
2092         ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
2093         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2094         if (vm_srb->scsi_status == SCSI_STATUS_OK) {
2095                 const struct scsi_generic *cmd;
2096
2097                 /*
2098                  * Check whether the data for INQUIRY cmd is valid or
2099                  * not.  Windows 10 and Windows 2016 send all zero
2100                  * inquiry data to VM even for unpopulated slots.
2101                  */
2102                 cmd = (const struct scsi_generic *)
2103                     ((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
2104                      csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
2105                 if (cmd->opcode == INQUIRY &&
2106                     /*
2107                      * XXX: Temporary work around disk hot plugin on win2k12r2,
2108                      * only filtering the invalid disk on win10 or 2016 server.
2109                      * So, the hot plugin on win10 and 2016 server needs
2110                      * to be fixed.
2111                      */
2112                     vmstor_proto_version == VMSTOR_PROTOCOL_VERSION_WIN10 &&
2113                     is_inquiry_valid(
2114                     (const struct scsi_inquiry_data *)csio->data_ptr) == 0) {
2115                         ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
2116                         if (bootverbose) {
2117                                 mtx_lock(&sc->hs_lock);
2118                                 xpt_print(ccb->ccb_h.path,
2119                                     "storvsc uninstalled device\n");
2120                                 mtx_unlock(&sc->hs_lock);
2121                         }
2122                 } else {
2123                         ccb->ccb_h.status |= CAM_REQ_CMP;
2124                 }
2125         } else {
2126                 mtx_lock(&sc->hs_lock);
2127                 xpt_print(ccb->ccb_h.path,
2128                         "storvsc scsi_status = %d\n",
2129                         vm_srb->scsi_status);
2130                 mtx_unlock(&sc->hs_lock);
2131                 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
2132         }
2133
2134         ccb->csio.scsi_status = (vm_srb->scsi_status & 0xFF);
2135         ccb->csio.resid = ccb->csio.dxfer_len - vm_srb->transfer_len;
2136
2137         if (reqp->sense_info_len != 0) {
2138                 csio->sense_resid = csio->sense_len - reqp->sense_info_len;
2139                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2140         }
2141
2142         mtx_lock(&sc->hs_lock);
2143         if (reqp->softc->hs_frozen == 1) {
2144                 xpt_print(ccb->ccb_h.path,
2145                         "%u: storvsc unfreezing softc 0x%p.\n",
2146                         ticks, reqp->softc);
2147                 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2148                 reqp->softc->hs_frozen = 0;
2149         }
2150         storvsc_free_request(sc, reqp);
2151         xpt_done(ccb);
2152         mtx_unlock(&sc->hs_lock);
2153 }
2154
2155 /**
2156  * @brief Free a request structure
2157  *
2158  * Free a request structure by returning it to the free list
2159  *
2160  * @param sc pointer to a softc
2161  * @param reqp pointer to a request structure
2162  */
2163 static void
2164 storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp)
2165 {
2166
2167         LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
2168 }
2169
2170 /**
2171  * @brief Determine type of storage device from GUID
2172  *
2173  * Using the type GUID, determine if this is a StorVSC (paravirtual
2174  * SCSI or BlkVSC (paravirtual IDE) device.
2175  *
2176  * @param dev a device
2177  * returns an enum
2178  */
2179 static enum hv_storage_type
2180 storvsc_get_storage_type(device_t dev)
2181 {
2182         const char *p = vmbus_get_type(dev);
2183
2184         if (!memcmp(p, &gBlkVscDeviceType, sizeof(hv_guid))) {
2185                 return DRIVER_BLKVSC;
2186         } else if (!memcmp(p, &gStorVscDeviceType, sizeof(hv_guid))) {
2187                 return DRIVER_STORVSC;
2188         }
2189         return (DRIVER_UNKNOWN);
2190 }
2191