sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c

   1 /*-
   2  * Copyright (c) 2009-2012 Microsoft Corp.
   3  * Copyright (c) 2012 NetApp Inc.
   4  * Copyright (c) 2012 Citrix Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /**
  30  * StorVSC driver for Hyper-V.  This driver presents a SCSI HBA interface
  31  * to the Comman Access Method (CAM) layer.  CAM control blocks (CCBs) are
  32  * converted into VSCSI protocol messages which are delivered to the parent
  33  * partition StorVSP driver over the Hyper-V VMBUS.
  34  */
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/proc.h>
  40 #include <sys/condvar.h>
  41 #include <sys/time.h>
  42 #include <sys/systm.h>
  43 #include <sys/sockio.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/malloc.h>
  46 #include <sys/module.h>
  47 #include <sys/kernel.h>
  48 #include <sys/queue.h>
  49 #include <sys/lock.h>
  50 #include <sys/sx.h>
  51 #include <sys/taskqueue.h>
  52 #include <sys/bus.h>
  53 #include <sys/mutex.h>
  54 #include <sys/callout.h>
  55 #include <vm/vm.h>
  56 #include <vm/pmap.h>
  57 #include <vm/uma.h>
  58 #include <sys/lock.h>
  59 #include <sys/sema.h>
  60 #include <sys/sglist.h>
  61 #include <machine/bus.h>
  62 #include <sys/bus_dma.h>
  63
  64 #include <cam/cam.h>
  65 #include <cam/cam_ccb.h>
  66 #include <cam/cam_periph.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_xpt_sim.h>
  69 #include <cam/cam_xpt_internal.h>
  70 #include <cam/cam_debug.h>
  71 #include <cam/scsi/scsi_all.h>
  72 #include <cam/scsi/scsi_message.h>
  73
  74 #include <dev/hyperv/include/hyperv.h>
  75 #include "hv_vstorage.h"
  76
  77 #define STORVSC_RINGBUFFER_SIZE         (20*PAGE_SIZE)
  78 #define STORVSC_MAX_LUNS_PER_TARGET     (64)
  79 #define STORVSC_MAX_IO_REQUESTS         (STORVSC_MAX_LUNS_PER_TARGET * 2)
  80 #define BLKVSC_MAX_IDE_DISKS_PER_TARGET (1)
  81 #define BLKVSC_MAX_IO_REQUESTS          STORVSC_MAX_IO_REQUESTS
  82 #define STORVSC_MAX_TARGETS             (2)
  83
  84 #define VSTOR_PKT_SIZE  (sizeof(struct vstor_packet) - vmscsi_size_delta)
  85
  86 #define HV_ALIGN(x, a) roundup2(x, a)
  87
  88 struct storvsc_softc;
  89
  90 struct hv_sgl_node {
  91         LIST_ENTRY(hv_sgl_node) link;
  92         struct sglist *sgl_data;
  93 };
  94
  95 struct hv_sgl_page_pool{
  96         LIST_HEAD(, hv_sgl_node) in_use_sgl_list;
  97         LIST_HEAD(, hv_sgl_node) free_sgl_list;
  98         boolean_t                is_init;
  99 } g_hv_sgl_page_pool;
 100
 101 #define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * HV_MAX_MULTIPAGE_BUFFER_COUNT
 102
 103 enum storvsc_request_type {
 104         WRITE_TYPE,
 105         READ_TYPE,
 106         UNKNOWN_TYPE
 107 };
 108
 109 struct hv_storvsc_request {
 110         LIST_ENTRY(hv_storvsc_request) link;
 111         struct vstor_packet     vstor_packet;
 112         hv_vmbus_multipage_buffer data_buf;
 113         void *sense_data;
 114         uint8_t sense_info_len;
 115         uint8_t retries;
 116         union ccb *ccb;
 117         struct storvsc_softc *softc;
 118         struct callout callout;
 119         struct sema synch_sema; /*Synchronize the request/response if needed */
 120         struct sglist *bounce_sgl;
 121         unsigned int bounce_sgl_count;
 122         uint64_t not_aligned_seg_bits;
 123 };
 124
 125 struct storvsc_softc {
 126         struct hv_device                *hs_dev;
 127         LIST_HEAD(, hv_storvsc_request) hs_free_list;
 128         struct mtx                      hs_lock;
 129         struct storvsc_driver_props     *hs_drv_props;
 130         int                             hs_unit;
 131         uint32_t                        hs_frozen;
 132         struct cam_sim                  *hs_sim;
 133         struct cam_path                 *hs_path;
 134         uint32_t                        hs_num_out_reqs;
 135         boolean_t                       hs_destroy;
 136         boolean_t                       hs_drain_notify;
 137         boolean_t                       hs_open_multi_channel;
 138         struct sema                     hs_drain_sema;
 139         struct hv_storvsc_request       hs_init_req;
 140         struct hv_storvsc_request       hs_reset_req;
 141 };
 142
 143
 144 /**
 145  * HyperV storvsc timeout testing cases:
 146  * a. IO returned after first timeout;
 147  * b. IO returned after second timeout and queue freeze;
 148  * c. IO returned while timer handler is running
 149  * The first can be tested by "sg_senddiag -vv /dev/daX",
 150  * and the second and third can be done by
 151  * "sg_wr_mode -v -p 08 -c 0,1a -m 0,ff /dev/daX".
 152  */
 153 #define HVS_TIMEOUT_TEST 0
 154
 155 /*
 156  * Bus/adapter reset functionality on the Hyper-V host is
 157  * buggy and it will be disabled until
 158  * it can be further tested.
 159  */
 160 #define HVS_HOST_RESET 0
 161
 162 struct storvsc_driver_props {
 163         char            *drv_name;
 164         char            *drv_desc;
 165         uint8_t         drv_max_luns_per_target;
 166         uint8_t         drv_max_ios_per_target;
 167         uint32_t        drv_ringbuffer_size;
 168 };
 169
 170 enum hv_storage_type {
 171         DRIVER_BLKVSC,
 172         DRIVER_STORVSC,
 173         DRIVER_UNKNOWN
 174 };
 175
 176 #define HS_MAX_ADAPTERS 10
 177
 178 #define HV_STORAGE_SUPPORTS_MULTI_CHANNEL 0x1
 179
 180 /* {ba6163d9-04a1-4d29-b605-72e2ffb1dc7f} */
 181 static const hv_guid gStorVscDeviceType={
 182         .data = {0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d,
 183                  0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f}
 184 };
 185
 186 /* {32412632-86cb-44a2-9b5c-50d1417354f5} */
 187 static const hv_guid gBlkVscDeviceType={
 188         .data = {0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44,
 189                  0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5}
 190 };
 191
 192 static struct storvsc_driver_props g_drv_props_table[] = {
 193         {"blkvsc", "Hyper-V IDE Storage Interface",
 194          BLKVSC_MAX_IDE_DISKS_PER_TARGET, BLKVSC_MAX_IO_REQUESTS,
 195          STORVSC_RINGBUFFER_SIZE},
 196         {"storvsc", "Hyper-V SCSI Storage Interface",
 197          STORVSC_MAX_LUNS_PER_TARGET, STORVSC_MAX_IO_REQUESTS,
 198          STORVSC_RINGBUFFER_SIZE}
 199 };
 200
 201 /*
 202  * Sense buffer size changed in win8; have a run-time
 203  * variable to track the size we should use.
 204  */
 205 static int sense_buffer_size = PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE;
 206
 207 /*
 208  * The size of the vmscsi_request has changed in win8. The
 209  * additional size is for the newly added elements in the
 210  * structure. These elements are valid only when we are talking
 211  * to a win8 host.
 212  * Track the correct size we need to apply.
 213  */
 214 static int vmscsi_size_delta;
 215 /*
 216  * The storage protocol version is determined during the
 217  * initial exchange with the host.  It will indicate which
 218  * storage functionality is available in the host.
 219 */
 220 static int vmstor_proto_version;
 221
 222 struct vmstor_proto {
 223         int proto_version;
 224         int sense_buffer_size;
 225         int vmscsi_size_delta;
 226 };
 227
 228 static const struct vmstor_proto vmstor_proto_list[] = {
 229         {
 230                 VMSTOR_PROTOCOL_VERSION_WIN10,
 231                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 232                 0
 233         },
 234         {
 235                 VMSTOR_PROTOCOL_VERSION_WIN8_1,
 236                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 237                 0
 238         },
 239         {
 240                 VMSTOR_PROTOCOL_VERSION_WIN8,
 241                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 242                 0
 243         },
 244         {
 245                 VMSTOR_PROTOCOL_VERSION_WIN7,
 246                 PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE,
 247                 sizeof(struct vmscsi_win8_extension),
 248         },
 249         {
 250                 VMSTOR_PROTOCOL_VERSION_WIN6,
 251                 PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE,
 252                 sizeof(struct vmscsi_win8_extension),
 253         }
 254 };
 255
 256 /* static functions */
 257 static int storvsc_probe(device_t dev);
 258 static int storvsc_attach(device_t dev);
 259 static int storvsc_detach(device_t dev);
 260 static void storvsc_poll(struct cam_sim * sim);
 261 static void storvsc_action(struct cam_sim * sim, union ccb * ccb);
 262 static int create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp);
 263 static void storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp);
 264 static enum hv_storage_type storvsc_get_storage_type(device_t dev);
 265 static void hv_storvsc_rescan_target(struct storvsc_softc *sc);
 266 static void hv_storvsc_on_channel_callback(void *context);
 267 static void hv_storvsc_on_iocompletion( struct storvsc_softc *sc,
 268                                         struct vstor_packet *vstor_packet,
 269                                         struct hv_storvsc_request *request);
 270 static int hv_storvsc_connect_vsp(struct hv_device *device);
 271 static void storvsc_io_done(struct hv_storvsc_request *reqp);
 272 static void storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
 273                                 bus_dma_segment_t *orig_sgl,
 274                                 unsigned int orig_sgl_count,
 275                                 uint64_t seg_bits);
 276 void storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
 277                                 unsigned int dest_sgl_count,
 278                                 struct sglist* src_sgl,
 279                                 uint64_t seg_bits);
 280
 281 static device_method_t storvsc_methods[] = {
 282         /* Device interface */
 283         DEVMETHOD(device_probe,         storvsc_probe),
 284         DEVMETHOD(device_attach,        storvsc_attach),
 285         DEVMETHOD(device_detach,        storvsc_detach),
 286         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 287         DEVMETHOD_END
 288 };
 289
 290 static driver_t storvsc_driver = {
 291         "storvsc", storvsc_methods, sizeof(struct storvsc_softc),
 292 };
 293
 294 static devclass_t storvsc_devclass;
 295 DRIVER_MODULE(storvsc, vmbus, storvsc_driver, storvsc_devclass, 0, 0);
 296 MODULE_VERSION(storvsc, 1);
 297 MODULE_DEPEND(storvsc, vmbus, 1, 1, 1);
 298
 299
 300 /**
 301  * The host is capable of sending messages to us that are
 302  * completely unsolicited. So, we need to address the race
 303  * condition where we may be in the process of unloading the
 304  * driver when the host may send us an unsolicited message.
 305  * We address this issue by implementing a sequentially
 306  * consistent protocol:
 307  *
 308  * 1. Channel callback is invoked while holding the the channel lock
 309  *    and an unloading driver will reset the channel callback under
 310  *    the protection of this channel lock.
 311  *
 312  * 2. To ensure bounded wait time for unloading a driver, we don't
 313  *    permit outgoing traffic once the device is marked as being
 314  *    destroyed.
 315  *
 316  * 3. Once the device is marked as being destroyed, we only
 317  *    permit incoming traffic to properly account for
 318  *    packets already sent out.
 319  */
 320 static inline struct storvsc_softc *
 321 get_stor_device(struct hv_device *device,
 322                                 boolean_t outbound)
 323 {
 324         struct storvsc_softc *sc;
 325
 326         sc = device_get_softc(device->device);
 327         if (sc == NULL) {
 328                 return NULL;
 329         }
 330
 331         if (outbound) {
 332                 /*
 333                  * Here we permit outgoing I/O only
 334                  * if the device is not being destroyed.
 335                  */
 336
 337                 if (sc->hs_destroy) {
 338                         sc = NULL;
 339                 }
 340         } else {
 341                 /*
 342                  * inbound case; if being destroyed
 343                  * only permit to account for
 344                  * messages already sent out.
 345                  */
 346                 if (sc->hs_destroy && (sc->hs_num_out_reqs == 0)) {
 347                         sc = NULL;
 348                 }
 349         }
 350         return sc;
 351 }
 352
 353 /**
 354  * @brief Callback handler, will be invoked when receive mutil-channel offer
 355  *
 356  * @param context  new multi-channel
 357  */
 358 static void
 359 storvsc_handle_sc_creation(void *context)
 360 {
 361         hv_vmbus_channel *new_channel;
 362         struct hv_device *device;
 363         struct storvsc_softc *sc;
 364         struct vmstor_chan_props props;
 365         int ret = 0;
 366
 367         new_channel = (hv_vmbus_channel *)context;
 368         device = new_channel->primary_channel->device;
 369         sc = get_stor_device(device, TRUE);
 370         if (sc == NULL)
 371                 return;
 372
 373         if (FALSE == sc->hs_open_multi_channel)
 374                 return;
 375
 376         memset(&props, 0, sizeof(props));
 377
 378         ret = hv_vmbus_channel_open(new_channel,
 379             sc->hs_drv_props->drv_ringbuffer_size,
 380             sc->hs_drv_props->drv_ringbuffer_size,
 381             (void *)&props,
 382             sizeof(struct vmstor_chan_props),
 383             hv_storvsc_on_channel_callback,
 384             new_channel);
 385
 386         return;
 387 }
 388
 389 /**
 390  * @brief Send multi-channel creation request to host
 391  *
 392  * @param device  a Hyper-V device pointer
 393  * @param max_chans  the max channels supported by vmbus
 394  */
 395 static void
 396 storvsc_send_multichannel_request(struct hv_device *dev, int max_chans)
 397 {
 398         struct storvsc_softc *sc;
 399         struct hv_storvsc_request *request;
 400         struct vstor_packet *vstor_packet;
 401         int request_channels_cnt = 0;
 402         int ret;
 403
 404         /* get multichannels count that need to create */
 405         request_channels_cnt = MIN(max_chans, mp_ncpus);
 406
 407         sc = get_stor_device(dev, TRUE);
 408         if (sc == NULL) {
 409                 printf("Storvsc_error: get sc failed while send mutilchannel "
 410                     "request\n");
 411                 return;
 412         }
 413
 414         request = &sc->hs_init_req;
 415
 416         /* Establish a handler for multi-channel */
 417         dev->channel->sc_creation_callback = storvsc_handle_sc_creation;
 418
 419         /* request the host to create multi-channel */
 420         memset(request, 0, sizeof(struct hv_storvsc_request));
 421
 422         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 423
 424         vstor_packet = &request->vstor_packet;
 425
 426         vstor_packet->operation = VSTOR_OPERATION_CREATE_MULTI_CHANNELS;
 427         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 428         vstor_packet->u.multi_channels_cnt = request_channels_cnt;
 429
 430         ret = hv_vmbus_channel_send_packet(
 431             dev->channel,
 432             vstor_packet,
 433             VSTOR_PKT_SIZE,
 434             (uint64_t)(uintptr_t)request,
 435             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 436             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 437
 438         /* wait for 5 seconds */
 439         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 440         if (ret != 0) {
 441                 printf("Storvsc_error: create multi-channel timeout, %d\n",
 442                     ret);
 443                 return;
 444         }
 445
 446         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 447             vstor_packet->status != 0) {
 448                 printf("Storvsc_error: create multi-channel invalid operation "
 449                     "(%d) or statue (%u)\n",
 450                     vstor_packet->operation, vstor_packet->status);
 451                 return;
 452         }
 453
 454         sc->hs_open_multi_channel = TRUE;
 455
 456         if (bootverbose)
 457                 printf("Storvsc create multi-channel success!\n");
 458 }
 459
 460 /**
 461  * @brief initialize channel connection to parent partition
 462  *
 463  * @param dev  a Hyper-V device pointer
 464  * @returns  0 on success, non-zero error on failure
 465  */
 466 static int
 467 hv_storvsc_channel_init(struct hv_device *dev)
 468 {
 469         int ret = 0, i;
 470         struct hv_storvsc_request *request;
 471         struct vstor_packet *vstor_packet;
 472         struct storvsc_softc *sc;
 473         uint16_t max_chans = 0;
 474         boolean_t support_multichannel = FALSE;
 475
 476         max_chans = 0;
 477         support_multichannel = FALSE;
 478
 479         sc = get_stor_device(dev, TRUE);
 480         if (sc == NULL)
 481                 return (ENODEV);
 482
 483         request = &sc->hs_init_req;
 484         memset(request, 0, sizeof(struct hv_storvsc_request));
 485         vstor_packet = &request->vstor_packet;
 486         request->softc = sc;
 487
 488         /**
 489          * Initiate the vsc/vsp initialization protocol on the open channel
 490          */
 491         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 492
 493         vstor_packet->operation = VSTOR_OPERATION_BEGININITIALIZATION;
 494         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 495
 496
 497         ret = hv_vmbus_channel_send_packet(
 498                         dev->channel,
 499                         vstor_packet,
 500                         VSTOR_PKT_SIZE,
 501                         (uint64_t)(uintptr_t)request,
 502                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 503                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 504
 505         if (ret != 0)
 506                 goto cleanup;
 507
 508         /* wait 5 seconds */
 509         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 510         if (ret != 0)
 511                 goto cleanup;
 512
 513         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 514                 vstor_packet->status != 0) {
 515                 goto cleanup;
 516         }
 517
 518         for (i = 0; i < nitems(vmstor_proto_list); i++) {
 519                 /* reuse the packet for version range supported */
 520
 521                 memset(vstor_packet, 0, sizeof(struct vstor_packet));
 522                 vstor_packet->operation = VSTOR_OPERATION_QUERYPROTOCOLVERSION;
 523                 vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 524
 525                 vstor_packet->u.version.major_minor =
 526                         vmstor_proto_list[i].proto_version;
 527
 528                 /* revision is only significant for Windows guests */
 529                 vstor_packet->u.version.revision = 0;
 530
 531                 ret = hv_vmbus_channel_send_packet(
 532                         dev->channel,
 533                         vstor_packet,
 534                         VSTOR_PKT_SIZE,
 535                         (uint64_t)(uintptr_t)request,
 536                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 537                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 538
 539                 if (ret != 0)
 540                         goto cleanup;
 541
 542                 /* wait 5 seconds */
 543                 ret = sema_timedwait(&request->synch_sema, 5 * hz);
 544
 545                 if (ret)
 546                         goto cleanup;
 547
 548                 if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO) {
 549                         ret = EINVAL;
 550                         goto cleanup;
 551                 }
 552                 if (vstor_packet->status == 0) {
 553                         vmstor_proto_version =
 554                                 vmstor_proto_list[i].proto_version;
 555                         sense_buffer_size =
 556                                 vmstor_proto_list[i].sense_buffer_size;
 557                         vmscsi_size_delta =
 558                                 vmstor_proto_list[i].vmscsi_size_delta;
 559                         break;
 560                 }
 561         }
 562
 563         if (vstor_packet->status != 0) {
 564                 ret = EINVAL;
 565                 goto cleanup;
 566         }
 567         /**
 568          * Query channel properties
 569          */
 570         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 571         vstor_packet->operation = VSTOR_OPERATION_QUERYPROPERTIES;
 572         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 573
 574         ret = hv_vmbus_channel_send_packet(
 575                                 dev->channel,
 576                                 vstor_packet,
 577                                 VSTOR_PKT_SIZE,
 578                                 (uint64_t)(uintptr_t)request,
 579                                 HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 580                                 HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 581
 582         if ( ret != 0)
 583                 goto cleanup;
 584
 585         /* wait 5 seconds */
 586         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 587
 588         if (ret != 0)
 589                 goto cleanup;
 590
 591         /* TODO: Check returned version */
 592         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 593             vstor_packet->status != 0) {
 594                 goto cleanup;
 595         }
 596
 597         /* multi-channels feature is supported by WIN8 and above version */
 598         max_chans = vstor_packet->u.chan_props.max_channel_cnt;
 599         if ((hv_vmbus_protocal_version != HV_VMBUS_VERSION_WIN7) &&
 600             (hv_vmbus_protocal_version != HV_VMBUS_VERSION_WS2008) &&
 601             (vstor_packet->u.chan_props.flags &
 602              HV_STORAGE_SUPPORTS_MULTI_CHANNEL)) {
 603                 support_multichannel = TRUE;
 604         }
 605
 606         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 607         vstor_packet->operation = VSTOR_OPERATION_ENDINITIALIZATION;
 608         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 609
 610         ret = hv_vmbus_channel_send_packet(
 611                         dev->channel,
 612                         vstor_packet,
 613                         VSTOR_PKT_SIZE,
 614                         (uint64_t)(uintptr_t)request,
 615                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 616                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 617
 618         if (ret != 0) {
 619                 goto cleanup;
 620         }
 621
 622         /* wait 5 seconds */
 623         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 624
 625         if (ret != 0)
 626                 goto cleanup;
 627
 628         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 629             vstor_packet->status != 0)
 630                 goto cleanup;
 631
 632         /*
 633          * If multi-channel is supported, send multichannel create
 634          * request to host.
 635          */
 636         if (support_multichannel)
 637                 storvsc_send_multichannel_request(dev, max_chans);
 638
 639 cleanup:
 640         sema_destroy(&request->synch_sema);
 641         return (ret);
 642 }
 643
 644 /**
 645  * @brief Open channel connection to paraent partition StorVSP driver
 646  *
 647  * Open and initialize channel connection to parent partition StorVSP driver.
 648  *
 649  * @param pointer to a Hyper-V device
 650  * @returns 0 on success, non-zero error on failure
 651  */
 652 static int
 653 hv_storvsc_connect_vsp(struct hv_device *dev)
 654 {
 655         int ret = 0;
 656         struct vmstor_chan_props props;
 657         struct storvsc_softc *sc;
 658
 659         sc = device_get_softc(dev->device);
 660
 661         memset(&props, 0, sizeof(struct vmstor_chan_props));
 662
 663         /*
 664          * Open the channel
 665          */
 666
 667         ret = hv_vmbus_channel_open(
 668                 dev->channel,
 669                 sc->hs_drv_props->drv_ringbuffer_size,
 670                 sc->hs_drv_props->drv_ringbuffer_size,
 671                 (void *)&props,
 672                 sizeof(struct vmstor_chan_props),
 673                 hv_storvsc_on_channel_callback,
 674                 dev->channel);
 675
 676         if (ret != 0) {
 677                 return ret;
 678         }
 679
 680         ret = hv_storvsc_channel_init(dev);
 681
 682         return (ret);
 683 }
 684
 685 #if HVS_HOST_RESET
 686 static int
 687 hv_storvsc_host_reset(struct hv_device *dev)
 688 {
 689         int ret = 0;
 690         struct storvsc_softc *sc;
 691
 692         struct hv_storvsc_request *request;
 693         struct vstor_packet *vstor_packet;
 694
 695         sc = get_stor_device(dev, TRUE);
 696         if (sc == NULL) {
 697                 return ENODEV;
 698         }
 699
 700         request = &sc->hs_reset_req;
 701         request->softc = sc;
 702         vstor_packet = &request->vstor_packet;
 703
 704         sema_init(&request->synch_sema, 0, "stor synch sema");
 705
 706         vstor_packet->operation = VSTOR_OPERATION_RESETBUS;
 707         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 708
 709         ret = hv_vmbus_channel_send_packet(dev->channel,
 710                         vstor_packet,
 711                         VSTOR_PKT_SIZE,
 712                         (uint64_t)(uintptr_t)&sc->hs_reset_req,
 713                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 714                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 715
 716         if (ret != 0) {
 717                 goto cleanup;
 718         }
 719
 720         ret = sema_timedwait(&request->synch_sema, 5 * hz); /* KYS 5 seconds */
 721
 722         if (ret) {
 723                 goto cleanup;
 724         }
 725
 726
 727         /*
 728          * At this point, all outstanding requests in the adapter
 729          * should have been flushed out and return to us
 730          */
 731
 732 cleanup:
 733         sema_destroy(&request->synch_sema);
 734         return (ret);
 735 }
 736 #endif /* HVS_HOST_RESET */
 737
 738 /**
 739  * @brief Function to initiate an I/O request
 740  *
 741  * @param device Hyper-V device pointer
 742  * @param request pointer to a request structure
 743  * @returns 0 on success, non-zero error on failure
 744  */
 745 static int
 746 hv_storvsc_io_request(struct hv_device *device,
 747                                           struct hv_storvsc_request *request)
 748 {
 749         struct storvsc_softc *sc;
 750         struct vstor_packet *vstor_packet = &request->vstor_packet;
 751         struct hv_vmbus_channel* outgoing_channel = NULL;
 752         int ret = 0;
 753
 754         sc = get_stor_device(device, TRUE);
 755
 756         if (sc == NULL) {
 757                 return ENODEV;
 758         }
 759
 760         vstor_packet->flags |= REQUEST_COMPLETION_FLAG;
 761
 762         vstor_packet->u.vm_srb.length = VSTOR_PKT_SIZE;
 763
 764         vstor_packet->u.vm_srb.sense_info_len = sense_buffer_size;
 765
 766         vstor_packet->u.vm_srb.transfer_len = request->data_buf.length;
 767
 768         vstor_packet->operation = VSTOR_OPERATION_EXECUTESRB;
 769
 770         outgoing_channel = vmbus_select_outgoing_channel(device->channel);
 771
 772         mtx_unlock(&request->softc->hs_lock);
 773         if (request->data_buf.length) {
 774                 ret = hv_vmbus_channel_send_packet_multipagebuffer(
 775                                 outgoing_channel,
 776                                 &request->data_buf,
 777                                 vstor_packet,
 778                                 VSTOR_PKT_SIZE,
 779                                 (uint64_t)(uintptr_t)request);
 780
 781         } else {
 782                 ret = hv_vmbus_channel_send_packet(
 783                         outgoing_channel,
 784                         vstor_packet,
 785                         VSTOR_PKT_SIZE,
 786                         (uint64_t)(uintptr_t)request,
 787                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 788                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 789         }
 790         mtx_lock(&request->softc->hs_lock);
 791
 792         if (ret != 0) {
 793                 printf("Unable to send packet %p ret %d", vstor_packet, ret);
 794         } else {
 795                 atomic_add_int(&sc->hs_num_out_reqs, 1);
 796         }
 797
 798         return (ret);
 799 }
 800
 801
 802 /**
 803  * Process IO_COMPLETION_OPERATION and ready
 804  * the result to be completed for upper layer
 805  * processing by the CAM layer.
 806  */
 807 static void
 808 hv_storvsc_on_iocompletion(struct storvsc_softc *sc,
 809                            struct vstor_packet *vstor_packet,
 810                            struct hv_storvsc_request *request)
 811 {
 812         struct vmscsi_req *vm_srb;
 813
 814         vm_srb = &vstor_packet->u.vm_srb;
 815
 816         /*
 817          * Copy some fields of the host's response into the request structure,
 818          * because the fields will be used later in storvsc_io_done().
 819          */
 820         request->vstor_packet.u.vm_srb.scsi_status = vm_srb->scsi_status;
 821         request->vstor_packet.u.vm_srb.transfer_len = vm_srb->transfer_len;
 822
 823         if (((vm_srb->scsi_status & 0xFF) == SCSI_STATUS_CHECK_COND) &&
 824                         (vm_srb->srb_status & SRB_STATUS_AUTOSENSE_VALID)) {
 825                 /* Autosense data available */
 826
 827                 KASSERT(vm_srb->sense_info_len <= request->sense_info_len,
 828                                 ("vm_srb->sense_info_len <= "
 829                                  "request->sense_info_len"));
 830
 831                 memcpy(request->sense_data, vm_srb->u.sense_data,
 832                         vm_srb->sense_info_len);
 833
 834                 request->sense_info_len = vm_srb->sense_info_len;
 835         }
 836
 837         /* Complete request by passing to the CAM layer */
 838         storvsc_io_done(request);
 839         atomic_subtract_int(&sc->hs_num_out_reqs, 1);
 840         if (sc->hs_drain_notify && (sc->hs_num_out_reqs == 0)) {
 841                 sema_post(&sc->hs_drain_sema);
 842         }
 843 }
 844
 845 static void
 846 hv_storvsc_rescan_target(struct storvsc_softc *sc)
 847 {
 848         path_id_t pathid;
 849         target_id_t targetid;
 850         union ccb *ccb;
 851
 852         pathid = cam_sim_path(sc->hs_sim);
 853         targetid = CAM_TARGET_WILDCARD;
 854
 855         /*
 856          * Allocate a CCB and schedule a rescan.
 857          */
 858         ccb = xpt_alloc_ccb_nowait();
 859         if (ccb == NULL) {
 860                 printf("unable to alloc CCB for rescan\n");
 861                 return;
 862         }
 863
 864         if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid,
 865             CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 866                 printf("unable to create path for rescan, pathid: %d,"
 867                     "targetid: %d\n", pathid, targetid);
 868                 xpt_free_ccb(ccb);
 869                 return;
 870         }
 871
 872         if (targetid == CAM_TARGET_WILDCARD)
 873                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 874         else
 875                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 876
 877         xpt_rescan(ccb);
 878 }
 879
 880 static void
 881 hv_storvsc_on_channel_callback(void *context)
 882 {
 883         int ret = 0;
 884         hv_vmbus_channel *channel = (hv_vmbus_channel *)context;
 885         struct hv_device *device = NULL;
 886         struct storvsc_softc *sc;
 887         uint32_t bytes_recvd;
 888         uint64_t request_id;
 889         uint8_t packet[roundup2(sizeof(struct vstor_packet), 8)];
 890         struct hv_storvsc_request *request;
 891         struct vstor_packet *vstor_packet;
 892
 893         if (channel->primary_channel != NULL){
 894                 device = channel->primary_channel->device;
 895         } else {
 896                 device = channel->device;
 897         }
 898
 899         KASSERT(device, ("device is NULL"));
 900
 901         sc = get_stor_device(device, FALSE);
 902         if (sc == NULL) {
 903                 printf("Storvsc_error: get stor device failed.\n");
 904                 return;
 905         }
 906
 907         ret = hv_vmbus_channel_recv_packet(
 908                         channel,
 909                         packet,
 910                         roundup2(VSTOR_PKT_SIZE, 8),
 911                         &bytes_recvd,
 912                         &request_id);
 913
 914         while ((ret == 0) && (bytes_recvd > 0)) {
 915                 request = (struct hv_storvsc_request *)(uintptr_t)request_id;
 916
 917                 if ((request == &sc->hs_init_req) ||
 918                         (request == &sc->hs_reset_req)) {
 919                         memcpy(&request->vstor_packet, packet,
 920                                    sizeof(struct vstor_packet));
 921                         sema_post(&request->synch_sema);
 922                 } else {
 923                         vstor_packet = (struct vstor_packet *)packet;
 924                         switch(vstor_packet->operation) {
 925                         case VSTOR_OPERATION_COMPLETEIO:
 926                                 if (request == NULL)
 927                                         panic("VMBUS: storvsc received a "
 928                                             "packet with NULL request id in "
 929                                             "COMPLETEIO operation.");
 930
 931                                 hv_storvsc_on_iocompletion(sc,
 932                                                         vstor_packet, request);
 933                                 break;
 934                         case VSTOR_OPERATION_REMOVEDEVICE:
 935                                 printf("VMBUS: storvsc operation %d not "
 936                                     "implemented.\n", vstor_packet->operation);
 937                                 /* TODO: implement */
 938                                 break;
 939                         case VSTOR_OPERATION_ENUMERATE_BUS:
 940                                 hv_storvsc_rescan_target(sc);
 941                                 break;
 942                         default:
 943                                 break;
 944                         }
 945                 }
 946                 ret = hv_vmbus_channel_recv_packet(
 947                                 channel,
 948                                 packet,
 949                                 roundup2(VSTOR_PKT_SIZE, 8),
 950                                 &bytes_recvd,
 951                                 &request_id);
 952         }
 953 }
 954
 955 /**
 956  * @brief StorVSC probe function
 957  *
 958  * Device probe function.  Returns 0 if the input device is a StorVSC
 959  * device.  Otherwise, a ENXIO is returned.  If the input device is
 960  * for BlkVSC (paravirtual IDE) device and this support is disabled in
 961  * favor of the emulated ATA/IDE device, return ENXIO.
 962  *
 963  * @param a device
 964  * @returns 0 on success, ENXIO if not a matcing StorVSC device
 965  */
 966 static int
 967 storvsc_probe(device_t dev)
 968 {
 969         int ata_disk_enable = 0;
 970         int ret = ENXIO;
 971
 972         switch (storvsc_get_storage_type(dev)) {
 973         case DRIVER_BLKVSC:
 974                 if(bootverbose)
 975                         device_printf(dev, "DRIVER_BLKVSC-Emulated ATA/IDE probe\n");
 976                 if (!getenv_int("hw.ata.disk_enable", &ata_disk_enable)) {
 977                         if(bootverbose)
 978                                 device_printf(dev,
 979                                         "Enlightened ATA/IDE detected\n");
 980                         ret = BUS_PROBE_DEFAULT;
 981                 } else if(bootverbose)
 982                         device_printf(dev, "Emulated ATA/IDE set (hw.ata.disk_enable set)\n");
 983                 break;
 984         case DRIVER_STORVSC:
 985                 if(bootverbose)
 986                         device_printf(dev, "Enlightened SCSI device detected\n");
 987                 ret = BUS_PROBE_DEFAULT;
 988                 break;
 989         default:
 990                 ret = ENXIO;
 991         }
 992         return (ret);
 993 }
 994
 995 /**
 996  * @brief StorVSC attach function
 997  *
 998  * Function responsible for allocating per-device structures,
 999  * setting up CAM interfaces and scanning for available LUNs to
1000  * be used for SCSI device peripherals.
1001  *
1002  * @param a device
1003  * @returns 0 on success or an error on failure
1004  */
1005 static int
1006 storvsc_attach(device_t dev)
1007 {
1008         struct hv_device *hv_dev = vmbus_get_devctx(dev);
1009         enum hv_storage_type stor_type;
1010         struct storvsc_softc *sc;
1011         struct cam_devq *devq;
1012         int ret, i, j;
1013         struct hv_storvsc_request *reqp;
1014         struct root_hold_token *root_mount_token = NULL;
1015         struct hv_sgl_node *sgl_node = NULL;
1016         void *tmp_buff = NULL;
1017
1018         /*
1019          * We need to serialize storvsc attach calls.
1020          */
1021         root_mount_token = root_mount_hold("storvsc");
1022
1023         sc = device_get_softc(dev);
1024         if (sc == NULL) {
1025                 ret = ENOMEM;
1026                 goto cleanup;
1027         }
1028
1029         stor_type = storvsc_get_storage_type(dev);
1030
1031         if (stor_type == DRIVER_UNKNOWN) {
1032                 ret = ENODEV;
1033                 goto cleanup;
1034         }
1035
1036         bzero(sc, sizeof(struct storvsc_softc));
1037
1038         /* fill in driver specific properties */
1039         sc->hs_drv_props = &g_drv_props_table[stor_type];
1040
1041         /* fill in device specific properties */
1042         sc->hs_unit     = device_get_unit(dev);
1043         sc->hs_dev      = hv_dev;
1044         device_set_desc(dev, g_drv_props_table[stor_type].drv_desc);
1045
1046         LIST_INIT(&sc->hs_free_list);
1047         mtx_init(&sc->hs_lock, "hvslck", NULL, MTX_DEF);
1048
1049         for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; ++i) {
1050                 reqp = malloc(sizeof(struct hv_storvsc_request),
1051                                  M_DEVBUF, M_WAITOK|M_ZERO);
1052                 reqp->softc = sc;
1053
1054                 LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
1055         }
1056
1057         /* create sg-list page pool */
1058         if (FALSE == g_hv_sgl_page_pool.is_init) {
1059                 g_hv_sgl_page_pool.is_init = TRUE;
1060                 LIST_INIT(&g_hv_sgl_page_pool.in_use_sgl_list);
1061                 LIST_INIT(&g_hv_sgl_page_pool.free_sgl_list);
1062
1063                 /*
1064                  * Pre-create SG list, each SG list with
1065                  * HV_MAX_MULTIPAGE_BUFFER_COUNT segments, each
1066                  * segment has one page buffer
1067                  */
1068                 for (i = 0; i < STORVSC_MAX_IO_REQUESTS; i++) {
1069                         sgl_node = malloc(sizeof(struct hv_sgl_node),
1070                             M_DEVBUF, M_WAITOK|M_ZERO);
1071
1072                         sgl_node->sgl_data =
1073                             sglist_alloc(HV_MAX_MULTIPAGE_BUFFER_COUNT,
1074                             M_WAITOK|M_ZERO);
1075
1076                         for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
1077                                 tmp_buff = malloc(PAGE_SIZE,
1078                                     M_DEVBUF, M_WAITOK|M_ZERO);
1079
1080                                 sgl_node->sgl_data->sg_segs[j].ss_paddr =
1081                                     (vm_paddr_t)tmp_buff;
1082                         }
1083
1084                         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list,
1085                             sgl_node, link);
1086                 }
1087         }
1088
1089         sc->hs_destroy = FALSE;
1090         sc->hs_drain_notify = FALSE;
1091         sc->hs_open_multi_channel = FALSE;
1092         sema_init(&sc->hs_drain_sema, 0, "Store Drain Sema");
1093
1094         ret = hv_storvsc_connect_vsp(hv_dev);
1095         if (ret != 0) {
1096                 goto cleanup;
1097         }
1098
1099         /*
1100          * Create the device queue.
1101          * Hyper-V maps each target to one SCSI HBA
1102          */
1103         devq = cam_simq_alloc(sc->hs_drv_props->drv_max_ios_per_target);
1104         if (devq == NULL) {
1105                 device_printf(dev, "Failed to alloc device queue\n");
1106                 ret = ENOMEM;
1107                 goto cleanup;
1108         }
1109
1110         sc->hs_sim = cam_sim_alloc(storvsc_action,
1111                                 storvsc_poll,
1112                                 sc->hs_drv_props->drv_name,
1113                                 sc,
1114                                 sc->hs_unit,
1115                                 &sc->hs_lock, 1,
1116                                 sc->hs_drv_props->drv_max_ios_per_target,
1117                                 devq);
1118
1119         if (sc->hs_sim == NULL) {
1120                 device_printf(dev, "Failed to alloc sim\n");
1121                 cam_simq_free(devq);
1122                 ret = ENOMEM;
1123                 goto cleanup;
1124         }
1125
1126         mtx_lock(&sc->hs_lock);
1127         /* bus_id is set to 0, need to get it from VMBUS channel query? */
1128         if (xpt_bus_register(sc->hs_sim, dev, 0) != CAM_SUCCESS) {
1129                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1130                 mtx_unlock(&sc->hs_lock);
1131                 device_printf(dev, "Unable to register SCSI bus\n");
1132                 ret = ENXIO;
1133                 goto cleanup;
1134         }
1135
1136         if (xpt_create_path(&sc->hs_path, /*periph*/NULL,
1137                  cam_sim_path(sc->hs_sim),
1138                 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1139                 xpt_bus_deregister(cam_sim_path(sc->hs_sim));
1140                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1141                 mtx_unlock(&sc->hs_lock);
1142                 device_printf(dev, "Unable to create path\n");
1143                 ret = ENXIO;
1144                 goto cleanup;
1145         }
1146
1147         mtx_unlock(&sc->hs_lock);
1148
1149         root_mount_rel(root_mount_token);
1150         return (0);
1151
1152
1153 cleanup:
1154         root_mount_rel(root_mount_token);
1155         while (!LIST_EMPTY(&sc->hs_free_list)) {
1156                 reqp = LIST_FIRST(&sc->hs_free_list);
1157                 LIST_REMOVE(reqp, link);
1158                 free(reqp, M_DEVBUF);
1159         }
1160
1161         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1162                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1163                 LIST_REMOVE(sgl_node, link);
1164                 for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
1165                         if (NULL !=
1166                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1167                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1168                         }
1169                 }
1170                 sglist_free(sgl_node->sgl_data);
1171                 free(sgl_node, M_DEVBUF);
1172         }
1173
1174         return (ret);
1175 }
1176
1177 /**
1178  * @brief StorVSC device detach function
1179  *
1180  * This function is responsible for safely detaching a
1181  * StorVSC device.  This includes waiting for inbound responses
1182  * to complete and freeing associated per-device structures.
1183  *
1184  * @param dev a device
1185  * returns 0 on success
1186  */
1187 static int
1188 storvsc_detach(device_t dev)
1189 {
1190         struct storvsc_softc *sc = device_get_softc(dev);
1191         struct hv_storvsc_request *reqp = NULL;
1192         struct hv_device *hv_device = vmbus_get_devctx(dev);
1193         struct hv_sgl_node *sgl_node = NULL;
1194         int j = 0;
1195
1196         mtx_lock(&hv_device->channel->inbound_lock);
1197         sc->hs_destroy = TRUE;
1198         mtx_unlock(&hv_device->channel->inbound_lock);
1199
1200         /*
1201          * At this point, all outbound traffic should be disabled. We
1202          * only allow inbound traffic (responses) to proceed so that
1203          * outstanding requests can be completed.
1204          */
1205
1206         sc->hs_drain_notify = TRUE;
1207         sema_wait(&sc->hs_drain_sema);
1208         sc->hs_drain_notify = FALSE;
1209
1210         /*
1211          * Since we have already drained, we don't need to busy wait.
1212          * The call to close the channel will reset the callback
1213          * under the protection of the incoming channel lock.
1214          */
1215
1216         hv_vmbus_channel_close(hv_device->channel);
1217
1218         mtx_lock(&sc->hs_lock);
1219         while (!LIST_EMPTY(&sc->hs_free_list)) {
1220                 reqp = LIST_FIRST(&sc->hs_free_list);
1221                 LIST_REMOVE(reqp, link);
1222
1223                 free(reqp, M_DEVBUF);
1224         }
1225         mtx_unlock(&sc->hs_lock);
1226
1227         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1228                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1229                 LIST_REMOVE(sgl_node, link);
1230                 for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++){
1231                         if (NULL !=
1232                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1233                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1234                         }
1235                 }
1236                 sglist_free(sgl_node->sgl_data);
1237                 free(sgl_node, M_DEVBUF);
1238         }
1239
1240         return (0);
1241 }
1242
1243 #if HVS_TIMEOUT_TEST
1244 /**
1245  * @brief unit test for timed out operations
1246  *
1247  * This function provides unit testing capability to simulate
1248  * timed out operations.  Recompilation with HV_TIMEOUT_TEST=1
1249  * is required.
1250  *
1251  * @param reqp pointer to a request structure
1252  * @param opcode SCSI operation being performed
1253  * @param wait if 1, wait for I/O to complete
1254  */
1255 static void
1256 storvsc_timeout_test(struct hv_storvsc_request *reqp,
1257                 uint8_t opcode, int wait)
1258 {
1259         int ret;
1260         union ccb *ccb = reqp->ccb;
1261         struct storvsc_softc *sc = reqp->softc;
1262
1263         if (reqp->vstor_packet.vm_srb.cdb[0] != opcode) {
1264                 return;
1265         }
1266
1267         if (wait) {
1268                 mtx_lock(&reqp->event.mtx);
1269         }
1270         ret = hv_storvsc_io_request(sc->hs_dev, reqp);
1271         if (ret != 0) {
1272                 if (wait) {
1273                         mtx_unlock(&reqp->event.mtx);
1274                 }
1275                 printf("%s: io_request failed with %d.\n",
1276                                 __func__, ret);
1277                 ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1278                 mtx_lock(&sc->hs_lock);
1279                 storvsc_free_request(sc, reqp);
1280                 xpt_done(ccb);
1281                 mtx_unlock(&sc->hs_lock);
1282                 return;
1283         }
1284
1285         if (wait) {
1286                 xpt_print(ccb->ccb_h.path,
1287                                 "%u: %s: waiting for IO return.\n",
1288                                 ticks, __func__);
1289                 ret = cv_timedwait(&reqp->event.cv, &reqp->event.mtx, 60*hz);
1290                 mtx_unlock(&reqp->event.mtx);
1291                 xpt_print(ccb->ccb_h.path, "%u: %s: %s.\n",
1292                                 ticks, __func__, (ret == 0)?
1293                                 "IO return detected" :
1294                                 "IO return not detected");
1295                 /*
1296                  * Now both the timer handler and io done are running
1297                  * simultaneously. We want to confirm the io done always
1298                  * finishes after the timer handler exits. So reqp used by
1299                  * timer handler is not freed or stale. Do busy loop for
1300                  * another 1/10 second to make sure io done does
1301                  * wait for the timer handler to complete.
1302                  */
1303                 DELAY(100*1000);
1304                 mtx_lock(&sc->hs_lock);
1305                 xpt_print(ccb->ccb_h.path,
1306                                 "%u: %s: finishing, queue frozen %d, "
1307                                 "ccb status 0x%x scsi_status 0x%x.\n",
1308                                 ticks, __func__, sc->hs_frozen,
1309                                 ccb->ccb_h.status,
1310                                 ccb->csio.scsi_status);
1311                 mtx_unlock(&sc->hs_lock);
1312         }
1313 }
1314 #endif /* HVS_TIMEOUT_TEST */
1315
1316 #ifdef notyet
1317 /**
1318  * @brief timeout handler for requests
1319  *
1320  * This function is called as a result of a callout expiring.
1321  *
1322  * @param arg pointer to a request
1323  */
1324 static void
1325 storvsc_timeout(void *arg)
1326 {
1327         struct hv_storvsc_request *reqp = arg;
1328         struct storvsc_softc *sc = reqp->softc;
1329         union ccb *ccb = reqp->ccb;
1330
1331         if (reqp->retries == 0) {
1332                 mtx_lock(&sc->hs_lock);
1333                 xpt_print(ccb->ccb_h.path,
1334                     "%u: IO timed out (req=0x%p), wait for another %u secs.\n",
1335                     ticks, reqp, ccb->ccb_h.timeout / 1000);
1336                 cam_error_print(ccb, CAM_ESF_ALL, CAM_EPF_ALL);
1337                 mtx_unlock(&sc->hs_lock);
1338
1339                 reqp->retries++;
1340                 callout_reset_sbt(&reqp->callout, SBT_1MS * ccb->ccb_h.timeout,
1341                     0, storvsc_timeout, reqp, 0);
1342 #if HVS_TIMEOUT_TEST
1343                 storvsc_timeout_test(reqp, SEND_DIAGNOSTIC, 0);
1344 #endif
1345                 return;
1346         }
1347
1348         mtx_lock(&sc->hs_lock);
1349         xpt_print(ccb->ccb_h.path,
1350                 "%u: IO (reqp = 0x%p) did not return for %u seconds, %s.\n",
1351                 ticks, reqp, ccb->ccb_h.timeout * (reqp->retries+1) / 1000,
1352                 (sc->hs_frozen == 0)?
1353                 "freezing the queue" : "the queue is already frozen");
1354         if (sc->hs_frozen == 0) {
1355                 sc->hs_frozen = 1;
1356                 xpt_freeze_simq(xpt_path_sim(ccb->ccb_h.path), 1);
1357         }
1358         mtx_unlock(&sc->hs_lock);
1359
1360 #if HVS_TIMEOUT_TEST
1361         storvsc_timeout_test(reqp, MODE_SELECT_10, 1);
1362 #endif
1363 }
1364 #endif
1365
1366 /**
1367  * @brief StorVSC device poll function
1368  *
1369  * This function is responsible for servicing requests when
1370  * interrupts are disabled (i.e when we are dumping core.)
1371  *
1372  * @param sim a pointer to a CAM SCSI interface module
1373  */
1374 static void
1375 storvsc_poll(struct cam_sim *sim)
1376 {
1377         struct storvsc_softc *sc = cam_sim_softc(sim);
1378
1379         mtx_assert(&sc->hs_lock, MA_OWNED);
1380         mtx_unlock(&sc->hs_lock);
1381         hv_storvsc_on_channel_callback(sc->hs_dev->channel);
1382         mtx_lock(&sc->hs_lock);
1383 }
1384
1385 /**
1386  * @brief StorVSC device action function
1387  *
1388  * This function is responsible for handling SCSI operations which
1389  * are passed from the CAM layer.  The requests are in the form of
1390  * CAM control blocks which indicate the action being performed.
1391  * Not all actions require converting the request to a VSCSI protocol
1392  * message - these actions can be responded to by this driver.
1393  * Requests which are destined for a backend storage device are converted
1394  * to a VSCSI protocol message and sent on the channel connection associated
1395  * with this device.
1396  *
1397  * @param sim pointer to a CAM SCSI interface module
1398  * @param ccb pointer to a CAM control block
1399  */
1400 static void
1401 storvsc_action(struct cam_sim *sim, union ccb *ccb)
1402 {
1403         struct storvsc_softc *sc = cam_sim_softc(sim);
1404         int res;
1405
1406         mtx_assert(&sc->hs_lock, MA_OWNED);
1407         switch (ccb->ccb_h.func_code) {
1408         case XPT_PATH_INQ: {
1409                 struct ccb_pathinq *cpi = &ccb->cpi;
1410
1411                 cpi->version_num = 1;
1412                 cpi->hba_inquiry = PI_TAG_ABLE|PI_SDTR_ABLE;
1413                 cpi->target_sprt = 0;
1414                 cpi->hba_misc = PIM_NOBUSRESET;
1415                 cpi->hba_eng_cnt = 0;
1416                 cpi->max_target = STORVSC_MAX_TARGETS;
1417                 cpi->max_lun = sc->hs_drv_props->drv_max_luns_per_target;
1418                 cpi->initiator_id = cpi->max_target;
1419                 cpi->bus_id = cam_sim_bus(sim);
1420                 cpi->base_transfer_speed = 300000;
1421                 cpi->transport = XPORT_SAS;
1422                 cpi->transport_version = 0;
1423                 cpi->protocol = PROTO_SCSI;
1424                 cpi->protocol_version = SCSI_REV_SPC2;
1425                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1426                 strncpy(cpi->hba_vid, sc->hs_drv_props->drv_name, HBA_IDLEN);
1427                 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1428                 cpi->unit_number = cam_sim_unit(sim);
1429
1430                 ccb->ccb_h.status = CAM_REQ_CMP;
1431                 xpt_done(ccb);
1432                 return;
1433         }
1434         case XPT_GET_TRAN_SETTINGS: {
1435                 struct  ccb_trans_settings *cts = &ccb->cts;
1436
1437                 cts->transport = XPORT_SAS;
1438                 cts->transport_version = 0;
1439                 cts->protocol = PROTO_SCSI;
1440                 cts->protocol_version = SCSI_REV_SPC2;
1441
1442                 /* enable tag queuing and disconnected mode */
1443                 cts->proto_specific.valid = CTS_SCSI_VALID_TQ;
1444                 cts->proto_specific.scsi.valid = CTS_SCSI_VALID_TQ;
1445                 cts->proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB;
1446                 cts->xport_specific.valid = CTS_SPI_VALID_DISC;
1447                 cts->xport_specific.spi.flags = CTS_SPI_FLAGS_DISC_ENB;
1448
1449                 ccb->ccb_h.status = CAM_REQ_CMP;
1450                 xpt_done(ccb);
1451                 return;
1452         }
1453         case XPT_SET_TRAN_SETTINGS:     {
1454                 ccb->ccb_h.status = CAM_REQ_CMP;
1455                 xpt_done(ccb);
1456                 return;
1457         }
1458         case XPT_CALC_GEOMETRY:{
1459                 cam_calc_geometry(&ccb->ccg, 1);
1460                 xpt_done(ccb);
1461                 return;
1462         }
1463         case  XPT_RESET_BUS:
1464         case  XPT_RESET_DEV:{
1465 #if HVS_HOST_RESET
1466                 if ((res = hv_storvsc_host_reset(sc->hs_dev)) != 0) {
1467                         xpt_print(ccb->ccb_h.path,
1468                                 "hv_storvsc_host_reset failed with %d\n", res);
1469                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1470                         xpt_done(ccb);
1471                         return;
1472                 }
1473                 ccb->ccb_h.status = CAM_REQ_CMP;
1474                 xpt_done(ccb);
1475                 return;
1476 #else
1477                 xpt_print(ccb->ccb_h.path,
1478                                   "%s reset not supported.\n",
1479                                   (ccb->ccb_h.func_code == XPT_RESET_BUS)?
1480                                   "bus" : "dev");
1481                 ccb->ccb_h.status = CAM_REQ_INVALID;
1482                 xpt_done(ccb);
1483                 return;
1484 #endif  /* HVS_HOST_RESET */
1485         }
1486         case XPT_SCSI_IO:
1487         case XPT_IMMED_NOTIFY: {
1488                 struct hv_storvsc_request *reqp = NULL;
1489
1490                 if (ccb->csio.cdb_len == 0) {
1491                         panic("cdl_len is 0\n");
1492                 }
1493
1494                 if (LIST_EMPTY(&sc->hs_free_list)) {
1495                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
1496                         if (sc->hs_frozen == 0) {
1497                                 sc->hs_frozen = 1;
1498                                 xpt_freeze_simq(sim, /* count*/1);
1499                         }
1500                         xpt_done(ccb);
1501                         return;
1502                 }
1503
1504                 reqp = LIST_FIRST(&sc->hs_free_list);
1505                 LIST_REMOVE(reqp, link);
1506
1507                 bzero(reqp, sizeof(struct hv_storvsc_request));
1508                 reqp->softc = sc;
1509
1510                 ccb->ccb_h.status |= CAM_SIM_QUEUED;
1511                 if ((res = create_storvsc_request(ccb, reqp)) != 0) {
1512                         ccb->ccb_h.status = CAM_REQ_INVALID;
1513                         xpt_done(ccb);
1514                         return;
1515                 }
1516
1517 #ifdef notyet
1518                 if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
1519                         callout_init(&reqp->callout, CALLOUT_MPSAFE);
1520                         callout_reset_sbt(&reqp->callout,
1521                             SBT_1MS * ccb->ccb_h.timeout, 0,
1522                             storvsc_timeout, reqp, 0);
1523 #if HVS_TIMEOUT_TEST
1524                         cv_init(&reqp->event.cv, "storvsc timeout cv");
1525                         mtx_init(&reqp->event.mtx, "storvsc timeout mutex",
1526                                         NULL, MTX_DEF);
1527                         switch (reqp->vstor_packet.vm_srb.cdb[0]) {
1528                                 case MODE_SELECT_10:
1529                                 case SEND_DIAGNOSTIC:
1530                                         /* To have timer send the request. */
1531                                         return;
1532                                 default:
1533                                         break;
1534                         }
1535 #endif /* HVS_TIMEOUT_TEST */
1536                 }
1537 #endif
1538
1539                 if ((res = hv_storvsc_io_request(sc->hs_dev, reqp)) != 0) {
1540                         xpt_print(ccb->ccb_h.path,
1541                                 "hv_storvsc_io_request failed with %d\n", res);
1542                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1543                         storvsc_free_request(sc, reqp);
1544                         xpt_done(ccb);
1545                         return;
1546                 }
1547                 return;
1548         }
1549
1550         default:
1551                 ccb->ccb_h.status = CAM_REQ_INVALID;
1552                 xpt_done(ccb);
1553                 return;
1554         }
1555 }
1556
1557 /**
1558  * @brief destroy bounce buffer
1559  *
1560  * This function is responsible for destroy a Scatter/Gather list
1561  * that create by storvsc_create_bounce_buffer()
1562  *
1563  * @param sgl- the Scatter/Gather need be destroy
1564  * @param sg_count- page count of the SG list.
1565  *
1566  */
1567 static void
1568 storvsc_destroy_bounce_buffer(struct sglist *sgl)
1569 {
1570         struct hv_sgl_node *sgl_node = NULL;
1571
1572         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.in_use_sgl_list);
1573         LIST_REMOVE(sgl_node, link);
1574         if (NULL == sgl_node) {
1575                 printf("storvsc error: not enough in use sgl\n");
1576                 return;
1577         }
1578         sgl_node->sgl_data = sgl;
1579         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list, sgl_node, link);
1580 }
1581
1582 /**
1583  * @brief create bounce buffer
1584  *
1585  * This function is responsible for create a Scatter/Gather list,
1586  * which hold several pages that can be aligned with page size.
1587  *
1588  * @param seg_count- SG-list segments count
1589  * @param write - if WRITE_TYPE, set SG list page used size to 0,
1590  * otherwise set used size to page size.
1591  *
1592  * return NULL if create failed
1593  */
1594 static struct sglist *
1595 storvsc_create_bounce_buffer(uint16_t seg_count, int write)
1596 {
1597         int i = 0;
1598         struct sglist *bounce_sgl = NULL;
1599         unsigned int buf_len = ((write == WRITE_TYPE) ? 0 : PAGE_SIZE);
1600         struct hv_sgl_node *sgl_node = NULL;
1601
1602         /* get struct sglist from free_sgl_list */
1603         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1604         LIST_REMOVE(sgl_node, link);
1605         if (NULL == sgl_node) {
1606                 printf("storvsc error: not enough free sgl\n");
1607                 return NULL;
1608         }
1609         bounce_sgl = sgl_node->sgl_data;
1610         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.in_use_sgl_list, sgl_node, link);
1611
1612         bounce_sgl->sg_maxseg = seg_count;
1613
1614         if (write == WRITE_TYPE)
1615                 bounce_sgl->sg_nseg = 0;
1616         else
1617                 bounce_sgl->sg_nseg = seg_count;
1618
1619         for (i = 0; i < seg_count; i++)
1620                 bounce_sgl->sg_segs[i].ss_len = buf_len;
1621
1622         return bounce_sgl;
1623 }
1624
1625 /**
1626  * @brief copy data from SG list to bounce buffer
1627  *
1628  * This function is responsible for copy data from one SG list's segments
1629  * to another SG list which used as bounce buffer.
1630  *
1631  * @param bounce_sgl - the destination SG list
1632  * @param orig_sgl - the segment of the source SG list.
1633  * @param orig_sgl_count - the count of segments.
1634  * @param orig_sgl_count - indicate which segment need bounce buffer,
1635  *  set 1 means need.
1636  *
1637  */
1638 static void
1639 storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
1640                                bus_dma_segment_t *orig_sgl,
1641                                unsigned int orig_sgl_count,
1642                                uint64_t seg_bits)
1643 {
1644         int src_sgl_idx = 0;
1645
1646         for (src_sgl_idx = 0; src_sgl_idx < orig_sgl_count; src_sgl_idx++) {
1647                 if (seg_bits & (1 << src_sgl_idx)) {
1648                         memcpy((void*)bounce_sgl->sg_segs[src_sgl_idx].ss_paddr,
1649                             (void*)orig_sgl[src_sgl_idx].ds_addr,
1650                             orig_sgl[src_sgl_idx].ds_len);
1651
1652                         bounce_sgl->sg_segs[src_sgl_idx].ss_len =
1653                             orig_sgl[src_sgl_idx].ds_len;
1654                 }
1655         }
1656 }
1657
1658 /**
1659  * @brief copy data from SG list which used as bounce to another SG list
1660  *
1661  * This function is responsible for copy data from one SG list with bounce
1662  * buffer to another SG list's segments.
1663  *
1664  * @param dest_sgl - the destination SG list's segments
1665  * @param dest_sgl_count - the count of destination SG list's segment.
1666  * @param src_sgl - the source SG list.
1667  * @param seg_bits - indicate which segment used bounce buffer of src SG-list.
1668  *
1669  */
1670 void
1671 storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
1672                                     unsigned int dest_sgl_count,
1673                                     struct sglist* src_sgl,
1674                                     uint64_t seg_bits)
1675 {
1676         int sgl_idx = 0;
1677
1678         for (sgl_idx = 0; sgl_idx < dest_sgl_count; sgl_idx++) {
1679                 if (seg_bits & (1 << sgl_idx)) {
1680                         memcpy((void*)(dest_sgl[sgl_idx].ds_addr),
1681                             (void*)(src_sgl->sg_segs[sgl_idx].ss_paddr),
1682                             src_sgl->sg_segs[sgl_idx].ss_len);
1683                 }
1684         }
1685 }
1686
1687 /**
1688  * @brief check SG list with bounce buffer or not
1689  *
1690  * This function is responsible for check if need bounce buffer for SG list.
1691  *
1692  * @param sgl - the SG list's segments
1693  * @param sg_count - the count of SG list's segment.
1694  * @param bits - segmengs number that need bounce buffer
1695  *
1696  * return -1 if SG list needless bounce buffer
1697  */
1698 static int
1699 storvsc_check_bounce_buffer_sgl(bus_dma_segment_t *sgl,
1700                                 unsigned int sg_count,
1701                                 uint64_t *bits)
1702 {
1703         int i = 0;
1704         int offset = 0;
1705         uint64_t phys_addr = 0;
1706         uint64_t tmp_bits = 0;
1707         boolean_t found_hole = FALSE;
1708         boolean_t pre_aligned = TRUE;
1709
1710         if (sg_count < 2){
1711                 return -1;
1712         }
1713
1714         *bits = 0;
1715
1716         phys_addr = vtophys(sgl[0].ds_addr);
1717         offset =  phys_addr - trunc_page(phys_addr);
1718
1719         if (offset != 0) {
1720                 pre_aligned = FALSE;
1721                 tmp_bits |= 1;
1722         }
1723
1724         for (i = 1; i < sg_count; i++) {
1725                 phys_addr = vtophys(sgl[i].ds_addr);
1726                 offset =  phys_addr - trunc_page(phys_addr);
1727
1728                 if (offset == 0) {
1729                         if (FALSE == pre_aligned){
1730                                 /*
1731                                  * This segment is aligned, if the previous
1732                                  * one is not aligned, find a hole
1733                                  */
1734                                 found_hole = TRUE;
1735                         }
1736                         pre_aligned = TRUE;
1737                 } else {
1738                         tmp_bits |= 1 << i;
1739                         if (!pre_aligned) {
1740                                 if (phys_addr != vtophys(sgl[i-1].ds_addr +
1741                                     sgl[i-1].ds_len)) {
1742                                         /*
1743                                          * Check whether connect to previous
1744                                          * segment,if not, find the hole
1745                                          */
1746                                         found_hole = TRUE;
1747                                 }
1748                         } else {
1749                                 found_hole = TRUE;
1750                         }
1751                         pre_aligned = FALSE;
1752                 }
1753         }
1754
1755         if (!found_hole) {
1756                 return (-1);
1757         } else {
1758                 *bits = tmp_bits;
1759                 return 0;
1760         }
1761 }
1762
1763 /**
1764  * @brief Fill in a request structure based on a CAM control block
1765  *
1766  * Fills in a request structure based on the contents of a CAM control
1767  * block.  The request structure holds the payload information for
1768  * VSCSI protocol request.
1769  *
1770  * @param ccb pointer to a CAM contorl block
1771  * @param reqp pointer to a request structure
1772  */
1773 static int
1774 create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
1775 {
1776         struct ccb_scsiio *csio = &ccb->csio;
1777         uint64_t phys_addr;
1778         uint32_t bytes_to_copy = 0;
1779         uint32_t pfn_num = 0;
1780         uint32_t pfn;
1781         uint64_t not_aligned_seg_bits = 0;
1782
1783         /* refer to struct vmscsi_req for meanings of these two fields */
1784         reqp->vstor_packet.u.vm_srb.port =
1785                 cam_sim_unit(xpt_path_sim(ccb->ccb_h.path));
1786         reqp->vstor_packet.u.vm_srb.path_id =
1787                 cam_sim_bus(xpt_path_sim(ccb->ccb_h.path));
1788
1789         reqp->vstor_packet.u.vm_srb.target_id = ccb->ccb_h.target_id;
1790         reqp->vstor_packet.u.vm_srb.lun = ccb->ccb_h.target_lun;
1791
1792         reqp->vstor_packet.u.vm_srb.cdb_len = csio->cdb_len;
1793         if(ccb->ccb_h.flags & CAM_CDB_POINTER) {
1794                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_ptr,
1795                         csio->cdb_len);
1796         } else {
1797                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_bytes,
1798                         csio->cdb_len);
1799         }
1800
1801         switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
1802         case CAM_DIR_OUT:
1803                 reqp->vstor_packet.u.vm_srb.data_in = WRITE_TYPE;
1804                 break;
1805         case CAM_DIR_IN:
1806                 reqp->vstor_packet.u.vm_srb.data_in = READ_TYPE;
1807                 break;
1808         case CAM_DIR_NONE:
1809                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1810                 break;
1811         default:
1812                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1813                 break;
1814         }
1815
1816         reqp->sense_data     = &csio->sense_data;
1817         reqp->sense_info_len = csio->sense_len;
1818
1819         reqp->ccb = ccb;
1820
1821         if (0 == csio->dxfer_len) {
1822                 return (0);
1823         }
1824
1825         reqp->data_buf.length = csio->dxfer_len;
1826
1827         switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
1828         case CAM_DATA_VADDR:
1829         {
1830                 bytes_to_copy = csio->dxfer_len;
1831                 phys_addr = vtophys(csio->data_ptr);
1832                 reqp->data_buf.offset = phys_addr & PAGE_MASK;
1833
1834                 while (bytes_to_copy != 0) {
1835                         int bytes, page_offset;
1836                         phys_addr =
1837                             vtophys(&csio->data_ptr[reqp->data_buf.length -
1838                             bytes_to_copy]);
1839                         pfn = phys_addr >> PAGE_SHIFT;
1840                         reqp->data_buf.pfn_array[pfn_num] = pfn;
1841                         page_offset = phys_addr & PAGE_MASK;
1842
1843                         bytes = min(PAGE_SIZE - page_offset, bytes_to_copy);
1844
1845                         bytes_to_copy -= bytes;
1846                         pfn_num++;
1847                 }
1848                 break;
1849         }
1850
1851         case CAM_DATA_SG:
1852         {
1853                 int i = 0;
1854                 int offset = 0;
1855                 int ret;
1856
1857                 bus_dma_segment_t *storvsc_sglist =
1858                     (bus_dma_segment_t *)ccb->csio.data_ptr;
1859                 u_int16_t storvsc_sg_count = ccb->csio.sglist_cnt;
1860
1861                 printf("Storvsc: get SG I/O operation, %d\n",
1862                     reqp->vstor_packet.u.vm_srb.data_in);
1863
1864                 if (storvsc_sg_count > HV_MAX_MULTIPAGE_BUFFER_COUNT){
1865                         printf("Storvsc: %d segments is too much, "
1866                             "only support %d segments\n",
1867                             storvsc_sg_count, HV_MAX_MULTIPAGE_BUFFER_COUNT);
1868                         return (EINVAL);
1869                 }
1870
1871                 /*
1872                  * We create our own bounce buffer function currently. Idealy
1873                  * we should use BUS_DMA(9) framework. But with current BUS_DMA
1874                  * code there is no callback API to check the page alignment of
1875                  * middle segments before busdma can decide if a bounce buffer
1876                  * is needed for particular segment. There is callback,
1877                  * "bus_dma_filter_t *filter", but the parrameters are not
1878                  * sufficient for storvsc driver.
1879                  * TODO:
1880                  *      Add page alignment check in BUS_DMA(9) callback. Once
1881                  *      this is complete, switch the following code to use
1882                  *      BUS_DMA(9) for storvsc bounce buffer support.
1883                  */
1884                 /* check if we need to create bounce buffer */
1885                 ret = storvsc_check_bounce_buffer_sgl(storvsc_sglist,
1886                     storvsc_sg_count, &not_aligned_seg_bits);
1887                 if (ret != -1) {
1888                         reqp->bounce_sgl =
1889                             storvsc_create_bounce_buffer(storvsc_sg_count,
1890                             reqp->vstor_packet.u.vm_srb.data_in);
1891                         if (NULL == reqp->bounce_sgl) {
1892                                 printf("Storvsc_error: "
1893                                     "create bounce buffer failed.\n");
1894                                 return (ENOMEM);
1895                         }
1896
1897                         reqp->bounce_sgl_count = storvsc_sg_count;
1898                         reqp->not_aligned_seg_bits = not_aligned_seg_bits;
1899
1900                         /*
1901                          * if it is write, we need copy the original data
1902                          *to bounce buffer
1903                          */
1904                         if (WRITE_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
1905                                 storvsc_copy_sgl_to_bounce_buf(
1906                                     reqp->bounce_sgl,
1907                                     storvsc_sglist,
1908                                     storvsc_sg_count,
1909                                     reqp->not_aligned_seg_bits);
1910                         }
1911
1912                         /* transfer virtual address to physical frame number */
1913                         if (reqp->not_aligned_seg_bits & 0x1){
1914                                 phys_addr =
1915                                     vtophys(reqp->bounce_sgl->sg_segs[0].ss_paddr);
1916                         }else{
1917                                 phys_addr =
1918                                         vtophys(storvsc_sglist[0].ds_addr);
1919                         }
1920                         reqp->data_buf.offset = phys_addr & PAGE_MASK;
1921
1922                         pfn = phys_addr >> PAGE_SHIFT;
1923                         reqp->data_buf.pfn_array[0] = pfn;
1924
1925                         for (i = 1; i < storvsc_sg_count; i++) {
1926                                 if (reqp->not_aligned_seg_bits & (1 << i)) {
1927                                         phys_addr =
1928                                             vtophys(reqp->bounce_sgl->sg_segs[i].ss_paddr);
1929                                 } else {
1930                                         phys_addr =
1931                                             vtophys(storvsc_sglist[i].ds_addr);
1932                                 }
1933
1934                                 pfn = phys_addr >> PAGE_SHIFT;
1935                                 reqp->data_buf.pfn_array[i] = pfn;
1936                         }
1937                 } else {
1938                         phys_addr = vtophys(storvsc_sglist[0].ds_addr);
1939
1940                         reqp->data_buf.offset = phys_addr & PAGE_MASK;
1941
1942                         for (i = 0; i < storvsc_sg_count; i++) {
1943                                 phys_addr = vtophys(storvsc_sglist[i].ds_addr);
1944                                 pfn = phys_addr >> PAGE_SHIFT;
1945                                 reqp->data_buf.pfn_array[i] = pfn;
1946                         }
1947
1948                         /* check the last segment cross boundary or not */
1949                         offset = phys_addr & PAGE_MASK;
1950                         if (offset) {
1951                                 phys_addr =
1952                                     vtophys(storvsc_sglist[i-1].ds_addr +
1953                                     PAGE_SIZE - offset);
1954                                 pfn = phys_addr >> PAGE_SHIFT;
1955                                 reqp->data_buf.pfn_array[i] = pfn;
1956                         }
1957
1958                         reqp->bounce_sgl_count = 0;
1959                 }
1960                 break;
1961         }
1962         default:
1963                 printf("Unknow flags: %d\n", ccb->ccb_h.flags);
1964                 return(EINVAL);
1965         }
1966
1967         return(0);
1968 }
1969
1970 /*
1971  * SCSI Inquiry checks qualifier and type.
1972  * If qualifier is 011b, means the device server is not capable
1973  * of supporting a peripheral device on this logical unit, and
1974  * the type should be set to 1Fh.
1975  *
1976  * Return 1 if it is valid, 0 otherwise.
1977  */
1978 static inline int
1979 is_inquiry_valid(const struct scsi_inquiry_data *inq_data)
1980 {
1981         uint8_t type;
1982         if (SID_QUAL(inq_data) != SID_QUAL_LU_CONNECTED) {
1983                 return (0);
1984         }
1985         type = SID_TYPE(inq_data);
1986         if (type == T_NODEVICE) {
1987                 return (0);
1988         }
1989         return (1);
1990 }
1991
1992 /**
1993  * @brief completion function before returning to CAM
1994  *
1995  * I/O process has been completed and the result needs
1996  * to be passed to the CAM layer.
1997  * Free resources related to this request.
1998  *
1999  * @param reqp pointer to a request structure
2000  */
2001 static void
2002 storvsc_io_done(struct hv_storvsc_request *reqp)
2003 {
2004         union ccb *ccb = reqp->ccb;
2005         struct ccb_scsiio *csio = &ccb->csio;
2006         struct storvsc_softc *sc = reqp->softc;
2007         struct vmscsi_req *vm_srb = &reqp->vstor_packet.u.vm_srb;
2008         bus_dma_segment_t *ori_sglist = NULL;
2009         int ori_sg_count = 0;
2010
2011         /* destroy bounce buffer if it is used */
2012         if (reqp->bounce_sgl_count) {
2013                 ori_sglist = (bus_dma_segment_t *)ccb->csio.data_ptr;
2014                 ori_sg_count = ccb->csio.sglist_cnt;
2015
2016                 /*
2017                  * If it is READ operation, we should copy back the data
2018                  * to original SG list.
2019                  */
2020                 if (READ_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
2021                         storvsc_copy_from_bounce_buf_to_sgl(ori_sglist,
2022                             ori_sg_count,
2023                             reqp->bounce_sgl,
2024                             reqp->not_aligned_seg_bits);
2025                 }
2026
2027                 storvsc_destroy_bounce_buffer(reqp->bounce_sgl);
2028                 reqp->bounce_sgl_count = 0;
2029         }
2030
2031         if (reqp->retries > 0) {
2032                 mtx_lock(&sc->hs_lock);
2033 #if HVS_TIMEOUT_TEST
2034                 xpt_print(ccb->ccb_h.path,
2035                         "%u: IO returned after timeout, "
2036                         "waking up timer handler if any.\n", ticks);
2037                 mtx_lock(&reqp->event.mtx);
2038                 cv_signal(&reqp->event.cv);
2039                 mtx_unlock(&reqp->event.mtx);
2040 #endif
2041                 reqp->retries = 0;
2042                 xpt_print(ccb->ccb_h.path,
2043                         "%u: IO returned after timeout, "
2044                         "stopping timer if any.\n", ticks);
2045                 mtx_unlock(&sc->hs_lock);
2046         }
2047
2048 #ifdef notyet
2049         /*
2050          * callout_drain() will wait for the timer handler to finish
2051          * if it is running. So we don't need any lock to synchronize
2052          * between this routine and the timer handler.
2053          * Note that we need to make sure reqp is not freed when timer
2054          * handler is using or will use it.
2055          */
2056         if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
2057                 callout_drain(&reqp->callout);
2058         }
2059 #endif
2060
2061         ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
2062         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2063         if (vm_srb->scsi_status == SCSI_STATUS_OK) {
2064                 const struct scsi_generic *cmd;
2065                 /*
2066                  * Check whether the data for INQUIRY cmd is valid or
2067                  * not.  Windows 10 and Windows 2016 send all zero
2068                  * inquiry data to VM even for unpopulated slots.
2069                  */
2070                 cmd = (const struct scsi_generic *)
2071                     ((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
2072                      csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
2073                 if (cmd->opcode == INQUIRY) {
2074                     /*
2075                      * The host of Windows 10 or 2016 server will response
2076                      * the inquiry request with invalid data for unexisted device:
2077                         [0x7f 0x0 0x5 0x2 0x1f ... ]
2078                      * But on windows 2012 R2, the response is:
2079                         [0x7f 0x0 0x0 0x0 0x0 ]
2080                      * That is why here wants to validate the inquiry response.
2081                      * The validation will skip the INQUIRY whose response is short,
2082                      * which is less than SHORT_INQUIRY_LENGTH (36).
2083                      *
2084                      * For more information about INQUIRY, please refer to:
2085                      *  ftp://ftp.avc-pioneer.com/Mtfuji_7/Proposal/Jun09/INQUIRY.pdf
2086                      */
2087                     const struct scsi_inquiry_data *inq_data =
2088                         (const struct scsi_inquiry_data *)csio->data_ptr;
2089                     uint8_t* resp_buf = (uint8_t*)csio->data_ptr;
2090                     /* Get the buffer length reported by host */
2091                     int resp_xfer_len = vm_srb->transfer_len;
2092                     /* Get the available buffer length */
2093                     int resp_buf_len = resp_xfer_len >= 5 ? resp_buf[4] + 5 : 0;
2094                     int data_len = (resp_buf_len < resp_xfer_len) ? resp_buf_len : resp_xfer_len;
2095                     if (data_len < SHORT_INQUIRY_LENGTH) {
2096                         ccb->ccb_h.status |= CAM_REQ_CMP;
2097                         if (bootverbose && data_len >= 5) {
2098                                 mtx_lock(&sc->hs_lock);
2099                                 xpt_print(ccb->ccb_h.path,
2100                                     "storvsc skips the validation for short inquiry (%d)"
2101                                     " [%x %x %x %x %x]\n",
2102                                     data_len,resp_buf[0],resp_buf[1],resp_buf[2],
2103                                     resp_buf[3],resp_buf[4]);
2104                                 mtx_unlock(&sc->hs_lock);
2105                         }
2106                     } else if (is_inquiry_valid(inq_data) == 0) {
2107                         ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
2108                         if (bootverbose && data_len >= 5) {
2109                                 mtx_lock(&sc->hs_lock);
2110                                 xpt_print(ccb->ccb_h.path,
2111                                     "storvsc uninstalled invalid device"
2112                                     " [%x %x %x %x %x]\n",
2113                                 resp_buf[0],resp_buf[1],resp_buf[2],resp_buf[3],resp_buf[4]);
2114                                 mtx_unlock(&sc->hs_lock);
2115                         }
2116                     } else {
2117                         ccb->ccb_h.status |= CAM_REQ_CMP;
2118                         if (bootverbose) {
2119                                 mtx_lock(&sc->hs_lock);
2120                                 xpt_print(ccb->ccb_h.path,
2121                                     "storvsc has passed inquiry response (%d) validation\n",
2122                                     data_len);
2123                                 mtx_unlock(&sc->hs_lock);
2124                         }
2125                     }
2126                 } else {
2127                         ccb->ccb_h.status |= CAM_REQ_CMP;
2128                 }
2129         } else {
2130                 mtx_lock(&sc->hs_lock);
2131                 xpt_print(ccb->ccb_h.path,
2132                         "storvsc scsi_status = %d\n",
2133                         vm_srb->scsi_status);
2134                 mtx_unlock(&sc->hs_lock);
2135                 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
2136         }
2137
2138         ccb->csio.scsi_status = (vm_srb->scsi_status & 0xFF);
2139         ccb->csio.resid = ccb->csio.dxfer_len - vm_srb->transfer_len;
2140
2141         if (reqp->sense_info_len != 0) {
2142                 csio->sense_resid = csio->sense_len - reqp->sense_info_len;
2143                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2144         }
2145
2146         mtx_lock(&sc->hs_lock);
2147         if (reqp->softc->hs_frozen == 1) {
2148                 xpt_print(ccb->ccb_h.path,
2149                         "%u: storvsc unfreezing softc 0x%p.\n",
2150                         ticks, reqp->softc);
2151                 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2152                 reqp->softc->hs_frozen = 0;
2153         }
2154         storvsc_free_request(sc, reqp);
2155         xpt_done(ccb);
2156         mtx_unlock(&sc->hs_lock);
2157 }
2158
2159 /**
2160  * @brief Free a request structure
2161  *
2162  * Free a request structure by returning it to the free list
2163  *
2164  * @param sc pointer to a softc
2165  * @param reqp pointer to a request structure
2166  */
2167 static void
2168 storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp)
2169 {
2170
2171         LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
2172 }
2173
2174 /**
2175  * @brief Determine type of storage device from GUID
2176  *
2177  * Using the type GUID, determine if this is a StorVSC (paravirtual
2178  * SCSI or BlkVSC (paravirtual IDE) device.
2179  *
2180  * @param dev a device
2181  * returns an enum
2182  */
2183 static enum hv_storage_type
2184 storvsc_get_storage_type(device_t dev)
2185 {
2186         const char *p = vmbus_get_type(dev);
2187
2188         if (!memcmp(p, &gBlkVscDeviceType, sizeof(hv_guid))) {
2189                 return DRIVER_BLKVSC;
2190         } else if (!memcmp(p, &gStorVscDeviceType, sizeof(hv_guid))) {
2191                 return DRIVER_STORVSC;
2192         }
2193         return (DRIVER_UNKNOWN);
2194 }
2195