sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c

   1 /*-
   2  * Copyright (c) 2009-2012 Microsoft Corp.
   3  * Copyright (c) 2012 NetApp Inc.
   4  * Copyright (c) 2012 Citrix Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /**
  30  * StorVSC driver for Hyper-V.  This driver presents a SCSI HBA interface
  31  * to the Comman Access Method (CAM) layer.  CAM control blocks (CCBs) are
  32  * converted into VSCSI protocol messages which are delivered to the parent
  33  * partition StorVSP driver over the Hyper-V VMBUS.
  34  */
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/proc.h>
  40 #include <sys/condvar.h>
  41 #include <sys/time.h>
  42 #include <sys/systm.h>
  43 #include <sys/sockio.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/malloc.h>
  46 #include <sys/module.h>
  47 #include <sys/kernel.h>
  48 #include <sys/queue.h>
  49 #include <sys/lock.h>
  50 #include <sys/sx.h>
  51 #include <sys/taskqueue.h>
  52 #include <sys/bus.h>
  53 #include <sys/mutex.h>
  54 #include <sys/callout.h>
  55 #include <vm/vm.h>
  56 #include <vm/pmap.h>
  57 #include <vm/uma.h>
  58 #include <sys/lock.h>
  59 #include <sys/sema.h>
  60 #include <sys/sglist.h>
  61 #include <machine/bus.h>
  62 #include <sys/bus_dma.h>
  63
  64 #include <cam/cam.h>
  65 #include <cam/cam_ccb.h>
  66 #include <cam/cam_periph.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_xpt_sim.h>
  69 #include <cam/cam_xpt_internal.h>
  70 #include <cam/cam_debug.h>
  71 #include <cam/scsi/scsi_all.h>
  72 #include <cam/scsi/scsi_message.h>
  73
  74 #include <dev/hyperv/include/hyperv.h>
  75 #include "hv_vstorage.h"
  76
  77 #define STORVSC_RINGBUFFER_SIZE         (20*PAGE_SIZE)
  78 #define STORVSC_MAX_LUNS_PER_TARGET     (64)
  79 #define STORVSC_MAX_IO_REQUESTS         (STORVSC_MAX_LUNS_PER_TARGET * 2)
  80 #define BLKVSC_MAX_IDE_DISKS_PER_TARGET (1)
  81 #define BLKVSC_MAX_IO_REQUESTS          STORVSC_MAX_IO_REQUESTS
  82 #define STORVSC_MAX_TARGETS             (2)
  83
  84 #define STORVSC_WIN7_MAJOR 4
  85 #define STORVSC_WIN7_MINOR 2
  86
  87 #define STORVSC_WIN8_MAJOR 5
  88 #define STORVSC_WIN8_MINOR 1
  89
  90 #define HV_ALIGN(x, a) roundup2(x, a)
  91
  92 struct storvsc_softc;
  93
  94 struct hv_sgl_node {
  95         LIST_ENTRY(hv_sgl_node) link;
  96         struct sglist *sgl_data;
  97 };
  98
  99 struct hv_sgl_page_pool{
 100         LIST_HEAD(, hv_sgl_node) in_use_sgl_list;
 101         LIST_HEAD(, hv_sgl_node) free_sgl_list;
 102         boolean_t                is_init;
 103 } g_hv_sgl_page_pool;
 104
 105 #define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * HV_MAX_MULTIPAGE_BUFFER_COUNT
 106
 107 enum storvsc_request_type {
 108         WRITE_TYPE,
 109         READ_TYPE,
 110         UNKNOWN_TYPE
 111 };
 112
 113 struct hv_storvsc_request {
 114         LIST_ENTRY(hv_storvsc_request) link;
 115         struct vstor_packet     vstor_packet;
 116         hv_vmbus_multipage_buffer data_buf;
 117         void *sense_data;
 118         uint8_t sense_info_len;
 119         uint8_t retries;
 120         union ccb *ccb;
 121         struct storvsc_softc *softc;
 122         struct callout callout;
 123         struct sema synch_sema; /*Synchronize the request/response if needed */
 124         struct sglist *bounce_sgl;
 125         unsigned int bounce_sgl_count;
 126         uint64_t not_aligned_seg_bits;
 127 };
 128
 129 struct storvsc_softc {
 130         struct hv_device                *hs_dev;
 131         LIST_HEAD(, hv_storvsc_request) hs_free_list;
 132         struct mtx                      hs_lock;
 133         struct storvsc_driver_props     *hs_drv_props;
 134         int                             hs_unit;
 135         uint32_t                        hs_frozen;
 136         struct cam_sim                  *hs_sim;
 137         struct cam_path                 *hs_path;
 138         uint32_t                        hs_num_out_reqs;
 139         boolean_t                       hs_destroy;
 140         boolean_t                       hs_drain_notify;
 141         boolean_t                       hs_open_multi_channel;
 142         struct sema                     hs_drain_sema;
 143         struct hv_storvsc_request       hs_init_req;
 144         struct hv_storvsc_request       hs_reset_req;
 145 };
 146
 147
 148 /**
 149  * HyperV storvsc timeout testing cases:
 150  * a. IO returned after first timeout;
 151  * b. IO returned after second timeout and queue freeze;
 152  * c. IO returned while timer handler is running
 153  * The first can be tested by "sg_senddiag -vv /dev/daX",
 154  * and the second and third can be done by
 155  * "sg_wr_mode -v -p 08 -c 0,1a -m 0,ff /dev/daX".
 156  */
 157 #define HVS_TIMEOUT_TEST 0
 158
 159 /*
 160  * Bus/adapter reset functionality on the Hyper-V host is
 161  * buggy and it will be disabled until
 162  * it can be further tested.
 163  */
 164 #define HVS_HOST_RESET 0
 165
 166 struct storvsc_driver_props {
 167         char            *drv_name;
 168         char            *drv_desc;
 169         uint8_t         drv_max_luns_per_target;
 170         uint8_t         drv_max_ios_per_target;
 171         uint32_t        drv_ringbuffer_size;
 172 };
 173
 174 enum hv_storage_type {
 175         DRIVER_BLKVSC,
 176         DRIVER_STORVSC,
 177         DRIVER_UNKNOWN
 178 };
 179
 180 #define HS_MAX_ADAPTERS 10
 181
 182 #define HV_STORAGE_SUPPORTS_MULTI_CHANNEL 0x1
 183
 184 /* {ba6163d9-04a1-4d29-b605-72e2ffb1dc7f} */
 185 static const hv_guid gStorVscDeviceType={
 186         .data = {0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d,
 187                  0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f}
 188 };
 189
 190 /* {32412632-86cb-44a2-9b5c-50d1417354f5} */
 191 static const hv_guid gBlkVscDeviceType={
 192         .data = {0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44,
 193                  0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5}
 194 };
 195
 196 static struct storvsc_driver_props g_drv_props_table[] = {
 197         {"blkvsc", "Hyper-V IDE Storage Interface",
 198          BLKVSC_MAX_IDE_DISKS_PER_TARGET, BLKVSC_MAX_IO_REQUESTS,
 199          STORVSC_RINGBUFFER_SIZE},
 200         {"storvsc", "Hyper-V SCSI Storage Interface",
 201          STORVSC_MAX_LUNS_PER_TARGET, STORVSC_MAX_IO_REQUESTS,
 202          STORVSC_RINGBUFFER_SIZE}
 203 };
 204
 205 static int storvsc_current_major;
 206 static int storvsc_current_minor;
 207
 208 /* static functions */
 209 static int storvsc_probe(device_t dev);
 210 static int storvsc_attach(device_t dev);
 211 static int storvsc_detach(device_t dev);
 212 static void storvsc_poll(struct cam_sim * sim);
 213 static void storvsc_action(struct cam_sim * sim, union ccb * ccb);
 214 static int create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp);
 215 static void storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp);
 216 static enum hv_storage_type storvsc_get_storage_type(device_t dev);
 217 static void hv_storvsc_on_channel_callback(void *context);
 218 static void hv_storvsc_on_iocompletion( struct storvsc_softc *sc,
 219                                         struct vstor_packet *vstor_packet,
 220                                         struct hv_storvsc_request *request);
 221 static int hv_storvsc_connect_vsp(struct hv_device *device);
 222 static void storvsc_io_done(struct hv_storvsc_request *reqp);
 223 static void storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
 224                                 bus_dma_segment_t *orig_sgl,
 225                                 unsigned int orig_sgl_count,
 226                                 uint64_t seg_bits);
 227 void storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
 228                                 unsigned int dest_sgl_count,
 229                                 struct sglist* src_sgl,
 230                                 uint64_t seg_bits);
 231
 232 static device_method_t storvsc_methods[] = {
 233         /* Device interface */
 234         DEVMETHOD(device_probe,         storvsc_probe),
 235         DEVMETHOD(device_attach,        storvsc_attach),
 236         DEVMETHOD(device_detach,        storvsc_detach),
 237         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 238         DEVMETHOD_END
 239 };
 240
 241 static driver_t storvsc_driver = {
 242         "storvsc", storvsc_methods, sizeof(struct storvsc_softc),
 243 };
 244
 245 static devclass_t storvsc_devclass;
 246 DRIVER_MODULE(storvsc, vmbus, storvsc_driver, storvsc_devclass, 0, 0);
 247 MODULE_VERSION(storvsc, 1);
 248 MODULE_DEPEND(storvsc, vmbus, 1, 1, 1);
 249
 250
 251 /**
 252  * The host is capable of sending messages to us that are
 253  * completely unsolicited. So, we need to address the race
 254  * condition where we may be in the process of unloading the
 255  * driver when the host may send us an unsolicited message.
 256  * We address this issue by implementing a sequentially
 257  * consistent protocol:
 258  *
 259  * 1. Channel callback is invoked while holding the the channel lock
 260  *    and an unloading driver will reset the channel callback under
 261  *    the protection of this channel lock.
 262  *
 263  * 2. To ensure bounded wait time for unloading a driver, we don't
 264  *    permit outgoing traffic once the device is marked as being
 265  *    destroyed.
 266  *
 267  * 3. Once the device is marked as being destroyed, we only
 268  *    permit incoming traffic to properly account for
 269  *    packets already sent out.
 270  */
 271 static inline struct storvsc_softc *
 272 get_stor_device(struct hv_device *device,
 273                                 boolean_t outbound)
 274 {
 275         struct storvsc_softc *sc;
 276
 277         sc = device_get_softc(device->device);
 278         if (sc == NULL) {
 279                 return NULL;
 280         }
 281
 282         if (outbound) {
 283                 /*
 284                  * Here we permit outgoing I/O only
 285                  * if the device is not being destroyed.
 286                  */
 287
 288                 if (sc->hs_destroy) {
 289                         sc = NULL;
 290                 }
 291         } else {
 292                 /*
 293                  * inbound case; if being destroyed
 294                  * only permit to account for
 295                  * messages already sent out.
 296                  */
 297                 if (sc->hs_destroy && (sc->hs_num_out_reqs == 0)) {
 298                         sc = NULL;
 299                 }
 300         }
 301         return sc;
 302 }
 303
 304 /**
 305  * @brief Callback handler, will be invoked when receive mutil-channel offer
 306  *
 307  * @param context  new multi-channel
 308  */
 309 static void
 310 storvsc_handle_sc_creation(void *context)
 311 {
 312         hv_vmbus_channel *new_channel;
 313         struct hv_device *device;
 314         struct storvsc_softc *sc;
 315         struct vmstor_chan_props props;
 316         int ret = 0;
 317
 318         new_channel = (hv_vmbus_channel *)context;
 319         device = new_channel->primary_channel->device;
 320         sc = get_stor_device(device, TRUE);
 321         if (sc == NULL)
 322                 return;
 323
 324         if (FALSE == sc->hs_open_multi_channel)
 325                 return;
 326
 327         memset(&props, 0, sizeof(props));
 328
 329         ret = hv_vmbus_channel_open(new_channel,
 330             sc->hs_drv_props->drv_ringbuffer_size,
 331             sc->hs_drv_props->drv_ringbuffer_size,
 332             (void *)&props,
 333             sizeof(struct vmstor_chan_props),
 334             hv_storvsc_on_channel_callback,
 335             new_channel);
 336
 337         return;
 338 }
 339
 340 /**
 341  * @brief Send multi-channel creation request to host
 342  *
 343  * @param device  a Hyper-V device pointer
 344  * @param max_chans  the max channels supported by vmbus
 345  */
 346 static void
 347 storvsc_send_multichannel_request(struct hv_device *dev, int max_chans)
 348 {
 349         struct storvsc_softc *sc;
 350         struct hv_storvsc_request *request;
 351         struct vstor_packet *vstor_packet;
 352         int request_channels_cnt = 0;
 353         int ret;
 354
 355         /* get multichannels count that need to create */
 356         request_channels_cnt = MIN(max_chans, mp_ncpus);
 357
 358         sc = get_stor_device(dev, TRUE);
 359         if (sc == NULL) {
 360                 printf("Storvsc_error: get sc failed while send mutilchannel "
 361                     "request\n");
 362                 return;
 363         }
 364
 365         request = &sc->hs_init_req;
 366
 367         /* Establish a handler for multi-channel */
 368         dev->channel->sc_creation_callback = storvsc_handle_sc_creation;
 369
 370         /* request the host to create multi-channel */
 371         memset(request, 0, sizeof(struct hv_storvsc_request));
 372
 373         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 374
 375         vstor_packet = &request->vstor_packet;
 376
 377         vstor_packet->operation = VSTOR_OPERATION_CREATE_MULTI_CHANNELS;
 378         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 379         vstor_packet->u.multi_channels_cnt = request_channels_cnt;
 380
 381         ret = hv_vmbus_channel_send_packet(
 382             dev->channel,
 383             vstor_packet,
 384             sizeof(struct vstor_packet),
 385             (uint64_t)(uintptr_t)request,
 386             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 387             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 388
 389         /* wait for 5 seconds */
 390         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 391         if (ret != 0) {
 392                 printf("Storvsc_error: create multi-channel timeout, %d\n",
 393                     ret);
 394                 return;
 395         }
 396
 397         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 398             vstor_packet->status != 0) {
 399                 printf("Storvsc_error: create multi-channel invalid operation "
 400                     "(%d) or statue (%u)\n",
 401                     vstor_packet->operation, vstor_packet->status);
 402                 return;
 403         }
 404
 405         sc->hs_open_multi_channel = TRUE;
 406
 407         if (bootverbose)
 408                 printf("Storvsc create multi-channel success!\n");
 409 }
 410
 411 /**
 412  * @brief initialize channel connection to parent partition
 413  *
 414  * @param dev  a Hyper-V device pointer
 415  * @returns  0 on success, non-zero error on failure
 416  */
 417 static int
 418 hv_storvsc_channel_init(struct hv_device *dev)
 419 {
 420         int ret = 0;
 421         struct hv_storvsc_request *request;
 422         struct vstor_packet *vstor_packet;
 423         struct storvsc_softc *sc;
 424         uint16_t max_chans = 0;
 425         boolean_t support_multichannel = FALSE;
 426
 427         max_chans = 0;
 428         support_multichannel = FALSE;
 429
 430         sc = get_stor_device(dev, TRUE);
 431         if (sc == NULL)
 432                 return (ENODEV);
 433
 434         request = &sc->hs_init_req;
 435         memset(request, 0, sizeof(struct hv_storvsc_request));
 436         vstor_packet = &request->vstor_packet;
 437         request->softc = sc;
 438
 439         /**
 440          * Initiate the vsc/vsp initialization protocol on the open channel
 441          */
 442         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 443
 444         vstor_packet->operation = VSTOR_OPERATION_BEGININITIALIZATION;
 445         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 446
 447
 448         ret = hv_vmbus_channel_send_packet(
 449                         dev->channel,
 450                         vstor_packet,
 451                         sizeof(struct vstor_packet),
 452                         (uint64_t)(uintptr_t)request,
 453                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 454                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 455
 456         if (ret != 0)
 457                 goto cleanup;
 458
 459         /* wait 5 seconds */
 460         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 461         if (ret != 0)
 462                 goto cleanup;
 463
 464         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 465                 vstor_packet->status != 0) {
 466                 goto cleanup;
 467         }
 468
 469         /* reuse the packet for version range supported */
 470
 471         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 472         vstor_packet->operation = VSTOR_OPERATION_QUERYPROTOCOLVERSION;
 473         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 474
 475         vstor_packet->u.version.major_minor =
 476             VMSTOR_PROTOCOL_VERSION(storvsc_current_major, storvsc_current_minor);
 477
 478         /* revision is only significant for Windows guests */
 479         vstor_packet->u.version.revision = 0;
 480
 481         ret = hv_vmbus_channel_send_packet(
 482                         dev->channel,
 483                         vstor_packet,
 484                         sizeof(struct vstor_packet),
 485                         (uint64_t)(uintptr_t)request,
 486                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 487                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 488
 489         if (ret != 0)
 490                 goto cleanup;
 491
 492         /* wait 5 seconds */
 493         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 494
 495         if (ret)
 496                 goto cleanup;
 497
 498         /* TODO: Check returned version */
 499         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 500                 vstor_packet->status != 0)
 501                 goto cleanup;
 502
 503         /**
 504          * Query channel properties
 505          */
 506         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 507         vstor_packet->operation = VSTOR_OPERATION_QUERYPROPERTIES;
 508         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 509
 510         ret = hv_vmbus_channel_send_packet(
 511                                 dev->channel,
 512                                 vstor_packet,
 513                                 sizeof(struct vstor_packet),
 514                                 (uint64_t)(uintptr_t)request,
 515                                 HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 516                                 HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 517
 518         if ( ret != 0)
 519                 goto cleanup;
 520
 521         /* wait 5 seconds */
 522         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 523
 524         if (ret != 0)
 525                 goto cleanup;
 526
 527         /* TODO: Check returned version */
 528         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 529             vstor_packet->status != 0) {
 530                 goto cleanup;
 531         }
 532
 533         /* multi-channels feature is supported by WIN8 and above version */
 534         max_chans = vstor_packet->u.chan_props.max_channel_cnt;
 535         if ((hv_vmbus_protocal_version != HV_VMBUS_VERSION_WIN7) &&
 536             (hv_vmbus_protocal_version != HV_VMBUS_VERSION_WS2008) &&
 537             (vstor_packet->u.chan_props.flags &
 538              HV_STORAGE_SUPPORTS_MULTI_CHANNEL)) {
 539                 support_multichannel = TRUE;
 540         }
 541
 542         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 543         vstor_packet->operation = VSTOR_OPERATION_ENDINITIALIZATION;
 544         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 545
 546         ret = hv_vmbus_channel_send_packet(
 547                         dev->channel,
 548                         vstor_packet,
 549                         sizeof(struct vstor_packet),
 550                         (uint64_t)(uintptr_t)request,
 551                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 552                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 553
 554         if (ret != 0) {
 555                 goto cleanup;
 556         }
 557
 558         /* wait 5 seconds */
 559         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 560
 561         if (ret != 0)
 562                 goto cleanup;
 563
 564         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 565             vstor_packet->status != 0)
 566                 goto cleanup;
 567
 568         /*
 569          * If multi-channel is supported, send multichannel create
 570          * request to host.
 571          */
 572         if (support_multichannel)
 573                 storvsc_send_multichannel_request(dev, max_chans);
 574
 575 cleanup:
 576         sema_destroy(&request->synch_sema);
 577         return (ret);
 578 }
 579
 580 /**
 581  * @brief Open channel connection to paraent partition StorVSP driver
 582  *
 583  * Open and initialize channel connection to parent partition StorVSP driver.
 584  *
 585  * @param pointer to a Hyper-V device
 586  * @returns 0 on success, non-zero error on failure
 587  */
 588 static int
 589 hv_storvsc_connect_vsp(struct hv_device *dev)
 590 {
 591         int ret = 0;
 592         struct vmstor_chan_props props;
 593         struct storvsc_softc *sc;
 594
 595         sc = device_get_softc(dev->device);
 596
 597         memset(&props, 0, sizeof(struct vmstor_chan_props));
 598
 599         /*
 600          * Open the channel
 601          */
 602
 603         ret = hv_vmbus_channel_open(
 604                 dev->channel,
 605                 sc->hs_drv_props->drv_ringbuffer_size,
 606                 sc->hs_drv_props->drv_ringbuffer_size,
 607                 (void *)&props,
 608                 sizeof(struct vmstor_chan_props),
 609                 hv_storvsc_on_channel_callback,
 610                 dev->channel);
 611
 612         if (ret != 0) {
 613                 return ret;
 614         }
 615
 616         ret = hv_storvsc_channel_init(dev);
 617
 618         return (ret);
 619 }
 620
 621 #if HVS_HOST_RESET
 622 static int
 623 hv_storvsc_host_reset(struct hv_device *dev)
 624 {
 625         int ret = 0;
 626         struct storvsc_softc *sc;
 627
 628         struct hv_storvsc_request *request;
 629         struct vstor_packet *vstor_packet;
 630
 631         sc = get_stor_device(dev, TRUE);
 632         if (sc == NULL) {
 633                 return ENODEV;
 634         }
 635
 636         request = &sc->hs_reset_req;
 637         request->softc = sc;
 638         vstor_packet = &request->vstor_packet;
 639
 640         sema_init(&request->synch_sema, 0, "stor synch sema");
 641
 642         vstor_packet->operation = VSTOR_OPERATION_RESETBUS;
 643         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 644
 645         ret = hv_vmbus_channel_send_packet(dev->channel,
 646                         vstor_packet,
 647                         sizeof(struct vstor_packet),
 648                         (uint64_t)(uintptr_t)&sc->hs_reset_req,
 649                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 650                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 651
 652         if (ret != 0) {
 653                 goto cleanup;
 654         }
 655
 656         ret = sema_timedwait(&request->synch_sema, 5 * hz); /* KYS 5 seconds */
 657
 658         if (ret) {
 659                 goto cleanup;
 660         }
 661
 662
 663         /*
 664          * At this point, all outstanding requests in the adapter
 665          * should have been flushed out and return to us
 666          */
 667
 668 cleanup:
 669         sema_destroy(&request->synch_sema);
 670         return (ret);
 671 }
 672 #endif /* HVS_HOST_RESET */
 673
 674 /**
 675  * @brief Function to initiate an I/O request
 676  *
 677  * @param device Hyper-V device pointer
 678  * @param request pointer to a request structure
 679  * @returns 0 on success, non-zero error on failure
 680  */
 681 static int
 682 hv_storvsc_io_request(struct hv_device *device,
 683                                           struct hv_storvsc_request *request)
 684 {
 685         struct storvsc_softc *sc;
 686         struct vstor_packet *vstor_packet = &request->vstor_packet;
 687         struct hv_vmbus_channel* outgoing_channel = NULL;
 688         int ret = 0;
 689
 690         sc = get_stor_device(device, TRUE);
 691
 692         if (sc == NULL) {
 693                 return ENODEV;
 694         }
 695
 696         vstor_packet->flags |= REQUEST_COMPLETION_FLAG;
 697
 698         vstor_packet->u.vm_srb.length = sizeof(struct vmscsi_req);
 699
 700         vstor_packet->u.vm_srb.sense_info_len = SENSE_BUFFER_SIZE;
 701
 702         vstor_packet->u.vm_srb.transfer_len = request->data_buf.length;
 703
 704         vstor_packet->operation = VSTOR_OPERATION_EXECUTESRB;
 705
 706         outgoing_channel = vmbus_select_outgoing_channel(device->channel);
 707
 708         mtx_unlock(&request->softc->hs_lock);
 709         if (request->data_buf.length) {
 710                 ret = hv_vmbus_channel_send_packet_multipagebuffer(
 711                                 outgoing_channel,
 712                                 &request->data_buf,
 713                                 vstor_packet,
 714                                 sizeof(struct vstor_packet),
 715                                 (uint64_t)(uintptr_t)request);
 716
 717         } else {
 718                 ret = hv_vmbus_channel_send_packet(
 719                         outgoing_channel,
 720                         vstor_packet,
 721                         sizeof(struct vstor_packet),
 722                         (uint64_t)(uintptr_t)request,
 723                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 724                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 725         }
 726         mtx_lock(&request->softc->hs_lock);
 727
 728         if (ret != 0) {
 729                 printf("Unable to send packet %p ret %d", vstor_packet, ret);
 730         } else {
 731                 atomic_add_int(&sc->hs_num_out_reqs, 1);
 732         }
 733
 734         return (ret);
 735 }
 736
 737
 738 /**
 739  * Process IO_COMPLETION_OPERATION and ready
 740  * the result to be completed for upper layer
 741  * processing by the CAM layer.
 742  */
 743 static void
 744 hv_storvsc_on_iocompletion(struct storvsc_softc *sc,
 745                            struct vstor_packet *vstor_packet,
 746                            struct hv_storvsc_request *request)
 747 {
 748         struct vmscsi_req *vm_srb;
 749
 750         vm_srb = &vstor_packet->u.vm_srb;
 751
 752         if (((vm_srb->scsi_status & 0xFF) == SCSI_STATUS_CHECK_COND) &&
 753                         (vm_srb->srb_status & SRB_STATUS_AUTOSENSE_VALID)) {
 754                 /* Autosense data available */
 755
 756                 KASSERT(vm_srb->sense_info_len <= request->sense_info_len,
 757                                 ("vm_srb->sense_info_len <= "
 758                                  "request->sense_info_len"));
 759
 760                 memcpy(request->sense_data, vm_srb->u.sense_data,
 761                         vm_srb->sense_info_len);
 762
 763                 request->sense_info_len = vm_srb->sense_info_len;
 764         }
 765
 766         /* Complete request by passing to the CAM layer */
 767         storvsc_io_done(request);
 768         atomic_subtract_int(&sc->hs_num_out_reqs, 1);
 769         if (sc->hs_drain_notify && (sc->hs_num_out_reqs == 0)) {
 770                 sema_post(&sc->hs_drain_sema);
 771         }
 772 }
 773
 774 static void
 775 hv_storvsc_on_channel_callback(void *context)
 776 {
 777         int ret = 0;
 778         hv_vmbus_channel *channel = (hv_vmbus_channel *)context;
 779         struct hv_device *device = NULL;
 780         struct storvsc_softc *sc;
 781         uint32_t bytes_recvd;
 782         uint64_t request_id;
 783         uint8_t packet[roundup2(sizeof(struct vstor_packet), 8)];
 784         struct hv_storvsc_request *request;
 785         struct vstor_packet *vstor_packet;
 786
 787         if (channel->primary_channel != NULL){
 788                 device = channel->primary_channel->device;
 789         } else {
 790                 device = channel->device;
 791         }
 792
 793         KASSERT(device, ("device is NULL"));
 794
 795         sc = get_stor_device(device, FALSE);
 796         if (sc == NULL) {
 797                 printf("Storvsc_error: get stor device failed.\n");
 798                 return;
 799         }
 800
 801         ret = hv_vmbus_channel_recv_packet(
 802                         channel,
 803                         packet,
 804                         roundup2(sizeof(struct vstor_packet), 8),
 805                         &bytes_recvd,
 806                         &request_id);
 807
 808         while ((ret == 0) && (bytes_recvd > 0)) {
 809                 request = (struct hv_storvsc_request *)(uintptr_t)request_id;
 810
 811                 if ((request == &sc->hs_init_req) ||
 812                         (request == &sc->hs_reset_req)) {
 813                         memcpy(&request->vstor_packet, packet,
 814                                    sizeof(struct vstor_packet));
 815                         sema_post(&request->synch_sema);
 816                 } else {
 817                         vstor_packet = (struct vstor_packet *)packet;
 818                         switch(vstor_packet->operation) {
 819                         case VSTOR_OPERATION_COMPLETEIO:
 820                                 if (request == NULL)
 821                                         panic("VMBUS: storvsc received a "
 822                                             "packet with NULL request id in "
 823                                             "COMPLETEIO operation.");
 824
 825                                 hv_storvsc_on_iocompletion(sc,
 826                                                         vstor_packet, request);
 827                                 break;
 828                         case VSTOR_OPERATION_REMOVEDEVICE:
 829                         case VSTOR_OPERATION_ENUMERATE_BUS:
 830                                 printf("VMBUS: storvsc operation %d not "
 831                                     "implemented.\n", vstor_packet->operation);
 832                                 /* TODO: implement */
 833                                 break;
 834                         default:
 835                                 break;
 836                         }
 837                 }
 838                 ret = hv_vmbus_channel_recv_packet(
 839                                 channel,
 840                                 packet,
 841                                 roundup2(sizeof(struct vstor_packet), 8),
 842                                 &bytes_recvd,
 843                                 &request_id);
 844         }
 845 }
 846
 847 /**
 848  * @brief StorVSC probe function
 849  *
 850  * Device probe function.  Returns 0 if the input device is a StorVSC
 851  * device.  Otherwise, a ENXIO is returned.  If the input device is
 852  * for BlkVSC (paravirtual IDE) device and this support is disabled in
 853  * favor of the emulated ATA/IDE device, return ENXIO.
 854  *
 855  * @param a device
 856  * @returns 0 on success, ENXIO if not a matcing StorVSC device
 857  */
 858 static int
 859 storvsc_probe(device_t dev)
 860 {
 861         int ata_disk_enable = 0;
 862         int ret = ENXIO;
 863
 864         if ((HV_VMBUS_VERSION_WIN8 == hv_vmbus_protocal_version) ||
 865             (HV_VMBUS_VERSION_WIN8_1 == hv_vmbus_protocal_version)){
 866                 storvsc_current_major = STORVSC_WIN8_MAJOR;
 867                 storvsc_current_minor = STORVSC_WIN8_MINOR;
 868         } else {
 869                 storvsc_current_major = STORVSC_WIN7_MAJOR;
 870                 storvsc_current_minor = STORVSC_WIN7_MINOR;
 871         }
 872
 873         switch (storvsc_get_storage_type(dev)) {
 874         case DRIVER_BLKVSC:
 875                 if(bootverbose)
 876                         device_printf(dev, "DRIVER_BLKVSC-Emulated ATA/IDE probe\n");
 877                 if (!getenv_int("hw.ata.disk_enable", &ata_disk_enable)) {
 878                         if(bootverbose)
 879                                 device_printf(dev,
 880                                         "Enlightened ATA/IDE detected\n");
 881                         ret = BUS_PROBE_DEFAULT;
 882                 } else if(bootverbose)
 883                         device_printf(dev, "Emulated ATA/IDE set (hw.ata.disk_enable set)\n");
 884                 break;
 885         case DRIVER_STORVSC:
 886                 if(bootverbose)
 887                         device_printf(dev, "Enlightened SCSI device detected\n");
 888                 ret = BUS_PROBE_DEFAULT;
 889                 break;
 890         default:
 891                 ret = ENXIO;
 892         }
 893         return (ret);
 894 }
 895
 896 /**
 897  * @brief StorVSC attach function
 898  *
 899  * Function responsible for allocating per-device structures,
 900  * setting up CAM interfaces and scanning for available LUNs to
 901  * be used for SCSI device peripherals.
 902  *
 903  * @param a device
 904  * @returns 0 on success or an error on failure
 905  */
 906 static int
 907 storvsc_attach(device_t dev)
 908 {
 909         struct hv_device *hv_dev = vmbus_get_devctx(dev);
 910         enum hv_storage_type stor_type;
 911         struct storvsc_softc *sc;
 912         struct cam_devq *devq;
 913         int ret, i, j;
 914         struct hv_storvsc_request *reqp;
 915         struct root_hold_token *root_mount_token = NULL;
 916         struct hv_sgl_node *sgl_node = NULL;
 917         void *tmp_buff = NULL;
 918
 919         /*
 920          * We need to serialize storvsc attach calls.
 921          */
 922         root_mount_token = root_mount_hold("storvsc");
 923
 924         sc = device_get_softc(dev);
 925         if (sc == NULL) {
 926                 ret = ENOMEM;
 927                 goto cleanup;
 928         }
 929
 930         stor_type = storvsc_get_storage_type(dev);
 931
 932         if (stor_type == DRIVER_UNKNOWN) {
 933                 ret = ENODEV;
 934                 goto cleanup;
 935         }
 936
 937         bzero(sc, sizeof(struct storvsc_softc));
 938
 939         /* fill in driver specific properties */
 940         sc->hs_drv_props = &g_drv_props_table[stor_type];
 941
 942         /* fill in device specific properties */
 943         sc->hs_unit     = device_get_unit(dev);
 944         sc->hs_dev      = hv_dev;
 945         device_set_desc(dev, g_drv_props_table[stor_type].drv_desc);
 946
 947         LIST_INIT(&sc->hs_free_list);
 948         mtx_init(&sc->hs_lock, "hvslck", NULL, MTX_DEF);
 949
 950         for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; ++i) {
 951                 reqp = malloc(sizeof(struct hv_storvsc_request),
 952                                  M_DEVBUF, M_WAITOK|M_ZERO);
 953                 reqp->softc = sc;
 954
 955                 LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
 956         }
 957
 958         /* create sg-list page pool */
 959         if (FALSE == g_hv_sgl_page_pool.is_init) {
 960                 g_hv_sgl_page_pool.is_init = TRUE;
 961                 LIST_INIT(&g_hv_sgl_page_pool.in_use_sgl_list);
 962                 LIST_INIT(&g_hv_sgl_page_pool.free_sgl_list);
 963
 964                 /*
 965                  * Pre-create SG list, each SG list with
 966                  * HV_MAX_MULTIPAGE_BUFFER_COUNT segments, each
 967                  * segment has one page buffer
 968                  */
 969                 for (i = 0; i < STORVSC_MAX_IO_REQUESTS; i++) {
 970                         sgl_node = malloc(sizeof(struct hv_sgl_node),
 971                             M_DEVBUF, M_WAITOK|M_ZERO);
 972
 973                         sgl_node->sgl_data =
 974                             sglist_alloc(HV_MAX_MULTIPAGE_BUFFER_COUNT,
 975                             M_WAITOK|M_ZERO);
 976
 977                         for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
 978                                 tmp_buff = malloc(PAGE_SIZE,
 979                                     M_DEVBUF, M_WAITOK|M_ZERO);
 980
 981                                 sgl_node->sgl_data->sg_segs[j].ss_paddr =
 982                                     (vm_paddr_t)tmp_buff;
 983                         }
 984
 985                         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list,
 986                             sgl_node, link);
 987                 }
 988         }
 989
 990         sc->hs_destroy = FALSE;
 991         sc->hs_drain_notify = FALSE;
 992         sc->hs_open_multi_channel = FALSE;
 993         sema_init(&sc->hs_drain_sema, 0, "Store Drain Sema");
 994
 995         ret = hv_storvsc_connect_vsp(hv_dev);
 996         if (ret != 0) {
 997                 goto cleanup;
 998         }
 999
1000         /*
1001          * Create the device queue.
1002          * Hyper-V maps each target to one SCSI HBA
1003          */
1004         devq = cam_simq_alloc(sc->hs_drv_props->drv_max_ios_per_target);
1005         if (devq == NULL) {
1006                 device_printf(dev, "Failed to alloc device queue\n");
1007                 ret = ENOMEM;
1008                 goto cleanup;
1009         }
1010
1011         sc->hs_sim = cam_sim_alloc(storvsc_action,
1012                                 storvsc_poll,
1013                                 sc->hs_drv_props->drv_name,
1014                                 sc,
1015                                 sc->hs_unit,
1016                                 &sc->hs_lock, 1,
1017                                 sc->hs_drv_props->drv_max_ios_per_target,
1018                                 devq);
1019
1020         if (sc->hs_sim == NULL) {
1021                 device_printf(dev, "Failed to alloc sim\n");
1022                 cam_simq_free(devq);
1023                 ret = ENOMEM;
1024                 goto cleanup;
1025         }
1026
1027         mtx_lock(&sc->hs_lock);
1028         /* bus_id is set to 0, need to get it from VMBUS channel query? */
1029         if (xpt_bus_register(sc->hs_sim, dev, 0) != CAM_SUCCESS) {
1030                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1031                 mtx_unlock(&sc->hs_lock);
1032                 device_printf(dev, "Unable to register SCSI bus\n");
1033                 ret = ENXIO;
1034                 goto cleanup;
1035         }
1036
1037         if (xpt_create_path(&sc->hs_path, /*periph*/NULL,
1038                  cam_sim_path(sc->hs_sim),
1039                 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1040                 xpt_bus_deregister(cam_sim_path(sc->hs_sim));
1041                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1042                 mtx_unlock(&sc->hs_lock);
1043                 device_printf(dev, "Unable to create path\n");
1044                 ret = ENXIO;
1045                 goto cleanup;
1046         }
1047
1048         mtx_unlock(&sc->hs_lock);
1049
1050         root_mount_rel(root_mount_token);
1051         return (0);
1052
1053
1054 cleanup:
1055         root_mount_rel(root_mount_token);
1056         while (!LIST_EMPTY(&sc->hs_free_list)) {
1057                 reqp = LIST_FIRST(&sc->hs_free_list);
1058                 LIST_REMOVE(reqp, link);
1059                 free(reqp, M_DEVBUF);
1060         }
1061
1062         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1063                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1064                 LIST_REMOVE(sgl_node, link);
1065                 for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
1066                         if (NULL !=
1067                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1068                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1069                         }
1070                 }
1071                 sglist_free(sgl_node->sgl_data);
1072                 free(sgl_node, M_DEVBUF);
1073         }
1074
1075         return (ret);
1076 }
1077
1078 /**
1079  * @brief StorVSC device detach function
1080  *
1081  * This function is responsible for safely detaching a
1082  * StorVSC device.  This includes waiting for inbound responses
1083  * to complete and freeing associated per-device structures.
1084  *
1085  * @param dev a device
1086  * returns 0 on success
1087  */
1088 static int
1089 storvsc_detach(device_t dev)
1090 {
1091         struct storvsc_softc *sc = device_get_softc(dev);
1092         struct hv_storvsc_request *reqp = NULL;
1093         struct hv_device *hv_device = vmbus_get_devctx(dev);
1094         struct hv_sgl_node *sgl_node = NULL;
1095         int j = 0;
1096
1097         mtx_lock(&hv_device->channel->inbound_lock);
1098         sc->hs_destroy = TRUE;
1099         mtx_unlock(&hv_device->channel->inbound_lock);
1100
1101         /*
1102          * At this point, all outbound traffic should be disabled. We
1103          * only allow inbound traffic (responses) to proceed so that
1104          * outstanding requests can be completed.
1105          */
1106
1107         sc->hs_drain_notify = TRUE;
1108         sema_wait(&sc->hs_drain_sema);
1109         sc->hs_drain_notify = FALSE;
1110
1111         /*
1112          * Since we have already drained, we don't need to busy wait.
1113          * The call to close the channel will reset the callback
1114          * under the protection of the incoming channel lock.
1115          */
1116
1117         hv_vmbus_channel_close(hv_device->channel);
1118
1119         mtx_lock(&sc->hs_lock);
1120         while (!LIST_EMPTY(&sc->hs_free_list)) {
1121                 reqp = LIST_FIRST(&sc->hs_free_list);
1122                 LIST_REMOVE(reqp, link);
1123
1124                 free(reqp, M_DEVBUF);
1125         }
1126         mtx_unlock(&sc->hs_lock);
1127
1128         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1129                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1130                 LIST_REMOVE(sgl_node, link);
1131                 for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++){
1132                         if (NULL !=
1133                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1134                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1135                         }
1136                 }
1137                 sglist_free(sgl_node->sgl_data);
1138                 free(sgl_node, M_DEVBUF);
1139         }
1140
1141         return (0);
1142 }
1143
1144 #if HVS_TIMEOUT_TEST
1145 /**
1146  * @brief unit test for timed out operations
1147  *
1148  * This function provides unit testing capability to simulate
1149  * timed out operations.  Recompilation with HV_TIMEOUT_TEST=1
1150  * is required.
1151  *
1152  * @param reqp pointer to a request structure
1153  * @param opcode SCSI operation being performed
1154  * @param wait if 1, wait for I/O to complete
1155  */
1156 static void
1157 storvsc_timeout_test(struct hv_storvsc_request *reqp,
1158                 uint8_t opcode, int wait)
1159 {
1160         int ret;
1161         union ccb *ccb = reqp->ccb;
1162         struct storvsc_softc *sc = reqp->softc;
1163
1164         if (reqp->vstor_packet.vm_srb.cdb[0] != opcode) {
1165                 return;
1166         }
1167
1168         if (wait) {
1169                 mtx_lock(&reqp->event.mtx);
1170         }
1171         ret = hv_storvsc_io_request(sc->hs_dev, reqp);
1172         if (ret != 0) {
1173                 if (wait) {
1174                         mtx_unlock(&reqp->event.mtx);
1175                 }
1176                 printf("%s: io_request failed with %d.\n",
1177                                 __func__, ret);
1178                 ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1179                 mtx_lock(&sc->hs_lock);
1180                 storvsc_free_request(sc, reqp);
1181                 xpt_done(ccb);
1182                 mtx_unlock(&sc->hs_lock);
1183                 return;
1184         }
1185
1186         if (wait) {
1187                 xpt_print(ccb->ccb_h.path,
1188                                 "%u: %s: waiting for IO return.\n",
1189                                 ticks, __func__);
1190                 ret = cv_timedwait(&reqp->event.cv, &reqp->event.mtx, 60*hz);
1191                 mtx_unlock(&reqp->event.mtx);
1192                 xpt_print(ccb->ccb_h.path, "%u: %s: %s.\n",
1193                                 ticks, __func__, (ret == 0)?
1194                                 "IO return detected" :
1195                                 "IO return not detected");
1196                 /*
1197                  * Now both the timer handler and io done are running
1198                  * simultaneously. We want to confirm the io done always
1199                  * finishes after the timer handler exits. So reqp used by
1200                  * timer handler is not freed or stale. Do busy loop for
1201                  * another 1/10 second to make sure io done does
1202                  * wait for the timer handler to complete.
1203                  */
1204                 DELAY(100*1000);
1205                 mtx_lock(&sc->hs_lock);
1206                 xpt_print(ccb->ccb_h.path,
1207                                 "%u: %s: finishing, queue frozen %d, "
1208                                 "ccb status 0x%x scsi_status 0x%x.\n",
1209                                 ticks, __func__, sc->hs_frozen,
1210                                 ccb->ccb_h.status,
1211                                 ccb->csio.scsi_status);
1212                 mtx_unlock(&sc->hs_lock);
1213         }
1214 }
1215 #endif /* HVS_TIMEOUT_TEST */
1216
1217 /**
1218  * @brief timeout handler for requests
1219  *
1220  * This function is called as a result of a callout expiring.
1221  *
1222  * @param arg pointer to a request
1223  */
1224 static void
1225 storvsc_timeout(void *arg)
1226 {
1227         struct hv_storvsc_request *reqp = arg;
1228         struct storvsc_softc *sc = reqp->softc;
1229         union ccb *ccb = reqp->ccb;
1230
1231         if (reqp->retries == 0) {
1232                 mtx_lock(&sc->hs_lock);
1233                 xpt_print(ccb->ccb_h.path,
1234                     "%u: IO timed out (req=0x%p), wait for another %u secs.\n",
1235                     ticks, reqp, ccb->ccb_h.timeout / 1000);
1236                 cam_error_print(ccb, CAM_ESF_ALL, CAM_EPF_ALL);
1237                 mtx_unlock(&sc->hs_lock);
1238
1239                 reqp->retries++;
1240                 callout_reset_sbt(&reqp->callout, SBT_1MS * ccb->ccb_h.timeout,
1241                     0, storvsc_timeout, reqp, 0);
1242 #if HVS_TIMEOUT_TEST
1243                 storvsc_timeout_test(reqp, SEND_DIAGNOSTIC, 0);
1244 #endif
1245                 return;
1246         }
1247
1248         mtx_lock(&sc->hs_lock);
1249         xpt_print(ccb->ccb_h.path,
1250                 "%u: IO (reqp = 0x%p) did not return for %u seconds, %s.\n",
1251                 ticks, reqp, ccb->ccb_h.timeout * (reqp->retries+1) / 1000,
1252                 (sc->hs_frozen == 0)?
1253                 "freezing the queue" : "the queue is already frozen");
1254         if (sc->hs_frozen == 0) {
1255                 sc->hs_frozen = 1;
1256                 xpt_freeze_simq(xpt_path_sim(ccb->ccb_h.path), 1);
1257         }
1258         mtx_unlock(&sc->hs_lock);
1259
1260 #if HVS_TIMEOUT_TEST
1261         storvsc_timeout_test(reqp, MODE_SELECT_10, 1);
1262 #endif
1263 }
1264
1265 /**
1266  * @brief StorVSC device poll function
1267  *
1268  * This function is responsible for servicing requests when
1269  * interrupts are disabled (i.e when we are dumping core.)
1270  *
1271  * @param sim a pointer to a CAM SCSI interface module
1272  */
1273 static void
1274 storvsc_poll(struct cam_sim *sim)
1275 {
1276         struct storvsc_softc *sc = cam_sim_softc(sim);
1277
1278         mtx_assert(&sc->hs_lock, MA_OWNED);
1279         mtx_unlock(&sc->hs_lock);
1280         hv_storvsc_on_channel_callback(sc->hs_dev->channel);
1281         mtx_lock(&sc->hs_lock);
1282 }
1283
1284 /**
1285  * @brief StorVSC device action function
1286  *
1287  * This function is responsible for handling SCSI operations which
1288  * are passed from the CAM layer.  The requests are in the form of
1289  * CAM control blocks which indicate the action being performed.
1290  * Not all actions require converting the request to a VSCSI protocol
1291  * message - these actions can be responded to by this driver.
1292  * Requests which are destined for a backend storage device are converted
1293  * to a VSCSI protocol message and sent on the channel connection associated
1294  * with this device.
1295  *
1296  * @param sim pointer to a CAM SCSI interface module
1297  * @param ccb pointer to a CAM control block
1298  */
1299 static void
1300 storvsc_action(struct cam_sim *sim, union ccb *ccb)
1301 {
1302         struct storvsc_softc *sc = cam_sim_softc(sim);
1303         int res;
1304
1305         mtx_assert(&sc->hs_lock, MA_OWNED);
1306         switch (ccb->ccb_h.func_code) {
1307         case XPT_PATH_INQ: {
1308                 struct ccb_pathinq *cpi = &ccb->cpi;
1309
1310                 cpi->version_num = 1;
1311                 cpi->hba_inquiry = PI_TAG_ABLE|PI_SDTR_ABLE;
1312                 cpi->target_sprt = 0;
1313                 cpi->hba_misc = PIM_NOBUSRESET;
1314                 cpi->hba_eng_cnt = 0;
1315                 cpi->max_target = STORVSC_MAX_TARGETS;
1316                 cpi->max_lun = sc->hs_drv_props->drv_max_luns_per_target;
1317                 cpi->initiator_id = cpi->max_target;
1318                 cpi->bus_id = cam_sim_bus(sim);
1319                 cpi->base_transfer_speed = 300000;
1320                 cpi->transport = XPORT_SAS;
1321                 cpi->transport_version = 0;
1322                 cpi->protocol = PROTO_SCSI;
1323                 cpi->protocol_version = SCSI_REV_SPC2;
1324                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1325                 strncpy(cpi->hba_vid, sc->hs_drv_props->drv_name, HBA_IDLEN);
1326                 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1327                 cpi->unit_number = cam_sim_unit(sim);
1328
1329                 ccb->ccb_h.status = CAM_REQ_CMP;
1330                 xpt_done(ccb);
1331                 return;
1332         }
1333         case XPT_GET_TRAN_SETTINGS: {
1334                 struct  ccb_trans_settings *cts = &ccb->cts;
1335
1336                 cts->transport = XPORT_SAS;
1337                 cts->transport_version = 0;
1338                 cts->protocol = PROTO_SCSI;
1339                 cts->protocol_version = SCSI_REV_SPC2;
1340
1341                 /* enable tag queuing and disconnected mode */
1342                 cts->proto_specific.valid = CTS_SCSI_VALID_TQ;
1343                 cts->proto_specific.scsi.valid = CTS_SCSI_VALID_TQ;
1344                 cts->proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB;
1345                 cts->xport_specific.valid = CTS_SPI_VALID_DISC;
1346                 cts->xport_specific.spi.flags = CTS_SPI_FLAGS_DISC_ENB;
1347
1348                 ccb->ccb_h.status = CAM_REQ_CMP;
1349                 xpt_done(ccb);
1350                 return;
1351         }
1352         case XPT_SET_TRAN_SETTINGS:     {
1353                 ccb->ccb_h.status = CAM_REQ_CMP;
1354                 xpt_done(ccb);
1355                 return;
1356         }
1357         case XPT_CALC_GEOMETRY:{
1358                 cam_calc_geometry(&ccb->ccg, 1);
1359                 xpt_done(ccb);
1360                 return;
1361         }
1362         case  XPT_RESET_BUS:
1363         case  XPT_RESET_DEV:{
1364 #if HVS_HOST_RESET
1365                 if ((res = hv_storvsc_host_reset(sc->hs_dev)) != 0) {
1366                         xpt_print(ccb->ccb_h.path,
1367                                 "hv_storvsc_host_reset failed with %d\n", res);
1368                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1369                         xpt_done(ccb);
1370                         return;
1371                 }
1372                 ccb->ccb_h.status = CAM_REQ_CMP;
1373                 xpt_done(ccb);
1374                 return;
1375 #else
1376                 xpt_print(ccb->ccb_h.path,
1377                                   "%s reset not supported.\n",
1378                                   (ccb->ccb_h.func_code == XPT_RESET_BUS)?
1379                                   "bus" : "dev");
1380                 ccb->ccb_h.status = CAM_REQ_INVALID;
1381                 xpt_done(ccb);
1382                 return;
1383 #endif  /* HVS_HOST_RESET */
1384         }
1385         case XPT_SCSI_IO:
1386         case XPT_IMMED_NOTIFY: {
1387                 struct hv_storvsc_request *reqp = NULL;
1388
1389                 if (ccb->csio.cdb_len == 0) {
1390                         panic("cdl_len is 0\n");
1391                 }
1392
1393                 if (LIST_EMPTY(&sc->hs_free_list)) {
1394                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
1395                         if (sc->hs_frozen == 0) {
1396                                 sc->hs_frozen = 1;
1397                                 xpt_freeze_simq(sim, /* count*/1);
1398                         }
1399                         xpt_done(ccb);
1400                         return;
1401                 }
1402
1403                 reqp = LIST_FIRST(&sc->hs_free_list);
1404                 LIST_REMOVE(reqp, link);
1405
1406                 bzero(reqp, sizeof(struct hv_storvsc_request));
1407                 reqp->softc = sc;
1408
1409                 ccb->ccb_h.status |= CAM_SIM_QUEUED;
1410                 if ((res = create_storvsc_request(ccb, reqp)) != 0) {
1411                         ccb->ccb_h.status = CAM_REQ_INVALID;
1412                         xpt_done(ccb);
1413                         return;
1414                 }
1415
1416                 if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
1417                         callout_init(&reqp->callout, CALLOUT_MPSAFE);
1418                         callout_reset_sbt(&reqp->callout,
1419                             SBT_1MS * ccb->ccb_h.timeout, 0,
1420                             storvsc_timeout, reqp, 0);
1421 #if HVS_TIMEOUT_TEST
1422                         cv_init(&reqp->event.cv, "storvsc timeout cv");
1423                         mtx_init(&reqp->event.mtx, "storvsc timeout mutex",
1424                                         NULL, MTX_DEF);
1425                         switch (reqp->vstor_packet.vm_srb.cdb[0]) {
1426                                 case MODE_SELECT_10:
1427                                 case SEND_DIAGNOSTIC:
1428                                         /* To have timer send the request. */
1429                                         return;
1430                                 default:
1431                                         break;
1432                         }
1433 #endif /* HVS_TIMEOUT_TEST */
1434                 }
1435
1436                 if ((res = hv_storvsc_io_request(sc->hs_dev, reqp)) != 0) {
1437                         xpt_print(ccb->ccb_h.path,
1438                                 "hv_storvsc_io_request failed with %d\n", res);
1439                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1440                         storvsc_free_request(sc, reqp);
1441                         xpt_done(ccb);
1442                         return;
1443                 }
1444                 return;
1445         }
1446
1447         default:
1448                 ccb->ccb_h.status = CAM_REQ_INVALID;
1449                 xpt_done(ccb);
1450                 return;
1451         }
1452 }
1453
1454 /**
1455  * @brief destroy bounce buffer
1456  *
1457  * This function is responsible for destroy a Scatter/Gather list
1458  * that create by storvsc_create_bounce_buffer()
1459  *
1460  * @param sgl- the Scatter/Gather need be destroy
1461  * @param sg_count- page count of the SG list.
1462  *
1463  */
1464 static void
1465 storvsc_destroy_bounce_buffer(struct sglist *sgl)
1466 {
1467         struct hv_sgl_node *sgl_node = NULL;
1468
1469         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.in_use_sgl_list);
1470         LIST_REMOVE(sgl_node, link);
1471         if (NULL == sgl_node) {
1472                 printf("storvsc error: not enough in use sgl\n");
1473                 return;
1474         }
1475         sgl_node->sgl_data = sgl;
1476         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list, sgl_node, link);
1477 }
1478
1479 /**
1480  * @brief create bounce buffer
1481  *
1482  * This function is responsible for create a Scatter/Gather list,
1483  * which hold several pages that can be aligned with page size.
1484  *
1485  * @param seg_count- SG-list segments count
1486  * @param write - if WRITE_TYPE, set SG list page used size to 0,
1487  * otherwise set used size to page size.
1488  *
1489  * return NULL if create failed
1490  */
1491 static struct sglist *
1492 storvsc_create_bounce_buffer(uint16_t seg_count, int write)
1493 {
1494         int i = 0;
1495         struct sglist *bounce_sgl = NULL;
1496         unsigned int buf_len = ((write == WRITE_TYPE) ? 0 : PAGE_SIZE);
1497         struct hv_sgl_node *sgl_node = NULL;
1498
1499         /* get struct sglist from free_sgl_list */
1500         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1501         LIST_REMOVE(sgl_node, link);
1502         if (NULL == sgl_node) {
1503                 printf("storvsc error: not enough free sgl\n");
1504                 return NULL;
1505         }
1506         bounce_sgl = sgl_node->sgl_data;
1507         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.in_use_sgl_list, sgl_node, link);
1508
1509         bounce_sgl->sg_maxseg = seg_count;
1510
1511         if (write == WRITE_TYPE)
1512                 bounce_sgl->sg_nseg = 0;
1513         else
1514                 bounce_sgl->sg_nseg = seg_count;
1515
1516         for (i = 0; i < seg_count; i++)
1517                 bounce_sgl->sg_segs[i].ss_len = buf_len;
1518
1519         return bounce_sgl;
1520 }
1521
1522 /**
1523  * @brief copy data from SG list to bounce buffer
1524  *
1525  * This function is responsible for copy data from one SG list's segments
1526  * to another SG list which used as bounce buffer.
1527  *
1528  * @param bounce_sgl - the destination SG list
1529  * @param orig_sgl - the segment of the source SG list.
1530  * @param orig_sgl_count - the count of segments.
1531  * @param orig_sgl_count - indicate which segment need bounce buffer,
1532  *  set 1 means need.
1533  *
1534  */
1535 static void
1536 storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
1537                                bus_dma_segment_t *orig_sgl,
1538                                unsigned int orig_sgl_count,
1539                                uint64_t seg_bits)
1540 {
1541         int src_sgl_idx = 0;
1542
1543         for (src_sgl_idx = 0; src_sgl_idx < orig_sgl_count; src_sgl_idx++) {
1544                 if (seg_bits & (1 << src_sgl_idx)) {
1545                         memcpy((void*)bounce_sgl->sg_segs[src_sgl_idx].ss_paddr,
1546                             (void*)orig_sgl[src_sgl_idx].ds_addr,
1547                             orig_sgl[src_sgl_idx].ds_len);
1548
1549                         bounce_sgl->sg_segs[src_sgl_idx].ss_len =
1550                             orig_sgl[src_sgl_idx].ds_len;
1551                 }
1552         }
1553 }
1554
1555 /**
1556  * @brief copy data from SG list which used as bounce to another SG list
1557  *
1558  * This function is responsible for copy data from one SG list with bounce
1559  * buffer to another SG list's segments.
1560  *
1561  * @param dest_sgl - the destination SG list's segments
1562  * @param dest_sgl_count - the count of destination SG list's segment.
1563  * @param src_sgl - the source SG list.
1564  * @param seg_bits - indicate which segment used bounce buffer of src SG-list.
1565  *
1566  */
1567 void
1568 storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
1569                                     unsigned int dest_sgl_count,
1570                                     struct sglist* src_sgl,
1571                                     uint64_t seg_bits)
1572 {
1573         int sgl_idx = 0;
1574
1575         for (sgl_idx = 0; sgl_idx < dest_sgl_count; sgl_idx++) {
1576                 if (seg_bits & (1 << sgl_idx)) {
1577                         memcpy((void*)(dest_sgl[sgl_idx].ds_addr),
1578                             (void*)(src_sgl->sg_segs[sgl_idx].ss_paddr),
1579                             src_sgl->sg_segs[sgl_idx].ss_len);
1580                 }
1581         }
1582 }
1583
1584 /**
1585  * @brief check SG list with bounce buffer or not
1586  *
1587  * This function is responsible for check if need bounce buffer for SG list.
1588  *
1589  * @param sgl - the SG list's segments
1590  * @param sg_count - the count of SG list's segment.
1591  * @param bits - segmengs number that need bounce buffer
1592  *
1593  * return -1 if SG list needless bounce buffer
1594  */
1595 static int
1596 storvsc_check_bounce_buffer_sgl(bus_dma_segment_t *sgl,
1597                                 unsigned int sg_count,
1598                                 uint64_t *bits)
1599 {
1600         int i = 0;
1601         int offset = 0;
1602         uint64_t phys_addr = 0;
1603         uint64_t tmp_bits = 0;
1604         boolean_t found_hole = FALSE;
1605         boolean_t pre_aligned = TRUE;
1606
1607         if (sg_count < 2){
1608                 return -1;
1609         }
1610
1611         *bits = 0;
1612
1613         phys_addr = vtophys(sgl[0].ds_addr);
1614         offset =  phys_addr - trunc_page(phys_addr);
1615
1616         if (offset != 0) {
1617                 pre_aligned = FALSE;
1618                 tmp_bits |= 1;
1619         }
1620
1621         for (i = 1; i < sg_count; i++) {
1622                 phys_addr = vtophys(sgl[i].ds_addr);
1623                 offset =  phys_addr - trunc_page(phys_addr);
1624
1625                 if (offset == 0) {
1626                         if (FALSE == pre_aligned){
1627                                 /*
1628                                  * This segment is aligned, if the previous
1629                                  * one is not aligned, find a hole
1630                                  */
1631                                 found_hole = TRUE;
1632                         }
1633                         pre_aligned = TRUE;
1634                 } else {
1635                         tmp_bits |= 1 << i;
1636                         if (!pre_aligned) {
1637                                 if (phys_addr != vtophys(sgl[i-1].ds_addr +
1638                                     sgl[i-1].ds_len)) {
1639                                         /*
1640                                          * Check whether connect to previous
1641                                          * segment,if not, find the hole
1642                                          */
1643                                         found_hole = TRUE;
1644                                 }
1645                         } else {
1646                                 found_hole = TRUE;
1647                         }
1648                         pre_aligned = FALSE;
1649                 }
1650         }
1651
1652         if (!found_hole) {
1653                 return (-1);
1654         } else {
1655                 *bits = tmp_bits;
1656                 return 0;
1657         }
1658 }
1659
1660 /**
1661  * @brief Fill in a request structure based on a CAM control block
1662  *
1663  * Fills in a request structure based on the contents of a CAM control
1664  * block.  The request structure holds the payload information for
1665  * VSCSI protocol request.
1666  *
1667  * @param ccb pointer to a CAM contorl block
1668  * @param reqp pointer to a request structure
1669  */
1670 static int
1671 create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
1672 {
1673         struct ccb_scsiio *csio = &ccb->csio;
1674         uint64_t phys_addr;
1675         uint32_t bytes_to_copy = 0;
1676         uint32_t pfn_num = 0;
1677         uint32_t pfn;
1678         uint64_t not_aligned_seg_bits = 0;
1679
1680         /* refer to struct vmscsi_req for meanings of these two fields */
1681         reqp->vstor_packet.u.vm_srb.port =
1682                 cam_sim_unit(xpt_path_sim(ccb->ccb_h.path));
1683         reqp->vstor_packet.u.vm_srb.path_id =
1684                 cam_sim_bus(xpt_path_sim(ccb->ccb_h.path));
1685
1686         reqp->vstor_packet.u.vm_srb.target_id = ccb->ccb_h.target_id;
1687         reqp->vstor_packet.u.vm_srb.lun = ccb->ccb_h.target_lun;
1688
1689         reqp->vstor_packet.u.vm_srb.cdb_len = csio->cdb_len;
1690         if(ccb->ccb_h.flags & CAM_CDB_POINTER) {
1691                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_ptr,
1692                         csio->cdb_len);
1693         } else {
1694                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_bytes,
1695                         csio->cdb_len);
1696         }
1697
1698         switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
1699         case CAM_DIR_OUT:
1700                 reqp->vstor_packet.u.vm_srb.data_in = WRITE_TYPE;
1701                 break;
1702         case CAM_DIR_IN:
1703                 reqp->vstor_packet.u.vm_srb.data_in = READ_TYPE;
1704                 break;
1705         case CAM_DIR_NONE:
1706                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1707                 break;
1708         default:
1709                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1710                 break;
1711         }
1712
1713         reqp->sense_data     = &csio->sense_data;
1714         reqp->sense_info_len = csio->sense_len;
1715
1716         reqp->ccb = ccb;
1717
1718         if (0 == csio->dxfer_len) {
1719                 return (0);
1720         }
1721
1722         reqp->data_buf.length = csio->dxfer_len;
1723
1724         switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
1725         case CAM_DATA_VADDR:
1726         {
1727                 bytes_to_copy = csio->dxfer_len;
1728                 phys_addr = vtophys(csio->data_ptr);
1729                 reqp->data_buf.offset = phys_addr & PAGE_MASK;
1730
1731                 while (bytes_to_copy != 0) {
1732                         int bytes, page_offset;
1733                         phys_addr =
1734                             vtophys(&csio->data_ptr[reqp->data_buf.length -
1735                             bytes_to_copy]);
1736                         pfn = phys_addr >> PAGE_SHIFT;
1737                         reqp->data_buf.pfn_array[pfn_num] = pfn;
1738                         page_offset = phys_addr & PAGE_MASK;
1739
1740                         bytes = min(PAGE_SIZE - page_offset, bytes_to_copy);
1741
1742                         bytes_to_copy -= bytes;
1743                         pfn_num++;
1744                 }
1745                 break;
1746         }
1747
1748         case CAM_DATA_SG:
1749         {
1750                 int i = 0;
1751                 int offset = 0;
1752                 int ret;
1753
1754                 bus_dma_segment_t *storvsc_sglist =
1755                     (bus_dma_segment_t *)ccb->csio.data_ptr;
1756                 u_int16_t storvsc_sg_count = ccb->csio.sglist_cnt;
1757
1758                 printf("Storvsc: get SG I/O operation, %d\n",
1759                     reqp->vstor_packet.u.vm_srb.data_in);
1760
1761                 if (storvsc_sg_count > HV_MAX_MULTIPAGE_BUFFER_COUNT){
1762                         printf("Storvsc: %d segments is too much, "
1763                             "only support %d segments\n",
1764                             storvsc_sg_count, HV_MAX_MULTIPAGE_BUFFER_COUNT);
1765                         return (EINVAL);
1766                 }
1767
1768                 /*
1769                  * We create our own bounce buffer function currently. Idealy
1770                  * we should use BUS_DMA(9) framework. But with current BUS_DMA
1771                  * code there is no callback API to check the page alignment of
1772                  * middle segments before busdma can decide if a bounce buffer
1773                  * is needed for particular segment. There is callback,
1774                  * "bus_dma_filter_t *filter", but the parrameters are not
1775                  * sufficient for storvsc driver.
1776                  * TODO:
1777                  *      Add page alignment check in BUS_DMA(9) callback. Once
1778                  *      this is complete, switch the following code to use
1779                  *      BUS_DMA(9) for storvsc bounce buffer support.
1780                  */
1781                 /* check if we need to create bounce buffer */
1782                 ret = storvsc_check_bounce_buffer_sgl(storvsc_sglist,
1783                     storvsc_sg_count, &not_aligned_seg_bits);
1784                 if (ret != -1) {
1785                         reqp->bounce_sgl =
1786                             storvsc_create_bounce_buffer(storvsc_sg_count,
1787                             reqp->vstor_packet.u.vm_srb.data_in);
1788                         if (NULL == reqp->bounce_sgl) {
1789                                 printf("Storvsc_error: "
1790                                     "create bounce buffer failed.\n");
1791                                 return (ENOMEM);
1792                         }
1793
1794                         reqp->bounce_sgl_count = storvsc_sg_count;
1795                         reqp->not_aligned_seg_bits = not_aligned_seg_bits;
1796
1797                         /*
1798                          * if it is write, we need copy the original data
1799                          *to bounce buffer
1800                          */
1801                         if (WRITE_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
1802                                 storvsc_copy_sgl_to_bounce_buf(
1803                                     reqp->bounce_sgl,
1804                                     storvsc_sglist,
1805                                     storvsc_sg_count,
1806                                     reqp->not_aligned_seg_bits);
1807                         }
1808
1809                         /* transfer virtual address to physical frame number */
1810                         if (reqp->not_aligned_seg_bits & 0x1){
1811                                 phys_addr =
1812                                     vtophys(reqp->bounce_sgl->sg_segs[0].ss_paddr);
1813                         }else{
1814                                 phys_addr =
1815                                         vtophys(storvsc_sglist[0].ds_addr);
1816                         }
1817                         reqp->data_buf.offset = phys_addr & PAGE_MASK;
1818
1819                         pfn = phys_addr >> PAGE_SHIFT;
1820                         reqp->data_buf.pfn_array[0] = pfn;
1821
1822                         for (i = 1; i < storvsc_sg_count; i++) {
1823                                 if (reqp->not_aligned_seg_bits & (1 << i)) {
1824                                         phys_addr =
1825                                             vtophys(reqp->bounce_sgl->sg_segs[i].ss_paddr);
1826                                 } else {
1827                                         phys_addr =
1828                                             vtophys(storvsc_sglist[i].ds_addr);
1829                                 }
1830
1831                                 pfn = phys_addr >> PAGE_SHIFT;
1832                                 reqp->data_buf.pfn_array[i] = pfn;
1833                         }
1834                 } else {
1835                         phys_addr = vtophys(storvsc_sglist[0].ds_addr);
1836
1837                         reqp->data_buf.offset = phys_addr & PAGE_MASK;
1838
1839                         for (i = 0; i < storvsc_sg_count; i++) {
1840                                 phys_addr = vtophys(storvsc_sglist[i].ds_addr);
1841                                 pfn = phys_addr >> PAGE_SHIFT;
1842                                 reqp->data_buf.pfn_array[i] = pfn;
1843                         }
1844
1845                         /* check the last segment cross boundary or not */
1846                         offset = phys_addr & PAGE_MASK;
1847                         if (offset) {
1848                                 phys_addr =
1849                                     vtophys(storvsc_sglist[i-1].ds_addr +
1850                                     PAGE_SIZE - offset);
1851                                 pfn = phys_addr >> PAGE_SHIFT;
1852                                 reqp->data_buf.pfn_array[i] = pfn;
1853                         }
1854
1855                         reqp->bounce_sgl_count = 0;
1856                 }
1857                 break;
1858         }
1859         default:
1860                 printf("Unknow flags: %d\n", ccb->ccb_h.flags);
1861                 return(EINVAL);
1862         }
1863
1864         return(0);
1865 }
1866
1867 /**
1868  * @brief completion function before returning to CAM
1869  *
1870  * I/O process has been completed and the result needs
1871  * to be passed to the CAM layer.
1872  * Free resources related to this request.
1873  *
1874  * @param reqp pointer to a request structure
1875  */
1876 static void
1877 storvsc_io_done(struct hv_storvsc_request *reqp)
1878 {
1879         union ccb *ccb = reqp->ccb;
1880         struct ccb_scsiio *csio = &ccb->csio;
1881         struct storvsc_softc *sc = reqp->softc;
1882         struct vmscsi_req *vm_srb = &reqp->vstor_packet.u.vm_srb;
1883         bus_dma_segment_t *ori_sglist = NULL;
1884         int ori_sg_count = 0;
1885
1886         /* destroy bounce buffer if it is used */
1887         if (reqp->bounce_sgl_count) {
1888                 ori_sglist = (bus_dma_segment_t *)ccb->csio.data_ptr;
1889                 ori_sg_count = ccb->csio.sglist_cnt;
1890
1891                 /*
1892                  * If it is READ operation, we should copy back the data
1893                  * to original SG list.
1894                  */
1895                 if (READ_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
1896                         storvsc_copy_from_bounce_buf_to_sgl(ori_sglist,
1897                             ori_sg_count,
1898                             reqp->bounce_sgl,
1899                             reqp->not_aligned_seg_bits);
1900                 }
1901
1902                 storvsc_destroy_bounce_buffer(reqp->bounce_sgl);
1903                 reqp->bounce_sgl_count = 0;
1904         }
1905
1906         if (reqp->retries > 0) {
1907                 mtx_lock(&sc->hs_lock);
1908 #if HVS_TIMEOUT_TEST
1909                 xpt_print(ccb->ccb_h.path,
1910                         "%u: IO returned after timeout, "
1911                         "waking up timer handler if any.\n", ticks);
1912                 mtx_lock(&reqp->event.mtx);
1913                 cv_signal(&reqp->event.cv);
1914                 mtx_unlock(&reqp->event.mtx);
1915 #endif
1916                 reqp->retries = 0;
1917                 xpt_print(ccb->ccb_h.path,
1918                         "%u: IO returned after timeout, "
1919                         "stopping timer if any.\n", ticks);
1920                 mtx_unlock(&sc->hs_lock);
1921         }
1922
1923         /*
1924          * callout_drain() will wait for the timer handler to finish
1925          * if it is running. So we don't need any lock to synchronize
1926          * between this routine and the timer handler.
1927          * Note that we need to make sure reqp is not freed when timer
1928          * handler is using or will use it.
1929          */
1930         if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
1931                 callout_drain(&reqp->callout);
1932         }
1933
1934         ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1935         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1936         if (vm_srb->scsi_status == SCSI_STATUS_OK) {
1937                 ccb->ccb_h.status |= CAM_REQ_CMP;
1938          } else {
1939                 mtx_lock(&sc->hs_lock);
1940                 xpt_print(ccb->ccb_h.path,
1941                         "srovsc scsi_status = %d\n",
1942                         vm_srb->scsi_status);
1943                 mtx_unlock(&sc->hs_lock);
1944                 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
1945         }
1946
1947         ccb->csio.scsi_status = (vm_srb->scsi_status & 0xFF);
1948         ccb->csio.resid = ccb->csio.dxfer_len - vm_srb->transfer_len;
1949
1950         if (reqp->sense_info_len != 0) {
1951                 csio->sense_resid = csio->sense_len - reqp->sense_info_len;
1952                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
1953         }
1954
1955         mtx_lock(&sc->hs_lock);
1956         if (reqp->softc->hs_frozen == 1) {
1957                 xpt_print(ccb->ccb_h.path,
1958                         "%u: storvsc unfreezing softc 0x%p.\n",
1959                         ticks, reqp->softc);
1960                 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1961                 reqp->softc->hs_frozen = 0;
1962         }
1963         storvsc_free_request(sc, reqp);
1964         xpt_done(ccb);
1965         mtx_unlock(&sc->hs_lock);
1966 }
1967
1968 /**
1969  * @brief Free a request structure
1970  *
1971  * Free a request structure by returning it to the free list
1972  *
1973  * @param sc pointer to a softc
1974  * @param reqp pointer to a request structure
1975  */
1976 static void
1977 storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp)
1978 {
1979
1980         LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
1981 }
1982
1983 /**
1984  * @brief Determine type of storage device from GUID
1985  *
1986  * Using the type GUID, determine if this is a StorVSC (paravirtual
1987  * SCSI or BlkVSC (paravirtual IDE) device.
1988  *
1989  * @param dev a device
1990  * returns an enum
1991  */
1992 static enum hv_storage_type
1993 storvsc_get_storage_type(device_t dev)
1994 {
1995         const char *p = vmbus_get_type(dev);
1996
1997         if (!memcmp(p, &gBlkVscDeviceType, sizeof(hv_guid))) {
1998                 return DRIVER_BLKVSC;
1999         } else if (!memcmp(p, &gStorVscDeviceType, sizeof(hv_guid))) {
2000                 return DRIVER_STORVSC;
2001         }
2002         return (DRIVER_UNKNOWN);
2003 }
2004