sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c

   1 /*-
   2  * Copyright (c) 2009-2012,2016 Microsoft Corp.
   3  * Copyright (c) 2012 NetApp Inc.
   4  * Copyright (c) 2012 Citrix Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /**
  30  * StorVSC driver for Hyper-V.  This driver presents a SCSI HBA interface
  31  * to the Comman Access Method (CAM) layer.  CAM control blocks (CCBs) are
  32  * converted into VSCSI protocol messages which are delivered to the parent
  33  * partition StorVSP driver over the Hyper-V VMBUS.
  34  */
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/proc.h>
  40 #include <sys/condvar.h>
  41 #include <sys/time.h>
  42 #include <sys/systm.h>
  43 #include <sys/sockio.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/malloc.h>
  46 #include <sys/module.h>
  47 #include <sys/kernel.h>
  48 #include <sys/queue.h>
  49 #include <sys/lock.h>
  50 #include <sys/sx.h>
  51 #include <sys/taskqueue.h>
  52 #include <sys/bus.h>
  53 #include <sys/mutex.h>
  54 #include <sys/callout.h>
  55 #include <vm/vm.h>
  56 #include <vm/pmap.h>
  57 #include <vm/uma.h>
  58 #include <sys/lock.h>
  59 #include <sys/sema.h>
  60 #include <sys/sglist.h>
  61 #include <machine/bus.h>
  62 #include <sys/bus_dma.h>
  63
  64 #include <cam/cam.h>
  65 #include <cam/cam_ccb.h>
  66 #include <cam/cam_periph.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_xpt_sim.h>
  69 #include <cam/cam_xpt_internal.h>
  70 #include <cam/cam_debug.h>
  71 #include <cam/scsi/scsi_all.h>
  72 #include <cam/scsi/scsi_message.h>
  73
  74 #include <dev/hyperv/include/hyperv.h>
  75
  76 #include "hv_vstorage.h"
  77 #include "vmbus_if.h"
  78
  79 #define STORVSC_RINGBUFFER_SIZE         (20*PAGE_SIZE)
  80 #define STORVSC_MAX_LUNS_PER_TARGET     (64)
  81 #define STORVSC_MAX_IO_REQUESTS         (STORVSC_MAX_LUNS_PER_TARGET * 2)
  82 #define BLKVSC_MAX_IDE_DISKS_PER_TARGET (1)
  83 #define BLKVSC_MAX_IO_REQUESTS          STORVSC_MAX_IO_REQUESTS
  84 #define STORVSC_MAX_TARGETS             (2)
  85
  86 #define VSTOR_PKT_SIZE  (sizeof(struct vstor_packet) - vmscsi_size_delta)
  87
  88 #define HV_ALIGN(x, a) roundup2(x, a)
  89
  90 struct storvsc_softc;
  91
  92 struct hv_sgl_node {
  93         LIST_ENTRY(hv_sgl_node) link;
  94         struct sglist *sgl_data;
  95 };
  96
  97 struct hv_sgl_page_pool{
  98         LIST_HEAD(, hv_sgl_node) in_use_sgl_list;
  99         LIST_HEAD(, hv_sgl_node) free_sgl_list;
 100         boolean_t                is_init;
 101 } g_hv_sgl_page_pool;
 102
 103 #define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * HV_MAX_MULTIPAGE_BUFFER_COUNT
 104
 105 enum storvsc_request_type {
 106         WRITE_TYPE,
 107         READ_TYPE,
 108         UNKNOWN_TYPE
 109 };
 110
 111 struct hv_storvsc_request {
 112         LIST_ENTRY(hv_storvsc_request) link;
 113         struct vstor_packet     vstor_packet;
 114         hv_vmbus_multipage_buffer data_buf;
 115         void *sense_data;
 116         uint8_t sense_info_len;
 117         uint8_t retries;
 118         union ccb *ccb;
 119         struct storvsc_softc *softc;
 120         struct callout callout;
 121         struct sema synch_sema; /*Synchronize the request/response if needed */
 122         struct sglist *bounce_sgl;
 123         unsigned int bounce_sgl_count;
 124         uint64_t not_aligned_seg_bits;
 125 };
 126
 127 struct storvsc_softc {
 128         struct hv_vmbus_channel         *hs_chan;
 129         LIST_HEAD(, hv_storvsc_request) hs_free_list;
 130         struct mtx                      hs_lock;
 131         struct storvsc_driver_props     *hs_drv_props;
 132         int                             hs_unit;
 133         uint32_t                        hs_frozen;
 134         struct cam_sim                  *hs_sim;
 135         struct cam_path                 *hs_path;
 136         uint32_t                        hs_num_out_reqs;
 137         boolean_t                       hs_destroy;
 138         boolean_t                       hs_drain_notify;
 139         struct sema                     hs_drain_sema;
 140         struct hv_storvsc_request       hs_init_req;
 141         struct hv_storvsc_request       hs_reset_req;
 142         device_t                        hs_dev;
 143 };
 144
 145
 146 /**
 147  * HyperV storvsc timeout testing cases:
 148  * a. IO returned after first timeout;
 149  * b. IO returned after second timeout and queue freeze;
 150  * c. IO returned while timer handler is running
 151  * The first can be tested by "sg_senddiag -vv /dev/daX",
 152  * and the second and third can be done by
 153  * "sg_wr_mode -v -p 08 -c 0,1a -m 0,ff /dev/daX".
 154  */
 155 #define HVS_TIMEOUT_TEST 0
 156
 157 /*
 158  * Bus/adapter reset functionality on the Hyper-V host is
 159  * buggy and it will be disabled until
 160  * it can be further tested.
 161  */
 162 #define HVS_HOST_RESET 0
 163
 164 struct storvsc_driver_props {
 165         char            *drv_name;
 166         char            *drv_desc;
 167         uint8_t         drv_max_luns_per_target;
 168         uint8_t         drv_max_ios_per_target;
 169         uint32_t        drv_ringbuffer_size;
 170 };
 171
 172 enum hv_storage_type {
 173         DRIVER_BLKVSC,
 174         DRIVER_STORVSC,
 175         DRIVER_UNKNOWN
 176 };
 177
 178 #define HS_MAX_ADAPTERS 10
 179
 180 #define HV_STORAGE_SUPPORTS_MULTI_CHANNEL 0x1
 181
 182 /* {ba6163d9-04a1-4d29-b605-72e2ffb1dc7f} */
 183 static const hv_guid gStorVscDeviceType={
 184         .data = {0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d,
 185                  0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f}
 186 };
 187
 188 /* {32412632-86cb-44a2-9b5c-50d1417354f5} */
 189 static const hv_guid gBlkVscDeviceType={
 190         .data = {0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44,
 191                  0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5}
 192 };
 193
 194 static struct storvsc_driver_props g_drv_props_table[] = {
 195         {"blkvsc", "Hyper-V IDE Storage Interface",
 196          BLKVSC_MAX_IDE_DISKS_PER_TARGET, BLKVSC_MAX_IO_REQUESTS,
 197          STORVSC_RINGBUFFER_SIZE},
 198         {"storvsc", "Hyper-V SCSI Storage Interface",
 199          STORVSC_MAX_LUNS_PER_TARGET, STORVSC_MAX_IO_REQUESTS,
 200          STORVSC_RINGBUFFER_SIZE}
 201 };
 202
 203 /*
 204  * Sense buffer size changed in win8; have a run-time
 205  * variable to track the size we should use.
 206  */
 207 static int sense_buffer_size = PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE;
 208
 209 /*
 210  * The size of the vmscsi_request has changed in win8. The
 211  * additional size is for the newly added elements in the
 212  * structure. These elements are valid only when we are talking
 213  * to a win8 host.
 214  * Track the correct size we need to apply.
 215  */
 216 static int vmscsi_size_delta;
 217 /*
 218  * The storage protocol version is determined during the
 219  * initial exchange with the host.  It will indicate which
 220  * storage functionality is available in the host.
 221 */
 222 static int vmstor_proto_version;
 223
 224 struct vmstor_proto {
 225         int proto_version;
 226         int sense_buffer_size;
 227         int vmscsi_size_delta;
 228 };
 229
 230 static const struct vmstor_proto vmstor_proto_list[] = {
 231         {
 232                 VMSTOR_PROTOCOL_VERSION_WIN10,
 233                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 234                 0
 235         },
 236         {
 237                 VMSTOR_PROTOCOL_VERSION_WIN8_1,
 238                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 239                 0
 240         },
 241         {
 242                 VMSTOR_PROTOCOL_VERSION_WIN8,
 243                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 244                 0
 245         },
 246         {
 247                 VMSTOR_PROTOCOL_VERSION_WIN7,
 248                 PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE,
 249                 sizeof(struct vmscsi_win8_extension),
 250         },
 251         {
 252                 VMSTOR_PROTOCOL_VERSION_WIN6,
 253                 PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE,
 254                 sizeof(struct vmscsi_win8_extension),
 255         }
 256 };
 257
 258 /* static functions */
 259 static int storvsc_probe(device_t dev);
 260 static int storvsc_attach(device_t dev);
 261 static int storvsc_detach(device_t dev);
 262 static void storvsc_poll(struct cam_sim * sim);
 263 static void storvsc_action(struct cam_sim * sim, union ccb * ccb);
 264 static int create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp);
 265 static void storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp);
 266 static enum hv_storage_type storvsc_get_storage_type(device_t dev);
 267 static void hv_storvsc_rescan_target(struct storvsc_softc *sc);
 268 static void hv_storvsc_on_channel_callback(void *xchan);
 269 static void hv_storvsc_on_iocompletion( struct storvsc_softc *sc,
 270                                         struct vstor_packet *vstor_packet,
 271                                         struct hv_storvsc_request *request);
 272 static int hv_storvsc_connect_vsp(struct storvsc_softc *);
 273 static void storvsc_io_done(struct hv_storvsc_request *reqp);
 274 static void storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
 275                                 bus_dma_segment_t *orig_sgl,
 276                                 unsigned int orig_sgl_count,
 277                                 uint64_t seg_bits);
 278 void storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
 279                                 unsigned int dest_sgl_count,
 280                                 struct sglist* src_sgl,
 281                                 uint64_t seg_bits);
 282
 283 static device_method_t storvsc_methods[] = {
 284         /* Device interface */
 285         DEVMETHOD(device_probe,         storvsc_probe),
 286         DEVMETHOD(device_attach,        storvsc_attach),
 287         DEVMETHOD(device_detach,        storvsc_detach),
 288         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 289         DEVMETHOD_END
 290 };
 291
 292 static driver_t storvsc_driver = {
 293         "storvsc", storvsc_methods, sizeof(struct storvsc_softc),
 294 };
 295
 296 static devclass_t storvsc_devclass;
 297 DRIVER_MODULE(storvsc, vmbus, storvsc_driver, storvsc_devclass, 0, 0);
 298 MODULE_VERSION(storvsc, 1);
 299 MODULE_DEPEND(storvsc, vmbus, 1, 1, 1);
 300
 301 static void
 302 storvsc_subchan_attach(struct storvsc_softc *sc,
 303     struct hv_vmbus_channel *new_channel)
 304 {
 305         struct vmstor_chan_props props;
 306         int ret = 0;
 307
 308         memset(&props, 0, sizeof(props));
 309
 310         new_channel->hv_chan_priv1 = sc;
 311         vmbus_channel_cpu_rr(new_channel);
 312         ret = hv_vmbus_channel_open(new_channel,
 313             sc->hs_drv_props->drv_ringbuffer_size,
 314             sc->hs_drv_props->drv_ringbuffer_size,
 315             (void *)&props,
 316             sizeof(struct vmstor_chan_props),
 317             hv_storvsc_on_channel_callback,
 318             new_channel);
 319 }
 320
 321 /**
 322  * @brief Send multi-channel creation request to host
 323  *
 324  * @param device  a Hyper-V device pointer
 325  * @param max_chans  the max channels supported by vmbus
 326  */
 327 static void
 328 storvsc_send_multichannel_request(struct storvsc_softc *sc, int max_chans)
 329 {
 330         struct hv_vmbus_channel **subchan;
 331         struct hv_storvsc_request *request;
 332         struct vstor_packet *vstor_packet;
 333         int request_channels_cnt = 0;
 334         int ret, i;
 335
 336         /* get multichannels count that need to create */
 337         request_channels_cnt = MIN(max_chans, mp_ncpus);
 338
 339         request = &sc->hs_init_req;
 340
 341         /* request the host to create multi-channel */
 342         memset(request, 0, sizeof(struct hv_storvsc_request));
 343
 344         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 345
 346         vstor_packet = &request->vstor_packet;
 347
 348         vstor_packet->operation = VSTOR_OPERATION_CREATE_MULTI_CHANNELS;
 349         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 350         vstor_packet->u.multi_channels_cnt = request_channels_cnt;
 351
 352         ret = hv_vmbus_channel_send_packet(
 353             sc->hs_chan,
 354             vstor_packet,
 355             VSTOR_PKT_SIZE,
 356             (uint64_t)(uintptr_t)request,
 357             HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 358             HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 359
 360         /* wait for 5 seconds */
 361         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 362         if (ret != 0) {
 363                 printf("Storvsc_error: create multi-channel timeout, %d\n",
 364                     ret);
 365                 return;
 366         }
 367
 368         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 369             vstor_packet->status != 0) {
 370                 printf("Storvsc_error: create multi-channel invalid operation "
 371                     "(%d) or statue (%u)\n",
 372                     vstor_packet->operation, vstor_packet->status);
 373                 return;
 374         }
 375
 376         /* Wait for sub-channels setup to complete. */
 377         subchan = vmbus_get_subchan(sc->hs_chan, request_channels_cnt);
 378
 379         /* Attach the sub-channels. */
 380         for (i = 0; i < request_channels_cnt; ++i)
 381                 storvsc_subchan_attach(sc, subchan[i]);
 382
 383         /* Release the sub-channels. */
 384         vmbus_rel_subchan(subchan, request_channels_cnt);
 385
 386         if (bootverbose)
 387                 printf("Storvsc create multi-channel success!\n");
 388 }
 389
 390 /**
 391  * @brief initialize channel connection to parent partition
 392  *
 393  * @param dev  a Hyper-V device pointer
 394  * @returns  0 on success, non-zero error on failure
 395  */
 396 static int
 397 hv_storvsc_channel_init(struct storvsc_softc *sc)
 398 {
 399         int ret = 0, i;
 400         struct hv_storvsc_request *request;
 401         struct vstor_packet *vstor_packet;
 402         uint16_t max_chans = 0;
 403         boolean_t support_multichannel = FALSE;
 404         uint32_t version;
 405
 406         max_chans = 0;
 407         support_multichannel = FALSE;
 408
 409         request = &sc->hs_init_req;
 410         memset(request, 0, sizeof(struct hv_storvsc_request));
 411         vstor_packet = &request->vstor_packet;
 412         request->softc = sc;
 413
 414         /**
 415          * Initiate the vsc/vsp initialization protocol on the open channel
 416          */
 417         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 418
 419         vstor_packet->operation = VSTOR_OPERATION_BEGININITIALIZATION;
 420         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 421
 422
 423         ret = hv_vmbus_channel_send_packet(
 424                         sc->hs_chan,
 425                         vstor_packet,
 426                         VSTOR_PKT_SIZE,
 427                         (uint64_t)(uintptr_t)request,
 428                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 429                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 430
 431         if (ret != 0)
 432                 goto cleanup;
 433
 434         /* wait 5 seconds */
 435         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 436         if (ret != 0)
 437                 goto cleanup;
 438
 439         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 440                 vstor_packet->status != 0) {
 441                 goto cleanup;
 442         }
 443
 444         for (i = 0; i < nitems(vmstor_proto_list); i++) {
 445                 /* reuse the packet for version range supported */
 446
 447                 memset(vstor_packet, 0, sizeof(struct vstor_packet));
 448                 vstor_packet->operation = VSTOR_OPERATION_QUERYPROTOCOLVERSION;
 449                 vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 450
 451                 vstor_packet->u.version.major_minor =
 452                         vmstor_proto_list[i].proto_version;
 453
 454                 /* revision is only significant for Windows guests */
 455                 vstor_packet->u.version.revision = 0;
 456
 457                 ret = hv_vmbus_channel_send_packet(
 458                         sc->hs_chan,
 459                         vstor_packet,
 460                         VSTOR_PKT_SIZE,
 461                         (uint64_t)(uintptr_t)request,
 462                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 463                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 464
 465                 if (ret != 0)
 466                         goto cleanup;
 467
 468                 /* wait 5 seconds */
 469                 ret = sema_timedwait(&request->synch_sema, 5 * hz);
 470
 471                 if (ret)
 472                         goto cleanup;
 473
 474                 if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO) {
 475                         ret = EINVAL;
 476                         goto cleanup;
 477                 }
 478                 if (vstor_packet->status == 0) {
 479                         vmstor_proto_version =
 480                                 vmstor_proto_list[i].proto_version;
 481                         sense_buffer_size =
 482                                 vmstor_proto_list[i].sense_buffer_size;
 483                         vmscsi_size_delta =
 484                                 vmstor_proto_list[i].vmscsi_size_delta;
 485                         break;
 486                 }
 487         }
 488
 489         if (vstor_packet->status != 0) {
 490                 ret = EINVAL;
 491                 goto cleanup;
 492         }
 493         /**
 494          * Query channel properties
 495          */
 496         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 497         vstor_packet->operation = VSTOR_OPERATION_QUERYPROPERTIES;
 498         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 499
 500         ret = hv_vmbus_channel_send_packet(
 501                                 sc->hs_chan,
 502                                 vstor_packet,
 503                                 VSTOR_PKT_SIZE,
 504                                 (uint64_t)(uintptr_t)request,
 505                                 HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 506                                 HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 507
 508         if ( ret != 0)
 509                 goto cleanup;
 510
 511         /* wait 5 seconds */
 512         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 513
 514         if (ret != 0)
 515                 goto cleanup;
 516
 517         /* TODO: Check returned version */
 518         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 519             vstor_packet->status != 0) {
 520                 goto cleanup;
 521         }
 522
 523         /* multi-channels feature is supported by WIN8 and above version */
 524         max_chans = vstor_packet->u.chan_props.max_channel_cnt;
 525         version = VMBUS_GET_VERSION(device_get_parent(sc->hs_dev), sc->hs_dev);
 526         if (version != VMBUS_VERSION_WIN7 && version != VMBUS_VERSION_WS2008 &&
 527             (vstor_packet->u.chan_props.flags &
 528              HV_STORAGE_SUPPORTS_MULTI_CHANNEL)) {
 529                 support_multichannel = TRUE;
 530         }
 531
 532         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 533         vstor_packet->operation = VSTOR_OPERATION_ENDINITIALIZATION;
 534         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 535
 536         ret = hv_vmbus_channel_send_packet(
 537                         sc->hs_chan,
 538                         vstor_packet,
 539                         VSTOR_PKT_SIZE,
 540                         (uint64_t)(uintptr_t)request,
 541                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 542                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 543
 544         if (ret != 0) {
 545                 goto cleanup;
 546         }
 547
 548         /* wait 5 seconds */
 549         ret = sema_timedwait(&request->synch_sema, 5 * hz);
 550
 551         if (ret != 0)
 552                 goto cleanup;
 553
 554         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 555             vstor_packet->status != 0)
 556                 goto cleanup;
 557
 558         /*
 559          * If multi-channel is supported, send multichannel create
 560          * request to host.
 561          */
 562         if (support_multichannel)
 563                 storvsc_send_multichannel_request(sc, max_chans);
 564
 565 cleanup:
 566         sema_destroy(&request->synch_sema);
 567         return (ret);
 568 }
 569
 570 /**
 571  * @brief Open channel connection to paraent partition StorVSP driver
 572  *
 573  * Open and initialize channel connection to parent partition StorVSP driver.
 574  *
 575  * @param pointer to a Hyper-V device
 576  * @returns 0 on success, non-zero error on failure
 577  */
 578 static int
 579 hv_storvsc_connect_vsp(struct storvsc_softc *sc)
 580 {
 581         int ret = 0;
 582         struct vmstor_chan_props props;
 583
 584         memset(&props, 0, sizeof(struct vmstor_chan_props));
 585
 586         /*
 587          * Open the channel
 588          */
 589         KASSERT(sc->hs_chan->hv_chan_priv1 == sc, ("invalid chan priv1"));
 590         vmbus_channel_cpu_rr(sc->hs_chan);
 591         ret = hv_vmbus_channel_open(
 592                 sc->hs_chan,
 593                 sc->hs_drv_props->drv_ringbuffer_size,
 594                 sc->hs_drv_props->drv_ringbuffer_size,
 595                 (void *)&props,
 596                 sizeof(struct vmstor_chan_props),
 597                 hv_storvsc_on_channel_callback,
 598                 sc->hs_chan);
 599
 600         if (ret != 0) {
 601                 return ret;
 602         }
 603
 604         ret = hv_storvsc_channel_init(sc);
 605
 606         return (ret);
 607 }
 608
 609 #if HVS_HOST_RESET
 610 static int
 611 hv_storvsc_host_reset(struct storvsc_softc *sc)
 612 {
 613         int ret = 0;
 614
 615         struct hv_storvsc_request *request;
 616         struct vstor_packet *vstor_packet;
 617
 618         request = &sc->hs_reset_req;
 619         request->softc = sc;
 620         vstor_packet = &request->vstor_packet;
 621
 622         sema_init(&request->synch_sema, 0, "stor synch sema");
 623
 624         vstor_packet->operation = VSTOR_OPERATION_RESETBUS;
 625         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 626
 627         ret = hv_vmbus_channel_send_packet(dev->channel,
 628                         vstor_packet,
 629                         VSTOR_PKT_SIZE,
 630                         (uint64_t)(uintptr_t)&sc->hs_reset_req,
 631                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 632                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 633
 634         if (ret != 0) {
 635                 goto cleanup;
 636         }
 637
 638         ret = sema_timedwait(&request->synch_sema, 5 * hz); /* KYS 5 seconds */
 639
 640         if (ret) {
 641                 goto cleanup;
 642         }
 643
 644
 645         /*
 646          * At this point, all outstanding requests in the adapter
 647          * should have been flushed out and return to us
 648          */
 649
 650 cleanup:
 651         sema_destroy(&request->synch_sema);
 652         return (ret);
 653 }
 654 #endif /* HVS_HOST_RESET */
 655
 656 /**
 657  * @brief Function to initiate an I/O request
 658  *
 659  * @param device Hyper-V device pointer
 660  * @param request pointer to a request structure
 661  * @returns 0 on success, non-zero error on failure
 662  */
 663 static int
 664 hv_storvsc_io_request(struct storvsc_softc *sc,
 665                                           struct hv_storvsc_request *request)
 666 {
 667         struct vstor_packet *vstor_packet = &request->vstor_packet;
 668         struct hv_vmbus_channel* outgoing_channel = NULL;
 669         int ret = 0;
 670
 671         vstor_packet->flags |= REQUEST_COMPLETION_FLAG;
 672
 673         vstor_packet->u.vm_srb.length =
 674             sizeof(struct vmscsi_req) - vmscsi_size_delta;
 675
 676         vstor_packet->u.vm_srb.sense_info_len = sense_buffer_size;
 677
 678         vstor_packet->u.vm_srb.transfer_len = request->data_buf.length;
 679
 680         vstor_packet->operation = VSTOR_OPERATION_EXECUTESRB;
 681
 682         outgoing_channel = vmbus_select_outgoing_channel(sc->hs_chan);
 683
 684         mtx_unlock(&request->softc->hs_lock);
 685         if (request->data_buf.length) {
 686                 ret = hv_vmbus_channel_send_packet_multipagebuffer(
 687                                 outgoing_channel,
 688                                 &request->data_buf,
 689                                 vstor_packet,
 690                                 VSTOR_PKT_SIZE,
 691                                 (uint64_t)(uintptr_t)request);
 692
 693         } else {
 694                 ret = hv_vmbus_channel_send_packet(
 695                         outgoing_channel,
 696                         vstor_packet,
 697                         VSTOR_PKT_SIZE,
 698                         (uint64_t)(uintptr_t)request,
 699                         HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
 700                         HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 701         }
 702         mtx_lock(&request->softc->hs_lock);
 703
 704         if (ret != 0) {
 705                 printf("Unable to send packet %p ret %d", vstor_packet, ret);
 706         } else {
 707                 atomic_add_int(&sc->hs_num_out_reqs, 1);
 708         }
 709
 710         return (ret);
 711 }
 712
 713
 714 /**
 715  * Process IO_COMPLETION_OPERATION and ready
 716  * the result to be completed for upper layer
 717  * processing by the CAM layer.
 718  */
 719 static void
 720 hv_storvsc_on_iocompletion(struct storvsc_softc *sc,
 721                            struct vstor_packet *vstor_packet,
 722                            struct hv_storvsc_request *request)
 723 {
 724         struct vmscsi_req *vm_srb;
 725
 726         vm_srb = &vstor_packet->u.vm_srb;
 727
 728         /*
 729          * Copy some fields of the host's response into the request structure,
 730          * because the fields will be used later in storvsc_io_done().
 731          */
 732         request->vstor_packet.u.vm_srb.scsi_status = vm_srb->scsi_status;
 733         request->vstor_packet.u.vm_srb.srb_status = vm_srb->srb_status;
 734         request->vstor_packet.u.vm_srb.transfer_len = vm_srb->transfer_len;
 735
 736         if (((vm_srb->scsi_status & 0xFF) == SCSI_STATUS_CHECK_COND) &&
 737                         (vm_srb->srb_status & SRB_STATUS_AUTOSENSE_VALID)) {
 738                 /* Autosense data available */
 739
 740                 KASSERT(vm_srb->sense_info_len <= request->sense_info_len,
 741                                 ("vm_srb->sense_info_len <= "
 742                                  "request->sense_info_len"));
 743
 744                 memcpy(request->sense_data, vm_srb->u.sense_data,
 745                         vm_srb->sense_info_len);
 746
 747                 request->sense_info_len = vm_srb->sense_info_len;
 748         }
 749
 750         /* Complete request by passing to the CAM layer */
 751         storvsc_io_done(request);
 752         atomic_subtract_int(&sc->hs_num_out_reqs, 1);
 753         if (sc->hs_drain_notify && (sc->hs_num_out_reqs == 0)) {
 754                 sema_post(&sc->hs_drain_sema);
 755         }
 756 }
 757
 758 static void
 759 hv_storvsc_rescan_target(struct storvsc_softc *sc)
 760 {
 761         path_id_t pathid;
 762         target_id_t targetid;
 763         union ccb *ccb;
 764
 765         pathid = cam_sim_path(sc->hs_sim);
 766         targetid = CAM_TARGET_WILDCARD;
 767
 768         /*
 769          * Allocate a CCB and schedule a rescan.
 770          */
 771         ccb = xpt_alloc_ccb_nowait();
 772         if (ccb == NULL) {
 773                 printf("unable to alloc CCB for rescan\n");
 774                 return;
 775         }
 776
 777         if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid,
 778             CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 779                 printf("unable to create path for rescan, pathid: %u,"
 780                     "targetid: %u\n", pathid, targetid);
 781                 xpt_free_ccb(ccb);
 782                 return;
 783         }
 784
 785         if (targetid == CAM_TARGET_WILDCARD)
 786                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 787         else
 788                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 789
 790         xpt_rescan(ccb);
 791 }
 792
 793 static void
 794 hv_storvsc_on_channel_callback(void *xchan)
 795 {
 796         int ret = 0;
 797         hv_vmbus_channel *channel = xchan;
 798         struct storvsc_softc *sc = channel->hv_chan_priv1;
 799         uint32_t bytes_recvd;
 800         uint64_t request_id;
 801         uint8_t packet[roundup2(sizeof(struct vstor_packet), 8)];
 802         struct hv_storvsc_request *request;
 803         struct vstor_packet *vstor_packet;
 804
 805         ret = hv_vmbus_channel_recv_packet(
 806                         channel,
 807                         packet,
 808                         roundup2(VSTOR_PKT_SIZE, 8),
 809                         &bytes_recvd,
 810                         &request_id);
 811
 812         while ((ret == 0) && (bytes_recvd > 0)) {
 813                 request = (struct hv_storvsc_request *)(uintptr_t)request_id;
 814
 815                 if ((request == &sc->hs_init_req) ||
 816                         (request == &sc->hs_reset_req)) {
 817                         memcpy(&request->vstor_packet, packet,
 818                                    sizeof(struct vstor_packet));
 819                         sema_post(&request->synch_sema);
 820                 } else {
 821                         vstor_packet = (struct vstor_packet *)packet;
 822                         switch(vstor_packet->operation) {
 823                         case VSTOR_OPERATION_COMPLETEIO:
 824                                 if (request == NULL)
 825                                         panic("VMBUS: storvsc received a "
 826                                             "packet with NULL request id in "
 827                                             "COMPLETEIO operation.");
 828
 829                                 hv_storvsc_on_iocompletion(sc,
 830                                                         vstor_packet, request);
 831                                 break;
 832                         case VSTOR_OPERATION_REMOVEDEVICE:
 833                                 printf("VMBUS: storvsc operation %d not "
 834                                     "implemented.\n", vstor_packet->operation);
 835                                 /* TODO: implement */
 836                                 break;
 837                         case VSTOR_OPERATION_ENUMERATE_BUS:
 838                                 hv_storvsc_rescan_target(sc);
 839                                 break;
 840                         default:
 841                                 break;
 842                         }
 843                 }
 844                 ret = hv_vmbus_channel_recv_packet(
 845                                 channel,
 846                                 packet,
 847                                 roundup2(VSTOR_PKT_SIZE, 8),
 848                                 &bytes_recvd,
 849                                 &request_id);
 850         }
 851 }
 852
 853 /**
 854  * @brief StorVSC probe function
 855  *
 856  * Device probe function.  Returns 0 if the input device is a StorVSC
 857  * device.  Otherwise, a ENXIO is returned.  If the input device is
 858  * for BlkVSC (paravirtual IDE) device and this support is disabled in
 859  * favor of the emulated ATA/IDE device, return ENXIO.
 860  *
 861  * @param a device
 862  * @returns 0 on success, ENXIO if not a matcing StorVSC device
 863  */
 864 static int
 865 storvsc_probe(device_t dev)
 866 {
 867         int ata_disk_enable = 0;
 868         int ret = ENXIO;
 869
 870         switch (storvsc_get_storage_type(dev)) {
 871         case DRIVER_BLKVSC:
 872                 if(bootverbose)
 873                         device_printf(dev, "DRIVER_BLKVSC-Emulated ATA/IDE probe\n");
 874                 if (!getenv_int("hw.ata.disk_enable", &ata_disk_enable)) {
 875                         if(bootverbose)
 876                                 device_printf(dev,
 877                                         "Enlightened ATA/IDE detected\n");
 878                         device_set_desc(dev, g_drv_props_table[DRIVER_BLKVSC].drv_desc);
 879                         ret = BUS_PROBE_DEFAULT;
 880                 } else if(bootverbose)
 881                         device_printf(dev, "Emulated ATA/IDE set (hw.ata.disk_enable set)\n");
 882                 break;
 883         case DRIVER_STORVSC:
 884                 if(bootverbose)
 885                         device_printf(dev, "Enlightened SCSI device detected\n");
 886                 device_set_desc(dev, g_drv_props_table[DRIVER_STORVSC].drv_desc);
 887                 ret = BUS_PROBE_DEFAULT;
 888                 break;
 889         default:
 890                 ret = ENXIO;
 891         }
 892         return (ret);
 893 }
 894
 895 /**
 896  * @brief StorVSC attach function
 897  *
 898  * Function responsible for allocating per-device structures,
 899  * setting up CAM interfaces and scanning for available LUNs to
 900  * be used for SCSI device peripherals.
 901  *
 902  * @param a device
 903  * @returns 0 on success or an error on failure
 904  */
 905 static int
 906 storvsc_attach(device_t dev)
 907 {
 908         enum hv_storage_type stor_type;
 909         struct storvsc_softc *sc;
 910         struct cam_devq *devq;
 911         int ret, i, j;
 912         struct hv_storvsc_request *reqp;
 913         struct root_hold_token *root_mount_token = NULL;
 914         struct hv_sgl_node *sgl_node = NULL;
 915         void *tmp_buff = NULL;
 916
 917         /*
 918          * We need to serialize storvsc attach calls.
 919          */
 920         root_mount_token = root_mount_hold("storvsc");
 921
 922         sc = device_get_softc(dev);
 923         sc->hs_chan = vmbus_get_channel(dev);
 924         sc->hs_chan->hv_chan_priv1 = sc;
 925
 926         stor_type = storvsc_get_storage_type(dev);
 927
 928         if (stor_type == DRIVER_UNKNOWN) {
 929                 ret = ENODEV;
 930                 goto cleanup;
 931         }
 932
 933         /* fill in driver specific properties */
 934         sc->hs_drv_props = &g_drv_props_table[stor_type];
 935
 936         /* fill in device specific properties */
 937         sc->hs_unit     = device_get_unit(dev);
 938         sc->hs_dev      = dev;
 939
 940         LIST_INIT(&sc->hs_free_list);
 941         mtx_init(&sc->hs_lock, "hvslck", NULL, MTX_DEF);
 942
 943         for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; ++i) {
 944                 reqp = malloc(sizeof(struct hv_storvsc_request),
 945                                  M_DEVBUF, M_WAITOK|M_ZERO);
 946                 reqp->softc = sc;
 947
 948                 LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
 949         }
 950
 951         /* create sg-list page pool */
 952         if (FALSE == g_hv_sgl_page_pool.is_init) {
 953                 g_hv_sgl_page_pool.is_init = TRUE;
 954                 LIST_INIT(&g_hv_sgl_page_pool.in_use_sgl_list);
 955                 LIST_INIT(&g_hv_sgl_page_pool.free_sgl_list);
 956
 957                 /*
 958                  * Pre-create SG list, each SG list with
 959                  * HV_MAX_MULTIPAGE_BUFFER_COUNT segments, each
 960                  * segment has one page buffer
 961                  */
 962                 for (i = 0; i < STORVSC_MAX_IO_REQUESTS; i++) {
 963                         sgl_node = malloc(sizeof(struct hv_sgl_node),
 964                             M_DEVBUF, M_WAITOK|M_ZERO);
 965
 966                         sgl_node->sgl_data =
 967                             sglist_alloc(HV_MAX_MULTIPAGE_BUFFER_COUNT,
 968                             M_WAITOK|M_ZERO);
 969
 970                         for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
 971                                 tmp_buff = malloc(PAGE_SIZE,
 972                                     M_DEVBUF, M_WAITOK|M_ZERO);
 973
 974                                 sgl_node->sgl_data->sg_segs[j].ss_paddr =
 975                                     (vm_paddr_t)tmp_buff;
 976                         }
 977
 978                         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list,
 979                             sgl_node, link);
 980                 }
 981         }
 982
 983         sc->hs_destroy = FALSE;
 984         sc->hs_drain_notify = FALSE;
 985         sema_init(&sc->hs_drain_sema, 0, "Store Drain Sema");
 986
 987         ret = hv_storvsc_connect_vsp(sc);
 988         if (ret != 0) {
 989                 goto cleanup;
 990         }
 991
 992         /*
 993          * Create the device queue.
 994          * Hyper-V maps each target to one SCSI HBA
 995          */
 996         devq = cam_simq_alloc(sc->hs_drv_props->drv_max_ios_per_target);
 997         if (devq == NULL) {
 998                 device_printf(dev, "Failed to alloc device queue\n");
 999                 ret = ENOMEM;
1000                 goto cleanup;
1001         }
1002
1003         sc->hs_sim = cam_sim_alloc(storvsc_action,
1004                                 storvsc_poll,
1005                                 sc->hs_drv_props->drv_name,
1006                                 sc,
1007                                 sc->hs_unit,
1008                                 &sc->hs_lock, 1,
1009                                 sc->hs_drv_props->drv_max_ios_per_target,
1010                                 devq);
1011
1012         if (sc->hs_sim == NULL) {
1013                 device_printf(dev, "Failed to alloc sim\n");
1014                 cam_simq_free(devq);
1015                 ret = ENOMEM;
1016                 goto cleanup;
1017         }
1018
1019         mtx_lock(&sc->hs_lock);
1020         /* bus_id is set to 0, need to get it from VMBUS channel query? */
1021         if (xpt_bus_register(sc->hs_sim, dev, 0) != CAM_SUCCESS) {
1022                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1023                 mtx_unlock(&sc->hs_lock);
1024                 device_printf(dev, "Unable to register SCSI bus\n");
1025                 ret = ENXIO;
1026                 goto cleanup;
1027         }
1028
1029         if (xpt_create_path(&sc->hs_path, /*periph*/NULL,
1030                  cam_sim_path(sc->hs_sim),
1031                 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1032                 xpt_bus_deregister(cam_sim_path(sc->hs_sim));
1033                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1034                 mtx_unlock(&sc->hs_lock);
1035                 device_printf(dev, "Unable to create path\n");
1036                 ret = ENXIO;
1037                 goto cleanup;
1038         }
1039
1040         mtx_unlock(&sc->hs_lock);
1041
1042         root_mount_rel(root_mount_token);
1043         return (0);
1044
1045
1046 cleanup:
1047         root_mount_rel(root_mount_token);
1048         while (!LIST_EMPTY(&sc->hs_free_list)) {
1049                 reqp = LIST_FIRST(&sc->hs_free_list);
1050                 LIST_REMOVE(reqp, link);
1051                 free(reqp, M_DEVBUF);
1052         }
1053
1054         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1055                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1056                 LIST_REMOVE(sgl_node, link);
1057                 for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
1058                         if (NULL !=
1059                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1060                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1061                         }
1062                 }
1063                 sglist_free(sgl_node->sgl_data);
1064                 free(sgl_node, M_DEVBUF);
1065         }
1066
1067         return (ret);
1068 }
1069
1070 /**
1071  * @brief StorVSC device detach function
1072  *
1073  * This function is responsible for safely detaching a
1074  * StorVSC device.  This includes waiting for inbound responses
1075  * to complete and freeing associated per-device structures.
1076  *
1077  * @param dev a device
1078  * returns 0 on success
1079  */
1080 static int
1081 storvsc_detach(device_t dev)
1082 {
1083         struct storvsc_softc *sc = device_get_softc(dev);
1084         struct hv_storvsc_request *reqp = NULL;
1085         struct hv_sgl_node *sgl_node = NULL;
1086         int j = 0;
1087
1088         sc->hs_destroy = TRUE;
1089
1090         /*
1091          * At this point, all outbound traffic should be disabled. We
1092          * only allow inbound traffic (responses) to proceed so that
1093          * outstanding requests can be completed.
1094          */
1095
1096         sc->hs_drain_notify = TRUE;
1097         sema_wait(&sc->hs_drain_sema);
1098         sc->hs_drain_notify = FALSE;
1099
1100         /*
1101          * Since we have already drained, we don't need to busy wait.
1102          * The call to close the channel will reset the callback
1103          * under the protection of the incoming channel lock.
1104          */
1105
1106         hv_vmbus_channel_close(sc->hs_chan);
1107
1108         mtx_lock(&sc->hs_lock);
1109         while (!LIST_EMPTY(&sc->hs_free_list)) {
1110                 reqp = LIST_FIRST(&sc->hs_free_list);
1111                 LIST_REMOVE(reqp, link);
1112
1113                 free(reqp, M_DEVBUF);
1114         }
1115         mtx_unlock(&sc->hs_lock);
1116
1117         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1118                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1119                 LIST_REMOVE(sgl_node, link);
1120                 for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++){
1121                         if (NULL !=
1122                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1123                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1124                         }
1125                 }
1126                 sglist_free(sgl_node->sgl_data);
1127                 free(sgl_node, M_DEVBUF);
1128         }
1129
1130         return (0);
1131 }
1132
1133 #if HVS_TIMEOUT_TEST
1134 /**
1135  * @brief unit test for timed out operations
1136  *
1137  * This function provides unit testing capability to simulate
1138  * timed out operations.  Recompilation with HV_TIMEOUT_TEST=1
1139  * is required.
1140  *
1141  * @param reqp pointer to a request structure
1142  * @param opcode SCSI operation being performed
1143  * @param wait if 1, wait for I/O to complete
1144  */
1145 static void
1146 storvsc_timeout_test(struct hv_storvsc_request *reqp,
1147                 uint8_t opcode, int wait)
1148 {
1149         int ret;
1150         union ccb *ccb = reqp->ccb;
1151         struct storvsc_softc *sc = reqp->softc;
1152
1153         if (reqp->vstor_packet.vm_srb.cdb[0] != opcode) {
1154                 return;
1155         }
1156
1157         if (wait) {
1158                 mtx_lock(&reqp->event.mtx);
1159         }
1160         ret = hv_storvsc_io_request(sc, reqp);
1161         if (ret != 0) {
1162                 if (wait) {
1163                         mtx_unlock(&reqp->event.mtx);
1164                 }
1165                 printf("%s: io_request failed with %d.\n",
1166                                 __func__, ret);
1167                 ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1168                 mtx_lock(&sc->hs_lock);
1169                 storvsc_free_request(sc, reqp);
1170                 xpt_done(ccb);
1171                 mtx_unlock(&sc->hs_lock);
1172                 return;
1173         }
1174
1175         if (wait) {
1176                 xpt_print(ccb->ccb_h.path,
1177                                 "%u: %s: waiting for IO return.\n",
1178                                 ticks, __func__);
1179                 ret = cv_timedwait(&reqp->event.cv, &reqp->event.mtx, 60*hz);
1180                 mtx_unlock(&reqp->event.mtx);
1181                 xpt_print(ccb->ccb_h.path, "%u: %s: %s.\n",
1182                                 ticks, __func__, (ret == 0)?
1183                                 "IO return detected" :
1184                                 "IO return not detected");
1185                 /*
1186                  * Now both the timer handler and io done are running
1187                  * simultaneously. We want to confirm the io done always
1188                  * finishes after the timer handler exits. So reqp used by
1189                  * timer handler is not freed or stale. Do busy loop for
1190                  * another 1/10 second to make sure io done does
1191                  * wait for the timer handler to complete.
1192                  */
1193                 DELAY(100*1000);
1194                 mtx_lock(&sc->hs_lock);
1195                 xpt_print(ccb->ccb_h.path,
1196                                 "%u: %s: finishing, queue frozen %d, "
1197                                 "ccb status 0x%x scsi_status 0x%x.\n",
1198                                 ticks, __func__, sc->hs_frozen,
1199                                 ccb->ccb_h.status,
1200                                 ccb->csio.scsi_status);
1201                 mtx_unlock(&sc->hs_lock);
1202         }
1203 }
1204 #endif /* HVS_TIMEOUT_TEST */
1205
1206 #ifdef notyet
1207 /**
1208  * @brief timeout handler for requests
1209  *
1210  * This function is called as a result of a callout expiring.
1211  *
1212  * @param arg pointer to a request
1213  */
1214 static void
1215 storvsc_timeout(void *arg)
1216 {
1217         struct hv_storvsc_request *reqp = arg;
1218         struct storvsc_softc *sc = reqp->softc;
1219         union ccb *ccb = reqp->ccb;
1220
1221         if (reqp->retries == 0) {
1222                 mtx_lock(&sc->hs_lock);
1223                 xpt_print(ccb->ccb_h.path,
1224                     "%u: IO timed out (req=0x%p), wait for another %u secs.\n",
1225                     ticks, reqp, ccb->ccb_h.timeout / 1000);
1226                 cam_error_print(ccb, CAM_ESF_ALL, CAM_EPF_ALL);
1227                 mtx_unlock(&sc->hs_lock);
1228
1229                 reqp->retries++;
1230                 callout_reset_sbt(&reqp->callout, SBT_1MS * ccb->ccb_h.timeout,
1231                     0, storvsc_timeout, reqp, 0);
1232 #if HVS_TIMEOUT_TEST
1233                 storvsc_timeout_test(reqp, SEND_DIAGNOSTIC, 0);
1234 #endif
1235                 return;
1236         }
1237
1238         mtx_lock(&sc->hs_lock);
1239         xpt_print(ccb->ccb_h.path,
1240                 "%u: IO (reqp = 0x%p) did not return for %u seconds, %s.\n",
1241                 ticks, reqp, ccb->ccb_h.timeout * (reqp->retries+1) / 1000,
1242                 (sc->hs_frozen == 0)?
1243                 "freezing the queue" : "the queue is already frozen");
1244         if (sc->hs_frozen == 0) {
1245                 sc->hs_frozen = 1;
1246                 xpt_freeze_simq(xpt_path_sim(ccb->ccb_h.path), 1);
1247         }
1248         mtx_unlock(&sc->hs_lock);
1249
1250 #if HVS_TIMEOUT_TEST
1251         storvsc_timeout_test(reqp, MODE_SELECT_10, 1);
1252 #endif
1253 }
1254 #endif
1255
1256 /**
1257  * @brief StorVSC device poll function
1258  *
1259  * This function is responsible for servicing requests when
1260  * interrupts are disabled (i.e when we are dumping core.)
1261  *
1262  * @param sim a pointer to a CAM SCSI interface module
1263  */
1264 static void
1265 storvsc_poll(struct cam_sim *sim)
1266 {
1267         struct storvsc_softc *sc = cam_sim_softc(sim);
1268
1269         mtx_assert(&sc->hs_lock, MA_OWNED);
1270         mtx_unlock(&sc->hs_lock);
1271         hv_storvsc_on_channel_callback(sc->hs_chan);
1272         mtx_lock(&sc->hs_lock);
1273 }
1274
1275 /**
1276  * @brief StorVSC device action function
1277  *
1278  * This function is responsible for handling SCSI operations which
1279  * are passed from the CAM layer.  The requests are in the form of
1280  * CAM control blocks which indicate the action being performed.
1281  * Not all actions require converting the request to a VSCSI protocol
1282  * message - these actions can be responded to by this driver.
1283  * Requests which are destined for a backend storage device are converted
1284  * to a VSCSI protocol message and sent on the channel connection associated
1285  * with this device.
1286  *
1287  * @param sim pointer to a CAM SCSI interface module
1288  * @param ccb pointer to a CAM control block
1289  */
1290 static void
1291 storvsc_action(struct cam_sim *sim, union ccb *ccb)
1292 {
1293         struct storvsc_softc *sc = cam_sim_softc(sim);
1294         int res;
1295
1296         mtx_assert(&sc->hs_lock, MA_OWNED);
1297         switch (ccb->ccb_h.func_code) {
1298         case XPT_PATH_INQ: {
1299                 struct ccb_pathinq *cpi = &ccb->cpi;
1300
1301                 cpi->version_num = 1;
1302                 cpi->hba_inquiry = PI_TAG_ABLE|PI_SDTR_ABLE;
1303                 cpi->target_sprt = 0;
1304                 cpi->hba_misc = PIM_NOBUSRESET;
1305                 cpi->hba_eng_cnt = 0;
1306                 cpi->max_target = STORVSC_MAX_TARGETS;
1307                 cpi->max_lun = sc->hs_drv_props->drv_max_luns_per_target;
1308                 cpi->initiator_id = cpi->max_target;
1309                 cpi->bus_id = cam_sim_bus(sim);
1310                 cpi->base_transfer_speed = 300000;
1311                 cpi->transport = XPORT_SAS;
1312                 cpi->transport_version = 0;
1313                 cpi->protocol = PROTO_SCSI;
1314                 cpi->protocol_version = SCSI_REV_SPC2;
1315                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1316                 strncpy(cpi->hba_vid, sc->hs_drv_props->drv_name, HBA_IDLEN);
1317                 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1318                 cpi->unit_number = cam_sim_unit(sim);
1319
1320                 ccb->ccb_h.status = CAM_REQ_CMP;
1321                 xpt_done(ccb);
1322                 return;
1323         }
1324         case XPT_GET_TRAN_SETTINGS: {
1325                 struct  ccb_trans_settings *cts = &ccb->cts;
1326
1327                 cts->transport = XPORT_SAS;
1328                 cts->transport_version = 0;
1329                 cts->protocol = PROTO_SCSI;
1330                 cts->protocol_version = SCSI_REV_SPC2;
1331
1332                 /* enable tag queuing and disconnected mode */
1333                 cts->proto_specific.valid = CTS_SCSI_VALID_TQ;
1334                 cts->proto_specific.scsi.valid = CTS_SCSI_VALID_TQ;
1335                 cts->proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB;
1336                 cts->xport_specific.valid = CTS_SPI_VALID_DISC;
1337                 cts->xport_specific.spi.flags = CTS_SPI_FLAGS_DISC_ENB;
1338
1339                 ccb->ccb_h.status = CAM_REQ_CMP;
1340                 xpt_done(ccb);
1341                 return;
1342         }
1343         case XPT_SET_TRAN_SETTINGS:     {
1344                 ccb->ccb_h.status = CAM_REQ_CMP;
1345                 xpt_done(ccb);
1346                 return;
1347         }
1348         case XPT_CALC_GEOMETRY:{
1349                 cam_calc_geometry(&ccb->ccg, 1);
1350                 xpt_done(ccb);
1351                 return;
1352         }
1353         case  XPT_RESET_BUS:
1354         case  XPT_RESET_DEV:{
1355 #if HVS_HOST_RESET
1356                 if ((res = hv_storvsc_host_reset(sc)) != 0) {
1357                         xpt_print(ccb->ccb_h.path,
1358                                 "hv_storvsc_host_reset failed with %d\n", res);
1359                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1360                         xpt_done(ccb);
1361                         return;
1362                 }
1363                 ccb->ccb_h.status = CAM_REQ_CMP;
1364                 xpt_done(ccb);
1365                 return;
1366 #else
1367                 xpt_print(ccb->ccb_h.path,
1368                                   "%s reset not supported.\n",
1369                                   (ccb->ccb_h.func_code == XPT_RESET_BUS)?
1370                                   "bus" : "dev");
1371                 ccb->ccb_h.status = CAM_REQ_INVALID;
1372                 xpt_done(ccb);
1373                 return;
1374 #endif  /* HVS_HOST_RESET */
1375         }
1376         case XPT_SCSI_IO:
1377         case XPT_IMMED_NOTIFY: {
1378                 struct hv_storvsc_request *reqp = NULL;
1379
1380                 if (ccb->csio.cdb_len == 0) {
1381                         panic("cdl_len is 0\n");
1382                 }
1383
1384                 if (LIST_EMPTY(&sc->hs_free_list)) {
1385                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
1386                         if (sc->hs_frozen == 0) {
1387                                 sc->hs_frozen = 1;
1388                                 xpt_freeze_simq(sim, /* count*/1);
1389                         }
1390                         xpt_done(ccb);
1391                         return;
1392                 }
1393
1394                 reqp = LIST_FIRST(&sc->hs_free_list);
1395                 LIST_REMOVE(reqp, link);
1396
1397                 bzero(reqp, sizeof(struct hv_storvsc_request));
1398                 reqp->softc = sc;
1399
1400                 ccb->ccb_h.status |= CAM_SIM_QUEUED;
1401                 if ((res = create_storvsc_request(ccb, reqp)) != 0) {
1402                         ccb->ccb_h.status = CAM_REQ_INVALID;
1403                         xpt_done(ccb);
1404                         return;
1405                 }
1406
1407 #ifdef notyet
1408                 if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
1409                         callout_init(&reqp->callout, CALLOUT_MPSAFE);
1410                         callout_reset_sbt(&reqp->callout,
1411                             SBT_1MS * ccb->ccb_h.timeout, 0,
1412                             storvsc_timeout, reqp, 0);
1413 #if HVS_TIMEOUT_TEST
1414                         cv_init(&reqp->event.cv, "storvsc timeout cv");
1415                         mtx_init(&reqp->event.mtx, "storvsc timeout mutex",
1416                                         NULL, MTX_DEF);
1417                         switch (reqp->vstor_packet.vm_srb.cdb[0]) {
1418                                 case MODE_SELECT_10:
1419                                 case SEND_DIAGNOSTIC:
1420                                         /* To have timer send the request. */
1421                                         return;
1422                                 default:
1423                                         break;
1424                         }
1425 #endif /* HVS_TIMEOUT_TEST */
1426                 }
1427 #endif
1428
1429                 if ((res = hv_storvsc_io_request(sc, reqp)) != 0) {
1430                         xpt_print(ccb->ccb_h.path,
1431                                 "hv_storvsc_io_request failed with %d\n", res);
1432                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1433                         storvsc_free_request(sc, reqp);
1434                         xpt_done(ccb);
1435                         return;
1436                 }
1437                 return;
1438         }
1439
1440         default:
1441                 ccb->ccb_h.status = CAM_REQ_INVALID;
1442                 xpt_done(ccb);
1443                 return;
1444         }
1445 }
1446
1447 /**
1448  * @brief destroy bounce buffer
1449  *
1450  * This function is responsible for destroy a Scatter/Gather list
1451  * that create by storvsc_create_bounce_buffer()
1452  *
1453  * @param sgl- the Scatter/Gather need be destroy
1454  * @param sg_count- page count of the SG list.
1455  *
1456  */
1457 static void
1458 storvsc_destroy_bounce_buffer(struct sglist *sgl)
1459 {
1460         struct hv_sgl_node *sgl_node = NULL;
1461         if (LIST_EMPTY(&g_hv_sgl_page_pool.in_use_sgl_list)) {
1462                 printf("storvsc error: not enough in use sgl\n");
1463                 return;
1464         }
1465         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.in_use_sgl_list);
1466         LIST_REMOVE(sgl_node, link);
1467         sgl_node->sgl_data = sgl;
1468         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list, sgl_node, link);
1469 }
1470
1471 /**
1472  * @brief create bounce buffer
1473  *
1474  * This function is responsible for create a Scatter/Gather list,
1475  * which hold several pages that can be aligned with page size.
1476  *
1477  * @param seg_count- SG-list segments count
1478  * @param write - if WRITE_TYPE, set SG list page used size to 0,
1479  * otherwise set used size to page size.
1480  *
1481  * return NULL if create failed
1482  */
1483 static struct sglist *
1484 storvsc_create_bounce_buffer(uint16_t seg_count, int write)
1485 {
1486         int i = 0;
1487         struct sglist *bounce_sgl = NULL;
1488         unsigned int buf_len = ((write == WRITE_TYPE) ? 0 : PAGE_SIZE);
1489         struct hv_sgl_node *sgl_node = NULL;
1490
1491         /* get struct sglist from free_sgl_list */
1492         if (LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1493                 printf("storvsc error: not enough free sgl\n");
1494                 return NULL;
1495         }
1496         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1497         LIST_REMOVE(sgl_node, link);
1498         bounce_sgl = sgl_node->sgl_data;
1499         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.in_use_sgl_list, sgl_node, link);
1500
1501         bounce_sgl->sg_maxseg = seg_count;
1502
1503         if (write == WRITE_TYPE)
1504                 bounce_sgl->sg_nseg = 0;
1505         else
1506                 bounce_sgl->sg_nseg = seg_count;
1507
1508         for (i = 0; i < seg_count; i++)
1509                 bounce_sgl->sg_segs[i].ss_len = buf_len;
1510
1511         return bounce_sgl;
1512 }
1513
1514 /**
1515  * @brief copy data from SG list to bounce buffer
1516  *
1517  * This function is responsible for copy data from one SG list's segments
1518  * to another SG list which used as bounce buffer.
1519  *
1520  * @param bounce_sgl - the destination SG list
1521  * @param orig_sgl - the segment of the source SG list.
1522  * @param orig_sgl_count - the count of segments.
1523  * @param orig_sgl_count - indicate which segment need bounce buffer,
1524  *  set 1 means need.
1525  *
1526  */
1527 static void
1528 storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
1529                                bus_dma_segment_t *orig_sgl,
1530                                unsigned int orig_sgl_count,
1531                                uint64_t seg_bits)
1532 {
1533         int src_sgl_idx = 0;
1534
1535         for (src_sgl_idx = 0; src_sgl_idx < orig_sgl_count; src_sgl_idx++) {
1536                 if (seg_bits & (1 << src_sgl_idx)) {
1537                         memcpy((void*)bounce_sgl->sg_segs[src_sgl_idx].ss_paddr,
1538                             (void*)orig_sgl[src_sgl_idx].ds_addr,
1539                             orig_sgl[src_sgl_idx].ds_len);
1540
1541                         bounce_sgl->sg_segs[src_sgl_idx].ss_len =
1542                             orig_sgl[src_sgl_idx].ds_len;
1543                 }
1544         }
1545 }
1546
1547 /**
1548  * @brief copy data from SG list which used as bounce to another SG list
1549  *
1550  * This function is responsible for copy data from one SG list with bounce
1551  * buffer to another SG list's segments.
1552  *
1553  * @param dest_sgl - the destination SG list's segments
1554  * @param dest_sgl_count - the count of destination SG list's segment.
1555  * @param src_sgl - the source SG list.
1556  * @param seg_bits - indicate which segment used bounce buffer of src SG-list.
1557  *
1558  */
1559 void
1560 storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
1561                                     unsigned int dest_sgl_count,
1562                                     struct sglist* src_sgl,
1563                                     uint64_t seg_bits)
1564 {
1565         int sgl_idx = 0;
1566
1567         for (sgl_idx = 0; sgl_idx < dest_sgl_count; sgl_idx++) {
1568                 if (seg_bits & (1 << sgl_idx)) {
1569                         memcpy((void*)(dest_sgl[sgl_idx].ds_addr),
1570                             (void*)(src_sgl->sg_segs[sgl_idx].ss_paddr),
1571                             src_sgl->sg_segs[sgl_idx].ss_len);
1572                 }
1573         }
1574 }
1575
1576 /**
1577  * @brief check SG list with bounce buffer or not
1578  *
1579  * This function is responsible for check if need bounce buffer for SG list.
1580  *
1581  * @param sgl - the SG list's segments
1582  * @param sg_count - the count of SG list's segment.
1583  * @param bits - segmengs number that need bounce buffer
1584  *
1585  * return -1 if SG list needless bounce buffer
1586  */
1587 static int
1588 storvsc_check_bounce_buffer_sgl(bus_dma_segment_t *sgl,
1589                                 unsigned int sg_count,
1590                                 uint64_t *bits)
1591 {
1592         int i = 0;
1593         int offset = 0;
1594         uint64_t phys_addr = 0;
1595         uint64_t tmp_bits = 0;
1596         boolean_t found_hole = FALSE;
1597         boolean_t pre_aligned = TRUE;
1598
1599         if (sg_count < 2){
1600                 return -1;
1601         }
1602
1603         *bits = 0;
1604
1605         phys_addr = vtophys(sgl[0].ds_addr);
1606         offset =  phys_addr - trunc_page(phys_addr);
1607
1608         if (offset != 0) {
1609                 pre_aligned = FALSE;
1610                 tmp_bits |= 1;
1611         }
1612
1613         for (i = 1; i < sg_count; i++) {
1614                 phys_addr = vtophys(sgl[i].ds_addr);
1615                 offset =  phys_addr - trunc_page(phys_addr);
1616
1617                 if (offset == 0) {
1618                         if (FALSE == pre_aligned){
1619                                 /*
1620                                  * This segment is aligned, if the previous
1621                                  * one is not aligned, find a hole
1622                                  */
1623                                 found_hole = TRUE;
1624                         }
1625                         pre_aligned = TRUE;
1626                 } else {
1627                         tmp_bits |= 1 << i;
1628                         if (!pre_aligned) {
1629                                 if (phys_addr != vtophys(sgl[i-1].ds_addr +
1630                                     sgl[i-1].ds_len)) {
1631                                         /*
1632                                          * Check whether connect to previous
1633                                          * segment,if not, find the hole
1634                                          */
1635                                         found_hole = TRUE;
1636                                 }
1637                         } else {
1638                                 found_hole = TRUE;
1639                         }
1640                         pre_aligned = FALSE;
1641                 }
1642         }
1643
1644         if (!found_hole) {
1645                 return (-1);
1646         } else {
1647                 *bits = tmp_bits;
1648                 return 0;
1649         }
1650 }
1651
1652 /**
1653  * @brief Fill in a request structure based on a CAM control block
1654  *
1655  * Fills in a request structure based on the contents of a CAM control
1656  * block.  The request structure holds the payload information for
1657  * VSCSI protocol request.
1658  *
1659  * @param ccb pointer to a CAM contorl block
1660  * @param reqp pointer to a request structure
1661  */
1662 static int
1663 create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
1664 {
1665         struct ccb_scsiio *csio = &ccb->csio;
1666         uint64_t phys_addr;
1667         uint32_t bytes_to_copy = 0;
1668         uint32_t pfn_num = 0;
1669         uint32_t pfn;
1670         uint64_t not_aligned_seg_bits = 0;
1671
1672         /* refer to struct vmscsi_req for meanings of these two fields */
1673         reqp->vstor_packet.u.vm_srb.port =
1674                 cam_sim_unit(xpt_path_sim(ccb->ccb_h.path));
1675         reqp->vstor_packet.u.vm_srb.path_id =
1676                 cam_sim_bus(xpt_path_sim(ccb->ccb_h.path));
1677
1678         reqp->vstor_packet.u.vm_srb.target_id = ccb->ccb_h.target_id;
1679         reqp->vstor_packet.u.vm_srb.lun = ccb->ccb_h.target_lun;
1680
1681         reqp->vstor_packet.u.vm_srb.cdb_len = csio->cdb_len;
1682         if(ccb->ccb_h.flags & CAM_CDB_POINTER) {
1683                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_ptr,
1684                         csio->cdb_len);
1685         } else {
1686                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_bytes,
1687                         csio->cdb_len);
1688         }
1689
1690         switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
1691         case CAM_DIR_OUT:
1692                 reqp->vstor_packet.u.vm_srb.data_in = WRITE_TYPE;
1693                 break;
1694         case CAM_DIR_IN:
1695                 reqp->vstor_packet.u.vm_srb.data_in = READ_TYPE;
1696                 break;
1697         case CAM_DIR_NONE:
1698                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1699                 break;
1700         default:
1701                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1702                 break;
1703         }
1704
1705         reqp->sense_data     = &csio->sense_data;
1706         reqp->sense_info_len = csio->sense_len;
1707
1708         reqp->ccb = ccb;
1709
1710         if (0 == csio->dxfer_len) {
1711                 return (0);
1712         }
1713
1714         reqp->data_buf.length = csio->dxfer_len;
1715
1716         switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
1717         case CAM_DATA_VADDR:
1718         {
1719                 bytes_to_copy = csio->dxfer_len;
1720                 phys_addr = vtophys(csio->data_ptr);
1721                 reqp->data_buf.offset = phys_addr & PAGE_MASK;
1722
1723                 while (bytes_to_copy != 0) {
1724                         int bytes, page_offset;
1725                         phys_addr =
1726                             vtophys(&csio->data_ptr[reqp->data_buf.length -
1727                             bytes_to_copy]);
1728                         pfn = phys_addr >> PAGE_SHIFT;
1729                         reqp->data_buf.pfn_array[pfn_num] = pfn;
1730                         page_offset = phys_addr & PAGE_MASK;
1731
1732                         bytes = min(PAGE_SIZE - page_offset, bytes_to_copy);
1733
1734                         bytes_to_copy -= bytes;
1735                         pfn_num++;
1736                 }
1737                 break;
1738         }
1739
1740         case CAM_DATA_SG:
1741         {
1742                 int i = 0;
1743                 int offset = 0;
1744                 int ret;
1745
1746                 bus_dma_segment_t *storvsc_sglist =
1747                     (bus_dma_segment_t *)ccb->csio.data_ptr;
1748                 u_int16_t storvsc_sg_count = ccb->csio.sglist_cnt;
1749
1750                 printf("Storvsc: get SG I/O operation, %d\n",
1751                     reqp->vstor_packet.u.vm_srb.data_in);
1752
1753                 if (storvsc_sg_count > HV_MAX_MULTIPAGE_BUFFER_COUNT){
1754                         printf("Storvsc: %d segments is too much, "
1755                             "only support %d segments\n",
1756                             storvsc_sg_count, HV_MAX_MULTIPAGE_BUFFER_COUNT);
1757                         return (EINVAL);
1758                 }
1759
1760                 /*
1761                  * We create our own bounce buffer function currently. Idealy
1762                  * we should use BUS_DMA(9) framework. But with current BUS_DMA
1763                  * code there is no callback API to check the page alignment of
1764                  * middle segments before busdma can decide if a bounce buffer
1765                  * is needed for particular segment. There is callback,
1766                  * "bus_dma_filter_t *filter", but the parrameters are not
1767                  * sufficient for storvsc driver.
1768                  * TODO:
1769                  *      Add page alignment check in BUS_DMA(9) callback. Once
1770                  *      this is complete, switch the following code to use
1771                  *      BUS_DMA(9) for storvsc bounce buffer support.
1772                  */
1773                 /* check if we need to create bounce buffer */
1774                 ret = storvsc_check_bounce_buffer_sgl(storvsc_sglist,
1775                     storvsc_sg_count, &not_aligned_seg_bits);
1776                 if (ret != -1) {
1777                         reqp->bounce_sgl =
1778                             storvsc_create_bounce_buffer(storvsc_sg_count,
1779                             reqp->vstor_packet.u.vm_srb.data_in);
1780                         if (NULL == reqp->bounce_sgl) {
1781                                 printf("Storvsc_error: "
1782                                     "create bounce buffer failed.\n");
1783                                 return (ENOMEM);
1784                         }
1785
1786                         reqp->bounce_sgl_count = storvsc_sg_count;
1787                         reqp->not_aligned_seg_bits = not_aligned_seg_bits;
1788
1789                         /*
1790                          * if it is write, we need copy the original data
1791                          *to bounce buffer
1792                          */
1793                         if (WRITE_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
1794                                 storvsc_copy_sgl_to_bounce_buf(
1795                                     reqp->bounce_sgl,
1796                                     storvsc_sglist,
1797                                     storvsc_sg_count,
1798                                     reqp->not_aligned_seg_bits);
1799                         }
1800
1801                         /* transfer virtual address to physical frame number */
1802                         if (reqp->not_aligned_seg_bits & 0x1){
1803                                 phys_addr =
1804                                     vtophys(reqp->bounce_sgl->sg_segs[0].ss_paddr);
1805                         }else{
1806                                 phys_addr =
1807                                         vtophys(storvsc_sglist[0].ds_addr);
1808                         }
1809                         reqp->data_buf.offset = phys_addr & PAGE_MASK;
1810
1811                         pfn = phys_addr >> PAGE_SHIFT;
1812                         reqp->data_buf.pfn_array[0] = pfn;
1813
1814                         for (i = 1; i < storvsc_sg_count; i++) {
1815                                 if (reqp->not_aligned_seg_bits & (1 << i)) {
1816                                         phys_addr =
1817                                             vtophys(reqp->bounce_sgl->sg_segs[i].ss_paddr);
1818                                 } else {
1819                                         phys_addr =
1820                                             vtophys(storvsc_sglist[i].ds_addr);
1821                                 }
1822
1823                                 pfn = phys_addr >> PAGE_SHIFT;
1824                                 reqp->data_buf.pfn_array[i] = pfn;
1825                         }
1826                 } else {
1827                         phys_addr = vtophys(storvsc_sglist[0].ds_addr);
1828
1829                         reqp->data_buf.offset = phys_addr & PAGE_MASK;
1830
1831                         for (i = 0; i < storvsc_sg_count; i++) {
1832                                 phys_addr = vtophys(storvsc_sglist[i].ds_addr);
1833                                 pfn = phys_addr >> PAGE_SHIFT;
1834                                 reqp->data_buf.pfn_array[i] = pfn;
1835                         }
1836
1837                         /* check the last segment cross boundary or not */
1838                         offset = phys_addr & PAGE_MASK;
1839                         if (offset) {
1840                                 phys_addr =
1841                                     vtophys(storvsc_sglist[i-1].ds_addr +
1842                                     PAGE_SIZE - offset);
1843                                 pfn = phys_addr >> PAGE_SHIFT;
1844                                 reqp->data_buf.pfn_array[i] = pfn;
1845                         }
1846
1847                         reqp->bounce_sgl_count = 0;
1848                 }
1849                 break;
1850         }
1851         default:
1852                 printf("Unknow flags: %d\n", ccb->ccb_h.flags);
1853                 return(EINVAL);
1854         }
1855
1856         return(0);
1857 }
1858
1859 /**
1860  * @brief completion function before returning to CAM
1861  *
1862  * I/O process has been completed and the result needs
1863  * to be passed to the CAM layer.
1864  * Free resources related to this request.
1865  *
1866  * @param reqp pointer to a request structure
1867  */
1868 static void
1869 storvsc_io_done(struct hv_storvsc_request *reqp)
1870 {
1871         union ccb *ccb = reqp->ccb;
1872         struct ccb_scsiio *csio = &ccb->csio;
1873         struct storvsc_softc *sc = reqp->softc;
1874         struct vmscsi_req *vm_srb = &reqp->vstor_packet.u.vm_srb;
1875         bus_dma_segment_t *ori_sglist = NULL;
1876         int ori_sg_count = 0;
1877         /* destroy bounce buffer if it is used */
1878         if (reqp->bounce_sgl_count) {
1879                 ori_sglist = (bus_dma_segment_t *)ccb->csio.data_ptr;
1880                 ori_sg_count = ccb->csio.sglist_cnt;
1881
1882                 /*
1883                  * If it is READ operation, we should copy back the data
1884                  * to original SG list.
1885                  */
1886                 if (READ_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
1887                         storvsc_copy_from_bounce_buf_to_sgl(ori_sglist,
1888                             ori_sg_count,
1889                             reqp->bounce_sgl,
1890                             reqp->not_aligned_seg_bits);
1891                 }
1892
1893                 storvsc_destroy_bounce_buffer(reqp->bounce_sgl);
1894                 reqp->bounce_sgl_count = 0;
1895         }
1896
1897         if (reqp->retries > 0) {
1898                 mtx_lock(&sc->hs_lock);
1899 #if HVS_TIMEOUT_TEST
1900                 xpt_print(ccb->ccb_h.path,
1901                         "%u: IO returned after timeout, "
1902                         "waking up timer handler if any.\n", ticks);
1903                 mtx_lock(&reqp->event.mtx);
1904                 cv_signal(&reqp->event.cv);
1905                 mtx_unlock(&reqp->event.mtx);
1906 #endif
1907                 reqp->retries = 0;
1908                 xpt_print(ccb->ccb_h.path,
1909                         "%u: IO returned after timeout, "
1910                         "stopping timer if any.\n", ticks);
1911                 mtx_unlock(&sc->hs_lock);
1912         }
1913
1914 #ifdef notyet
1915         /*
1916          * callout_drain() will wait for the timer handler to finish
1917          * if it is running. So we don't need any lock to synchronize
1918          * between this routine and the timer handler.
1919          * Note that we need to make sure reqp is not freed when timer
1920          * handler is using or will use it.
1921          */
1922         if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
1923                 callout_drain(&reqp->callout);
1924         }
1925 #endif
1926
1927         ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1928         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1929         if (vm_srb->scsi_status == SCSI_STATUS_OK) {
1930                 const struct scsi_generic *cmd;
1931
1932                 if (vm_srb->srb_status != SRB_STATUS_SUCCESS) {
1933                         if (vm_srb->srb_status == SRB_STATUS_INVALID_LUN) {
1934                                 xpt_print(ccb->ccb_h.path, "invalid LUN %d\n",
1935                                     vm_srb->lun);
1936                         } else {
1937                                 xpt_print(ccb->ccb_h.path, "Unknown SRB flag: %d\n",
1938                                     vm_srb->srb_status);
1939                         }
1940                         /*
1941                          * If there are errors, for example, invalid LUN,
1942                          * host will inform VM through SRB status.
1943                          */
1944                         ccb->ccb_h.status |= CAM_SEL_TIMEOUT;
1945                 } else {
1946                         ccb->ccb_h.status |= CAM_REQ_CMP;
1947                 }
1948
1949                 cmd = (const struct scsi_generic *)
1950                     ((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1951                      csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
1952                 if (cmd->opcode == INQUIRY) {
1953                         struct scsi_inquiry_data *inq_data =
1954                             (struct scsi_inquiry_data *)csio->data_ptr;
1955                         uint8_t *resp_buf = (uint8_t *)csio->data_ptr;
1956                         int resp_xfer_len, resp_buf_len, data_len;
1957
1958                         /* Get the buffer length reported by host */
1959                         resp_xfer_len = vm_srb->transfer_len;
1960                         /* Get the available buffer length */
1961                         resp_buf_len = resp_xfer_len >= 5 ? resp_buf[4] + 5 : 0;
1962                         data_len = (resp_buf_len < resp_xfer_len) ?
1963                             resp_buf_len : resp_xfer_len;
1964
1965                         if (bootverbose && data_len >= 5) {
1966                                 xpt_print(ccb->ccb_h.path, "storvsc inquiry "
1967                                     "(%d) [%x %x %x %x %x ... ]\n", data_len,
1968                                     resp_buf[0], resp_buf[1], resp_buf[2],
1969                                     resp_buf[3], resp_buf[4]);
1970                         }
1971                         if (vm_srb->srb_status == SRB_STATUS_SUCCESS &&
1972                             data_len > SHORT_INQUIRY_LENGTH) {
1973                                 char vendor[16];
1974
1975                                 cam_strvis(vendor, inq_data->vendor,
1976                                     sizeof(inq_data->vendor), sizeof(vendor));
1977
1978                                 /*
1979                                  * XXX: Upgrade SPC2 to SPC3 if host is WIN8 or
1980                                  * WIN2012 R2 in order to support UNMAP feature.
1981                                  */
1982                                 if (!strncmp(vendor, "Msft", 4) &&
1983                                     SID_ANSI_REV(inq_data) == SCSI_REV_SPC2 &&
1984                                     (vmstor_proto_version ==
1985                                      VMSTOR_PROTOCOL_VERSION_WIN8_1 ||
1986                                      vmstor_proto_version ==
1987                                      VMSTOR_PROTOCOL_VERSION_WIN8)) {
1988                                         inq_data->version = SCSI_REV_SPC3;
1989                                         if (bootverbose) {
1990                                                 xpt_print(ccb->ccb_h.path,
1991                                                     "storvsc upgrades "
1992                                                     "SPC2 to SPC3\n");
1993                                         }
1994                                 }
1995                         }
1996                 }
1997         } else {
1998                 mtx_lock(&sc->hs_lock);
1999                 xpt_print(ccb->ccb_h.path,
2000                         "storvsc scsi_status = %d\n",
2001                         vm_srb->scsi_status);
2002                 mtx_unlock(&sc->hs_lock);
2003                 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
2004         }
2005
2006         ccb->csio.scsi_status = (vm_srb->scsi_status & 0xFF);
2007         ccb->csio.resid = ccb->csio.dxfer_len - vm_srb->transfer_len;
2008
2009         if (reqp->sense_info_len != 0) {
2010                 csio->sense_resid = csio->sense_len - reqp->sense_info_len;
2011                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2012         }
2013
2014         mtx_lock(&sc->hs_lock);
2015         if (reqp->softc->hs_frozen == 1) {
2016                 xpt_print(ccb->ccb_h.path,
2017                         "%u: storvsc unfreezing softc 0x%p.\n",
2018                         ticks, reqp->softc);
2019                 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2020                 reqp->softc->hs_frozen = 0;
2021         }
2022         storvsc_free_request(sc, reqp);
2023         mtx_unlock(&sc->hs_lock);
2024
2025         xpt_done_direct(ccb);
2026 }
2027
2028 /**
2029  * @brief Free a request structure
2030  *
2031  * Free a request structure by returning it to the free list
2032  *
2033  * @param sc pointer to a softc
2034  * @param reqp pointer to a request structure
2035  */
2036 static void
2037 storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp)
2038 {
2039
2040         LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
2041 }
2042
2043 /**
2044  * @brief Determine type of storage device from GUID
2045  *
2046  * Using the type GUID, determine if this is a StorVSC (paravirtual
2047  * SCSI or BlkVSC (paravirtual IDE) device.
2048  *
2049  * @param dev a device
2050  * returns an enum
2051  */
2052 static enum hv_storage_type
2053 storvsc_get_storage_type(device_t dev)
2054 {
2055         device_t parent = device_get_parent(dev);
2056
2057         if (VMBUS_PROBE_GUID(parent, dev, &gBlkVscDeviceType) == 0)
2058                 return DRIVER_BLKVSC;
2059         if (VMBUS_PROBE_GUID(parent, dev, &gStorVscDeviceType) == 0)
2060                 return DRIVER_STORVSC;
2061         return DRIVER_UNKNOWN;
2062 }