sys/dev/isci/isci_controller.c

   1 /*-
   2  * BSD LICENSE
   3  *
   4  * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  *   * Redistributions of source code must retain the above copyright
  12  *     notice, this list of conditions and the following disclaimer.
  13  *   * Redistributions in binary form must reproduce the above copyright
  14  *     notice, this list of conditions and the following disclaimer in
  15  *     the documentation and/or other materials provided with the
  16  *     distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <dev/isci/isci.h>
  35
  36 #include <sys/conf.h>
  37 #include <sys/malloc.h>
  38
  39 #include <cam/cam_periph.h>
  40 #include <cam/cam_xpt_periph.h>
  41
  42 #include <dev/isci/scil/sci_memory_descriptor_list.h>
  43 #include <dev/isci/scil/sci_memory_descriptor_list_decorator.h>
  44
  45 #include <dev/isci/scil/scif_controller.h>
  46 #include <dev/isci/scil/scif_library.h>
  47 #include <dev/isci/scil/scif_io_request.h>
  48 #include <dev/isci/scil/scif_task_request.h>
  49 #include <dev/isci/scil/scif_remote_device.h>
  50 #include <dev/isci/scil/scif_domain.h>
  51 #include <dev/isci/scil/scif_user_callback.h>
  52 #include <dev/isci/scil/scic_sgpio.h>
  53
  54 #include <dev/led/led.h>
  55
  56 void isci_action(struct cam_sim *sim, union ccb *ccb);
  57 void isci_poll(struct cam_sim *sim);
  58
  59 #define ccb_sim_ptr sim_priv.entries[0].ptr
  60
  61 /**
  62  * @brief This user callback will inform the user that the controller has
  63  *        had a serious unexpected error.  The user should not the error,
  64  *        disable interrupts, and wait for current ongoing processing to
  65  *        complete.  Subsequently, the user should reset the controller.
  66  *
  67  * @param[in]  controller This parameter specifies the controller that had
  68  *                        an error.
  69  *
  70  * @return none
  71  */
  72 void scif_cb_controller_error(SCI_CONTROLLER_HANDLE_T controller,
  73     SCI_CONTROLLER_ERROR error)
  74 {
  75
  76         isci_log_message(0, "ISCI", "scif_cb_controller_error: 0x%x\n",
  77             error);
  78 }
  79
  80 /**
  81  * @brief This user callback will inform the user that the controller has
  82  *        finished the start process.
  83  *
  84  * @param[in]  controller This parameter specifies the controller that was
  85  *             started.
  86  * @param[in]  completion_status This parameter specifies the results of
  87  *             the start operation.  SCI_SUCCESS indicates successful
  88  *             completion.
  89  *
  90  * @return none
  91  */
  92 void scif_cb_controller_start_complete(SCI_CONTROLLER_HANDLE_T controller,
  93     SCI_STATUS completion_status)
  94 {
  95         uint32_t index;
  96         struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
  97             sci_object_get_association(controller);
  98
  99         isci_controller->is_started = TRUE;
 100
 101         /* Set bits for all domains.  We will clear them one-by-one once
 102          *  the domains complete discovery, or return error when calling
 103          *  scif_domain_discover.  Once all bits are clear, we will register
 104          *  the controller with CAM.
 105          */
 106         isci_controller->initial_discovery_mask = (1 << SCI_MAX_DOMAINS) - 1;
 107
 108         for(index = 0; index < SCI_MAX_DOMAINS; index++) {
 109                 SCI_STATUS status;
 110                 SCI_DOMAIN_HANDLE_T domain =
 111                     isci_controller->domain[index].sci_object;
 112
 113                 status = scif_domain_discover(
 114                         domain,
 115                         scif_domain_get_suggested_discover_timeout(domain),
 116                         DEVICE_TIMEOUT
 117                 );
 118
 119                 if (status != SCI_SUCCESS)
 120                 {
 121                         isci_controller_domain_discovery_complete(
 122                             isci_controller, &isci_controller->domain[index]);
 123                 }
 124         }
 125 }
 126
 127 /**
 128  * @brief This user callback will inform the user that the controller has
 129  *        finished the stop process. Note, after user calls
 130  *        scif_controller_stop(), before user receives this controller stop
 131  *        complete callback, user should not expect any callback from
 132  *        framework, such like scif_cb_domain_change_notification().
 133  *
 134  * @param[in]  controller This parameter specifies the controller that was
 135  *             stopped.
 136  * @param[in]  completion_status This parameter specifies the results of
 137  *             the stop operation.  SCI_SUCCESS indicates successful
 138  *             completion.
 139  *
 140  * @return none
 141  */
 142 void scif_cb_controller_stop_complete(SCI_CONTROLLER_HANDLE_T controller,
 143     SCI_STATUS completion_status)
 144 {
 145         struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
 146             sci_object_get_association(controller);
 147
 148         isci_controller->is_started = FALSE;
 149 }
 150
 151 static void
 152 isci_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
 153 {
 154         SCI_PHYSICAL_ADDRESS *phys_addr = arg;
 155
 156         *phys_addr = seg[0].ds_addr;
 157 }
 158
 159 /**
 160  * @brief This method will be invoked to allocate memory dynamically.
 161  *
 162  * @param[in]  controller This parameter represents the controller
 163  *             object for which to allocate memory.
 164  * @param[out] mde This parameter represents the memory descriptor to
 165  *             be filled in by the user that will reference the newly
 166  *             allocated memory.
 167  *
 168  * @return none
 169  */
 170 void scif_cb_controller_allocate_memory(SCI_CONTROLLER_HANDLE_T controller,
 171     SCI_PHYSICAL_MEMORY_DESCRIPTOR_T *mde)
 172 {
 173         struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
 174             sci_object_get_association(controller);
 175
 176         /*
 177          * Note this routine is only used for buffers needed to translate
 178          * SCSI UNMAP commands to ATA DSM commands for SATA disks.
 179          *
 180          * We first try to pull a buffer from the controller's pool, and only
 181          * call contigmalloc if one isn't there.
 182          */
 183         if (!sci_pool_empty(isci_controller->unmap_buffer_pool)) {
 184                 sci_pool_get(isci_controller->unmap_buffer_pool,
 185                     mde->virtual_address);
 186         } else
 187                 mde->virtual_address = contigmalloc(PAGE_SIZE,
 188                     M_ISCI, M_NOWAIT, 0, BUS_SPACE_MAXADDR,
 189                     mde->constant_memory_alignment, 0);
 190
 191         if (mde->virtual_address != NULL)
 192                 bus_dmamap_load(isci_controller->buffer_dma_tag,
 193                     NULL, mde->virtual_address, PAGE_SIZE,
 194                     isci_single_map, &mde->physical_address,
 195                     BUS_DMA_NOWAIT);
 196 }
 197
 198 /**
 199  * @brief This method will be invoked to allocate memory dynamically.
 200  *
 201  * @param[in]  controller This parameter represents the controller
 202  *             object for which to allocate memory.
 203  * @param[out] mde This parameter represents the memory descriptor to
 204  *             be filled in by the user that will reference the newly
 205  *             allocated memory.
 206  *
 207  * @return none
 208  */
 209 void scif_cb_controller_free_memory(SCI_CONTROLLER_HANDLE_T controller,
 210     SCI_PHYSICAL_MEMORY_DESCRIPTOR_T * mde)
 211 {
 212         struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
 213             sci_object_get_association(controller);
 214
 215         /*
 216          * Put the buffer back into the controller's buffer pool, rather
 217          * than invoking configfree.  This helps reduce chance we won't
 218          * have buffers available when system is under memory pressure.
 219          */
 220         sci_pool_put(isci_controller->unmap_buffer_pool,
 221             mde->virtual_address);
 222 }
 223
 224 void isci_controller_construct(struct ISCI_CONTROLLER *controller,
 225     struct isci_softc *isci)
 226 {
 227         SCI_CONTROLLER_HANDLE_T scif_controller_handle;
 228
 229         scif_library_allocate_controller(isci->sci_library_handle,
 230             &scif_controller_handle);
 231
 232         scif_controller_construct(isci->sci_library_handle,
 233             scif_controller_handle, NULL);
 234
 235         controller->isci = isci;
 236         controller->scif_controller_handle = scif_controller_handle;
 237
 238         /* This allows us to later use
 239          *  sci_object_get_association(scif_controller_handle)
 240          * inside of a callback routine to get our struct ISCI_CONTROLLER object
 241          */
 242         sci_object_set_association(scif_controller_handle, (void *)controller);
 243
 244         controller->is_started = FALSE;
 245         controller->is_frozen = FALSE;
 246         controller->release_queued_ccbs = FALSE;
 247         controller->sim = NULL;
 248         controller->initial_discovery_mask = 0;
 249
 250         sci_fast_list_init(&controller->pending_device_reset_list);
 251
 252         mtx_init(&controller->lock, "isci", NULL, MTX_DEF);
 253
 254         uint32_t domain_index;
 255
 256         for(domain_index = 0; domain_index < SCI_MAX_DOMAINS; domain_index++) {
 257                 isci_domain_construct( &controller->domain[domain_index],
 258                     domain_index, controller);
 259         }
 260
 261         controller->timer_memory = malloc(
 262             sizeof(struct ISCI_TIMER) * SCI_MAX_TIMERS, M_ISCI,
 263             M_NOWAIT | M_ZERO);
 264
 265         sci_pool_initialize(controller->timer_pool);
 266
 267         struct ISCI_TIMER *timer = (struct ISCI_TIMER *)
 268             controller->timer_memory;
 269
 270         for ( int i = 0; i < SCI_MAX_TIMERS; i++ ) {
 271                 sci_pool_put(controller->timer_pool, timer++);
 272         }
 273
 274         sci_pool_initialize(controller->unmap_buffer_pool);
 275 }
 276
 277 static void isci_led_fault_func(void *priv, int onoff)
 278 {
 279         struct ISCI_PHY *phy = priv;
 280
 281         /* map onoff to the fault LED */
 282         phy->led_fault = onoff;
 283         scic_sgpio_update_led_state(phy->handle, 1 << phy->index,
 284                 phy->led_fault, phy->led_locate, 0);
 285 }
 286
 287 static void isci_led_locate_func(void *priv, int onoff)
 288 {
 289         struct ISCI_PHY *phy = priv;
 290
 291         /* map onoff to the locate LED */
 292         phy->led_locate = onoff;
 293         scic_sgpio_update_led_state(phy->handle, 1 << phy->index,
 294                 phy->led_fault, phy->led_locate, 0);
 295 }
 296
 297 SCI_STATUS isci_controller_initialize(struct ISCI_CONTROLLER *controller)
 298 {
 299         SCIC_USER_PARAMETERS_T scic_user_parameters;
 300         SCI_CONTROLLER_HANDLE_T scic_controller_handle;
 301         char led_name[64];
 302         unsigned long tunable;
 303         uint32_t io_shortage;
 304         uint32_t fail_on_timeout;
 305         int i;
 306
 307         scic_controller_handle =
 308             scif_controller_get_scic_handle(controller->scif_controller_handle);
 309
 310         if (controller->isci->oem_parameters_found == TRUE)
 311         {
 312                 scic_oem_parameters_set(
 313                     scic_controller_handle,
 314                     &controller->oem_parameters,
 315                     (uint8_t)(controller->oem_parameters_version));
 316         }
 317
 318         scic_user_parameters_get(scic_controller_handle, &scic_user_parameters);
 319
 320         if (TUNABLE_ULONG_FETCH("hw.isci.no_outbound_task_timeout", &tunable))
 321                 scic_user_parameters.sds1.no_outbound_task_timeout =
 322                     (uint8_t)tunable;
 323
 324         if (TUNABLE_ULONG_FETCH("hw.isci.ssp_max_occupancy_timeout", &tunable))
 325                 scic_user_parameters.sds1.ssp_max_occupancy_timeout =
 326                     (uint16_t)tunable;
 327
 328         if (TUNABLE_ULONG_FETCH("hw.isci.stp_max_occupancy_timeout", &tunable))
 329                 scic_user_parameters.sds1.stp_max_occupancy_timeout =
 330                     (uint16_t)tunable;
 331
 332         if (TUNABLE_ULONG_FETCH("hw.isci.ssp_inactivity_timeout", &tunable))
 333                 scic_user_parameters.sds1.ssp_inactivity_timeout =
 334                     (uint16_t)tunable;
 335
 336         if (TUNABLE_ULONG_FETCH("hw.isci.stp_inactivity_timeout", &tunable))
 337                 scic_user_parameters.sds1.stp_inactivity_timeout =
 338                     (uint16_t)tunable;
 339
 340         if (TUNABLE_ULONG_FETCH("hw.isci.max_speed_generation", &tunable))
 341                 for (i = 0; i < SCI_MAX_PHYS; i++)
 342                         scic_user_parameters.sds1.phys[i].max_speed_generation =
 343                             (uint8_t)tunable;
 344
 345         scic_user_parameters_set(scic_controller_handle, &scic_user_parameters);
 346
 347         /* Scheduler bug in SCU requires SCIL to reserve some task contexts as a
 348          *  a workaround - one per domain.
 349          */
 350         controller->queue_depth = SCI_MAX_IO_REQUESTS - SCI_MAX_DOMAINS;
 351
 352         if (TUNABLE_INT_FETCH("hw.isci.controller_queue_depth",
 353             &controller->queue_depth)) {
 354                 controller->queue_depth = max(1, min(controller->queue_depth,
 355                     SCI_MAX_IO_REQUESTS - SCI_MAX_DOMAINS));
 356         }
 357
 358         /* Reserve one request so that we can ensure we have one available TC
 359          *  to do internal device resets.
 360          */
 361         controller->sim_queue_depth = controller->queue_depth - 1;
 362
 363         /* Although we save one TC to do internal device resets, it is possible
 364          *  we could end up using several TCs for simultaneous device resets
 365          *  while at the same time having CAM fill our controller queue.  To
 366          *  simulate this condition, and how our driver handles it, we can set
 367          *  this io_shortage parameter, which will tell CAM that we have a
 368          *  large queue depth than we really do.
 369          */
 370         io_shortage = 0;
 371         TUNABLE_INT_FETCH("hw.isci.io_shortage", &io_shortage);
 372         controller->sim_queue_depth += io_shortage;
 373
 374         fail_on_timeout = 1;
 375         TUNABLE_INT_FETCH("hw.isci.fail_on_task_timeout", &fail_on_timeout);
 376         controller->fail_on_task_timeout = fail_on_timeout;
 377
 378         /* Attach to CAM using xpt_bus_register now, then immediately freeze
 379          *  the simq.  It will get released later when initial domain discovery
 380          *  is complete.
 381          */
 382         controller->has_been_scanned = FALSE;
 383         mtx_lock(&controller->lock);
 384         isci_controller_attach_to_cam(controller);
 385         xpt_freeze_simq(controller->sim, 1);
 386         mtx_unlock(&controller->lock);
 387
 388         for (i = 0; i < SCI_MAX_PHYS; i++) {
 389                 controller->phys[i].handle = scic_controller_handle;
 390                 controller->phys[i].index = i;
 391
 392                 /* fault */
 393                 controller->phys[i].led_fault = 0;
 394                 sprintf(led_name, "isci.bus%d.port%d.fault", controller->index, i);
 395                 controller->phys[i].cdev_fault = led_create(isci_led_fault_func,
 396                     &controller->phys[i], led_name);
 397
 398                 /* locate */
 399                 controller->phys[i].led_locate = 0;
 400                 sprintf(led_name, "isci.bus%d.port%d.locate", controller->index, i);
 401                 controller->phys[i].cdev_locate = led_create(isci_led_locate_func,
 402                     &controller->phys[i], led_name);
 403         }
 404
 405         return (scif_controller_initialize(controller->scif_controller_handle));
 406 }
 407
 408 int isci_controller_allocate_memory(struct ISCI_CONTROLLER *controller)
 409 {
 410         int error;
 411         device_t device =  controller->isci->device;
 412         uint32_t max_segment_size = isci_io_request_get_max_io_size();
 413         uint32_t status = 0;
 414         struct ISCI_MEMORY *uncached_controller_memory =
 415             &controller->uncached_controller_memory;
 416         struct ISCI_MEMORY *cached_controller_memory =
 417             &controller->cached_controller_memory;
 418         struct ISCI_MEMORY *request_memory =
 419             &controller->request_memory;
 420         POINTER_UINT virtual_address;
 421         bus_addr_t physical_address;
 422
 423         controller->mdl = sci_controller_get_memory_descriptor_list_handle(
 424             controller->scif_controller_handle);
 425
 426         uncached_controller_memory->size = sci_mdl_decorator_get_memory_size(
 427             controller->mdl, SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS);
 428
 429         error = isci_allocate_dma_buffer(device, uncached_controller_memory);
 430
 431         if (error != 0)
 432             return (error);
 433
 434         sci_mdl_decorator_assign_memory( controller->mdl,
 435             SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS,
 436             uncached_controller_memory->virtual_address,
 437             uncached_controller_memory->physical_address);
 438
 439         cached_controller_memory->size = sci_mdl_decorator_get_memory_size(
 440             controller->mdl,
 441             SCI_MDE_ATTRIBUTE_CACHEABLE | SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS
 442         );
 443
 444         error = isci_allocate_dma_buffer(device, cached_controller_memory);
 445
 446         if (error != 0)
 447             return (error);
 448
 449         sci_mdl_decorator_assign_memory(controller->mdl,
 450             SCI_MDE_ATTRIBUTE_CACHEABLE | SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS,
 451             cached_controller_memory->virtual_address,
 452             cached_controller_memory->physical_address);
 453
 454         request_memory->size =
 455             controller->queue_depth * isci_io_request_get_object_size();
 456
 457         error = isci_allocate_dma_buffer(device, request_memory);
 458
 459         if (error != 0)
 460             return (error);
 461
 462         /* For STP PIO testing, we want to ensure we can force multiple SGLs
 463          *  since this has been a problem area in SCIL.  This tunable parameter
 464          *  will allow us to force DMA segments to a smaller size, ensuring
 465          *  that even if a physically contiguous buffer is attached to this
 466          *  I/O, the DMA subsystem will pass us multiple segments in our DMA
 467          *  load callback.
 468          */
 469         TUNABLE_INT_FETCH("hw.isci.max_segment_size", &max_segment_size);
 470
 471         /* Create DMA tag for our I/O requests.  Then we can create DMA maps based off
 472          *  of this tag and store them in each of our ISCI_IO_REQUEST objects.  This
 473          *  will enable better performance than creating the DMA maps everytime we get
 474          *  an I/O.
 475          */
 476         status = bus_dma_tag_create(bus_get_dma_tag(device), 0x1, 0x0,
 477             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
 478             isci_io_request_get_max_io_size(),
 479             SCI_MAX_SCATTER_GATHER_ELEMENTS, max_segment_size, 0, NULL, NULL,
 480             &controller->buffer_dma_tag);
 481
 482         sci_pool_initialize(controller->request_pool);
 483
 484         virtual_address = request_memory->virtual_address;
 485         physical_address = request_memory->physical_address;
 486
 487         for (int i = 0; i < controller->queue_depth; i++) {
 488                 struct ISCI_REQUEST *request =
 489                     (struct ISCI_REQUEST *)virtual_address;
 490
 491                 isci_request_construct(request,
 492                     controller->scif_controller_handle,
 493                     controller->buffer_dma_tag, physical_address);
 494
 495                 sci_pool_put(controller->request_pool, request);
 496
 497                 virtual_address += isci_request_get_object_size();
 498                 physical_address += isci_request_get_object_size();
 499         }
 500
 501         uint32_t remote_device_size = sizeof(struct ISCI_REMOTE_DEVICE) +
 502             scif_remote_device_get_object_size();
 503
 504         controller->remote_device_memory = (uint8_t *) malloc(
 505             remote_device_size * SCI_MAX_REMOTE_DEVICES, M_ISCI,
 506             M_NOWAIT | M_ZERO);
 507
 508         sci_pool_initialize(controller->remote_device_pool);
 509
 510         uint8_t *remote_device_memory_ptr = controller->remote_device_memory;
 511
 512         for (int i = 0; i < SCI_MAX_REMOTE_DEVICES; i++) {
 513                 struct ISCI_REMOTE_DEVICE *remote_device =
 514                     (struct ISCI_REMOTE_DEVICE *)remote_device_memory_ptr;
 515
 516                 controller->remote_device[i] = NULL;
 517                 remote_device->index = i;
 518                 remote_device->is_resetting = FALSE;
 519                 remote_device->frozen_lun_mask = 0;
 520                 sci_fast_list_element_init(remote_device,
 521                     &remote_device->pending_device_reset_element);
 522                 TAILQ_INIT(&remote_device->queued_ccbs);
 523                 remote_device->release_queued_ccb = FALSE;
 524                 remote_device->queued_ccb_in_progress = NULL;
 525
 526                 /*
 527                  * For the first SCI_MAX_DOMAINS device objects, do not put
 528                  *  them in the pool, rather assign them to each domain.  This
 529                  *  ensures that any device attached directly to port "i" will
 530                  *  always get CAM target id "i".
 531                  */
 532                 if (i < SCI_MAX_DOMAINS)
 533                         controller->domain[i].da_remote_device = remote_device;
 534                 else
 535                         sci_pool_put(controller->remote_device_pool,
 536                             remote_device);
 537                 remote_device_memory_ptr += remote_device_size;
 538         }
 539
 540         return (0);
 541 }
 542
 543 void isci_controller_start(void *controller_handle)
 544 {
 545         struct ISCI_CONTROLLER *controller =
 546             (struct ISCI_CONTROLLER *)controller_handle;
 547         SCI_CONTROLLER_HANDLE_T scif_controller_handle =
 548             controller->scif_controller_handle;
 549
 550         scif_controller_start(scif_controller_handle,
 551             scif_controller_get_suggested_start_timeout(scif_controller_handle));
 552
 553         scic_controller_enable_interrupts(
 554             scif_controller_get_scic_handle(controller->scif_controller_handle));
 555 }
 556
 557 void isci_controller_domain_discovery_complete(
 558     struct ISCI_CONTROLLER *isci_controller, struct ISCI_DOMAIN *isci_domain)
 559 {
 560         if (!isci_controller->has_been_scanned)
 561         {
 562                 /* Controller has not been scanned yet.  We'll clear
 563                  *  the discovery bit for this domain, then check if all bits
 564                  *  are now clear.  That would indicate that all domains are
 565                  *  done with discovery and we can then proceed with initial
 566                  *  scan.
 567                  */
 568
 569                 isci_controller->initial_discovery_mask &=
 570                     ~(1 << isci_domain->index);
 571
 572                 if (isci_controller->initial_discovery_mask == 0) {
 573                         struct isci_softc *driver = isci_controller->isci;
 574                         uint8_t next_index = isci_controller->index + 1;
 575
 576                         isci_controller->has_been_scanned = TRUE;
 577
 578                         /* Unfreeze simq to allow initial scan to proceed. */
 579                         xpt_release_simq(isci_controller->sim, TRUE);
 580
 581 #if __FreeBSD_version < 800000
 582                         /* When driver is loaded after boot, we need to
 583                          *  explicitly rescan here for versions <8.0, because
 584                          *  CAM only automatically scans new buses at boot
 585                          *  time.
 586                          */
 587                         union ccb *ccb = xpt_alloc_ccb_nowait();
 588
 589                         xpt_create_path(&ccb->ccb_h.path, NULL,
 590                             cam_sim_path(isci_controller->sim),
 591                             CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
 592
 593                         xpt_rescan(ccb);
 594 #endif
 595
 596                         if (next_index < driver->controller_count) {
 597                                 /*  There are more controllers that need to
 598                                  *   start.  So start the next one.
 599                                  */
 600                                 isci_controller_start(
 601                                     &driver->controllers[next_index]);
 602                         }
 603                         else
 604                         {
 605                                 /* All controllers have been started and completed discovery.
 606                                  *  Disestablish the config hook while will signal to the
 607                                  *  kernel during boot that it is safe to try to find and
 608                                  *  mount the root partition.
 609                                  */
 610                                 config_intrhook_disestablish(
 611                                     &driver->config_hook);
 612                         }
 613                 }
 614         }
 615 }
 616
 617 int isci_controller_attach_to_cam(struct ISCI_CONTROLLER *controller)
 618 {
 619         struct isci_softc *isci = controller->isci;
 620         device_t parent = device_get_parent(isci->device);
 621         int unit = device_get_unit(isci->device);
 622         struct cam_devq *isci_devq = cam_simq_alloc(controller->sim_queue_depth);
 623
 624         if(isci_devq == NULL) {
 625                 isci_log_message(0, "ISCI", "isci_devq is NULL \n");
 626                 return (-1);
 627         }
 628
 629         controller->sim = cam_sim_alloc(isci_action, isci_poll, "isci",
 630             controller, unit, &controller->lock, controller->sim_queue_depth,
 631             controller->sim_queue_depth, isci_devq);
 632
 633         if(controller->sim == NULL) {
 634                 isci_log_message(0, "ISCI", "cam_sim_alloc... fails\n");
 635                 cam_simq_free(isci_devq);
 636                 return (-1);
 637         }
 638
 639         if(xpt_bus_register(controller->sim, parent, controller->index)
 640             != CAM_SUCCESS) {
 641                 isci_log_message(0, "ISCI", "xpt_bus_register...fails \n");
 642                 cam_sim_free(controller->sim, TRUE);
 643                 mtx_unlock(&controller->lock);
 644                 return (-1);
 645         }
 646
 647         if(xpt_create_path(&controller->path, NULL,
 648             cam_sim_path(controller->sim), CAM_TARGET_WILDCARD,
 649             CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 650                 isci_log_message(0, "ISCI", "xpt_create_path....fails\n");
 651                 xpt_bus_deregister(cam_sim_path(controller->sim));
 652                 cam_sim_free(controller->sim, TRUE);
 653                 mtx_unlock(&controller->lock);
 654                 return (-1);
 655         }
 656
 657         return (0);
 658 }
 659
 660 void isci_poll(struct cam_sim *sim)
 661 {
 662         struct ISCI_CONTROLLER *controller =
 663             (struct ISCI_CONTROLLER *)cam_sim_softc(sim);
 664
 665         isci_interrupt_poll_handler(controller);
 666 }
 667
 668 void isci_action(struct cam_sim *sim, union ccb *ccb)
 669 {
 670         struct ISCI_CONTROLLER *controller =
 671             (struct ISCI_CONTROLLER *)cam_sim_softc(sim);
 672
 673         switch ( ccb->ccb_h.func_code ) {
 674         case XPT_PATH_INQ:
 675                 {
 676                         struct ccb_pathinq *cpi = &ccb->cpi;
 677                         int bus = cam_sim_bus(sim);
 678                         ccb->ccb_h.ccb_sim_ptr = sim;
 679                         cpi->version_num = 1;
 680                         cpi->hba_inquiry = PI_TAG_ABLE;
 681                         cpi->target_sprt = 0;
 682                         cpi->hba_misc = PIM_NOBUSRESET | PIM_SEQSCAN |
 683                             PIM_UNMAPPED;
 684                         cpi->hba_eng_cnt = 0;
 685                         cpi->max_target = SCI_MAX_REMOTE_DEVICES - 1;
 686                         cpi->max_lun = ISCI_MAX_LUN;
 687 #if __FreeBSD_version >= 800102
 688                         cpi->maxio = isci_io_request_get_max_io_size();
 689 #endif
 690                         cpi->unit_number = cam_sim_unit(sim);
 691                         cpi->bus_id = bus;
 692                         cpi->initiator_id = SCI_MAX_REMOTE_DEVICES;
 693                         cpi->base_transfer_speed = 300000;
 694                         strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 695                         strncpy(cpi->hba_vid, "Intel Corp.", HBA_IDLEN);
 696                         strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 697                         cpi->transport = XPORT_SAS;
 698                         cpi->transport_version = 0;
 699                         cpi->protocol = PROTO_SCSI;
 700                         cpi->protocol_version = SCSI_REV_SPC2;
 701                         cpi->ccb_h.status = CAM_REQ_CMP;
 702                         xpt_done(ccb);
 703                 }
 704                 break;
 705         case XPT_GET_TRAN_SETTINGS:
 706                 {
 707                         struct ccb_trans_settings *general_settings = &ccb->cts;
 708                         struct ccb_trans_settings_sas *sas_settings =
 709                             &general_settings->xport_specific.sas;
 710                         struct ccb_trans_settings_scsi *scsi_settings =
 711                             &general_settings->proto_specific.scsi;
 712                         struct ISCI_REMOTE_DEVICE *remote_device;
 713
 714                         remote_device = controller->remote_device[ccb->ccb_h.target_id];
 715
 716                         if (remote_device == NULL) {
 717                                 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 718                                 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 719                                 ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
 720                                 xpt_done(ccb);
 721                                 break;
 722                         }
 723
 724                         general_settings->protocol = PROTO_SCSI;
 725                         general_settings->transport = XPORT_SAS;
 726                         general_settings->protocol_version = SCSI_REV_SPC2;
 727                         general_settings->transport_version = 0;
 728                         scsi_settings->valid = CTS_SCSI_VALID_TQ;
 729                         scsi_settings->flags = CTS_SCSI_FLAGS_TAG_ENB;
 730                         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 731                         ccb->ccb_h.status |= CAM_REQ_CMP;
 732
 733                         sas_settings->bitrate =
 734                             isci_remote_device_get_bitrate(remote_device);
 735
 736                         if (sas_settings->bitrate != 0)
 737                                 sas_settings->valid = CTS_SAS_VALID_SPEED;
 738
 739                         xpt_done(ccb);
 740                 }
 741                 break;
 742         case XPT_SCSI_IO:
 743                 isci_io_request_execute_scsi_io(ccb, controller);
 744                 break;
 745 #if __FreeBSD_version >= 900026
 746         case XPT_SMP_IO:
 747                 isci_io_request_execute_smp_io(ccb, controller);
 748                 break;
 749 #endif
 750         case XPT_SET_TRAN_SETTINGS:
 751                 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 752                 ccb->ccb_h.status |= CAM_REQ_CMP;
 753                 xpt_done(ccb);
 754                 break;
 755         case XPT_CALC_GEOMETRY:
 756                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
 757                 xpt_done(ccb);
 758                 break;
 759         case XPT_RESET_DEV:
 760                 {
 761                         struct ISCI_REMOTE_DEVICE *remote_device =
 762                             controller->remote_device[ccb->ccb_h.target_id];
 763
 764                         if (remote_device != NULL)
 765                                 isci_remote_device_reset(remote_device, ccb);
 766                         else {
 767                                 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 768                                 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 769                                 ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
 770                                 xpt_done(ccb);
 771                         }
 772                 }
 773                 break;
 774         case XPT_RESET_BUS:
 775                 ccb->ccb_h.status = CAM_REQ_CMP;
 776                 xpt_done(ccb);
 777                 break;
 778         default:
 779                 isci_log_message(0, "ISCI", "Unhandled func_code 0x%x\n",
 780                     ccb->ccb_h.func_code);
 781                 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 782                 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 783                 ccb->ccb_h.status |= CAM_REQ_INVALID;
 784                 xpt_done(ccb);
 785                 break;
 786         }
 787 }
 788
 789 /*
 790  * Unfortunately, SCIL doesn't cleanly handle retry conditions.
 791  *  CAM_REQUEUE_REQ works only when no one is using the pass(4) interface.  So
 792  *  when SCIL denotes an I/O needs to be retried (typically because of mixing
 793  *  tagged/non-tagged ATA commands, or running out of NCQ slots), we queue
 794  *  these I/O internally.  Once SCIL completes an I/O to this device, or we get
 795  *  a ready notification, we will retry the first I/O on the queue.
 796  *  Unfortunately, SCIL also doesn't cleanly handle starting the new I/O within
 797  *  the context of the completion handler, so we need to retry these I/O after
 798  *  the completion handler is done executing.
 799  */
 800 void
 801 isci_controller_release_queued_ccbs(struct ISCI_CONTROLLER *controller)
 802 {
 803         struct ISCI_REMOTE_DEVICE *dev;
 804         struct ccb_hdr *ccb_h;
 805         int dev_idx;
 806
 807         KASSERT(mtx_owned(&controller->lock), ("controller lock not owned"));
 808
 809         controller->release_queued_ccbs = FALSE;
 810         for (dev_idx = 0;
 811              dev_idx < SCI_MAX_REMOTE_DEVICES;
 812              dev_idx++) {
 813
 814                 dev = controller->remote_device[dev_idx];
 815                 if (dev != NULL &&
 816                     dev->release_queued_ccb == TRUE &&
 817                     dev->queued_ccb_in_progress == NULL) {
 818                         dev->release_queued_ccb = FALSE;
 819                         ccb_h = TAILQ_FIRST(&dev->queued_ccbs);
 820
 821                         if (ccb_h == NULL)
 822                                 continue;
 823
 824                         isci_log_message(1, "ISCI", "release %p %x\n", ccb_h,
 825                             ((union ccb *)ccb_h)->csio.cdb_io.cdb_bytes[0]);
 826
 827                         dev->queued_ccb_in_progress = (union ccb *)ccb_h;
 828                         isci_io_request_execute_scsi_io(
 829                             (union ccb *)ccb_h, controller);
 830                 }
 831         }
 832 }