sys/dev/isci/isci_controller.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * BSD LICENSE
   5  *
   6  * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved.
   7  * All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  *
  13  *   * Redistributions of source code must retain the above copyright
  14  *     notice, this list of conditions and the following disclaimer.
  15  *   * Redistributions in binary form must reproduce the above copyright
  16  *     notice, this list of conditions and the following disclaimer in
  17  *     the documentation and/or other materials provided with the
  18  *     distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include <dev/isci/isci.h>
  37
  38 #include <sys/conf.h>
  39 #include <sys/malloc.h>
  40
  41 #include <cam/cam_periph.h>
  42 #include <cam/cam_xpt_periph.h>
  43
  44 #include <dev/isci/scil/sci_memory_descriptor_list.h>
  45 #include <dev/isci/scil/sci_memory_descriptor_list_decorator.h>
  46
  47 #include <dev/isci/scil/scif_controller.h>
  48 #include <dev/isci/scil/scif_library.h>
  49 #include <dev/isci/scil/scif_io_request.h>
  50 #include <dev/isci/scil/scif_task_request.h>
  51 #include <dev/isci/scil/scif_remote_device.h>
  52 #include <dev/isci/scil/scif_domain.h>
  53 #include <dev/isci/scil/scif_user_callback.h>
  54 #include <dev/isci/scil/scic_sgpio.h>
  55
  56 #include <dev/led/led.h>
  57
  58 void isci_action(struct cam_sim *sim, union ccb *ccb);
  59 void isci_poll(struct cam_sim *sim);
  60
  61 #define ccb_sim_ptr sim_priv.entries[0].ptr
  62
  63 /**
  64  * @brief This user callback will inform the user that the controller has
  65  *        had a serious unexpected error.  The user should not the error,
  66  *        disable interrupts, and wait for current ongoing processing to
  67  *        complete.  Subsequently, the user should reset the controller.
  68  *
  69  * @param[in]  controller This parameter specifies the controller that had
  70  *                        an error.
  71  *
  72  * @return none
  73  */
  74 void scif_cb_controller_error(SCI_CONTROLLER_HANDLE_T controller,
  75     SCI_CONTROLLER_ERROR error)
  76 {
  77
  78         isci_log_message(0, "ISCI", "scif_cb_controller_error: 0x%x\n",
  79             error);
  80 }
  81
  82 /**
  83  * @brief This user callback will inform the user that the controller has
  84  *        finished the start process.
  85  *
  86  * @param[in]  controller This parameter specifies the controller that was
  87  *             started.
  88  * @param[in]  completion_status This parameter specifies the results of
  89  *             the start operation.  SCI_SUCCESS indicates successful
  90  *             completion.
  91  *
  92  * @return none
  93  */
  94 void scif_cb_controller_start_complete(SCI_CONTROLLER_HANDLE_T controller,
  95     SCI_STATUS completion_status)
  96 {
  97         uint32_t index;
  98         struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
  99             sci_object_get_association(controller);
 100
 101         isci_controller->is_started = TRUE;
 102
 103         /* Set bits for all domains.  We will clear them one-by-one once
 104          *  the domains complete discovery, or return error when calling
 105          *  scif_domain_discover.  Once all bits are clear, we will register
 106          *  the controller with CAM.
 107          */
 108         isci_controller->initial_discovery_mask = (1 << SCI_MAX_DOMAINS) - 1;
 109
 110         for(index = 0; index < SCI_MAX_DOMAINS; index++) {
 111                 SCI_STATUS status;
 112                 SCI_DOMAIN_HANDLE_T domain =
 113                     isci_controller->domain[index].sci_object;
 114
 115                 status = scif_domain_discover(
 116                         domain,
 117                         scif_domain_get_suggested_discover_timeout(domain),
 118                         DEVICE_TIMEOUT
 119                 );
 120
 121                 if (status != SCI_SUCCESS)
 122                 {
 123                         isci_controller_domain_discovery_complete(
 124                             isci_controller, &isci_controller->domain[index]);
 125                 }
 126         }
 127 }
 128
 129 /**
 130  * @brief This user callback will inform the user that the controller has
 131  *        finished the stop process. Note, after user calls
 132  *        scif_controller_stop(), before user receives this controller stop
 133  *        complete callback, user should not expect any callback from
 134  *        framework, such like scif_cb_domain_change_notification().
 135  *
 136  * @param[in]  controller This parameter specifies the controller that was
 137  *             stopped.
 138  * @param[in]  completion_status This parameter specifies the results of
 139  *             the stop operation.  SCI_SUCCESS indicates successful
 140  *             completion.
 141  *
 142  * @return none
 143  */
 144 void scif_cb_controller_stop_complete(SCI_CONTROLLER_HANDLE_T controller,
 145     SCI_STATUS completion_status)
 146 {
 147         struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
 148             sci_object_get_association(controller);
 149
 150         isci_controller->is_started = FALSE;
 151 }
 152
 153 static void
 154 isci_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
 155 {
 156         SCI_PHYSICAL_ADDRESS *phys_addr = arg;
 157
 158         *phys_addr = seg[0].ds_addr;
 159 }
 160
 161 /**
 162  * @brief This method will be invoked to allocate memory dynamically.
 163  *
 164  * @param[in]  controller This parameter represents the controller
 165  *             object for which to allocate memory.
 166  * @param[out] mde This parameter represents the memory descriptor to
 167  *             be filled in by the user that will reference the newly
 168  *             allocated memory.
 169  *
 170  * @return none
 171  */
 172 void scif_cb_controller_allocate_memory(SCI_CONTROLLER_HANDLE_T controller,
 173     SCI_PHYSICAL_MEMORY_DESCRIPTOR_T *mde)
 174 {
 175         struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
 176             sci_object_get_association(controller);
 177
 178         /*
 179          * Note this routine is only used for buffers needed to translate
 180          * SCSI UNMAP commands to ATA DSM commands for SATA disks.
 181          *
 182          * We first try to pull a buffer from the controller's pool, and only
 183          * call contigmalloc if one isn't there.
 184          */
 185         if (!sci_pool_empty(isci_controller->unmap_buffer_pool)) {
 186                 sci_pool_get(isci_controller->unmap_buffer_pool,
 187                     mde->virtual_address);
 188         } else
 189                 mde->virtual_address = contigmalloc(PAGE_SIZE,
 190                     M_ISCI, M_NOWAIT, 0, BUS_SPACE_MAXADDR,
 191                     mde->constant_memory_alignment, 0);
 192
 193         if (mde->virtual_address != NULL)
 194                 bus_dmamap_load(isci_controller->buffer_dma_tag,
 195                     NULL, mde->virtual_address, PAGE_SIZE,
 196                     isci_single_map, &mde->physical_address,
 197                     BUS_DMA_NOWAIT);
 198 }
 199
 200 /**
 201  * @brief This method will be invoked to allocate memory dynamically.
 202  *
 203  * @param[in]  controller This parameter represents the controller
 204  *             object for which to allocate memory.
 205  * @param[out] mde This parameter represents the memory descriptor to
 206  *             be filled in by the user that will reference the newly
 207  *             allocated memory.
 208  *
 209  * @return none
 210  */
 211 void scif_cb_controller_free_memory(SCI_CONTROLLER_HANDLE_T controller,
 212     SCI_PHYSICAL_MEMORY_DESCRIPTOR_T * mde)
 213 {
 214         struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
 215             sci_object_get_association(controller);
 216
 217         /*
 218          * Put the buffer back into the controller's buffer pool, rather
 219          * than invoking configfree.  This helps reduce chance we won't
 220          * have buffers available when system is under memory pressure.
 221          */
 222         sci_pool_put(isci_controller->unmap_buffer_pool,
 223             mde->virtual_address);
 224 }
 225
 226 void isci_controller_construct(struct ISCI_CONTROLLER *controller,
 227     struct isci_softc *isci)
 228 {
 229         SCI_CONTROLLER_HANDLE_T scif_controller_handle;
 230
 231         scif_library_allocate_controller(isci->sci_library_handle,
 232             &scif_controller_handle);
 233
 234         scif_controller_construct(isci->sci_library_handle,
 235             scif_controller_handle, NULL);
 236
 237         controller->isci = isci;
 238         controller->scif_controller_handle = scif_controller_handle;
 239
 240         /* This allows us to later use
 241          *  sci_object_get_association(scif_controller_handle)
 242          * inside of a callback routine to get our struct ISCI_CONTROLLER object
 243          */
 244         sci_object_set_association(scif_controller_handle, (void *)controller);
 245
 246         controller->is_started = FALSE;
 247         controller->is_frozen = FALSE;
 248         controller->release_queued_ccbs = FALSE;
 249         controller->sim = NULL;
 250         controller->initial_discovery_mask = 0;
 251
 252         sci_fast_list_init(&controller->pending_device_reset_list);
 253
 254         mtx_init(&controller->lock, "isci", NULL, MTX_DEF);
 255
 256         uint32_t domain_index;
 257
 258         for(domain_index = 0; domain_index < SCI_MAX_DOMAINS; domain_index++) {
 259                 isci_domain_construct( &controller->domain[domain_index],
 260                     domain_index, controller);
 261         }
 262
 263         controller->timer_memory = malloc(
 264             sizeof(struct ISCI_TIMER) * SCI_MAX_TIMERS, M_ISCI,
 265             M_NOWAIT | M_ZERO);
 266
 267         sci_pool_initialize(controller->timer_pool);
 268
 269         struct ISCI_TIMER *timer = (struct ISCI_TIMER *)
 270             controller->timer_memory;
 271
 272         for ( int i = 0; i < SCI_MAX_TIMERS; i++ ) {
 273                 sci_pool_put(controller->timer_pool, timer++);
 274         }
 275
 276         sci_pool_initialize(controller->unmap_buffer_pool);
 277 }
 278
 279 static void isci_led_fault_func(void *priv, int onoff)
 280 {
 281         struct ISCI_PHY *phy = priv;
 282
 283         /* map onoff to the fault LED */
 284         phy->led_fault = onoff;
 285         scic_sgpio_update_led_state(phy->handle, 1 << phy->index,
 286                 phy->led_fault, phy->led_locate, 0);
 287 }
 288
 289 static void isci_led_locate_func(void *priv, int onoff)
 290 {
 291         struct ISCI_PHY *phy = priv;
 292
 293         /* map onoff to the locate LED */
 294         phy->led_locate = onoff;
 295         scic_sgpio_update_led_state(phy->handle, 1 << phy->index,
 296                 phy->led_fault, phy->led_locate, 0);
 297 }
 298
 299 SCI_STATUS isci_controller_initialize(struct ISCI_CONTROLLER *controller)
 300 {
 301         SCIC_USER_PARAMETERS_T scic_user_parameters;
 302         SCI_CONTROLLER_HANDLE_T scic_controller_handle;
 303         char led_name[64];
 304         unsigned long tunable;
 305         uint32_t io_shortage;
 306         uint32_t fail_on_timeout;
 307         int i;
 308
 309         scic_controller_handle =
 310             scif_controller_get_scic_handle(controller->scif_controller_handle);
 311
 312         if (controller->isci->oem_parameters_found == TRUE)
 313         {
 314                 scic_oem_parameters_set(
 315                     scic_controller_handle,
 316                     &controller->oem_parameters,
 317                     (uint8_t)(controller->oem_parameters_version));
 318         }
 319
 320         scic_user_parameters_get(scic_controller_handle, &scic_user_parameters);
 321
 322         if (TUNABLE_ULONG_FETCH("hw.isci.no_outbound_task_timeout", &tunable))
 323                 scic_user_parameters.sds1.no_outbound_task_timeout =
 324                     (uint8_t)tunable;
 325
 326         if (TUNABLE_ULONG_FETCH("hw.isci.ssp_max_occupancy_timeout", &tunable))
 327                 scic_user_parameters.sds1.ssp_max_occupancy_timeout =
 328                     (uint16_t)tunable;
 329
 330         if (TUNABLE_ULONG_FETCH("hw.isci.stp_max_occupancy_timeout", &tunable))
 331                 scic_user_parameters.sds1.stp_max_occupancy_timeout =
 332                     (uint16_t)tunable;
 333
 334         if (TUNABLE_ULONG_FETCH("hw.isci.ssp_inactivity_timeout", &tunable))
 335                 scic_user_parameters.sds1.ssp_inactivity_timeout =
 336                     (uint16_t)tunable;
 337
 338         if (TUNABLE_ULONG_FETCH("hw.isci.stp_inactivity_timeout", &tunable))
 339                 scic_user_parameters.sds1.stp_inactivity_timeout =
 340                     (uint16_t)tunable;
 341
 342         if (TUNABLE_ULONG_FETCH("hw.isci.max_speed_generation", &tunable))
 343                 for (i = 0; i < SCI_MAX_PHYS; i++)
 344                         scic_user_parameters.sds1.phys[i].max_speed_generation =
 345                             (uint8_t)tunable;
 346
 347         scic_user_parameters_set(scic_controller_handle, &scic_user_parameters);
 348
 349         /* Scheduler bug in SCU requires SCIL to reserve some task contexts as a
 350          *  a workaround - one per domain.
 351          */
 352         controller->queue_depth = SCI_MAX_IO_REQUESTS - SCI_MAX_DOMAINS;
 353
 354         if (TUNABLE_INT_FETCH("hw.isci.controller_queue_depth",
 355             &controller->queue_depth)) {
 356                 controller->queue_depth = max(1, min(controller->queue_depth,
 357                     SCI_MAX_IO_REQUESTS - SCI_MAX_DOMAINS));
 358         }
 359
 360         /* Reserve one request so that we can ensure we have one available TC
 361          *  to do internal device resets.
 362          */
 363         controller->sim_queue_depth = controller->queue_depth - 1;
 364
 365         /* Although we save one TC to do internal device resets, it is possible
 366          *  we could end up using several TCs for simultaneous device resets
 367          *  while at the same time having CAM fill our controller queue.  To
 368          *  simulate this condition, and how our driver handles it, we can set
 369          *  this io_shortage parameter, which will tell CAM that we have a
 370          *  large queue depth than we really do.
 371          */
 372         io_shortage = 0;
 373         TUNABLE_INT_FETCH("hw.isci.io_shortage", &io_shortage);
 374         controller->sim_queue_depth += io_shortage;
 375
 376         fail_on_timeout = 1;
 377         TUNABLE_INT_FETCH("hw.isci.fail_on_task_timeout", &fail_on_timeout);
 378         controller->fail_on_task_timeout = fail_on_timeout;
 379
 380         /* Attach to CAM using xpt_bus_register now, then immediately freeze
 381          *  the simq.  It will get released later when initial domain discovery
 382          *  is complete.
 383          */
 384         controller->has_been_scanned = FALSE;
 385         mtx_lock(&controller->lock);
 386         isci_controller_attach_to_cam(controller);
 387         xpt_freeze_simq(controller->sim, 1);
 388         mtx_unlock(&controller->lock);
 389
 390         for (i = 0; i < SCI_MAX_PHYS; i++) {
 391                 controller->phys[i].handle = scic_controller_handle;
 392                 controller->phys[i].index = i;
 393
 394                 /* fault */
 395                 controller->phys[i].led_fault = 0;
 396                 sprintf(led_name, "isci.bus%d.port%d.fault", controller->index, i);
 397                 controller->phys[i].cdev_fault = led_create(isci_led_fault_func,
 398                     &controller->phys[i], led_name);
 399
 400                 /* locate */
 401                 controller->phys[i].led_locate = 0;
 402                 sprintf(led_name, "isci.bus%d.port%d.locate", controller->index, i);
 403                 controller->phys[i].cdev_locate = led_create(isci_led_locate_func,
 404                     &controller->phys[i], led_name);
 405         }
 406
 407         return (scif_controller_initialize(controller->scif_controller_handle));
 408 }
 409
 410 int isci_controller_allocate_memory(struct ISCI_CONTROLLER *controller)
 411 {
 412         int error;
 413         device_t device =  controller->isci->device;
 414         uint32_t max_segment_size = isci_io_request_get_max_io_size();
 415         uint32_t status = 0;
 416         struct ISCI_MEMORY *uncached_controller_memory =
 417             &controller->uncached_controller_memory;
 418         struct ISCI_MEMORY *cached_controller_memory =
 419             &controller->cached_controller_memory;
 420         struct ISCI_MEMORY *request_memory =
 421             &controller->request_memory;
 422         POINTER_UINT virtual_address;
 423         bus_addr_t physical_address;
 424
 425         controller->mdl = sci_controller_get_memory_descriptor_list_handle(
 426             controller->scif_controller_handle);
 427
 428         uncached_controller_memory->size = sci_mdl_decorator_get_memory_size(
 429             controller->mdl, SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS);
 430
 431         error = isci_allocate_dma_buffer(device, uncached_controller_memory);
 432
 433         if (error != 0)
 434             return (error);
 435
 436         sci_mdl_decorator_assign_memory( controller->mdl,
 437             SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS,
 438             uncached_controller_memory->virtual_address,
 439             uncached_controller_memory->physical_address);
 440
 441         cached_controller_memory->size = sci_mdl_decorator_get_memory_size(
 442             controller->mdl,
 443             SCI_MDE_ATTRIBUTE_CACHEABLE | SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS
 444         );
 445
 446         error = isci_allocate_dma_buffer(device, cached_controller_memory);
 447
 448         if (error != 0)
 449             return (error);
 450
 451         sci_mdl_decorator_assign_memory(controller->mdl,
 452             SCI_MDE_ATTRIBUTE_CACHEABLE | SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS,
 453             cached_controller_memory->virtual_address,
 454             cached_controller_memory->physical_address);
 455
 456         request_memory->size =
 457             controller->queue_depth * isci_io_request_get_object_size();
 458
 459         error = isci_allocate_dma_buffer(device, request_memory);
 460
 461         if (error != 0)
 462             return (error);
 463
 464         /* For STP PIO testing, we want to ensure we can force multiple SGLs
 465          *  since this has been a problem area in SCIL.  This tunable parameter
 466          *  will allow us to force DMA segments to a smaller size, ensuring
 467          *  that even if a physically contiguous buffer is attached to this
 468          *  I/O, the DMA subsystem will pass us multiple segments in our DMA
 469          *  load callback.
 470          */
 471         TUNABLE_INT_FETCH("hw.isci.max_segment_size", &max_segment_size);
 472
 473         /* Create DMA tag for our I/O requests.  Then we can create DMA maps based off
 474          *  of this tag and store them in each of our ISCI_IO_REQUEST objects.  This
 475          *  will enable better performance than creating the DMA maps every time we get
 476          *  an I/O.
 477          */
 478         status = bus_dma_tag_create(bus_get_dma_tag(device), 0x1, 0x0,
 479             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
 480             isci_io_request_get_max_io_size(),
 481             SCI_MAX_SCATTER_GATHER_ELEMENTS, max_segment_size, 0, NULL, NULL,
 482             &controller->buffer_dma_tag);
 483
 484         sci_pool_initialize(controller->request_pool);
 485
 486         virtual_address = request_memory->virtual_address;
 487         physical_address = request_memory->physical_address;
 488
 489         for (int i = 0; i < controller->queue_depth; i++) {
 490                 struct ISCI_REQUEST *request =
 491                     (struct ISCI_REQUEST *)virtual_address;
 492
 493                 isci_request_construct(request,
 494                     controller->scif_controller_handle,
 495                     controller->buffer_dma_tag, physical_address);
 496
 497                 sci_pool_put(controller->request_pool, request);
 498
 499                 virtual_address += isci_request_get_object_size();
 500                 physical_address += isci_request_get_object_size();
 501         }
 502
 503         uint32_t remote_device_size = sizeof(struct ISCI_REMOTE_DEVICE) +
 504             scif_remote_device_get_object_size();
 505
 506         controller->remote_device_memory = (uint8_t *) malloc(
 507             remote_device_size * SCI_MAX_REMOTE_DEVICES, M_ISCI,
 508             M_NOWAIT | M_ZERO);
 509
 510         sci_pool_initialize(controller->remote_device_pool);
 511
 512         uint8_t *remote_device_memory_ptr = controller->remote_device_memory;
 513
 514         for (int i = 0; i < SCI_MAX_REMOTE_DEVICES; i++) {
 515                 struct ISCI_REMOTE_DEVICE *remote_device =
 516                     (struct ISCI_REMOTE_DEVICE *)remote_device_memory_ptr;
 517
 518                 controller->remote_device[i] = NULL;
 519                 remote_device->index = i;
 520                 remote_device->is_resetting = FALSE;
 521                 remote_device->frozen_lun_mask = 0;
 522                 sci_fast_list_element_init(remote_device,
 523                     &remote_device->pending_device_reset_element);
 524                 TAILQ_INIT(&remote_device->queued_ccbs);
 525                 remote_device->release_queued_ccb = FALSE;
 526                 remote_device->queued_ccb_in_progress = NULL;
 527
 528                 /*
 529                  * For the first SCI_MAX_DOMAINS device objects, do not put
 530                  *  them in the pool, rather assign them to each domain.  This
 531                  *  ensures that any device attached directly to port "i" will
 532                  *  always get CAM target id "i".
 533                  */
 534                 if (i < SCI_MAX_DOMAINS)
 535                         controller->domain[i].da_remote_device = remote_device;
 536                 else
 537                         sci_pool_put(controller->remote_device_pool,
 538                             remote_device);
 539                 remote_device_memory_ptr += remote_device_size;
 540         }
 541
 542         return (0);
 543 }
 544
 545 void isci_controller_start(void *controller_handle)
 546 {
 547         struct ISCI_CONTROLLER *controller =
 548             (struct ISCI_CONTROLLER *)controller_handle;
 549         SCI_CONTROLLER_HANDLE_T scif_controller_handle =
 550             controller->scif_controller_handle;
 551
 552         scif_controller_start(scif_controller_handle,
 553             scif_controller_get_suggested_start_timeout(scif_controller_handle));
 554
 555         scic_controller_enable_interrupts(
 556             scif_controller_get_scic_handle(controller->scif_controller_handle));
 557 }
 558
 559 void isci_controller_domain_discovery_complete(
 560     struct ISCI_CONTROLLER *isci_controller, struct ISCI_DOMAIN *isci_domain)
 561 {
 562         if (!isci_controller->has_been_scanned)
 563         {
 564                 /* Controller has not been scanned yet.  We'll clear
 565                  *  the discovery bit for this domain, then check if all bits
 566                  *  are now clear.  That would indicate that all domains are
 567                  *  done with discovery and we can then proceed with initial
 568                  *  scan.
 569                  */
 570
 571                 isci_controller->initial_discovery_mask &=
 572                     ~(1 << isci_domain->index);
 573
 574                 if (isci_controller->initial_discovery_mask == 0) {
 575                         struct isci_softc *driver = isci_controller->isci;
 576                         uint8_t next_index = isci_controller->index + 1;
 577
 578                         isci_controller->has_been_scanned = TRUE;
 579
 580                         /* Unfreeze simq to allow initial scan to proceed. */
 581                         xpt_release_simq(isci_controller->sim, TRUE);
 582
 583 #if __FreeBSD_version < 800000
 584                         /* When driver is loaded after boot, we need to
 585                          *  explicitly rescan here for versions <8.0, because
 586                          *  CAM only automatically scans new buses at boot
 587                          *  time.
 588                          */
 589                         union ccb *ccb = xpt_alloc_ccb_nowait();
 590
 591                         xpt_create_path(&ccb->ccb_h.path, NULL,
 592                             cam_sim_path(isci_controller->sim),
 593                             CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
 594
 595                         xpt_rescan(ccb);
 596 #endif
 597
 598                         if (next_index < driver->controller_count) {
 599                                 /*  There are more controllers that need to
 600                                  *   start.  So start the next one.
 601                                  */
 602                                 isci_controller_start(
 603                                     &driver->controllers[next_index]);
 604                         }
 605                         else
 606                         {
 607                                 /* All controllers have been started and completed discovery.
 608                                  *  Disestablish the config hook while will signal to the
 609                                  *  kernel during boot that it is safe to try to find and
 610                                  *  mount the root partition.
 611                                  */
 612                                 config_intrhook_disestablish(
 613                                     &driver->config_hook);
 614                         }
 615                 }
 616         }
 617 }
 618
 619 int isci_controller_attach_to_cam(struct ISCI_CONTROLLER *controller)
 620 {
 621         struct isci_softc *isci = controller->isci;
 622         device_t parent = device_get_parent(isci->device);
 623         int unit = device_get_unit(isci->device);
 624         struct cam_devq *isci_devq = cam_simq_alloc(controller->sim_queue_depth);
 625
 626         if(isci_devq == NULL) {
 627                 isci_log_message(0, "ISCI", "isci_devq is NULL \n");
 628                 return (-1);
 629         }
 630
 631         controller->sim = cam_sim_alloc(isci_action, isci_poll, "isci",
 632             controller, unit, &controller->lock, controller->sim_queue_depth,
 633             controller->sim_queue_depth, isci_devq);
 634
 635         if(controller->sim == NULL) {
 636                 isci_log_message(0, "ISCI", "cam_sim_alloc... fails\n");
 637                 cam_simq_free(isci_devq);
 638                 return (-1);
 639         }
 640
 641         if(xpt_bus_register(controller->sim, parent, controller->index)
 642             != CAM_SUCCESS) {
 643                 isci_log_message(0, "ISCI", "xpt_bus_register...fails \n");
 644                 cam_sim_free(controller->sim, TRUE);
 645                 mtx_unlock(&controller->lock);
 646                 return (-1);
 647         }
 648
 649         if(xpt_create_path(&controller->path, NULL,
 650             cam_sim_path(controller->sim), CAM_TARGET_WILDCARD,
 651             CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 652                 isci_log_message(0, "ISCI", "xpt_create_path....fails\n");
 653                 xpt_bus_deregister(cam_sim_path(controller->sim));
 654                 cam_sim_free(controller->sim, TRUE);
 655                 mtx_unlock(&controller->lock);
 656                 return (-1);
 657         }
 658
 659         return (0);
 660 }
 661
 662 void isci_poll(struct cam_sim *sim)
 663 {
 664         struct ISCI_CONTROLLER *controller =
 665             (struct ISCI_CONTROLLER *)cam_sim_softc(sim);
 666
 667         isci_interrupt_poll_handler(controller);
 668 }
 669
 670 void isci_action(struct cam_sim *sim, union ccb *ccb)
 671 {
 672         struct ISCI_CONTROLLER *controller =
 673             (struct ISCI_CONTROLLER *)cam_sim_softc(sim);
 674
 675         switch ( ccb->ccb_h.func_code ) {
 676         case XPT_PATH_INQ:
 677                 {
 678                         struct ccb_pathinq *cpi = &ccb->cpi;
 679                         int bus = cam_sim_bus(sim);
 680                         ccb->ccb_h.ccb_sim_ptr = sim;
 681                         cpi->version_num = 1;
 682                         cpi->hba_inquiry = PI_TAG_ABLE;
 683                         cpi->target_sprt = 0;
 684                         cpi->hba_misc = PIM_NOBUSRESET | PIM_SEQSCAN |
 685                             PIM_UNMAPPED;
 686                         cpi->hba_eng_cnt = 0;
 687                         cpi->max_target = SCI_MAX_REMOTE_DEVICES - 1;
 688                         cpi->max_lun = ISCI_MAX_LUN;
 689 #if __FreeBSD_version >= 800102
 690                         cpi->maxio = isci_io_request_get_max_io_size();
 691 #endif
 692                         cpi->unit_number = cam_sim_unit(sim);
 693                         cpi->bus_id = bus;
 694                         cpi->initiator_id = SCI_MAX_REMOTE_DEVICES;
 695                         cpi->base_transfer_speed = 300000;
 696                         strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 697                         strlcpy(cpi->hba_vid, "Intel Corp.", HBA_IDLEN);
 698                         strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 699                         cpi->transport = XPORT_SAS;
 700                         cpi->transport_version = 0;
 701                         cpi->protocol = PROTO_SCSI;
 702                         cpi->protocol_version = SCSI_REV_SPC2;
 703                         cpi->ccb_h.status = CAM_REQ_CMP;
 704                         xpt_done(ccb);
 705                 }
 706                 break;
 707         case XPT_GET_TRAN_SETTINGS:
 708                 {
 709                         struct ccb_trans_settings *general_settings = &ccb->cts;
 710                         struct ccb_trans_settings_sas *sas_settings =
 711                             &general_settings->xport_specific.sas;
 712                         struct ccb_trans_settings_scsi *scsi_settings =
 713                             &general_settings->proto_specific.scsi;
 714                         struct ISCI_REMOTE_DEVICE *remote_device;
 715
 716                         remote_device = controller->remote_device[ccb->ccb_h.target_id];
 717
 718                         if (remote_device == NULL) {
 719                                 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 720                                 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 721                                 ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
 722                                 xpt_done(ccb);
 723                                 break;
 724                         }
 725
 726                         general_settings->protocol = PROTO_SCSI;
 727                         general_settings->transport = XPORT_SAS;
 728                         general_settings->protocol_version = SCSI_REV_SPC2;
 729                         general_settings->transport_version = 0;
 730                         scsi_settings->valid = CTS_SCSI_VALID_TQ;
 731                         scsi_settings->flags = CTS_SCSI_FLAGS_TAG_ENB;
 732                         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 733                         ccb->ccb_h.status |= CAM_REQ_CMP;
 734
 735                         sas_settings->bitrate =
 736                             isci_remote_device_get_bitrate(remote_device);
 737
 738                         if (sas_settings->bitrate != 0)
 739                                 sas_settings->valid = CTS_SAS_VALID_SPEED;
 740
 741                         xpt_done(ccb);
 742                 }
 743                 break;
 744         case XPT_SCSI_IO:
 745                 if (ccb->ccb_h.flags & CAM_CDB_PHYS) {
 746                         ccb->ccb_h.status = CAM_REQ_INVALID;
 747                         xpt_done(ccb);
 748                         break;
 749                 }
 750                 isci_io_request_execute_scsi_io(ccb, controller);
 751                 break;
 752 #if __FreeBSD_version >= 900026
 753         case XPT_SMP_IO:
 754                 isci_io_request_execute_smp_io(ccb, controller);
 755                 break;
 756 #endif
 757         case XPT_SET_TRAN_SETTINGS:
 758                 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 759                 ccb->ccb_h.status |= CAM_REQ_CMP;
 760                 xpt_done(ccb);
 761                 break;
 762         case XPT_CALC_GEOMETRY:
 763                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
 764                 xpt_done(ccb);
 765                 break;
 766         case XPT_RESET_DEV:
 767                 {
 768                         struct ISCI_REMOTE_DEVICE *remote_device =
 769                             controller->remote_device[ccb->ccb_h.target_id];
 770
 771                         if (remote_device != NULL)
 772                                 isci_remote_device_reset(remote_device, ccb);
 773                         else {
 774                                 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 775                                 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 776                                 ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
 777                                 xpt_done(ccb);
 778                         }
 779                 }
 780                 break;
 781         case XPT_RESET_BUS:
 782                 ccb->ccb_h.status = CAM_REQ_CMP;
 783                 xpt_done(ccb);
 784                 break;
 785         default:
 786                 isci_log_message(0, "ISCI", "Unhandled func_code 0x%x\n",
 787                     ccb->ccb_h.func_code);
 788                 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 789                 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 790                 ccb->ccb_h.status |= CAM_REQ_INVALID;
 791                 xpt_done(ccb);
 792                 break;
 793         }
 794 }
 795
 796 /*
 797  * Unfortunately, SCIL doesn't cleanly handle retry conditions.
 798  *  CAM_REQUEUE_REQ works only when no one is using the pass(4) interface.  So
 799  *  when SCIL denotes an I/O needs to be retried (typically because of mixing
 800  *  tagged/non-tagged ATA commands, or running out of NCQ slots), we queue
 801  *  these I/O internally.  Once SCIL completes an I/O to this device, or we get
 802  *  a ready notification, we will retry the first I/O on the queue.
 803  *  Unfortunately, SCIL also doesn't cleanly handle starting the new I/O within
 804  *  the context of the completion handler, so we need to retry these I/O after
 805  *  the completion handler is done executing.
 806  */
 807 void
 808 isci_controller_release_queued_ccbs(struct ISCI_CONTROLLER *controller)
 809 {
 810         struct ISCI_REMOTE_DEVICE *dev;
 811         struct ccb_hdr *ccb_h;
 812         uint8_t *ptr;
 813         int dev_idx;
 814
 815         KASSERT(mtx_owned(&controller->lock), ("controller lock not owned"));
 816
 817         controller->release_queued_ccbs = FALSE;
 818         for (dev_idx = 0;
 819              dev_idx < SCI_MAX_REMOTE_DEVICES;
 820              dev_idx++) {
 821
 822                 dev = controller->remote_device[dev_idx];
 823                 if (dev != NULL &&
 824                     dev->release_queued_ccb == TRUE &&
 825                     dev->queued_ccb_in_progress == NULL) {
 826                         dev->release_queued_ccb = FALSE;
 827                         ccb_h = TAILQ_FIRST(&dev->queued_ccbs);
 828
 829                         if (ccb_h == NULL)
 830                                 continue;
 831
 832                         ptr = scsiio_cdb_ptr(&((union ccb *)ccb_h)->csio);
 833                         isci_log_message(1, "ISCI", "release %p %x\n", ccb_h, *ptr);
 834
 835                         dev->queued_ccb_in_progress = (union ccb *)ccb_h;
 836                         isci_io_request_execute_scsi_io(
 837                             (union ccb *)ccb_h, controller);
 838                 }
 839         }
 840 }