sys/cam/ctl/ctl.c

   1 /*-
   2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
   3  * Copyright (c) 2012 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Edward Tomasz Napierala
   7  * under sponsorship from the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  16  *    substantially similar to the "NO WARRANTY" disclaimer below
  17  *    ("Disclaimer") and any redistribution must be conditioned upon
  18  *    including a substantially similar Disclaimer requirement for further
  19  *    binary redistribution.
  20  *
  21  * NO WARRANTY
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  30  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  31  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGES.
  33  *
  34  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl.c#8 $
  35  */
  36 /*
  37  * CAM Target Layer, a SCSI device emulation subsystem.
  38  *
  39  * Author: Ken Merry <ken@FreeBSD.org>
  40  */
  41
  42 #define _CTL_C
  43
  44 #include <sys/cdefs.h>
  45 __FBSDID("$FreeBSD$");
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/types.h>
  51 #include <sys/kthread.h>
  52 #include <sys/bio.h>
  53 #include <sys/fcntl.h>
  54 #include <sys/lock.h>
  55 #include <sys/mutex.h>
  56 #include <sys/condvar.h>
  57 #include <sys/malloc.h>
  58 #include <sys/conf.h>
  59 #include <sys/ioccom.h>
  60 #include <sys/queue.h>
  61 #include <sys/sbuf.h>
  62 #include <sys/endian.h>
  63 #include <sys/sysctl.h>
  64
  65 #include <cam/cam.h>
  66 #include <cam/scsi/scsi_all.h>
  67 #include <cam/scsi/scsi_da.h>
  68 #include <cam/ctl/ctl_io.h>
  69 #include <cam/ctl/ctl.h>
  70 #include <cam/ctl/ctl_frontend.h>
  71 #include <cam/ctl/ctl_frontend_internal.h>
  72 #include <cam/ctl/ctl_util.h>
  73 #include <cam/ctl/ctl_backend.h>
  74 #include <cam/ctl/ctl_ioctl.h>
  75 #include <cam/ctl/ctl_ha.h>
  76 #include <cam/ctl/ctl_private.h>
  77 #include <cam/ctl/ctl_debug.h>
  78 #include <cam/ctl/ctl_scsi_all.h>
  79 #include <cam/ctl/ctl_error.h>
  80
  81 struct ctl_softc *control_softc = NULL;
  82
  83 /*
  84  * The default is to run with CTL_DONE_THREAD turned on.  Completed
  85  * transactions are queued for processing by the CTL work thread.  When
  86  * CTL_DONE_THREAD is not defined, completed transactions are processed in
  87  * the caller's context.
  88  */
  89 #define CTL_DONE_THREAD
  90
  91 /*
  92  *  * Use the serial number and device ID provided by the backend, rather than
  93  *   * making up our own.
  94  *    */
  95 #define CTL_USE_BACKEND_SN
  96
  97 /*
  98  * Size and alignment macros needed for Copan-specific HA hardware.  These
  99  * can go away when the HA code is re-written, and uses busdma for any
 100  * hardware.
 101  */
 102 #define CTL_ALIGN_8B(target, source, type)                              \
 103         if (((uint32_t)source & 0x7) != 0)                              \
 104                 target = (type)(source + (0x8 - ((uint32_t)source & 0x7)));\
 105         else                                                            \
 106                 target = (type)source;
 107
 108 #define CTL_SIZE_8B(target, size)                                       \
 109         if ((size & 0x7) != 0)                                          \
 110                 target = size + (0x8 - (size & 0x7));                   \
 111         else                                                            \
 112                 target = size;
 113
 114 #define CTL_ALIGN_8B_MARGIN     16
 115
 116 /*
 117  * Template mode pages.
 118  */
 119
 120 /*
 121  * Note that these are default values only.  The actual values will be
 122  * filled in when the user does a mode sense.
 123  */
 124 static struct copan_power_subpage power_page_default = {
 125         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 126         /*subpage*/ PWR_SUBPAGE_CODE,
 127         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 128                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 129         /*page_version*/ PWR_VERSION,
 130         /* total_luns */ 26,
 131         /* max_active_luns*/ PWR_DFLT_MAX_LUNS,
 132         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 133                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 134                       0, 0, 0, 0, 0, 0}
 135 };
 136
 137 static struct copan_power_subpage power_page_changeable = {
 138         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 139         /*subpage*/ PWR_SUBPAGE_CODE,
 140         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 141                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 142         /*page_version*/ 0,
 143         /* total_luns */ 0,
 144         /* max_active_luns*/ 0,
 145         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 146                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 147                       0, 0, 0, 0, 0, 0}
 148 };
 149
 150 static struct copan_aps_subpage aps_page_default = {
 151         APS_PAGE_CODE | SMPH_SPF, //page_code
 152         APS_SUBPAGE_CODE, //subpage
 153         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 154          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 155         APS_VERSION, //page_version
 156         0, //lock_active
 157         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 158         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 159         0, 0, 0, 0, 0} //reserved
 160 };
 161
 162 static struct copan_aps_subpage aps_page_changeable = {
 163         APS_PAGE_CODE | SMPH_SPF, //page_code
 164         APS_SUBPAGE_CODE, //subpage
 165         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 166          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 167         0, //page_version
 168         0, //lock_active
 169         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 170         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 171         0, 0, 0, 0, 0} //reserved
 172 };
 173
 174 static struct copan_debugconf_subpage debugconf_page_default = {
 175         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 176         DBGCNF_SUBPAGE_CODE,            /* subpage */
 177         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 178          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 179         DBGCNF_VERSION,                 /* page_version */
 180         {CTL_TIME_IO_DEFAULT_SECS>>8,
 181          CTL_TIME_IO_DEFAULT_SECS>>0},  /* ctl_time_io_secs */
 182 };
 183
 184 static struct copan_debugconf_subpage debugconf_page_changeable = {
 185         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 186         DBGCNF_SUBPAGE_CODE,            /* subpage */
 187         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 188          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 189         0,                              /* page_version */
 190         {0xff,0xff},                    /* ctl_time_io_secs */
 191 };
 192
 193 static struct scsi_format_page format_page_default = {
 194         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 195         /*page_length*/sizeof(struct scsi_format_page) - 2,
 196         /*tracks_per_zone*/ {0, 0},
 197         /*alt_sectors_per_zone*/ {0, 0},
 198         /*alt_tracks_per_zone*/ {0, 0},
 199         /*alt_tracks_per_lun*/ {0, 0},
 200         /*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
 201                                 CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
 202         /*bytes_per_sector*/ {0, 0},
 203         /*interleave*/ {0, 0},
 204         /*track_skew*/ {0, 0},
 205         /*cylinder_skew*/ {0, 0},
 206         /*flags*/ SFP_HSEC,
 207         /*reserved*/ {0, 0, 0}
 208 };
 209
 210 static struct scsi_format_page format_page_changeable = {
 211         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 212         /*page_length*/sizeof(struct scsi_format_page) - 2,
 213         /*tracks_per_zone*/ {0, 0},
 214         /*alt_sectors_per_zone*/ {0, 0},
 215         /*alt_tracks_per_zone*/ {0, 0},
 216         /*alt_tracks_per_lun*/ {0, 0},
 217         /*sectors_per_track*/ {0, 0},
 218         /*bytes_per_sector*/ {0, 0},
 219         /*interleave*/ {0, 0},
 220         /*track_skew*/ {0, 0},
 221         /*cylinder_skew*/ {0, 0},
 222         /*flags*/ 0,
 223         /*reserved*/ {0, 0, 0}
 224 };
 225
 226 static struct scsi_rigid_disk_page rigid_disk_page_default = {
 227         /*page_code*/SMS_RIGID_DISK_PAGE,
 228         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 229         /*cylinders*/ {0, 0, 0},
 230         /*heads*/ CTL_DEFAULT_HEADS,
 231         /*start_write_precomp*/ {0, 0, 0},
 232         /*start_reduced_current*/ {0, 0, 0},
 233         /*step_rate*/ {0, 0},
 234         /*landing_zone_cylinder*/ {0, 0, 0},
 235         /*rpl*/ SRDP_RPL_DISABLED,
 236         /*rotational_offset*/ 0,
 237         /*reserved1*/ 0,
 238         /*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
 239                            CTL_DEFAULT_ROTATION_RATE & 0xff},
 240         /*reserved2*/ {0, 0}
 241 };
 242
 243 static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
 244         /*page_code*/SMS_RIGID_DISK_PAGE,
 245         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 246         /*cylinders*/ {0, 0, 0},
 247         /*heads*/ 0,
 248         /*start_write_precomp*/ {0, 0, 0},
 249         /*start_reduced_current*/ {0, 0, 0},
 250         /*step_rate*/ {0, 0},
 251         /*landing_zone_cylinder*/ {0, 0, 0},
 252         /*rpl*/ 0,
 253         /*rotational_offset*/ 0,
 254         /*reserved1*/ 0,
 255         /*rotation_rate*/ {0, 0},
 256         /*reserved2*/ {0, 0}
 257 };
 258
 259 static struct scsi_caching_page caching_page_default = {
 260         /*page_code*/SMS_CACHING_PAGE,
 261         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 262         /*flags1*/ SCP_DISC | SCP_WCE,
 263         /*ret_priority*/ 0,
 264         /*disable_pf_transfer_len*/ {0xff, 0xff},
 265         /*min_prefetch*/ {0, 0},
 266         /*max_prefetch*/ {0xff, 0xff},
 267         /*max_pf_ceiling*/ {0xff, 0xff},
 268         /*flags2*/ 0,
 269         /*cache_segments*/ 0,
 270         /*cache_seg_size*/ {0, 0},
 271         /*reserved*/ 0,
 272         /*non_cache_seg_size*/ {0, 0, 0}
 273 };
 274
 275 static struct scsi_caching_page caching_page_changeable = {
 276         /*page_code*/SMS_CACHING_PAGE,
 277         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 278         /*flags1*/ 0,
 279         /*ret_priority*/ 0,
 280         /*disable_pf_transfer_len*/ {0, 0},
 281         /*min_prefetch*/ {0, 0},
 282         /*max_prefetch*/ {0, 0},
 283         /*max_pf_ceiling*/ {0, 0},
 284         /*flags2*/ 0,
 285         /*cache_segments*/ 0,
 286         /*cache_seg_size*/ {0, 0},
 287         /*reserved*/ 0,
 288         /*non_cache_seg_size*/ {0, 0, 0}
 289 };
 290
 291 static struct scsi_control_page control_page_default = {
 292         /*page_code*/SMS_CONTROL_MODE_PAGE,
 293         /*page_length*/sizeof(struct scsi_control_page) - 2,
 294         /*rlec*/0,
 295         /*queue_flags*/0,
 296         /*eca_and_aen*/0,
 297         /*reserved*/0,
 298         /*aen_holdoff_period*/{0, 0}
 299 };
 300
 301 static struct scsi_control_page control_page_changeable = {
 302         /*page_code*/SMS_CONTROL_MODE_PAGE,
 303         /*page_length*/sizeof(struct scsi_control_page) - 2,
 304         /*rlec*/SCP_DSENSE,
 305         /*queue_flags*/0,
 306         /*eca_and_aen*/0,
 307         /*reserved*/0,
 308         /*aen_holdoff_period*/{0, 0}
 309 };
 310
 311 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 312
 313 /*
 314  * XXX KDM move these into the softc.
 315  */
 316 static int rcv_sync_msg;
 317 static int persis_offset;
 318 static uint8_t ctl_pause_rtr;
 319 static int     ctl_is_single;
 320 static int     index_to_aps_page;
 321
 322
 323 /*
 324  * Serial number (0x80), device id (0x83), and supported pages (0x00)
 325  */
 326 #define SCSI_EVPD_NUM_SUPPORTED_PAGES   3
 327
 328 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 329                                   int param);
 330 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 331 static void ctl_init(void);
 332 void ctl_shutdown(void);
 333 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 334 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 335 static void ctl_ioctl_online(void *arg);
 336 static void ctl_ioctl_offline(void *arg);
 337 static int ctl_ioctl_targ_enable(void *arg, struct ctl_id targ_id);
 338 static int ctl_ioctl_targ_disable(void *arg, struct ctl_id targ_id);
 339 static int ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id);
 340 static int ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id);
 341 static int ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio);
 342 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio, int have_lock);
 343 static int ctl_ioctl_submit_wait(union ctl_io *io);
 344 static void ctl_ioctl_datamove(union ctl_io *io);
 345 static void ctl_ioctl_done(union ctl_io *io);
 346 static void ctl_ioctl_hard_startstop_callback(void *arg,
 347                                               struct cfi_metatask *metatask);
 348 static void ctl_ioctl_bbrread_callback(void *arg,struct cfi_metatask *metatask);
 349 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 350                               struct ctl_ooa *ooa_hdr);
 351 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 352                      struct thread *td);
 353 uint32_t ctl_get_resindex(struct ctl_nexus *nexus);
 354 uint32_t ctl_port_idx(int port_num);
 355 #ifdef unused
 356 static union ctl_io *ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port,
 357                                    uint32_t targ_target, uint32_t targ_lun,
 358                                    int can_wait);
 359 static void ctl_kfree_io(union ctl_io *io);
 360 #endif /* unused */
 361 static void ctl_free_io_internal(union ctl_io *io, int have_lock);
 362 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
 363                          struct ctl_be_lun *be_lun, struct ctl_id target_id);
 364 static int ctl_free_lun(struct ctl_lun *lun);
 365 static void ctl_create_lun(struct ctl_be_lun *be_lun);
 366 /**
 367 static void ctl_failover_change_pages(struct ctl_softc *softc,
 368                                       struct ctl_scsiio *ctsio, int master);
 369 **/
 370
 371 static int ctl_do_mode_select(union ctl_io *io);
 372 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
 373                            uint64_t res_key, uint64_t sa_res_key,
 374                            uint8_t type, uint32_t residx,
 375                            struct ctl_scsiio *ctsio,
 376                            struct scsi_per_res_out *cdb,
 377                            struct scsi_per_res_out_parms* param);
 378 static void ctl_pro_preempt_other(struct ctl_lun *lun,
 379                                   union ctl_ha_msg *msg);
 380 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
 381 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
 382 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
 383 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
 384 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
 385 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
 386 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len);
 387 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2);
 388 static ctl_action ctl_check_for_blockage(union ctl_io *pending_io,
 389                                          union ctl_io *ooa_io);
 390 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
 391                                 union ctl_io *starting_io);
 392 static int ctl_check_blocked(struct ctl_lun *lun);
 393 static int ctl_scsiio_lun_check(struct ctl_softc *ctl_softc,
 394                                 struct ctl_lun *lun,
 395                                 struct ctl_cmd_entry *entry,
 396                                 struct ctl_scsiio *ctsio);
 397 //static int ctl_check_rtr(union ctl_io *pending_io, struct ctl_softc *softc);
 398 static void ctl_failover(void);
 399 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 400                                struct ctl_scsiio *ctsio);
 401 static int ctl_scsiio(struct ctl_scsiio *ctsio);
 402
 403 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 404 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 405                             ctl_ua_type ua_type);
 406 static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 407                          ctl_ua_type ua_type);
 408 static int ctl_abort_task(union ctl_io *io);
 409 static void ctl_run_task_queue(struct ctl_softc *ctl_softc);
 410 #ifdef CTL_IO_DELAY
 411 static void ctl_datamove_timer_wakeup(void *arg);
 412 static void ctl_done_timer_wakeup(void *arg);
 413 #endif /* CTL_IO_DELAY */
 414
 415 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 416 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
 417 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
 418 static void ctl_datamove_remote_write(union ctl_io *io);
 419 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
 420 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 421 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 422 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 423                                     ctl_ha_dt_cb callback);
 424 static void ctl_datamove_remote_read(union ctl_io *io);
 425 static void ctl_datamove_remote(union ctl_io *io);
 426 static int ctl_process_done(union ctl_io *io, int have_lock);
 427 static void ctl_work_thread(void *arg);
 428
 429 /*
 430  * Load the serialization table.  This isn't very pretty, but is probably
 431  * the easiest way to do it.
 432  */
 433 #include "ctl_ser_table.c"
 434
 435 /*
 436  * We only need to define open, close and ioctl routines for this driver.
 437  */
 438 static struct cdevsw ctl_cdevsw = {
 439         .d_version =    D_VERSION,
 440         .d_flags =      0,
 441         .d_open =       ctl_open,
 442         .d_close =      ctl_close,
 443         .d_ioctl =      ctl_ioctl,
 444         .d_name =       "ctl",
 445 };
 446
 447
 448 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 449
 450 /*
 451  * If we have the CAM SIM, we may or may not have another SIM that will
 452  * cause CTL to get initialized.  If not, we need to initialize it.
 453  */
 454 SYSINIT(ctl_init, SI_SUB_CONFIGURE, SI_ORDER_THIRD, ctl_init, NULL);
 455
 456 static void
 457 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 458                             union ctl_ha_msg *msg_info)
 459 {
 460         struct ctl_scsiio *ctsio;
 461
 462         if (msg_info->hdr.original_sc == NULL) {
 463                 printf("%s: original_sc == NULL!\n", __func__);
 464                 /* XXX KDM now what? */
 465                 return;
 466         }
 467
 468         ctsio = &msg_info->hdr.original_sc->scsiio;
 469         ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 470         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 471         ctsio->io_hdr.status = msg_info->hdr.status;
 472         ctsio->scsi_status = msg_info->scsi.scsi_status;
 473         ctsio->sense_len = msg_info->scsi.sense_len;
 474         ctsio->sense_residual = msg_info->scsi.sense_residual;
 475         ctsio->residual = msg_info->scsi.residual;
 476         memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 477                sizeof(ctsio->sense_data));
 478         memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
 479                &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));;
 480         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue, &ctsio->io_hdr, links);
 481         ctl_wakeup_thread();
 482 }
 483
 484 static void
 485 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
 486                                 union ctl_ha_msg *msg_info)
 487 {
 488         struct ctl_scsiio *ctsio;
 489
 490         if (msg_info->hdr.serializing_sc == NULL) {
 491                 printf("%s: serializing_sc == NULL!\n", __func__);
 492                 /* XXX KDM now what? */
 493                 return;
 494         }
 495
 496         ctsio = &msg_info->hdr.serializing_sc->scsiio;
 497 #if 0
 498         /*
 499          * Attempt to catch the situation where an I/O has
 500          * been freed, and we're using it again.
 501          */
 502         if (ctsio->io_hdr.io_type == 0xff) {
 503                 union ctl_io *tmp_io;
 504                 tmp_io = (union ctl_io *)ctsio;
 505                 printf("%s: %p use after free!\n", __func__,
 506                        ctsio);
 507                 printf("%s: type %d msg %d cdb %x iptl: "
 508                        "%d:%d:%d:%d tag 0x%04x "
 509                        "flag %#x status %x\n",
 510                         __func__,
 511                         tmp_io->io_hdr.io_type,
 512                         tmp_io->io_hdr.msg_type,
 513                         tmp_io->scsiio.cdb[0],
 514                         tmp_io->io_hdr.nexus.initid.id,
 515                         tmp_io->io_hdr.nexus.targ_port,
 516                         tmp_io->io_hdr.nexus.targ_target.id,
 517                         tmp_io->io_hdr.nexus.targ_lun,
 518                         (tmp_io->io_hdr.io_type ==
 519                         CTL_IO_TASK) ?
 520                         tmp_io->taskio.tag_num :
 521                         tmp_io->scsiio.tag_num,
 522                         tmp_io->io_hdr.flags,
 523                         tmp_io->io_hdr.status);
 524         }
 525 #endif
 526         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 527         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue, &ctsio->io_hdr, links);
 528         ctl_wakeup_thread();
 529 }
 530
 531 /*
 532  * ISC (Inter Shelf Communication) event handler.  Events from the HA
 533  * subsystem come in here.
 534  */
 535 static void
 536 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 537 {
 538         struct ctl_softc *ctl_softc;
 539         union ctl_io *io;
 540         struct ctl_prio *presio;
 541         ctl_ha_status isc_status;
 542
 543         ctl_softc = control_softc;
 544         io = NULL;
 545
 546
 547 #if 0
 548         printf("CTL: Isc Msg event %d\n", event);
 549 #endif
 550         if (event == CTL_HA_EVT_MSG_RECV) {
 551                 union ctl_ha_msg msg_info;
 552
 553                 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
 554                                              sizeof(msg_info), /*wait*/ 0);
 555 #if 0
 556                 printf("CTL: msg_type %d\n", msg_info.msg_type);
 557 #endif
 558                 if (isc_status != 0) {
 559                         printf("Error receiving message, status = %d\n",
 560                                isc_status);
 561                         return;
 562                 }
 563                 mtx_lock(&ctl_softc->ctl_lock);
 564
 565                 switch (msg_info.hdr.msg_type) {
 566                 case CTL_MSG_SERIALIZE:
 567 #if 0
 568                         printf("Serialize\n");
 569 #endif
 570                         io = ctl_alloc_io((void *)ctl_softc->othersc_pool);
 571                         if (io == NULL) {
 572                                 printf("ctl_isc_event_handler: can't allocate "
 573                                        "ctl_io!\n");
 574                                 /* Bad Juju */
 575                                 /* Need to set busy and send msg back */
 576                                 mtx_unlock(&ctl_softc->ctl_lock);
 577                                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
 578                                 msg_info.hdr.status = CTL_SCSI_ERROR;
 579                                 msg_info.scsi.scsi_status = SCSI_STATUS_BUSY;
 580                                 msg_info.scsi.sense_len = 0;
 581                                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
 582                                     sizeof(msg_info), 0) > CTL_HA_STATUS_SUCCESS){
 583                                 }
 584                                 goto bailout;
 585                         }
 586                         ctl_zero_io(io);
 587                         // populate ctsio from msg_info
 588                         io->io_hdr.io_type = CTL_IO_SCSI;
 589                         io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
 590                         io->io_hdr.original_sc = msg_info.hdr.original_sc;
 591 #if 0
 592                         printf("pOrig %x\n", (int)msg_info.original_sc);
 593 #endif
 594                         io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
 595                                             CTL_FLAG_IO_ACTIVE;
 596                         /*
 597                          * If we're in serialization-only mode, we don't
 598                          * want to go through full done processing.  Thus
 599                          * the COPY flag.
 600                          *
 601                          * XXX KDM add another flag that is more specific.
 602                          */
 603                         if (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)
 604                                 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
 605                         io->io_hdr.nexus = msg_info.hdr.nexus;
 606 #if 0
 607                         printf("targ %d, port %d, iid %d, lun %d\n",
 608                                io->io_hdr.nexus.targ_target.id,
 609                                io->io_hdr.nexus.targ_port,
 610                                io->io_hdr.nexus.initid.id,
 611                                io->io_hdr.nexus.targ_lun);
 612 #endif
 613                         io->scsiio.tag_num = msg_info.scsi.tag_num;
 614                         io->scsiio.tag_type = msg_info.scsi.tag_type;
 615                         memcpy(io->scsiio.cdb, msg_info.scsi.cdb,
 616                                CTL_MAX_CDBLEN);
 617                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
 618                                 struct ctl_cmd_entry *entry;
 619                                 uint8_t opcode;
 620
 621                                 opcode = io->scsiio.cdb[0];
 622                                 entry = &ctl_cmd_table[opcode];
 623                                 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
 624                                 io->io_hdr.flags |=
 625                                         entry->flags & CTL_FLAG_DATA_MASK;
 626                         }
 627                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 628                                            &io->io_hdr, links);
 629                         ctl_wakeup_thread();
 630                         break;
 631
 632                 /* Performed on the Originating SC, XFER mode only */
 633                 case CTL_MSG_DATAMOVE: {
 634                         struct ctl_sg_entry *sgl;
 635                         int i, j;
 636
 637                         io = msg_info.hdr.original_sc;
 638                         if (io == NULL) {
 639                                 printf("%s: original_sc == NULL!\n", __func__);
 640                                 /* XXX KDM do something here */
 641                                 break;
 642                         }
 643                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
 644                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 645                         /*
 646                          * Keep track of this, we need to send it back over
 647                          * when the datamove is complete.
 648                          */
 649                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 650
 651                         if (msg_info.dt.sg_sequence == 0) {
 652                                 /*
 653                                  * XXX KDM we use the preallocated S/G list
 654                                  * here, but we'll need to change this to
 655                                  * dynamic allocation if we need larger S/G
 656                                  * lists.
 657                                  */
 658                                 if (msg_info.dt.kern_sg_entries >
 659                                     sizeof(io->io_hdr.remote_sglist) /
 660                                     sizeof(io->io_hdr.remote_sglist[0])) {
 661                                         printf("%s: number of S/G entries "
 662                                             "needed %u > allocated num %zd\n",
 663                                             __func__,
 664                                             msg_info.dt.kern_sg_entries,
 665                                             sizeof(io->io_hdr.remote_sglist)/
 666                                             sizeof(io->io_hdr.remote_sglist[0]));
 667
 668                                         /*
 669                                          * XXX KDM send a message back to
 670                                          * the other side to shut down the
 671                                          * DMA.  The error will come back
 672                                          * through via the normal channel.
 673                                          */
 674                                         break;
 675                                 }
 676                                 sgl = io->io_hdr.remote_sglist;
 677                                 memset(sgl, 0,
 678                                        sizeof(io->io_hdr.remote_sglist));
 679
 680                                 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
 681
 682                                 io->scsiio.kern_sg_entries =
 683                                         msg_info.dt.kern_sg_entries;
 684                                 io->scsiio.rem_sg_entries =
 685                                         msg_info.dt.kern_sg_entries;
 686                                 io->scsiio.kern_data_len =
 687                                         msg_info.dt.kern_data_len;
 688                                 io->scsiio.kern_total_len =
 689                                         msg_info.dt.kern_total_len;
 690                                 io->scsiio.kern_data_resid =
 691                                         msg_info.dt.kern_data_resid;
 692                                 io->scsiio.kern_rel_offset =
 693                                         msg_info.dt.kern_rel_offset;
 694                                 /*
 695                                  * Clear out per-DMA flags.
 696                                  */
 697                                 io->io_hdr.flags &= ~CTL_FLAG_RDMA_MASK;
 698                                 /*
 699                                  * Add per-DMA flags that are set for this
 700                                  * particular DMA request.
 701                                  */
 702                                 io->io_hdr.flags |= msg_info.dt.flags &
 703                                                     CTL_FLAG_RDMA_MASK;
 704                         } else
 705                                 sgl = (struct ctl_sg_entry *)
 706                                         io->scsiio.kern_data_ptr;
 707
 708                         for (i = msg_info.dt.sent_sg_entries, j = 0;
 709                              i < (msg_info.dt.sent_sg_entries +
 710                              msg_info.dt.cur_sg_entries); i++, j++) {
 711                                 sgl[i].addr = msg_info.dt.sg_list[j].addr;
 712                                 sgl[i].len = msg_info.dt.sg_list[j].len;
 713
 714 #if 0
 715                                 printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
 716                                        __func__,
 717                                        msg_info.dt.sg_list[j].addr,
 718                                        msg_info.dt.sg_list[j].len,
 719                                        sgl[i].addr, sgl[i].len, j, i);
 720 #endif
 721                         }
 722 #if 0
 723                         memcpy(&sgl[msg_info.dt.sent_sg_entries],
 724                                msg_info.dt.sg_list,
 725                                sizeof(*sgl) * msg_info.dt.cur_sg_entries);
 726 #endif
 727
 728                         /*
 729                          * If this is the last piece of the I/O, we've got
 730                          * the full S/G list.  Queue processing in the thread.
 731                          * Otherwise wait for the next piece.
 732                          */
 733                         if (msg_info.dt.sg_last != 0) {
 734                                 STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 735                                                    &io->io_hdr, links);
 736                                 ctl_wakeup_thread();
 737                         }
 738                         break;
 739                 }
 740                 /* Performed on the Serializing (primary) SC, XFER mode only */
 741                 case CTL_MSG_DATAMOVE_DONE: {
 742                         if (msg_info.hdr.serializing_sc == NULL) {
 743                                 printf("%s: serializing_sc == NULL!\n",
 744                                        __func__);
 745                                 /* XXX KDM now what? */
 746                                 break;
 747                         }
 748                         /*
 749                          * We grab the sense information here in case
 750                          * there was a failure, so we can return status
 751                          * back to the initiator.
 752                          */
 753                         io = msg_info.hdr.serializing_sc;
 754                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
 755                         io->io_hdr.status = msg_info.hdr.status;
 756                         io->scsiio.scsi_status = msg_info.scsi.scsi_status;
 757                         io->scsiio.sense_len = msg_info.scsi.sense_len;
 758                         io->scsiio.sense_residual =msg_info.scsi.sense_residual;
 759                         io->io_hdr.port_status = msg_info.scsi.fetd_status;
 760                         io->scsiio.residual = msg_info.scsi.residual;
 761                         memcpy(&io->scsiio.sense_data,&msg_info.scsi.sense_data,
 762                                sizeof(io->scsiio.sense_data));
 763
 764                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 765                                            &io->io_hdr, links);
 766                         ctl_wakeup_thread();
 767                         break;
 768                 }
 769
 770                 /* Preformed on Originating SC, SER_ONLY mode */
 771                 case CTL_MSG_R2R:
 772                         io = msg_info.hdr.original_sc;
 773                         if (io == NULL) {
 774                                 printf("%s: Major Bummer\n", __func__);
 775                                 mtx_unlock(&ctl_softc->ctl_lock);
 776                                 return;
 777                         } else {
 778 #if 0
 779                                 printf("pOrig %x\n",(int) ctsio);
 780 #endif
 781                         }
 782                         io->io_hdr.msg_type = CTL_MSG_R2R;
 783                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 784                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 785                                            &io->io_hdr, links);
 786                         ctl_wakeup_thread();
 787                         break;
 788
 789                 /*
 790                  * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
 791                  * mode.
 792                  * Performed on the Originating (i.e. secondary) SC in XFER
 793                  * mode
 794                  */
 795                 case CTL_MSG_FINISH_IO:
 796                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER)
 797                                 ctl_isc_handler_finish_xfer(ctl_softc,
 798                                                             &msg_info);
 799                         else
 800                                 ctl_isc_handler_finish_ser_only(ctl_softc,
 801                                                                 &msg_info);
 802                         break;
 803
 804                 /* Preformed on Originating SC */
 805                 case CTL_MSG_BAD_JUJU:
 806                         io = msg_info.hdr.original_sc;
 807                         if (io == NULL) {
 808                                 printf("%s: Bad JUJU!, original_sc is NULL!\n",
 809                                        __func__);
 810                                 break;
 811                         }
 812                         ctl_copy_sense_data(&msg_info, io);
 813                         /*
 814                          * IO should have already been cleaned up on other
 815                          * SC so clear this flag so we won't send a message
 816                          * back to finish the IO there.
 817                          */
 818                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
 819                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 820
 821                         /* io = msg_info.hdr.serializing_sc; */
 822                         io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
 823                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 824                                            &io->io_hdr, links);
 825                         ctl_wakeup_thread();
 826                         break;
 827
 828                 /* Handle resets sent from the other side */
 829                 case CTL_MSG_MANAGE_TASKS: {
 830                         struct ctl_taskio *taskio;
 831                         taskio = (struct ctl_taskio *)ctl_alloc_io(
 832                                 (void *)ctl_softc->othersc_pool);
 833                         if (taskio == NULL) {
 834                                 printf("ctl_isc_event_handler: can't allocate "
 835                                        "ctl_io!\n");
 836                                 /* Bad Juju */
 837                                 /* should I just call the proper reset func
 838                                    here??? */
 839                                 mtx_unlock(&ctl_softc->ctl_lock);
 840                                 goto bailout;
 841                         }
 842                         ctl_zero_io((union ctl_io *)taskio);
 843                         taskio->io_hdr.io_type = CTL_IO_TASK;
 844                         taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
 845                         taskio->io_hdr.nexus = msg_info.hdr.nexus;
 846                         taskio->task_action = msg_info.task.task_action;
 847                         taskio->tag_num = msg_info.task.tag_num;
 848                         taskio->tag_type = msg_info.task.tag_type;
 849 #ifdef CTL_TIME_IO
 850                         taskio->io_hdr.start_time = time_uptime;
 851                         getbintime(&taskio->io_hdr.start_bt);
 852 #if 0
 853                         cs_prof_gettime(&taskio->io_hdr.start_ticks);
 854 #endif
 855 #endif /* CTL_TIME_IO */
 856                         STAILQ_INSERT_TAIL(&ctl_softc->task_queue,
 857                                            &taskio->io_hdr, links);
 858                         ctl_softc->flags |= CTL_FLAG_TASK_PENDING;
 859                         ctl_wakeup_thread();
 860                         break;
 861                 }
 862                 /* Persistent Reserve action which needs attention */
 863                 case CTL_MSG_PERS_ACTION:
 864                         presio = (struct ctl_prio *)ctl_alloc_io(
 865                                 (void *)ctl_softc->othersc_pool);
 866                         if (presio == NULL) {
 867                                 printf("ctl_isc_event_handler: can't allocate "
 868                                        "ctl_io!\n");
 869                                 /* Bad Juju */
 870                                 /* Need to set busy and send msg back */
 871                                 mtx_unlock(&ctl_softc->ctl_lock);
 872                                 goto bailout;
 873                         }
 874                         ctl_zero_io((union ctl_io *)presio);
 875                         presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
 876                         presio->pr_msg = msg_info.pr;
 877                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 878                                            &presio->io_hdr, links);
 879                         ctl_wakeup_thread();
 880                         break;
 881                 case CTL_MSG_SYNC_FE:
 882                         rcv_sync_msg = 1;
 883                         break;
 884                 case CTL_MSG_APS_LOCK: {
 885                         // It's quicker to execute this then to
 886                         // queue it.
 887                         struct ctl_lun *lun;
 888                         struct ctl_page_index *page_index;
 889                         struct copan_aps_subpage *current_sp;
 890
 891                         lun = ctl_softc->ctl_luns[msg_info.hdr.nexus.targ_lun];
 892                         page_index = &lun->mode_pages.index[index_to_aps_page];
 893                         current_sp = (struct copan_aps_subpage *)
 894                                      (page_index->page_data +
 895                                      (page_index->page_len * CTL_PAGE_CURRENT));
 896
 897                         current_sp->lock_active = msg_info.aps.lock_flag;
 898                         break;
 899                 }
 900                 default:
 901                         printf("How did I get here?\n");
 902                 }
 903                 mtx_unlock(&ctl_softc->ctl_lock);
 904         } else if (event == CTL_HA_EVT_MSG_SENT) {
 905                 if (param != CTL_HA_STATUS_SUCCESS) {
 906                         printf("Bad status from ctl_ha_msg_send status %d\n",
 907                                param);
 908                 }
 909                 return;
 910         } else if (event == CTL_HA_EVT_DISCONNECT) {
 911                 printf("CTL: Got a disconnect from Isc\n");
 912                 return;
 913         } else {
 914                 printf("ctl_isc_event_handler: Unknown event %d\n", event);
 915                 return;
 916         }
 917
 918 bailout:
 919         return;
 920 }
 921
 922 static void
 923 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
 924 {
 925         struct scsi_sense_data *sense;
 926
 927         sense = &dest->scsiio.sense_data;
 928         bcopy(&src->scsi.sense_data, sense, sizeof(*sense));
 929         dest->scsiio.scsi_status = src->scsi.scsi_status;
 930         dest->scsiio.sense_len = src->scsi.sense_len;
 931         dest->io_hdr.status = src->hdr.status;
 932 }
 933
 934 static void
 935 ctl_init(void)
 936 {
 937         struct ctl_softc *softc;
 938         struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
 939         struct ctl_frontend *fe;
 940         struct ctl_lun *lun;
 941         uint8_t sc_id =0;
 942 #if 0
 943         int i;
 944 #endif
 945         int retval;
 946         //int isc_retval;
 947
 948         retval = 0;
 949         ctl_pause_rtr = 0;
 950         rcv_sync_msg = 0;
 951
 952         control_softc = malloc(sizeof(*control_softc), M_DEVBUF, M_WAITOK);
 953         softc = control_softc;
 954
 955         memset(softc, 0, sizeof(*softc));
 956
 957         softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
 958                               "cam/ctl");
 959
 960         softc->dev->si_drv1 = softc;
 961
 962         mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
 963         softc->open_count = 0;
 964
 965         /*
 966          * Default to actually sending a SYNCHRONIZE CACHE command down to
 967          * the drive.
 968          */
 969         softc->flags = CTL_FLAG_REAL_SYNC;
 970
 971         /*
 972          * In Copan's HA scheme, the "master" and "slave" roles are
 973          * figured out through the slot the controller is in.  Although it
 974          * is an active/active system, someone has to be in charge.
 975          */
 976 #ifdef NEEDTOPORT
 977         scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
 978 #endif
 979
 980         if (sc_id == 0) {
 981                 softc->flags |= CTL_FLAG_MASTER_SHELF;
 982                 persis_offset = 0;
 983         } else
 984                 persis_offset = CTL_MAX_INITIATORS;
 985
 986         /*
 987          * XXX KDM need to figure out where we want to get our target ID
 988          * and WWID.  Is it different on each port?
 989          */
 990         softc->target.id = 0;
 991         softc->target.wwid[0] = 0x12345678;
 992         softc->target.wwid[1] = 0x87654321;
 993         STAILQ_INIT(&softc->lun_list);
 994         STAILQ_INIT(&softc->pending_lun_queue);
 995         STAILQ_INIT(&softc->task_queue);
 996         STAILQ_INIT(&softc->incoming_queue);
 997         STAILQ_INIT(&softc->rtr_queue);
 998         STAILQ_INIT(&softc->done_queue);
 999         STAILQ_INIT(&softc->isc_queue);
1000         STAILQ_INIT(&softc->fe_list);
1001         STAILQ_INIT(&softc->be_list);
1002         STAILQ_INIT(&softc->io_pools);
1003
1004         lun = &softc->lun;
1005
1006         /*
1007          * We don't bother calling these with ctl_lock held here, because,
1008          * in theory, no one else can try to do anything while we're in our
1009          * module init routine.
1010          */
1011         if (ctl_pool_create(softc, CTL_POOL_INTERNAL, CTL_POOL_ENTRIES_INTERNAL,
1012                             &internal_pool)!= 0){
1013                 printf("ctl: can't allocate %d entry internal pool, "
1014                        "exiting\n", CTL_POOL_ENTRIES_INTERNAL);
1015                 return;
1016         }
1017
1018         if (ctl_pool_create(softc, CTL_POOL_EMERGENCY,
1019                             CTL_POOL_ENTRIES_EMERGENCY, &emergency_pool) != 0) {
1020                 printf("ctl: can't allocate %d entry emergency pool, "
1021                        "exiting\n", CTL_POOL_ENTRIES_EMERGENCY);
1022                 ctl_pool_free(softc, internal_pool);
1023                 return;
1024         }
1025
1026         if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC,
1027                             &other_pool) != 0)
1028         {
1029                 printf("ctl: can't allocate %d entry other SC pool, "
1030                        "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1031                 ctl_pool_free(softc, internal_pool);
1032                 ctl_pool_free(softc, emergency_pool);
1033                 return;
1034         }
1035
1036         softc->internal_pool = internal_pool;
1037         softc->emergency_pool = emergency_pool;
1038         softc->othersc_pool = other_pool;
1039
1040         ctl_pool_acquire(internal_pool);
1041         ctl_pool_acquire(emergency_pool);
1042         ctl_pool_acquire(other_pool);
1043
1044         /*
1045          * We used to allocate a processor LUN here.  The new scheme is to
1046          * just let the user allocate LUNs as he sees fit.
1047          */
1048 #if 0
1049         mtx_lock(&softc->ctl_lock);
1050         ctl_alloc_lun(softc, lun, /*be_lun*/NULL, /*target*/softc->target);
1051         mtx_unlock(&softc->ctl_lock);
1052 #endif
1053
1054         if (kproc_create(ctl_work_thread, softc, &softc->work_thread, 0, 0,
1055                          "ctl_thrd") != 0) {
1056                 printf("error creating CTL work thread!\n");
1057                 ctl_free_lun(lun);
1058                 ctl_pool_free(softc, internal_pool);
1059                 ctl_pool_free(softc, emergency_pool);
1060                 ctl_pool_free(softc, other_pool);
1061                 return;
1062         }
1063         printf("ctl: CAM Target Layer loaded\n");
1064
1065         /*
1066          * Initialize the initiator and portname mappings
1067          */
1068         memset(softc->wwpn_iid, 0, sizeof(softc->wwpn_iid));
1069
1070         /*
1071          * Initialize the ioctl front end.
1072          */
1073         fe = &softc->ioctl_info.fe;
1074         sprintf(softc->ioctl_info.port_name, "CTL ioctl");
1075         fe->port_type = CTL_PORT_IOCTL;
1076         fe->num_requested_ctl_io = 100;
1077         fe->port_name = softc->ioctl_info.port_name;
1078         fe->port_online = ctl_ioctl_online;
1079         fe->port_offline = ctl_ioctl_offline;
1080         fe->onoff_arg = &softc->ioctl_info;
1081         fe->targ_enable = ctl_ioctl_targ_enable;
1082         fe->targ_disable = ctl_ioctl_targ_disable;
1083         fe->lun_enable = ctl_ioctl_lun_enable;
1084         fe->lun_disable = ctl_ioctl_lun_disable;
1085         fe->targ_lun_arg = &softc->ioctl_info;
1086         fe->fe_datamove = ctl_ioctl_datamove;
1087         fe->fe_done = ctl_ioctl_done;
1088         fe->max_targets = 15;
1089         fe->max_target_id = 15;
1090
1091         if (ctl_frontend_register(&softc->ioctl_info.fe,
1092                           (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
1093                 printf("ctl: ioctl front end registration failed, will "
1094                        "continue anyway\n");
1095         }
1096
1097 #ifdef CTL_IO_DELAY
1098         if (sizeof(struct callout) > CTL_TIMER_BYTES) {
1099                 printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
1100                        sizeof(struct callout), CTL_TIMER_BYTES);
1101                 return;
1102         }
1103 #endif /* CTL_IO_DELAY */
1104
1105 }
1106
1107 void
1108 ctl_shutdown(void)
1109 {
1110         struct ctl_softc *softc;
1111         struct ctl_lun *lun, *next_lun;
1112         struct ctl_io_pool *pool, *next_pool;
1113
1114         softc = (struct ctl_softc *)control_softc;
1115
1116         if (ctl_frontend_deregister(&softc->ioctl_info.fe) != 0)
1117                 printf("ctl: ioctl front end deregistration failed\n");
1118
1119         mtx_lock(&softc->ctl_lock);
1120
1121         /*
1122          * Free up each LUN.
1123          */
1124         for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1125                 next_lun = STAILQ_NEXT(lun, links);
1126                 ctl_free_lun(lun);
1127         }
1128
1129         /*
1130          * This will rip the rug out from under any FETDs or anyone else
1131          * that has a pool allocated.  Since we increment our module
1132          * refcount any time someone outside the main CTL module allocates
1133          * a pool, we shouldn't have any problems here.  The user won't be
1134          * able to unload the CTL module until client modules have
1135          * successfully unloaded.
1136          */
1137         for (pool = STAILQ_FIRST(&softc->io_pools); pool != NULL;
1138              pool = next_pool) {
1139                 next_pool = STAILQ_NEXT(pool, links);
1140                 ctl_pool_free(softc, pool);
1141         }
1142
1143         mtx_unlock(&softc->ctl_lock);
1144
1145 #if 0
1146         ctl_shutdown_thread(softc->work_thread);
1147 #endif
1148
1149         mtx_destroy(&softc->ctl_lock);
1150
1151         destroy_dev(softc->dev);
1152
1153         printf("ctl: CAM Target Layer unloaded\n");
1154 }
1155
1156 /*
1157  * XXX KDM should we do some access checks here?  Bump a reference count to
1158  * prevent a CTL module from being unloaded while someone has it open?
1159  */
1160 static int
1161 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1162 {
1163         return (0);
1164 }
1165
1166 static int
1167 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1168 {
1169         return (0);
1170 }
1171
1172 int
1173 ctl_port_enable(ctl_port_type port_type)
1174 {
1175         struct ctl_softc *softc;
1176         struct ctl_frontend *fe;
1177
1178         if (ctl_is_single == 0) {
1179                 union ctl_ha_msg msg_info;
1180                 int isc_retval;
1181
1182 #if 0
1183                 printf("%s: HA mode, synchronizing frontend enable\n",
1184                         __func__);
1185 #endif
1186                 msg_info.hdr.msg_type = CTL_MSG_SYNC_FE;
1187                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1188                         sizeof(msg_info), 1 )) > CTL_HA_STATUS_SUCCESS) {
1189                         printf("Sync msg send error retval %d\n", isc_retval);
1190                 }
1191                 if (!rcv_sync_msg) {
1192                         isc_retval=ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
1193                                 sizeof(msg_info), 1);
1194                 }
1195 #if 0
1196                 printf("CTL:Frontend Enable\n");
1197         } else {
1198                 printf("%s: single mode, skipping frontend synchronization\n",
1199                         __func__);
1200 #endif
1201         }
1202
1203         softc = control_softc;
1204
1205         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1206                 if (port_type & fe->port_type)
1207                 {
1208 #if 0
1209                         printf("port %d\n", fe->targ_port);
1210 #endif
1211                         ctl_frontend_online(fe);
1212                 }
1213         }
1214
1215         return (0);
1216 }
1217
1218 int
1219 ctl_port_disable(ctl_port_type port_type)
1220 {
1221         struct ctl_softc *softc;
1222         struct ctl_frontend *fe;
1223
1224         softc = control_softc;
1225
1226         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1227                 if (port_type & fe->port_type)
1228                         ctl_frontend_offline(fe);
1229         }
1230
1231         return (0);
1232 }
1233
1234 /*
1235  * Returns 0 for success, 1 for failure.
1236  * Currently the only failure mode is if there aren't enough entries
1237  * allocated.  So, in case of a failure, look at num_entries_dropped,
1238  * reallocate and try again.
1239  */
1240 int
1241 ctl_port_list(struct ctl_port_entry *entries, int num_entries_alloced,
1242               int *num_entries_filled, int *num_entries_dropped,
1243               ctl_port_type port_type, int no_virtual)
1244 {
1245         struct ctl_softc *softc;
1246         struct ctl_frontend *fe;
1247         int entries_dropped, entries_filled;
1248         int retval;
1249         int i;
1250
1251         softc = control_softc;
1252
1253         retval = 0;
1254         entries_filled = 0;
1255         entries_dropped = 0;
1256
1257         i = 0;
1258         mtx_lock(&softc->ctl_lock);
1259         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1260                 struct ctl_port_entry *entry;
1261
1262                 if ((fe->port_type & port_type) == 0)
1263                         continue;
1264
1265                 if ((no_virtual != 0)
1266                  && (fe->virtual_port != 0))
1267                         continue;
1268
1269                 if (entries_filled >= num_entries_alloced) {
1270                         entries_dropped++;
1271                         continue;
1272                 }
1273                 entry = &entries[i];
1274
1275                 entry->port_type = fe->port_type;
1276                 strlcpy(entry->port_name, fe->port_name,
1277                         sizeof(entry->port_name));
1278                 entry->physical_port = fe->physical_port;
1279                 entry->virtual_port = fe->virtual_port;
1280                 entry->wwnn = fe->wwnn;
1281                 entry->wwpn = fe->wwpn;
1282
1283                 i++;
1284                 entries_filled++;
1285         }
1286
1287         mtx_unlock(&softc->ctl_lock);
1288
1289         if (entries_dropped > 0)
1290                 retval = 1;
1291
1292         *num_entries_dropped = entries_dropped;
1293         *num_entries_filled = entries_filled;
1294
1295         return (retval);
1296 }
1297
1298 static void
1299 ctl_ioctl_online(void *arg)
1300 {
1301         struct ctl_ioctl_info *ioctl_info;
1302
1303         ioctl_info = (struct ctl_ioctl_info *)arg;
1304
1305         ioctl_info->flags |= CTL_IOCTL_FLAG_ENABLED;
1306 }
1307
1308 static void
1309 ctl_ioctl_offline(void *arg)
1310 {
1311         struct ctl_ioctl_info *ioctl_info;
1312
1313         ioctl_info = (struct ctl_ioctl_info *)arg;
1314
1315         ioctl_info->flags &= ~CTL_IOCTL_FLAG_ENABLED;
1316 }
1317
1318 /*
1319  * Remove an initiator by port number and initiator ID.
1320  * Returns 0 for success, 1 for failure.
1321  * Assumes the caller does NOT hold the CTL lock.
1322  */
1323 int
1324 ctl_remove_initiator(int32_t targ_port, uint32_t iid)
1325 {
1326         struct ctl_softc *softc;
1327
1328         softc = control_softc;
1329
1330         if ((targ_port < 0)
1331          || (targ_port > CTL_MAX_PORTS)) {
1332                 printf("%s: invalid port number %d\n", __func__, targ_port);
1333                 return (1);
1334         }
1335         if (iid > CTL_MAX_INIT_PER_PORT) {
1336                 printf("%s: initiator ID %u > maximun %u!\n",
1337                        __func__, iid, CTL_MAX_INIT_PER_PORT);
1338                 return (1);
1339         }
1340
1341         mtx_lock(&softc->ctl_lock);
1342
1343         softc->wwpn_iid[targ_port][iid].in_use = 0;
1344
1345         mtx_unlock(&softc->ctl_lock);
1346
1347         return (0);
1348 }
1349
1350 /*
1351  * Add an initiator to the initiator map.
1352  * Returns 0 for success, 1 for failure.
1353  * Assumes the caller does NOT hold the CTL lock.
1354  */
1355 int
1356 ctl_add_initiator(uint64_t wwpn, int32_t targ_port, uint32_t iid)
1357 {
1358         struct ctl_softc *softc;
1359         int retval;
1360
1361         softc = control_softc;
1362
1363         retval = 0;
1364
1365         if ((targ_port < 0)
1366          || (targ_port > CTL_MAX_PORTS)) {
1367                 printf("%s: invalid port number %d\n", __func__, targ_port);
1368                 return (1);
1369         }
1370         if (iid > CTL_MAX_INIT_PER_PORT) {
1371                 printf("%s: WWPN %#jx initiator ID %u > maximun %u!\n",
1372                        __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1373                 return (1);
1374         }
1375
1376         mtx_lock(&softc->ctl_lock);
1377
1378         if (softc->wwpn_iid[targ_port][iid].in_use != 0) {
1379                 /*
1380                  * We don't treat this as an error.
1381                  */
1382                 if (softc->wwpn_iid[targ_port][iid].wwpn == wwpn) {
1383                         printf("%s: port %d iid %u WWPN %#jx arrived again?\n",
1384                                __func__, targ_port, iid, (uintmax_t)wwpn);
1385                         goto bailout;
1386                 }
1387
1388                 /*
1389                  * This is an error, but what do we do about it?  The
1390                  * driver is telling us we have a new WWPN for this
1391                  * initiator ID, so we pretty much need to use it.
1392                  */
1393                 printf("%s: port %d iid %u WWPN %#jx arrived, WWPN %#jx is "
1394                        "still at that address\n", __func__, targ_port, iid,
1395                        (uintmax_t)wwpn,
1396                        (uintmax_t)softc->wwpn_iid[targ_port][iid].wwpn);
1397
1398                 /*
1399                  * XXX KDM clear have_ca and ua_pending on each LUN for
1400                  * this initiator.
1401                  */
1402         }
1403         softc->wwpn_iid[targ_port][iid].in_use = 1;
1404         softc->wwpn_iid[targ_port][iid].iid = iid;
1405         softc->wwpn_iid[targ_port][iid].wwpn = wwpn;
1406         softc->wwpn_iid[targ_port][iid].port = targ_port;
1407
1408 bailout:
1409
1410         mtx_unlock(&softc->ctl_lock);
1411
1412         return (retval);
1413 }
1414
1415 /*
1416  * XXX KDM should we pretend to do something in the target/lun
1417  * enable/disable functions?
1418  */
1419 static int
1420 ctl_ioctl_targ_enable(void *arg, struct ctl_id targ_id)
1421 {
1422         return (0);
1423 }
1424
1425 static int
1426 ctl_ioctl_targ_disable(void *arg, struct ctl_id targ_id)
1427 {
1428         return (0);
1429 }
1430
1431 static int
1432 ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
1433 {
1434         return (0);
1435 }
1436
1437 static int
1438 ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
1439 {
1440         return (0);
1441 }
1442
1443 /*
1444  * Data movement routine for the CTL ioctl frontend port.
1445  */
1446 static int
1447 ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
1448 {
1449         struct ctl_sg_entry *ext_sglist, *kern_sglist;
1450         struct ctl_sg_entry ext_entry, kern_entry;
1451         int ext_sglen, ext_sg_entries, kern_sg_entries;
1452         int ext_sg_start, ext_offset;
1453         int len_to_copy, len_copied;
1454         int kern_watermark, ext_watermark;
1455         int ext_sglist_malloced;
1456         int i, j;
1457
1458         ext_sglist_malloced = 0;
1459         ext_sg_start = 0;
1460         ext_offset = 0;
1461
1462         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove\n"));
1463
1464         /*
1465          * If this flag is set, fake the data transfer.
1466          */
1467         if (ctsio->io_hdr.flags & CTL_FLAG_NO_DATAMOVE) {
1468                 ctsio->ext_data_filled = ctsio->ext_data_len;
1469                 goto bailout;
1470         }
1471
1472         /*
1473          * To simplify things here, if we have a single buffer, stick it in
1474          * a S/G entry and just make it a single entry S/G list.
1475          */
1476         if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
1477                 int len_seen;
1478
1479                 ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
1480
1481                 ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
1482                                                            M_WAITOK);
1483                 if (ext_sglist == NULL) {
1484                         ctl_set_internal_failure(ctsio,
1485                                                  /*sks_valid*/ 0,
1486                                                  /*retry_count*/ 0);
1487                         return (CTL_RETVAL_COMPLETE);
1488                 }
1489                 ext_sglist_malloced = 1;
1490                 if (copyin(ctsio->ext_data_ptr, ext_sglist,
1491                                    ext_sglen) != 0) {
1492                         ctl_set_internal_failure(ctsio,
1493                                                  /*sks_valid*/ 0,
1494                                                  /*retry_count*/ 0);
1495                         goto bailout;
1496                 }
1497                 ext_sg_entries = ctsio->ext_sg_entries;
1498                 len_seen = 0;
1499                 for (i = 0; i < ext_sg_entries; i++) {
1500                         if ((len_seen + ext_sglist[i].len) >=
1501                              ctsio->ext_data_filled) {
1502                                 ext_sg_start = i;
1503                                 ext_offset = ctsio->ext_data_filled - len_seen;
1504                                 break;
1505                         }
1506                         len_seen += ext_sglist[i].len;
1507                 }
1508         } else {
1509                 ext_sglist = &ext_entry;
1510                 ext_sglist->addr = ctsio->ext_data_ptr;
1511                 ext_sglist->len = ctsio->ext_data_len;
1512                 ext_sg_entries = 1;
1513                 ext_sg_start = 0;
1514                 ext_offset = ctsio->ext_data_filled;
1515         }
1516
1517         if (ctsio->kern_sg_entries > 0) {
1518                 kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
1519                 kern_sg_entries = ctsio->kern_sg_entries;
1520         } else {
1521                 kern_sglist = &kern_entry;
1522                 kern_sglist->addr = ctsio->kern_data_ptr;
1523                 kern_sglist->len = ctsio->kern_data_len;
1524                 kern_sg_entries = 1;
1525         }
1526
1527
1528         kern_watermark = 0;
1529         ext_watermark = ext_offset;
1530         len_copied = 0;
1531         for (i = ext_sg_start, j = 0;
1532              i < ext_sg_entries && j < kern_sg_entries;) {
1533                 uint8_t *ext_ptr, *kern_ptr;
1534
1535                 len_to_copy = ctl_min(ext_sglist[i].len - ext_watermark,
1536                                       kern_sglist[j].len - kern_watermark);
1537
1538                 ext_ptr = (uint8_t *)ext_sglist[i].addr;
1539                 ext_ptr = ext_ptr + ext_watermark;
1540                 if (ctsio->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
1541                         /*
1542                          * XXX KDM fix this!
1543                          */
1544                         panic("need to implement bus address support");
1545 #if 0
1546                         kern_ptr = bus_to_virt(kern_sglist[j].addr);
1547 #endif
1548                 } else
1549                         kern_ptr = (uint8_t *)kern_sglist[j].addr;
1550                 kern_ptr = kern_ptr + kern_watermark;
1551
1552                 kern_watermark += len_to_copy;
1553                 ext_watermark += len_to_copy;
1554
1555                 if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
1556                      CTL_FLAG_DATA_IN) {
1557                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1558                                          "bytes to user\n", len_to_copy));
1559                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1560                                          "to %p\n", kern_ptr, ext_ptr));
1561                         if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
1562                                 ctl_set_internal_failure(ctsio,
1563                                                          /*sks_valid*/ 0,
1564                                                          /*retry_count*/ 0);
1565                                 goto bailout;
1566                         }
1567                 } else {
1568                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1569                                          "bytes from user\n", len_to_copy));
1570                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1571                                          "to %p\n", ext_ptr, kern_ptr));
1572                         if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
1573                                 ctl_set_internal_failure(ctsio,
1574                                                          /*sks_valid*/ 0,
1575                                                          /*retry_count*/0);
1576                                 goto bailout;
1577                         }
1578                 }
1579
1580                 len_copied += len_to_copy;
1581
1582                 if (ext_sglist[i].len == ext_watermark) {
1583                         i++;
1584                         ext_watermark = 0;
1585                 }
1586
1587                 if (kern_sglist[j].len == kern_watermark) {
1588                         j++;
1589                         kern_watermark = 0;
1590                 }
1591         }
1592
1593         ctsio->ext_data_filled += len_copied;
1594
1595         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_sg_entries: %d, "
1596                          "kern_sg_entries: %d\n", ext_sg_entries,
1597                          kern_sg_entries));
1598         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_data_len = %d, "
1599                          "kern_data_len = %d\n", ctsio->ext_data_len,
1600                          ctsio->kern_data_len));
1601
1602
1603         /* XXX KDM set residual?? */
1604 bailout:
1605
1606         if (ext_sglist_malloced != 0)
1607                 free(ext_sglist, M_CTL);
1608
1609         return (CTL_RETVAL_COMPLETE);
1610 }
1611
1612 /*
1613  * Serialize a command that went down the "wrong" side, and so was sent to
1614  * this controller for execution.  The logic is a little different than the
1615  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1616  * sent back to the other side, but in the success case, we execute the
1617  * command on this side (XFER mode) or tell the other side to execute it
1618  * (SER_ONLY mode).
1619  */
1620 static int
1621 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio, int have_lock)
1622 {
1623         struct ctl_softc *ctl_softc;
1624         union ctl_ha_msg msg_info;
1625         struct ctl_lun *lun;
1626         int retval = 0;
1627
1628         ctl_softc = control_softc;
1629         if (have_lock == 0)
1630                 mtx_lock(&ctl_softc->ctl_lock);
1631
1632         lun = ctl_softc->ctl_luns[ctsio->io_hdr.nexus.targ_lun];
1633         if (lun==NULL)
1634         {
1635                 /*
1636                  * Why isn't LUN defined? The other side wouldn't
1637                  * send a cmd if the LUN is undefined.
1638                  */
1639                 printf("%s: Bad JUJU!, LUN is NULL!\n", __func__);
1640
1641                 /* "Logical unit not supported" */
1642                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1643                                    lun,
1644                                    /*sense_format*/SSD_TYPE_NONE,
1645                                    /*current_error*/ 1,
1646                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1647                                    /*asc*/ 0x25,
1648                                    /*ascq*/ 0x00,
1649                                    SSD_ELEM_NONE);
1650
1651                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1652                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1653                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1654                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1655                 msg_info.hdr.serializing_sc = NULL;
1656                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1657                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1658                                 sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1659                 }
1660                 if (have_lock == 0)
1661                         mtx_unlock(&ctl_softc->ctl_lock);
1662                 return(1);
1663
1664         }
1665
1666         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1667
1668         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1669                 (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1670                  ooa_links))) {
1671         case CTL_ACTION_BLOCK:
1672                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
1673                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
1674                                   blocked_links);
1675                 break;
1676         case CTL_ACTION_PASS:
1677         case CTL_ACTION_SKIP:
1678                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
1679                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
1680                         STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
1681                                            &ctsio->io_hdr, links);
1682                 } else {
1683
1684                         /* send msg back to other side */
1685                         msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1686                         msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
1687                         msg_info.hdr.msg_type = CTL_MSG_R2R;
1688 #if 0
1689                         printf("2. pOrig %x\n", (int)msg_info.hdr.original_sc);
1690 #endif
1691                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1692                             sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1693                         }
1694                 }
1695                 break;
1696         case CTL_ACTION_OVERLAP:
1697                 /* OVERLAPPED COMMANDS ATTEMPTED */
1698                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1699                                    lun,
1700                                    /*sense_format*/SSD_TYPE_NONE,
1701                                    /*current_error*/ 1,
1702                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1703                                    /*asc*/ 0x4E,
1704                                    /*ascq*/ 0x00,
1705                                    SSD_ELEM_NONE);
1706
1707                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1708                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1709                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1710                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1711                 msg_info.hdr.serializing_sc = NULL;
1712                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1713 #if 0
1714                 printf("BAD JUJU:Major Bummer Overlap\n");
1715 #endif
1716                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1717                 retval = 1;
1718                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1719                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1720                 }
1721                 break;
1722         case CTL_ACTION_OVERLAP_TAG:
1723                 /* TAGGED OVERLAPPED COMMANDS (NN = QUEUE TAG) */
1724                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1725                                    lun,
1726                                    /*sense_format*/SSD_TYPE_NONE,
1727                                    /*current_error*/ 1,
1728                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1729                                    /*asc*/ 0x4D,
1730                                    /*ascq*/ ctsio->tag_num & 0xff,
1731                                    SSD_ELEM_NONE);
1732
1733                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1734                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1735                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1736                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1737                 msg_info.hdr.serializing_sc = NULL;
1738                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1739 #if 0
1740                 printf("BAD JUJU:Major Bummer Overlap Tag\n");
1741 #endif
1742                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1743                 retval = 1;
1744                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1745                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1746                 }
1747                 break;
1748         case CTL_ACTION_ERROR:
1749         default:
1750                 /* "Internal target failure" */
1751                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1752                                    lun,
1753                                    /*sense_format*/SSD_TYPE_NONE,
1754                                    /*current_error*/ 1,
1755                                    /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
1756                                    /*asc*/ 0x44,
1757                                    /*ascq*/ 0x00,
1758                                    SSD_ELEM_NONE);
1759
1760                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1761                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1762                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1763                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1764                 msg_info.hdr.serializing_sc = NULL;
1765                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1766 #if 0
1767                 printf("BAD JUJU:Major Bummer HW Error\n");
1768 #endif
1769                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1770                 retval = 1;
1771                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1772                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1773                 }
1774                 break;
1775         }
1776         if (have_lock == 0)
1777                 mtx_unlock(&ctl_softc->ctl_lock);
1778         return (retval);
1779 }
1780
1781 static int
1782 ctl_ioctl_submit_wait(union ctl_io *io)
1783 {
1784         struct ctl_fe_ioctl_params params;
1785         ctl_fe_ioctl_state last_state;
1786         int done, retval;
1787
1788         retval = 0;
1789
1790         bzero(&params, sizeof(params));
1791
1792         mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
1793         cv_init(&params.sem, "ctlioccv");
1794         params.state = CTL_IOCTL_INPROG;
1795         last_state = params.state;
1796
1797         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = &params;
1798
1799         CTL_DEBUG_PRINT(("ctl_ioctl_submit_wait\n"));
1800
1801         /* This shouldn't happen */
1802         if ((retval = ctl_queue(io)) != CTL_RETVAL_COMPLETE)
1803                 return (retval);
1804
1805         done = 0;
1806
1807         do {
1808                 mtx_lock(&params.ioctl_mtx);
1809                 /*
1810                  * Check the state here, and don't sleep if the state has
1811                  * already changed (i.e. wakeup has already occured, but we
1812                  * weren't waiting yet).
1813                  */
1814                 if (params.state == last_state) {
1815                         /* XXX KDM cv_wait_sig instead? */
1816                         cv_wait(&params.sem, &params.ioctl_mtx);
1817                 }
1818                 last_state = params.state;
1819
1820                 switch (params.state) {
1821                 case CTL_IOCTL_INPROG:
1822                         /* Why did we wake up? */
1823                         /* XXX KDM error here? */
1824                         mtx_unlock(&params.ioctl_mtx);
1825                         break;
1826                 case CTL_IOCTL_DATAMOVE:
1827                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DATAMOVE\n"));
1828
1829                         /*
1830                          * change last_state back to INPROG to avoid
1831                          * deadlock on subsequent data moves.
1832                          */
1833                         params.state = last_state = CTL_IOCTL_INPROG;
1834
1835                         mtx_unlock(&params.ioctl_mtx);
1836                         ctl_ioctl_do_datamove(&io->scsiio);
1837                         /*
1838                          * Note that in some cases, most notably writes,
1839                          * this will queue the I/O and call us back later.
1840                          * In other cases, generally reads, this routine
1841                          * will immediately call back and wake us up,
1842                          * probably using our own context.
1843                          */
1844                         io->scsiio.be_move_done(io);
1845                         break;
1846                 case CTL_IOCTL_DONE:
1847                         mtx_unlock(&params.ioctl_mtx);
1848                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DONE\n"));
1849                         done = 1;
1850                         break;
1851                 default:
1852                         mtx_unlock(&params.ioctl_mtx);
1853                         /* XXX KDM error here? */
1854                         break;
1855                 }
1856         } while (done == 0);
1857
1858         mtx_destroy(&params.ioctl_mtx);
1859         cv_destroy(&params.sem);
1860
1861         return (CTL_RETVAL_COMPLETE);
1862 }
1863
1864 static void
1865 ctl_ioctl_datamove(union ctl_io *io)
1866 {
1867         struct ctl_fe_ioctl_params *params;
1868
1869         params = (struct ctl_fe_ioctl_params *)
1870                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1871
1872         mtx_lock(&params->ioctl_mtx);
1873         params->state = CTL_IOCTL_DATAMOVE;
1874         cv_broadcast(&params->sem);
1875         mtx_unlock(&params->ioctl_mtx);
1876 }
1877
1878 static void
1879 ctl_ioctl_done(union ctl_io *io)
1880 {
1881         struct ctl_fe_ioctl_params *params;
1882
1883         params = (struct ctl_fe_ioctl_params *)
1884                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1885
1886         mtx_lock(&params->ioctl_mtx);
1887         params->state = CTL_IOCTL_DONE;
1888         cv_broadcast(&params->sem);
1889         mtx_unlock(&params->ioctl_mtx);
1890 }
1891
1892 static void
1893 ctl_ioctl_hard_startstop_callback(void *arg, struct cfi_metatask *metatask)
1894 {
1895         struct ctl_fe_ioctl_startstop_info *sd_info;
1896
1897         sd_info = (struct ctl_fe_ioctl_startstop_info *)arg;
1898
1899         sd_info->hs_info.status = metatask->status;
1900         sd_info->hs_info.total_luns = metatask->taskinfo.startstop.total_luns;
1901         sd_info->hs_info.luns_complete =
1902                 metatask->taskinfo.startstop.luns_complete;
1903         sd_info->hs_info.luns_failed = metatask->taskinfo.startstop.luns_failed;
1904
1905         cv_broadcast(&sd_info->sem);
1906 }
1907
1908 static void
1909 ctl_ioctl_bbrread_callback(void *arg, struct cfi_metatask *metatask)
1910 {
1911         struct ctl_fe_ioctl_bbrread_info *fe_bbr_info;
1912
1913         fe_bbr_info = (struct ctl_fe_ioctl_bbrread_info *)arg;
1914
1915         mtx_lock(fe_bbr_info->lock);
1916         fe_bbr_info->bbr_info->status = metatask->status;
1917         fe_bbr_info->bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
1918         fe_bbr_info->wakeup_done = 1;
1919         mtx_unlock(fe_bbr_info->lock);
1920
1921         cv_broadcast(&fe_bbr_info->sem);
1922 }
1923
1924 /*
1925  * Must be called with the ctl_lock held.
1926  * Returns 0 for success, errno for failure.
1927  */
1928 static int
1929 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
1930                    struct ctl_ooa *ooa_hdr)
1931 {
1932         union ctl_io *io;
1933         int retval;
1934
1935         retval = 0;
1936
1937         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
1938              (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
1939              ooa_links)) {
1940                 struct ctl_ooa_entry *cur_entry, entry;
1941
1942                 /*
1943                  * If we've got more than we can fit, just count the
1944                  * remaining entries.
1945                  */
1946                 if (*cur_fill_num >= ooa_hdr->alloc_num)
1947                         continue;
1948
1949                 cur_entry = &ooa_hdr->entries[*cur_fill_num];
1950
1951                 bzero(&entry, sizeof(entry));
1952
1953                 entry.tag_num = io->scsiio.tag_num;
1954                 entry.lun_num = lun->lun;
1955 #ifdef CTL_TIME_IO
1956                 entry.start_bt = io->io_hdr.start_bt;
1957 #endif
1958                 bcopy(io->scsiio.cdb, entry.cdb, io->scsiio.cdb_len);
1959                 entry.cdb_len = io->scsiio.cdb_len;
1960                 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
1961                         entry.cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
1962
1963                 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
1964                         entry.cmd_flags |= CTL_OOACMD_FLAG_DMA;
1965
1966                 if (io->io_hdr.flags & CTL_FLAG_ABORT)
1967                         entry.cmd_flags |= CTL_OOACMD_FLAG_ABORT;
1968
1969                 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
1970                         entry.cmd_flags |= CTL_OOACMD_FLAG_RTR;
1971
1972                 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
1973                         entry.cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
1974
1975                 retval = copyout(&entry, cur_entry, sizeof(entry));
1976
1977                 if (retval != 0)
1978                         break;
1979         }
1980
1981         return (retval);
1982 }
1983
1984 static void *
1985 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
1986                  size_t error_str_len)
1987 {
1988         void *kptr;
1989
1990         kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
1991         if (kptr == NULL) {
1992                 snprintf(error_str, error_str_len, "Cannot allocate %d bytes",
1993                          len);
1994                 return (NULL);
1995         }
1996
1997         if (copyin(user_addr, kptr, len) != 0) {
1998                 snprintf(error_str, error_str_len, "Error copying %d bytes "
1999                          "from user address %p to kernel address %p", len,
2000                          user_addr, kptr);
2001                 free(kptr, M_CTL);
2002                 return (NULL);
2003         }
2004
2005         return (kptr);
2006 }
2007
2008 static void
2009 ctl_free_args(int num_be_args, struct ctl_be_arg *be_args)
2010 {
2011         int i;
2012
2013         if (be_args == NULL)
2014                 return;
2015
2016         for (i = 0; i < num_be_args; i++) {
2017                 free(be_args[i].kname, M_CTL);
2018                 free(be_args[i].kvalue, M_CTL);
2019         }
2020
2021         free(be_args, M_CTL);
2022 }
2023
2024 static struct ctl_be_arg *
2025 ctl_copyin_args(int num_be_args, struct ctl_be_arg *be_args,
2026                 char *error_str, size_t error_str_len)
2027 {
2028         struct ctl_be_arg *args;
2029         int i;
2030
2031         args = ctl_copyin_alloc(be_args, num_be_args * sizeof(*be_args),
2032                                 error_str, error_str_len);
2033
2034         if (args == NULL)
2035                 goto bailout;
2036
2037         for (i = 0; i < num_be_args; i++) {
2038                 uint8_t *tmpptr;
2039
2040                 args[i].kname = ctl_copyin_alloc(args[i].name,
2041                         args[i].namelen, error_str, error_str_len);
2042                 if (args[i].kname == NULL)
2043                         goto bailout;
2044
2045                 if (args[i].kname[args[i].namelen - 1] != '\0') {
2046                         snprintf(error_str, error_str_len, "Argument %d "
2047                                  "name is not NUL-terminated", i);
2048                         goto bailout;
2049                 }
2050
2051                 args[i].kvalue = NULL;
2052
2053                 tmpptr = ctl_copyin_alloc(args[i].value,
2054                         args[i].vallen, error_str, error_str_len);
2055                 if (tmpptr == NULL)
2056                         goto bailout;
2057
2058                 args[i].kvalue = tmpptr;
2059
2060                 if ((args[i].flags & CTL_BEARG_ASCII)
2061                  && (tmpptr[args[i].vallen - 1] != '\0')) {
2062                         snprintf(error_str, error_str_len, "Argument %d "
2063                                  "value is not NUL-terminated", i);
2064                         goto bailout;
2065                 }
2066         }
2067
2068         return (args);
2069 bailout:
2070
2071         ctl_free_args(num_be_args, args);
2072
2073         return (NULL);
2074 }
2075
2076 /*
2077  * Escape characters that are illegal or not recommended in XML.
2078  */
2079 int
2080 ctl_sbuf_printf_esc(struct sbuf *sb, char *str)
2081 {
2082         int retval;
2083
2084         retval = 0;
2085
2086         for (; *str; str++) {
2087                 switch (*str) {
2088                 case '&':
2089                         retval = sbuf_printf(sb, "&amp;");
2090                         break;
2091                 case '>':
2092                         retval = sbuf_printf(sb, "&gt;");
2093                         break;
2094                 case '<':
2095                         retval = sbuf_printf(sb, "&lt;");
2096                         break;
2097                 default:
2098                         retval = sbuf_putc(sb, *str);
2099                         break;
2100                 }
2101
2102                 if (retval != 0)
2103                         break;
2104
2105         }
2106
2107         return (retval);
2108 }
2109
2110 static int
2111 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2112           struct thread *td)
2113 {
2114         struct ctl_softc *softc;
2115         int retval;
2116
2117         softc = control_softc;
2118
2119         retval = 0;
2120
2121         switch (cmd) {
2122         case CTL_IO: {
2123                 union ctl_io *io;
2124                 void *pool_tmp;
2125
2126                 /*
2127                  * If we haven't been "enabled", don't allow any SCSI I/O
2128                  * to this FETD.
2129                  */
2130                 if ((softc->ioctl_info.flags & CTL_IOCTL_FLAG_ENABLED) == 0) {
2131                         retval = -EPERM;
2132                         break;
2133                 }
2134
2135                 io = ctl_alloc_io(softc->ioctl_info.fe.ctl_pool_ref);
2136                 if (io == NULL) {
2137                         printf("ctl_ioctl: can't allocate ctl_io!\n");
2138                         retval = -ENOSPC;
2139                         break;
2140                 }
2141
2142                 /*
2143                  * Need to save the pool reference so it doesn't get
2144                  * spammed by the user's ctl_io.
2145                  */
2146                 pool_tmp = io->io_hdr.pool;
2147
2148                 memcpy(io, (void *)addr, sizeof(*io));
2149
2150                 io->io_hdr.pool = pool_tmp;
2151                 /*
2152                  * No status yet, so make sure the status is set properly.
2153                  */
2154                 io->io_hdr.status = CTL_STATUS_NONE;
2155
2156                 /*
2157                  * The user sets the initiator ID, target and LUN IDs.
2158                  */
2159                 io->io_hdr.nexus.targ_port = softc->ioctl_info.fe.targ_port;
2160                 io->io_hdr.flags |= CTL_FLAG_USER_REQ;
2161                 if ((io->io_hdr.io_type == CTL_IO_SCSI)
2162                  && (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
2163                         io->scsiio.tag_num = softc->ioctl_info.cur_tag_num++;
2164
2165                 retval = ctl_ioctl_submit_wait(io);
2166
2167                 if (retval != 0) {
2168                         ctl_free_io(io);
2169                         break;
2170                 }
2171
2172                 memcpy((void *)addr, io, sizeof(*io));
2173
2174                 /* return this to our pool */
2175                 ctl_free_io(io);
2176
2177                 break;
2178         }
2179         case CTL_ENABLE_PORT:
2180         case CTL_DISABLE_PORT:
2181         case CTL_SET_PORT_WWNS: {
2182                 struct ctl_frontend *fe;
2183                 struct ctl_port_entry *entry;
2184
2185                 entry = (struct ctl_port_entry *)addr;
2186
2187                 mtx_lock(&softc->ctl_lock);
2188                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2189                         int action, done;
2190
2191                         action = 0;
2192                         done = 0;
2193
2194                         if ((entry->port_type == CTL_PORT_NONE)
2195                          && (entry->targ_port == fe->targ_port)) {
2196                                 /*
2197                                  * If the user only wants to enable or
2198                                  * disable or set WWNs on a specific port,
2199                                  * do the operation and we're done.
2200                                  */
2201                                 action = 1;
2202                                 done = 1;
2203                         } else if (entry->port_type & fe->port_type) {
2204                                 /*
2205                                  * Compare the user's type mask with the
2206                                  * particular frontend type to see if we
2207                                  * have a match.
2208                                  */
2209                                 action = 1;
2210                                 done = 0;
2211
2212                                 /*
2213                                  * Make sure the user isn't trying to set
2214                                  * WWNs on multiple ports at the same time.
2215                                  */
2216                                 if (cmd == CTL_SET_PORT_WWNS) {
2217                                         printf("%s: Can't set WWNs on "
2218                                                "multiple ports\n", __func__);
2219                                         retval = EINVAL;
2220                                         break;
2221                                 }
2222                         }
2223                         if (action != 0) {
2224                                 /*
2225                                  * XXX KDM we have to drop the lock here,
2226                                  * because the online/offline operations
2227                                  * can potentially block.  We need to
2228                                  * reference count the frontends so they
2229                                  * can't go away,
2230                                  */
2231                                 mtx_unlock(&softc->ctl_lock);
2232
2233                                 if (cmd == CTL_ENABLE_PORT)
2234                                         ctl_frontend_online(fe);
2235                                 else if (cmd == CTL_DISABLE_PORT)
2236                                         ctl_frontend_offline(fe);
2237
2238                                 mtx_lock(&softc->ctl_lock);
2239
2240                                 if (cmd == CTL_SET_PORT_WWNS)
2241                                         ctl_frontend_set_wwns(fe,
2242                                             (entry->flags & CTL_PORT_WWNN_VALID) ?
2243                                             1 : 0, entry->wwnn,
2244                                             (entry->flags & CTL_PORT_WWPN_VALID) ?
2245                                             1 : 0, entry->wwpn);
2246                         }
2247                         if (done != 0)
2248                                 break;
2249                 }
2250                 mtx_unlock(&softc->ctl_lock);
2251                 break;
2252         }
2253         case CTL_GET_PORT_LIST: {
2254                 struct ctl_frontend *fe;
2255                 struct ctl_port_list *list;
2256                 int i;
2257
2258                 list = (struct ctl_port_list *)addr;
2259
2260                 if (list->alloc_len != (list->alloc_num *
2261                     sizeof(struct ctl_port_entry))) {
2262                         printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2263                                "alloc_num %u * sizeof(struct ctl_port_entry) "
2264                                "%zu\n", __func__, list->alloc_len,
2265                                list->alloc_num, sizeof(struct ctl_port_entry));
2266                         retval = EINVAL;
2267                         break;
2268                 }
2269                 list->fill_len = 0;
2270                 list->fill_num = 0;
2271                 list->dropped_num = 0;
2272                 i = 0;
2273                 mtx_lock(&softc->ctl_lock);
2274                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2275                         struct ctl_port_entry entry, *list_entry;
2276
2277                         if (list->fill_num >= list->alloc_num) {
2278                                 list->dropped_num++;
2279                                 continue;
2280                         }
2281
2282                         entry.port_type = fe->port_type;
2283                         strlcpy(entry.port_name, fe->port_name,
2284                                 sizeof(entry.port_name));
2285                         entry.targ_port = fe->targ_port;
2286                         entry.physical_port = fe->physical_port;
2287                         entry.virtual_port = fe->virtual_port;
2288                         entry.wwnn = fe->wwnn;
2289                         entry.wwpn = fe->wwpn;
2290                         if (fe->status & CTL_PORT_STATUS_ONLINE)
2291                                 entry.online = 1;
2292                         else
2293                                 entry.online = 0;
2294
2295                         list_entry = &list->entries[i];
2296
2297                         retval = copyout(&entry, list_entry, sizeof(entry));
2298                         if (retval != 0) {
2299                                 printf("%s: CTL_GET_PORT_LIST: copyout "
2300                                        "returned %d\n", __func__, retval);
2301                                 break;
2302                         }
2303                         i++;
2304                         list->fill_num++;
2305                         list->fill_len += sizeof(entry);
2306                 }
2307                 mtx_unlock(&softc->ctl_lock);
2308
2309                 /*
2310                  * If this is non-zero, we had a copyout fault, so there's
2311                  * probably no point in attempting to set the status inside
2312                  * the structure.
2313                  */
2314                 if (retval != 0)
2315                         break;
2316
2317                 if (list->dropped_num > 0)
2318                         list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2319                 else
2320                         list->status = CTL_PORT_LIST_OK;
2321                 break;
2322         }
2323         case CTL_DUMP_OOA: {
2324                 struct ctl_lun *lun;
2325                 union ctl_io *io;
2326                 char printbuf[128];
2327                 struct sbuf sb;
2328
2329                 mtx_lock(&softc->ctl_lock);
2330                 printf("Dumping OOA queues:\n");
2331                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2332                         for (io = (union ctl_io *)TAILQ_FIRST(
2333                              &lun->ooa_queue); io != NULL;
2334                              io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2335                              ooa_links)) {
2336                                 sbuf_new(&sb, printbuf, sizeof(printbuf),
2337                                          SBUF_FIXEDLEN);
2338                                 sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2339                                             (intmax_t)lun->lun,
2340                                             io->scsiio.tag_num,
2341                                             (io->io_hdr.flags &
2342                                             CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2343                                             (io->io_hdr.flags &
2344                                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2345                                             (io->io_hdr.flags &
2346                                             CTL_FLAG_ABORT) ? " ABORT" : "",
2347                                             (io->io_hdr.flags &
2348                                         CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2349                                 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2350                                 sbuf_finish(&sb);
2351                                 printf("%s\n", sbuf_data(&sb));
2352                         }
2353                 }
2354                 printf("OOA queues dump done\n");
2355                 mtx_unlock(&softc->ctl_lock);
2356                 break;
2357         }
2358         case CTL_GET_OOA: {
2359                 struct ctl_lun *lun;
2360                 struct ctl_ooa *ooa_hdr;
2361                 uint32_t cur_fill_num;
2362
2363                 ooa_hdr = (struct ctl_ooa *)addr;
2364
2365                 if ((ooa_hdr->alloc_len == 0)
2366                  || (ooa_hdr->alloc_num == 0)) {
2367                         printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2368                                "must be non-zero\n", __func__,
2369                                ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2370                         retval = EINVAL;
2371                         break;
2372                 }
2373
2374                 if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2375                     sizeof(struct ctl_ooa_entry))) {
2376                         printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2377                                "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2378                                __func__, ooa_hdr->alloc_len,
2379                                ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2380                         retval = EINVAL;
2381                         break;
2382                 }
2383
2384                 mtx_lock(&softc->ctl_lock);
2385                 if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2386                  && ((ooa_hdr->lun_num > CTL_MAX_LUNS)
2387                   || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2388                         mtx_unlock(&softc->ctl_lock);
2389                         printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2390                                __func__, (uintmax_t)ooa_hdr->lun_num);
2391                         retval = EINVAL;
2392                         break;
2393                 }
2394
2395                 cur_fill_num = 0;
2396
2397                 if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2398                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
2399                                 retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2400                                         ooa_hdr);
2401                                 if (retval != 0)
2402                                         break;
2403                         }
2404                         if (retval != 0) {
2405                                 mtx_unlock(&softc->ctl_lock);
2406                                 break;
2407                         }
2408                 } else {
2409                         lun = softc->ctl_luns[ooa_hdr->lun_num];
2410
2411                         retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr);
2412                 }
2413                 mtx_unlock(&softc->ctl_lock);
2414
2415                 ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2416                 ooa_hdr->fill_len = ooa_hdr->fill_num *
2417                         sizeof(struct ctl_ooa_entry);
2418
2419                 getbintime(&ooa_hdr->cur_bt);
2420
2421                 if (cur_fill_num > ooa_hdr->alloc_num) {
2422                         ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2423                         ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2424                 } else {
2425                         ooa_hdr->dropped_num = 0;
2426                         ooa_hdr->status = CTL_OOA_OK;
2427                 }
2428                 break;
2429         }
2430         case CTL_CHECK_OOA: {
2431                 union ctl_io *io;
2432                 struct ctl_lun *lun;
2433                 struct ctl_ooa_info *ooa_info;
2434
2435
2436                 ooa_info = (struct ctl_ooa_info *)addr;
2437
2438                 if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2439                         ooa_info->status = CTL_OOA_INVALID_LUN;
2440                         break;
2441                 }
2442                 mtx_lock(&softc->ctl_lock);
2443                 lun = softc->ctl_luns[ooa_info->lun_id];
2444                 if (lun == NULL) {
2445                         mtx_unlock(&softc->ctl_lock);
2446                         ooa_info->status = CTL_OOA_INVALID_LUN;
2447                         break;
2448                 }
2449
2450                 ooa_info->num_entries = 0;
2451                 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2452                      io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2453                      &io->io_hdr, ooa_links)) {
2454                         ooa_info->num_entries++;
2455                 }
2456
2457                 mtx_unlock(&softc->ctl_lock);
2458                 ooa_info->status = CTL_OOA_SUCCESS;
2459
2460                 break;
2461         }
2462         case CTL_HARD_START:
2463         case CTL_HARD_STOP: {
2464                 struct ctl_fe_ioctl_startstop_info ss_info;
2465                 struct cfi_metatask *metatask;
2466                 struct mtx hs_mtx;
2467
2468                 mtx_init(&hs_mtx, "HS Mutex", NULL, MTX_DEF);
2469
2470                 cv_init(&ss_info.sem, "hard start/stop cv" );
2471
2472                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2473                 if (metatask == NULL) {
2474                         retval = ENOMEM;
2475                         mtx_destroy(&hs_mtx);
2476                         break;
2477                 }
2478
2479                 if (cmd == CTL_HARD_START)
2480                         metatask->tasktype = CFI_TASK_STARTUP;
2481                 else
2482                         metatask->tasktype = CFI_TASK_SHUTDOWN;
2483
2484                 metatask->callback = ctl_ioctl_hard_startstop_callback;
2485                 metatask->callback_arg = &ss_info;
2486
2487                 cfi_action(metatask);
2488
2489                 /* Wait for the callback */
2490                 mtx_lock(&hs_mtx);
2491                 cv_wait_sig(&ss_info.sem, &hs_mtx);
2492                 mtx_unlock(&hs_mtx);
2493
2494                 /*
2495                  * All information has been copied from the metatask by the
2496                  * time cv_broadcast() is called, so we free the metatask here.
2497                  */
2498                 cfi_free_metatask(metatask);
2499
2500                 memcpy((void *)addr, &ss_info.hs_info, sizeof(ss_info.hs_info));
2501
2502                 mtx_destroy(&hs_mtx);
2503                 break;
2504         }
2505         case CTL_BBRREAD: {
2506                 struct ctl_bbrread_info *bbr_info;
2507                 struct ctl_fe_ioctl_bbrread_info fe_bbr_info;
2508                 struct mtx bbr_mtx;
2509                 struct cfi_metatask *metatask;
2510
2511                 bbr_info = (struct ctl_bbrread_info *)addr;
2512
2513                 bzero(&fe_bbr_info, sizeof(fe_bbr_info));
2514
2515                 bzero(&bbr_mtx, sizeof(bbr_mtx));
2516                 mtx_init(&bbr_mtx, "BBR Mutex", NULL, MTX_DEF);
2517
2518                 fe_bbr_info.bbr_info = bbr_info;
2519                 fe_bbr_info.lock = &bbr_mtx;
2520
2521                 cv_init(&fe_bbr_info.sem, "BBR read cv");
2522                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2523
2524                 if (metatask == NULL) {
2525                         mtx_destroy(&bbr_mtx);
2526                         cv_destroy(&fe_bbr_info.sem);
2527                         retval = ENOMEM;
2528                         break;
2529                 }
2530                 metatask->tasktype = CFI_TASK_BBRREAD;
2531                 metatask->callback = ctl_ioctl_bbrread_callback;
2532                 metatask->callback_arg = &fe_bbr_info;
2533                 metatask->taskinfo.bbrread.lun_num = bbr_info->lun_num;
2534                 metatask->taskinfo.bbrread.lba = bbr_info->lba;
2535                 metatask->taskinfo.bbrread.len = bbr_info->len;
2536
2537                 cfi_action(metatask);
2538
2539                 mtx_lock(&bbr_mtx);
2540                 while (fe_bbr_info.wakeup_done == 0)
2541                         cv_wait_sig(&fe_bbr_info.sem, &bbr_mtx);
2542                 mtx_unlock(&bbr_mtx);
2543
2544                 bbr_info->status = metatask->status;
2545                 bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2546                 bbr_info->scsi_status = metatask->taskinfo.bbrread.scsi_status;
2547                 memcpy(&bbr_info->sense_data,
2548                        &metatask->taskinfo.bbrread.sense_data,
2549                        ctl_min(sizeof(bbr_info->sense_data),
2550                                sizeof(metatask->taskinfo.bbrread.sense_data)));
2551
2552                 cfi_free_metatask(metatask);
2553
2554                 mtx_destroy(&bbr_mtx);
2555                 cv_destroy(&fe_bbr_info.sem);
2556
2557                 break;
2558         }
2559         case CTL_DELAY_IO: {
2560                 struct ctl_io_delay_info *delay_info;
2561 #ifdef CTL_IO_DELAY
2562                 struct ctl_lun *lun;
2563 #endif /* CTL_IO_DELAY */
2564
2565                 delay_info = (struct ctl_io_delay_info *)addr;
2566
2567 #ifdef CTL_IO_DELAY
2568                 mtx_lock(&softc->ctl_lock);
2569
2570                 if ((delay_info->lun_id > CTL_MAX_LUNS)
2571                  || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2572                         delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2573                 } else {
2574                         lun = softc->ctl_luns[delay_info->lun_id];
2575
2576                         delay_info->status = CTL_DELAY_STATUS_OK;
2577
2578                         switch (delay_info->delay_type) {
2579                         case CTL_DELAY_TYPE_CONT:
2580                                 break;
2581                         case CTL_DELAY_TYPE_ONESHOT:
2582                                 break;
2583                         default:
2584                                 delay_info->status =
2585                                         CTL_DELAY_STATUS_INVALID_TYPE;
2586                                 break;
2587                         }
2588
2589                         switch (delay_info->delay_loc) {
2590                         case CTL_DELAY_LOC_DATAMOVE:
2591                                 lun->delay_info.datamove_type =
2592                                         delay_info->delay_type;
2593                                 lun->delay_info.datamove_delay =
2594                                         delay_info->delay_secs;
2595                                 break;
2596                         case CTL_DELAY_LOC_DONE:
2597                                 lun->delay_info.done_type =
2598                                         delay_info->delay_type;
2599                                 lun->delay_info.done_delay =
2600                                         delay_info->delay_secs;
2601                                 break;
2602                         default:
2603                                 delay_info->status =
2604                                         CTL_DELAY_STATUS_INVALID_LOC;
2605                                 break;
2606                         }
2607                 }
2608
2609                 mtx_unlock(&softc->ctl_lock);
2610 #else
2611                 delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2612 #endif /* CTL_IO_DELAY */
2613                 break;
2614         }
2615         case CTL_REALSYNC_SET: {
2616                 int *syncstate;
2617
2618                 syncstate = (int *)addr;
2619
2620                 mtx_lock(&softc->ctl_lock);
2621                 switch (*syncstate) {
2622                 case 0:
2623                         softc->flags &= ~CTL_FLAG_REAL_SYNC;
2624                         break;
2625                 case 1:
2626                         softc->flags |= CTL_FLAG_REAL_SYNC;
2627                         break;
2628                 default:
2629                         retval = -EINVAL;
2630                         break;
2631                 }
2632                 mtx_unlock(&softc->ctl_lock);
2633                 break;
2634         }
2635         case CTL_REALSYNC_GET: {
2636                 int *syncstate;
2637
2638                 syncstate = (int*)addr;
2639
2640                 mtx_lock(&softc->ctl_lock);
2641                 if (softc->flags & CTL_FLAG_REAL_SYNC)
2642                         *syncstate = 1;
2643                 else
2644                         *syncstate = 0;
2645                 mtx_unlock(&softc->ctl_lock);
2646
2647                 break;
2648         }
2649         case CTL_SETSYNC:
2650         case CTL_GETSYNC: {
2651                 struct ctl_sync_info *sync_info;
2652                 struct ctl_lun *lun;
2653
2654                 sync_info = (struct ctl_sync_info *)addr;
2655
2656                 mtx_lock(&softc->ctl_lock);
2657                 lun = softc->ctl_luns[sync_info->lun_id];
2658                 if (lun == NULL) {
2659                         mtx_unlock(&softc->ctl_lock);
2660                         sync_info->status = CTL_GS_SYNC_NO_LUN;
2661                 }
2662                 /*
2663                  * Get or set the sync interval.  We're not bounds checking
2664                  * in the set case, hopefully the user won't do something
2665                  * silly.
2666                  */
2667                 if (cmd == CTL_GETSYNC)
2668                         sync_info->sync_interval = lun->sync_interval;
2669                 else
2670                         lun->sync_interval = sync_info->sync_interval;
2671
2672                 mtx_unlock(&softc->ctl_lock);
2673
2674                 sync_info->status = CTL_GS_SYNC_OK;
2675
2676                 break;
2677         }
2678         case CTL_GETSTATS: {
2679                 struct ctl_stats *stats;
2680                 struct ctl_lun *lun;
2681                 int i;
2682
2683                 stats = (struct ctl_stats *)addr;
2684
2685                 if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2686                      stats->alloc_len) {
2687                         stats->status = CTL_SS_NEED_MORE_SPACE;
2688                         stats->num_luns = softc->num_luns;
2689                         break;
2690                 }
2691                 /*
2692                  * XXX KDM no locking here.  If the LUN list changes,
2693                  * things can blow up.
2694                  */
2695                 for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2696                      i++, lun = STAILQ_NEXT(lun, links)) {
2697                         retval = copyout(&lun->stats, &stats->lun_stats[i],
2698                                          sizeof(lun->stats));
2699                         if (retval != 0)
2700                                 break;
2701                 }
2702                 stats->num_luns = softc->num_luns;
2703                 stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2704                                  softc->num_luns;
2705                 stats->status = CTL_SS_OK;
2706 #ifdef CTL_TIME_IO
2707                 stats->flags = CTL_STATS_FLAG_TIME_VALID;
2708 #else
2709                 stats->flags = CTL_STATS_FLAG_NONE;
2710 #endif
2711                 getnanouptime(&stats->timestamp);
2712                 break;
2713         }
2714         case CTL_ERROR_INJECT: {
2715                 struct ctl_error_desc *err_desc, *new_err_desc;
2716                 struct ctl_lun *lun;
2717
2718                 err_desc = (struct ctl_error_desc *)addr;
2719
2720                 new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2721                                       M_WAITOK | M_ZERO);
2722                 if (new_err_desc == NULL) {
2723                         printf("%s: CTL_ERROR_INJECT: error allocating %zu "
2724                                "bytes\n", __func__, sizeof(*new_err_desc));
2725                         retval = ENOMEM;
2726                         break;
2727                 }
2728                 bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2729
2730                 mtx_lock(&softc->ctl_lock);
2731                 lun = softc->ctl_luns[err_desc->lun_id];
2732                 if (lun == NULL) {
2733                         mtx_unlock(&softc->ctl_lock);
2734                         printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2735                                __func__, (uintmax_t)err_desc->lun_id);
2736                         retval = EINVAL;
2737                         break;
2738                 }
2739
2740                 /*
2741                  * We could do some checking here to verify the validity
2742                  * of the request, but given the complexity of error
2743                  * injection requests, the checking logic would be fairly
2744                  * complex.
2745                  *
2746                  * For now, if the request is invalid, it just won't get
2747                  * executed and might get deleted.
2748                  */
2749                 STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2750
2751                 /*
2752                  * XXX KDM check to make sure the serial number is unique,
2753                  * in case we somehow manage to wrap.  That shouldn't
2754                  * happen for a very long time, but it's the right thing to
2755                  * do.
2756                  */
2757                 new_err_desc->serial = lun->error_serial;
2758                 err_desc->serial = lun->error_serial;
2759                 lun->error_serial++;
2760
2761                 mtx_unlock(&softc->ctl_lock);
2762                 break;
2763         }
2764         case CTL_ERROR_INJECT_DELETE: {
2765                 struct ctl_error_desc *delete_desc, *desc, *desc2;
2766                 struct ctl_lun *lun;
2767                 int delete_done;
2768
2769                 delete_desc = (struct ctl_error_desc *)addr;
2770                 delete_done = 0;
2771
2772                 mtx_lock(&softc->ctl_lock);
2773                 lun = softc->ctl_luns[delete_desc->lun_id];
2774                 if (lun == NULL) {
2775                         mtx_unlock(&softc->ctl_lock);
2776                         printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
2777                                __func__, (uintmax_t)delete_desc->lun_id);
2778                         retval = EINVAL;
2779                         break;
2780                 }
2781                 STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
2782                         if (desc->serial != delete_desc->serial)
2783                                 continue;
2784
2785                         STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
2786                                       links);
2787                         free(desc, M_CTL);
2788                         delete_done = 1;
2789                 }
2790                 mtx_unlock(&softc->ctl_lock);
2791                 if (delete_done == 0) {
2792                         printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
2793                                "error serial %ju on LUN %u\n", __func__,
2794                                delete_desc->serial, delete_desc->lun_id);
2795                         retval = EINVAL;
2796                         break;
2797                 }
2798                 break;
2799         }
2800         case CTL_DUMP_STRUCTS: {
2801                 int i, j, k;
2802                 struct ctl_frontend *fe;
2803
2804                 printf("CTL IID to WWPN map start:\n");
2805                 for (i = 0; i < CTL_MAX_PORTS; i++) {
2806                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
2807                                 if (softc->wwpn_iid[i][j].in_use == 0)
2808                                         continue;
2809
2810                                 printf("port %d iid %u WWPN %#jx\n",
2811                                        softc->wwpn_iid[i][j].port,
2812                                        softc->wwpn_iid[i][j].iid,
2813                                        (uintmax_t)softc->wwpn_iid[i][j].wwpn);
2814                         }
2815                 }
2816                 printf("CTL IID to WWPN map end\n");
2817                 printf("CTL Persistent Reservation information start:\n");
2818                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2819                         struct ctl_lun *lun;
2820
2821                         lun = softc->ctl_luns[i];
2822
2823                         if ((lun == NULL)
2824                          || ((lun->flags & CTL_LUN_DISABLED) != 0))
2825                                 continue;
2826
2827                         for (j = 0; j < (CTL_MAX_PORTS * 2); j++) {
2828                                 for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
2829                                         if (lun->per_res[j+k].registered == 0)
2830                                                 continue;
2831                                         printf("LUN %d port %d iid %d key "
2832                                                "%#jx\n", i, j, k,
2833                                                (uintmax_t)scsi_8btou64(
2834                                                lun->per_res[j+k].res_key.key));
2835                                 }
2836                         }
2837                 }
2838                 printf("CTL Persistent Reservation information end\n");
2839                 printf("CTL Frontends:\n");
2840                 /*
2841                  * XXX KDM calling this without a lock.  We'd likely want
2842                  * to drop the lock before calling the frontend's dump
2843                  * routine anyway.
2844                  */
2845                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2846                         printf("Frontend %s Type %u pport %d vport %d WWNN "
2847                                "%#jx WWPN %#jx\n", fe->port_name, fe->port_type,
2848                                fe->physical_port, fe->virtual_port,
2849                                (uintmax_t)fe->wwnn, (uintmax_t)fe->wwpn);
2850
2851                         /*
2852                          * Frontends are not required to support the dump
2853                          * routine.
2854                          */
2855                         if (fe->fe_dump == NULL)
2856                                 continue;
2857
2858                         fe->fe_dump();
2859                 }
2860                 printf("CTL Frontend information end\n");
2861                 break;
2862         }
2863         case CTL_LUN_REQ: {
2864                 struct ctl_lun_req *lun_req;
2865                 struct ctl_backend_driver *backend;
2866
2867                 lun_req = (struct ctl_lun_req *)addr;
2868
2869                 backend = ctl_backend_find(lun_req->backend);
2870                 if (backend == NULL) {
2871                         lun_req->status = CTL_LUN_ERROR;
2872                         snprintf(lun_req->error_str,
2873                                  sizeof(lun_req->error_str),
2874                                  "Backend \"%s\" not found.",
2875                                  lun_req->backend);
2876                         break;
2877                 }
2878                 if (lun_req->num_be_args > 0) {
2879                         lun_req->kern_be_args = ctl_copyin_args(
2880                                 lun_req->num_be_args,
2881                                 lun_req->be_args,
2882                                 lun_req->error_str,
2883                                 sizeof(lun_req->error_str));
2884                         if (lun_req->kern_be_args == NULL) {
2885                                 lun_req->status = CTL_LUN_ERROR;
2886                                 break;
2887                         }
2888                 }
2889
2890                 retval = backend->ioctl(dev, cmd, addr, flag, td);
2891
2892                 if (lun_req->num_be_args > 0) {
2893                         ctl_free_args(lun_req->num_be_args,
2894                                       lun_req->kern_be_args);
2895                 }
2896                 break;
2897         }
2898         case CTL_LUN_LIST: {
2899                 struct sbuf *sb;
2900                 struct ctl_lun *lun;
2901                 struct ctl_lun_list *list;
2902
2903                 list = (struct ctl_lun_list *)addr;
2904
2905                 /*
2906                  * Allocate a fixed length sbuf here, based on the length
2907                  * of the user's buffer.  We could allocate an auto-extending
2908                  * buffer, and then tell the user how much larger our
2909                  * amount of data is than his buffer, but that presents
2910                  * some problems:
2911                  *
2912                  * 1.  The sbuf(9) routines use a blocking malloc, and so
2913                  *     we can't hold a lock while calling them with an
2914                  *     auto-extending buffer.
2915                  *
2916                  * 2.  There is not currently a LUN reference counting
2917                  *     mechanism, outside of outstanding transactions on
2918                  *     the LUN's OOA queue.  So a LUN could go away on us
2919                  *     while we're getting the LUN number, backend-specific
2920                  *     information, etc.  Thus, given the way things
2921                  *     currently work, we need to hold the CTL lock while
2922                  *     grabbing LUN information.
2923                  *
2924                  * So, from the user's standpoint, the best thing to do is
2925                  * allocate what he thinks is a reasonable buffer length,
2926                  * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
2927                  * double the buffer length and try again.  (And repeat
2928                  * that until he succeeds.)
2929                  */
2930                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
2931                 if (sb == NULL) {
2932                         list->status = CTL_LUN_LIST_ERROR;
2933                         snprintf(list->error_str, sizeof(list->error_str),
2934                                  "Unable to allocate %d bytes for LUN list",
2935                                  list->alloc_len);
2936                         break;
2937                 }
2938
2939                 sbuf_printf(sb, "<ctllunlist>\n");
2940
2941                 mtx_lock(&softc->ctl_lock);
2942
2943                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2944                         retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
2945                                              (uintmax_t)lun->lun);
2946
2947                         /*
2948                          * Bail out as soon as we see that we've overfilled
2949                          * the buffer.
2950                          */
2951                         if (retval != 0)
2952                                 break;
2953
2954                         retval = sbuf_printf(sb, "<backend_type>%s"
2955                                              "</backend_type>\n",
2956                                              (lun->backend == NULL) ?  "none" :
2957                                              lun->backend->name);
2958
2959                         if (retval != 0)
2960                                 break;
2961
2962                         retval = sbuf_printf(sb, "<lun_type>%d</lun_type>\n",
2963                                              lun->be_lun->lun_type);
2964
2965                         if (retval != 0)
2966                                 break;
2967
2968                         if (lun->backend == NULL) {
2969                                 retval = sbuf_printf(sb, "</lun>\n");
2970                                 if (retval != 0)
2971                                         break;
2972                                 continue;
2973                         }
2974
2975                         retval = sbuf_printf(sb, "<size>%ju</size>\n",
2976                                              (lun->be_lun->maxlba > 0) ?
2977                                              lun->be_lun->maxlba + 1 : 0);
2978
2979                         if (retval != 0)
2980                                 break;
2981
2982                         retval = sbuf_printf(sb, "<blocksize>%u</blocksize>\n",
2983                                              lun->be_lun->blocksize);
2984
2985                         if (retval != 0)
2986                                 break;
2987
2988                         retval = sbuf_printf(sb, "<serial_number>");
2989
2990                         if (retval != 0)
2991                                 break;
2992
2993                         retval = ctl_sbuf_printf_esc(sb,
2994                                                      lun->be_lun->serial_num);
2995
2996                         if (retval != 0)
2997                                 break;
2998
2999                         retval = sbuf_printf(sb, "</serial_number>\n");
3000
3001                         if (retval != 0)
3002                                 break;
3003
3004                         retval = sbuf_printf(sb, "<device_id>");
3005
3006                         if (retval != 0)
3007                                 break;
3008
3009                         retval = ctl_sbuf_printf_esc(sb,lun->be_lun->device_id);
3010
3011                         if (retval != 0)
3012                                 break;
3013
3014                         retval = sbuf_printf(sb, "</device_id>\n");
3015
3016                         if (retval != 0)
3017                                 break;
3018
3019                         if (lun->backend->lun_info == NULL) {
3020                                 retval = sbuf_printf(sb, "</lun>\n");
3021                                 if (retval != 0)
3022                                         break;
3023                                 continue;
3024                         }
3025
3026                         retval =lun->backend->lun_info(lun->be_lun->be_lun, sb);
3027
3028                         if (retval != 0)
3029                                 break;
3030
3031                         retval = sbuf_printf(sb, "</lun>\n");
3032
3033                         if (retval != 0)
3034                                 break;
3035                 }
3036                 mtx_unlock(&softc->ctl_lock);
3037
3038                 if ((retval != 0)
3039                  || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3040                         retval = 0;
3041                         sbuf_delete(sb);
3042                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3043                         snprintf(list->error_str, sizeof(list->error_str),
3044                                  "Out of space, %d bytes is too small",
3045                                  list->alloc_len);
3046                         break;
3047                 }
3048
3049                 sbuf_finish(sb);
3050
3051                 retval = copyout(sbuf_data(sb), list->lun_xml,
3052                                  sbuf_len(sb) + 1);
3053
3054                 list->fill_len = sbuf_len(sb) + 1;
3055                 list->status = CTL_LUN_LIST_OK;
3056                 sbuf_delete(sb);
3057                 break;
3058         }
3059         default: {
3060                 /* XXX KDM should we fix this? */
3061 #if 0
3062                 struct ctl_backend_driver *backend;
3063                 unsigned int type;
3064                 int found;
3065
3066                 found = 0;
3067
3068                 /*
3069                  * We encode the backend type as the ioctl type for backend
3070                  * ioctls.  So parse it out here, and then search for a
3071                  * backend of this type.
3072                  */
3073                 type = _IOC_TYPE(cmd);
3074
3075                 STAILQ_FOREACH(backend, &softc->be_list, links) {
3076                         if (backend->type == type) {
3077                                 found = 1;
3078                                 break;
3079                         }
3080                 }
3081                 if (found == 0) {
3082                         printf("ctl: unknown ioctl command %#lx or backend "
3083                                "%d\n", cmd, type);
3084                         retval = -EINVAL;
3085                         break;
3086                 }
3087                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3088 #endif
3089                 retval = ENOTTY;
3090                 break;
3091         }
3092         }
3093         return (retval);
3094 }
3095
3096 uint32_t
3097 ctl_get_initindex(struct ctl_nexus *nexus)
3098 {
3099         if (nexus->targ_port < CTL_MAX_PORTS)
3100                 return (nexus->initid.id +
3101                         (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3102         else
3103                 return (nexus->initid.id +
3104                        ((nexus->targ_port - CTL_MAX_PORTS) *
3105                         CTL_MAX_INIT_PER_PORT));
3106 }
3107
3108 uint32_t
3109 ctl_get_resindex(struct ctl_nexus *nexus)
3110 {
3111         return (nexus->initid.id + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3112 }
3113
3114 uint32_t
3115 ctl_port_idx(int port_num)
3116 {
3117         if (port_num < CTL_MAX_PORTS)
3118                 return(port_num);
3119         else
3120                 return(port_num - CTL_MAX_PORTS);
3121 }
3122
3123 /*
3124  * Note:  This only works for bitmask sizes that are at least 32 bits, and
3125  * that are a power of 2.
3126  */
3127 int
3128 ctl_ffz(uint32_t *mask, uint32_t size)
3129 {
3130         uint32_t num_chunks, num_pieces;
3131         int i, j;
3132
3133         num_chunks = (size >> 5);
3134         if (num_chunks == 0)
3135                 num_chunks++;
3136         num_pieces = ctl_min((sizeof(uint32_t) * 8), size);
3137
3138         for (i = 0; i < num_chunks; i++) {
3139                 for (j = 0; j < num_pieces; j++) {
3140                         if ((mask[i] & (1 << j)) == 0)
3141                                 return ((i << 5) + j);
3142                 }
3143         }
3144
3145         return (-1);
3146 }
3147
3148 int
3149 ctl_set_mask(uint32_t *mask, uint32_t bit)
3150 {
3151         uint32_t chunk, piece;
3152
3153         chunk = bit >> 5;
3154         piece = bit % (sizeof(uint32_t) * 8);
3155
3156         if ((mask[chunk] & (1 << piece)) != 0)
3157                 return (-1);
3158         else
3159                 mask[chunk] |= (1 << piece);
3160
3161         return (0);
3162 }
3163
3164 int
3165 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3166 {
3167         uint32_t chunk, piece;
3168
3169         chunk = bit >> 5;
3170         piece = bit % (sizeof(uint32_t) * 8);
3171
3172         if ((mask[chunk] & (1 << piece)) == 0)
3173                 return (-1);
3174         else
3175                 mask[chunk] &= ~(1 << piece);
3176
3177         return (0);
3178 }
3179
3180 int
3181 ctl_is_set(uint32_t *mask, uint32_t bit)
3182 {
3183         uint32_t chunk, piece;
3184
3185         chunk = bit >> 5;
3186         piece = bit % (sizeof(uint32_t) * 8);
3187
3188         if ((mask[chunk] & (1 << piece)) == 0)
3189                 return (0);
3190         else
3191                 return (1);
3192 }
3193
3194 #ifdef unused
3195 /*
3196  * The bus, target and lun are optional, they can be filled in later.
3197  * can_wait is used to determine whether we can wait on the malloc or not.
3198  */
3199 union ctl_io*
3200 ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port, uint32_t targ_target,
3201               uint32_t targ_lun, int can_wait)
3202 {
3203         union ctl_io *io;
3204
3205         if (can_wait)
3206                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_WAITOK);
3207         else
3208                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3209
3210         if (io != NULL) {
3211                 io->io_hdr.io_type = io_type;
3212                 io->io_hdr.targ_port = targ_port;
3213                 /*
3214                  * XXX KDM this needs to change/go away.  We need to move
3215                  * to a preallocated pool of ctl_scsiio structures.
3216                  */
3217                 io->io_hdr.nexus.targ_target.id = targ_target;
3218                 io->io_hdr.nexus.targ_lun = targ_lun;
3219         }
3220
3221         return (io);
3222 }
3223
3224 void
3225 ctl_kfree_io(union ctl_io *io)
3226 {
3227         free(io, M_CTL);
3228 }
3229 #endif /* unused */
3230
3231 /*
3232  * ctl_softc, pool_type, total_ctl_io are passed in.
3233  * npool is passed out.
3234  */
3235 int
3236 ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type,
3237                 uint32_t total_ctl_io, struct ctl_io_pool **npool)
3238 {
3239         uint32_t i;
3240         union ctl_io *cur_io, *next_io;
3241         struct ctl_io_pool *pool;
3242         int retval;
3243
3244         retval = 0;
3245
3246         pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL, M_NOWAIT);
3247         if (pool == NULL) {
3248                 retval = -ENOMEM;
3249                 goto bailout;
3250         }
3251
3252         memset(pool, 0, sizeof(*pool));
3253
3254         pool->type = pool_type;
3255         pool->ctl_softc = ctl_softc;
3256
3257         mtx_lock(&ctl_softc->ctl_lock);
3258         pool->id = ctl_softc->cur_pool_id++;
3259         mtx_unlock(&ctl_softc->ctl_lock);
3260
3261         pool->flags = CTL_POOL_FLAG_NONE;
3262         STAILQ_INIT(&pool->free_queue);
3263
3264         /*
3265          * XXX KDM other options here:
3266          * - allocate a page at a time
3267          * - allocate one big chunk of memory.
3268          * Page allocation might work well, but would take a little more
3269          * tracking.
3270          */
3271         for (i = 0; i < total_ctl_io; i++) {
3272                 cur_io = (union ctl_io *)malloc(sizeof(*cur_io), M_CTL,
3273                                                 M_NOWAIT);
3274                 if (cur_io == NULL) {
3275                         retval = ENOMEM;
3276                         break;
3277                 }
3278                 cur_io->io_hdr.pool = pool;
3279                 STAILQ_INSERT_TAIL(&pool->free_queue, &cur_io->io_hdr, links);
3280                 pool->total_ctl_io++;
3281                 pool->free_ctl_io++;
3282         }
3283
3284         if (retval != 0) {
3285                 for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3286                      cur_io != NULL; cur_io = next_io) {
3287                         next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr,
3288                                                               links);
3289                         STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr,
3290                                       ctl_io_hdr, links);
3291                         free(cur_io, M_CTL);
3292                 }
3293
3294                 free(pool, M_CTL);
3295                 goto bailout;
3296         }
3297         mtx_lock(&ctl_softc->ctl_lock);
3298         ctl_softc->num_pools++;
3299         STAILQ_INSERT_TAIL(&ctl_softc->io_pools, pool, links);
3300         /*
3301          * Increment our usage count if this is an external consumer, so we
3302          * can't get unloaded until the external consumer (most likely a
3303          * FETD) unloads and frees his pool.
3304          *
3305          * XXX KDM will this increment the caller's module use count, or
3306          * mine?
3307          */
3308 #if 0
3309         if ((pool_type != CTL_POOL_EMERGENCY)
3310          && (pool_type != CTL_POOL_INTERNAL)
3311          && (pool_type != CTL_POOL_IOCTL)
3312          && (pool_type != CTL_POOL_4OTHERSC))
3313                 MOD_INC_USE_COUNT;
3314 #endif
3315
3316         mtx_unlock(&ctl_softc->ctl_lock);
3317
3318         *npool = pool;
3319
3320 bailout:
3321
3322         return (retval);
3323 }
3324
3325 /*
3326  * Caller must hold ctl_softc->ctl_lock.
3327  */
3328 int
3329 ctl_pool_acquire(struct ctl_io_pool *pool)
3330 {
3331         if (pool == NULL)
3332                 return (-EINVAL);
3333
3334         if (pool->flags & CTL_POOL_FLAG_INVALID)
3335                 return (-EINVAL);
3336
3337         pool->refcount++;
3338
3339         return (0);
3340 }
3341
3342 /*
3343  * Caller must hold ctl_softc->ctl_lock.
3344  */
3345 int
3346 ctl_pool_invalidate(struct ctl_io_pool *pool)
3347 {
3348         if (pool == NULL)
3349                 return (-EINVAL);
3350
3351         pool->flags |= CTL_POOL_FLAG_INVALID;
3352
3353         return (0);
3354 }
3355
3356 /*
3357  * Caller must hold ctl_softc->ctl_lock.
3358  */
3359 int
3360 ctl_pool_release(struct ctl_io_pool *pool)
3361 {
3362         if (pool == NULL)
3363                 return (-EINVAL);
3364
3365         if ((--pool->refcount == 0)
3366          && (pool->flags & CTL_POOL_FLAG_INVALID)) {
3367                 ctl_pool_free(pool->ctl_softc, pool);
3368         }
3369
3370         return (0);
3371 }
3372
3373 /*
3374  * Must be called with ctl_softc->ctl_lock held.
3375  */
3376 void
3377 ctl_pool_free(struct ctl_softc *ctl_softc, struct ctl_io_pool *pool)
3378 {
3379         union ctl_io *cur_io, *next_io;
3380
3381         for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3382              cur_io != NULL; cur_io = next_io) {
3383                 next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr,
3384                                                       links);
3385                 STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr, ctl_io_hdr,
3386                               links);
3387                 free(cur_io, M_CTL);
3388         }
3389
3390         STAILQ_REMOVE(&ctl_softc->io_pools, pool, ctl_io_pool, links);
3391         ctl_softc->num_pools--;
3392
3393         /*
3394          * XXX KDM will this decrement the caller's usage count or mine?
3395          */
3396 #if 0
3397         if ((pool->type != CTL_POOL_EMERGENCY)
3398          && (pool->type != CTL_POOL_INTERNAL)
3399          && (pool->type != CTL_POOL_IOCTL))
3400                 MOD_DEC_USE_COUNT;
3401 #endif
3402
3403         free(pool, M_CTL);
3404 }
3405
3406 /*
3407  * This routine does not block (except for spinlocks of course).
3408  * It tries to allocate a ctl_io union from the caller's pool as quickly as
3409  * possible.
3410  */
3411 union ctl_io *
3412 ctl_alloc_io(void *pool_ref)
3413 {
3414         union ctl_io *io;
3415         struct ctl_softc *ctl_softc;
3416         struct ctl_io_pool *pool, *npool;
3417         struct ctl_io_pool *emergency_pool;
3418
3419         pool = (struct ctl_io_pool *)pool_ref;
3420
3421         if (pool == NULL) {
3422                 printf("%s: pool is NULL\n", __func__);
3423                 return (NULL);
3424         }
3425
3426         emergency_pool = NULL;
3427
3428         ctl_softc = pool->ctl_softc;
3429
3430         mtx_lock(&ctl_softc->ctl_lock);
3431         /*
3432          * First, try to get the io structure from the user's pool.
3433          */
3434         if (ctl_pool_acquire(pool) == 0) {
3435                 io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3436                 if (io != NULL) {
3437                         STAILQ_REMOVE_HEAD(&pool->free_queue, links);
3438                         pool->total_allocated++;
3439                         pool->free_ctl_io--;
3440                         mtx_unlock(&ctl_softc->ctl_lock);
3441                         return (io);
3442                 } else
3443                         ctl_pool_release(pool);
3444         }
3445         /*
3446          * If he doesn't have any io structures left, search for an
3447          * emergency pool and grab one from there.
3448          */
3449         STAILQ_FOREACH(npool, &ctl_softc->io_pools, links) {
3450                 if (npool->type != CTL_POOL_EMERGENCY)
3451                         continue;
3452
3453                 if (ctl_pool_acquire(npool) != 0)
3454                         continue;
3455
3456                 emergency_pool = npool;
3457
3458                 io = (union ctl_io *)STAILQ_FIRST(&npool->free_queue);
3459                 if (io != NULL) {
3460                         STAILQ_REMOVE_HEAD(&npool->free_queue, links);
3461                         npool->total_allocated++;
3462                         npool->free_ctl_io--;
3463                         mtx_unlock(&ctl_softc->ctl_lock);
3464                         return (io);
3465                 } else
3466                         ctl_pool_release(npool);
3467         }
3468
3469         /* Drop the spinlock before we malloc */
3470         mtx_unlock(&ctl_softc->ctl_lock);
3471
3472         /*
3473          * The emergency pool (if it exists) didn't have one, so try an
3474          * atomic (i.e. nonblocking) malloc and see if we get lucky.
3475          */
3476         io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3477         if (io != NULL) {
3478                 /*
3479                  * If the emergency pool exists but is empty, add this
3480                  * ctl_io to its list when it gets freed.
3481                  */
3482                 if (emergency_pool != NULL) {
3483                         mtx_lock(&ctl_softc->ctl_lock);
3484                         if (ctl_pool_acquire(emergency_pool) == 0) {
3485                                 io->io_hdr.pool = emergency_pool;
3486                                 emergency_pool->total_ctl_io++;
3487                                 /*
3488                                  * Need to bump this, otherwise
3489                                  * total_allocated and total_freed won't
3490                                  * match when we no longer have anything
3491                                  * outstanding.
3492                                  */
3493                                 emergency_pool->total_allocated++;
3494                         }
3495                         mtx_unlock(&ctl_softc->ctl_lock);
3496                 } else
3497                         io->io_hdr.pool = NULL;
3498         }
3499
3500         return (io);
3501 }
3502
3503 static void
3504 ctl_free_io_internal(union ctl_io *io, int have_lock)
3505 {
3506         if (io == NULL)
3507                 return;
3508
3509         /*
3510          * If this ctl_io has a pool, return it to that pool.
3511          */
3512         if (io->io_hdr.pool != NULL) {
3513                 struct ctl_io_pool *pool;
3514 #if 0
3515                 struct ctl_softc *ctl_softc;
3516                 union ctl_io *tmp_io;
3517                 unsigned long xflags;
3518                 int i;
3519
3520                 ctl_softc = control_softc;
3521 #endif
3522
3523                 pool = (struct ctl_io_pool *)io->io_hdr.pool;
3524
3525                 if (have_lock == 0)
3526                         mtx_lock(&pool->ctl_softc->ctl_lock);
3527 #if 0
3528                 save_flags(xflags);
3529
3530                 for (i = 0, tmp_io = (union ctl_io *)STAILQ_FIRST(
3531                      &ctl_softc->task_queue); tmp_io != NULL; i++,
3532                      tmp_io = (union ctl_io *)STAILQ_NEXT(&tmp_io->io_hdr,
3533                      links)) {
3534                         if (tmp_io == io) {
3535                                 printf("%s: %p is still on the task queue!\n",
3536                                        __func__, tmp_io);
3537                                 printf("%s: (%d): type %d "
3538                                        "msg %d cdb %x iptl: "
3539                                        "%d:%d:%d:%d tag 0x%04x "
3540                                        "flg %#lx\n",
3541                                         __func__, i,
3542                                         tmp_io->io_hdr.io_type,
3543                                         tmp_io->io_hdr.msg_type,
3544                                         tmp_io->scsiio.cdb[0],
3545                                         tmp_io->io_hdr.nexus.initid.id,
3546                                         tmp_io->io_hdr.nexus.targ_port,
3547                                         tmp_io->io_hdr.nexus.targ_target.id,
3548                                         tmp_io->io_hdr.nexus.targ_lun,
3549                                         (tmp_io->io_hdr.io_type ==
3550                                         CTL_IO_TASK) ?
3551                                         tmp_io->taskio.tag_num :
3552                                         tmp_io->scsiio.tag_num,
3553                                         xflags);
3554                                 panic("I/O still on the task queue!");
3555                         }
3556                 }
3557 #endif
3558                 io->io_hdr.io_type = 0xff;
3559                 STAILQ_INSERT_TAIL(&pool->free_queue, &io->io_hdr, links);
3560                 pool->total_freed++;
3561                 pool->free_ctl_io++;
3562                 ctl_pool_release(pool);
3563                 if (have_lock == 0)
3564                         mtx_unlock(&pool->ctl_softc->ctl_lock);
3565         } else {
3566                 /*
3567                  * Otherwise, just free it.  We probably malloced it and
3568                  * the emergency pool wasn't available.
3569                  */
3570                 free(io, M_CTL);
3571         }
3572
3573 }
3574
3575 void
3576 ctl_free_io(union ctl_io *io)
3577 {
3578         ctl_free_io_internal(io, /*have_lock*/ 0);
3579 }
3580
3581 void
3582 ctl_zero_io(union ctl_io *io)
3583 {
3584         void *pool_ref;
3585
3586         if (io == NULL)
3587                 return;
3588
3589         /*
3590          * May need to preserve linked list pointers at some point too.
3591          */
3592         pool_ref = io->io_hdr.pool;
3593
3594         memset(io, 0, sizeof(*io));
3595
3596         io->io_hdr.pool = pool_ref;
3597 }
3598
3599 /*
3600  * This routine is currently used for internal copies of ctl_ios that need
3601  * to persist for some reason after we've already returned status to the
3602  * FETD.  (Thus the flag set.)
3603  *
3604  * XXX XXX
3605  * Note that this makes a blind copy of all fields in the ctl_io, except
3606  * for the pool reference.  This includes any memory that has been
3607  * allocated!  That memory will no longer be valid after done has been
3608  * called, so this would be VERY DANGEROUS for command that actually does
3609  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3610  * start and stop commands, which don't transfer any data, so this is not a
3611  * problem.  If it is used for anything else, the caller would also need to
3612  * allocate data buffer space and this routine would need to be modified to
3613  * copy the data buffer(s) as well.
3614  */
3615 void
3616 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3617 {
3618         void *pool_ref;
3619
3620         if ((src == NULL)
3621          || (dest == NULL))
3622                 return;
3623
3624         /*
3625          * May need to preserve linked list pointers at some point too.
3626          */
3627         pool_ref = dest->io_hdr.pool;
3628
3629         memcpy(dest, src, ctl_min(sizeof(*src), sizeof(*dest)));
3630
3631         dest->io_hdr.pool = pool_ref;
3632         /*
3633          * We need to know that this is an internal copy, and doesn't need
3634          * to get passed back to the FETD that allocated it.
3635          */
3636         dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3637 }
3638
3639 #ifdef NEEDTOPORT
3640 static void
3641 ctl_update_power_subpage(struct copan_power_subpage *page)
3642 {
3643         int num_luns, num_partitions, config_type;
3644         struct ctl_softc *softc;
3645         cs_BOOL_t aor_present, shelf_50pct_power;
3646         cs_raidset_personality_t rs_type;
3647         int max_active_luns;
3648
3649         softc = control_softc;
3650
3651         /* subtract out the processor LUN */
3652         num_luns = softc->num_luns - 1;
3653         /*
3654          * Default to 7 LUNs active, which was the only number we allowed
3655          * in the past.
3656          */
3657         max_active_luns = 7;
3658
3659         num_partitions = config_GetRsPartitionInfo();
3660         config_type = config_GetConfigType();
3661         shelf_50pct_power = config_GetShelfPowerMode();
3662         aor_present = config_IsAorRsPresent();
3663
3664         rs_type = ddb_GetRsRaidType(1);
3665         if ((rs_type != CS_RAIDSET_PERSONALITY_RAID5)
3666          && (rs_type != CS_RAIDSET_PERSONALITY_RAID1)) {
3667                 EPRINT(0, "Unsupported RS type %d!", rs_type);
3668         }
3669
3670
3671         page->total_luns = num_luns;
3672
3673         switch (config_type) {
3674         case 40:
3675                 /*
3676                  * In a 40 drive configuration, it doesn't matter what DC
3677                  * cards we have, whether we have AOR enabled or not,
3678                  * partitioning or not, or what type of RAIDset we have.
3679                  * In that scenario, we can power up every LUN we present
3680                  * to the user.
3681                  */
3682                 max_active_luns = num_luns;
3683
3684                 break;
3685         case 64:
3686                 if (shelf_50pct_power == CS_FALSE) {
3687                         /* 25% power */
3688                         if (aor_present == CS_TRUE) {
3689                                 if (rs_type ==
3690                                      CS_RAIDSET_PERSONALITY_RAID5) {
3691                                         max_active_luns = 7;
3692                                 } else if (rs_type ==
3693                                          CS_RAIDSET_PERSONALITY_RAID1){
3694                                         max_active_luns = 14;
3695                                 } else {
3696                                         /* XXX KDM now what?? */
3697                                 }
3698                         } else {
3699                                 if (rs_type ==
3700                                      CS_RAIDSET_PERSONALITY_RAID5) {
3701                                         max_active_luns = 8;
3702                                 } else if (rs_type ==
3703                                          CS_RAIDSET_PERSONALITY_RAID1){
3704                                         max_active_luns = 16;
3705                                 } else {
3706                                         /* XXX KDM now what?? */
3707                                 }
3708                         }
3709                 } else {
3710                         /* 50% power */
3711                         /*
3712                          * With 50% power in a 64 drive configuration, we
3713                          * can power all LUNs we present.
3714                          */
3715                         max_active_luns = num_luns;
3716                 }
3717                 break;
3718         case 112:
3719                 if (shelf_50pct_power == CS_FALSE) {
3720                         /* 25% power */
3721                         if (aor_present == CS_TRUE) {
3722                                 if (rs_type ==
3723                                      CS_RAIDSET_PERSONALITY_RAID5) {
3724                                         max_active_luns = 7;
3725                                 } else if (rs_type ==
3726                                          CS_RAIDSET_PERSONALITY_RAID1){
3727                                         max_active_luns = 14;
3728                                 } else {
3729                                         /* XXX KDM now what?? */
3730                                 }
3731                         } else {
3732                                 if (rs_type ==
3733                                      CS_RAIDSET_PERSONALITY_RAID5) {
3734                                         max_active_luns = 8;
3735                                 } else if (rs_type ==
3736                                          CS_RAIDSET_PERSONALITY_RAID1){
3737                                         max_active_luns = 16;
3738                                 } else {
3739                                         /* XXX KDM now what?? */
3740                                 }
3741                         }
3742                 } else {
3743                         /* 50% power */
3744                         if (aor_present == CS_TRUE) {
3745                                 if (rs_type ==
3746                                      CS_RAIDSET_PERSONALITY_RAID5) {
3747                                         max_active_luns = 14;
3748                                 } else if (rs_type ==
3749                                          CS_RAIDSET_PERSONALITY_RAID1){
3750                                         /*
3751                                          * We're assuming here that disk
3752                                          * caching is enabled, and so we're
3753                                          * able to power up half of each
3754                                          * LUN, and cache all writes.
3755                                          */
3756                                         max_active_luns = num_luns;
3757                                 } else {
3758                                         /* XXX KDM now what?? */
3759                                 }
3760                         } else {
3761                                 if (rs_type ==
3762                                      CS_RAIDSET_PERSONALITY_RAID5) {
3763                                         max_active_luns = 15;
3764                                 } else if (rs_type ==
3765                                          CS_RAIDSET_PERSONALITY_RAID1){
3766                                         max_active_luns = 30;
3767                                 } else {
3768                                         /* XXX KDM now what?? */
3769                                 }
3770                         }
3771                 }
3772                 break;
3773         default:
3774                 /*
3775                  * In this case, we have an unknown configuration, so we
3776                  * just use the default from above.
3777                  */
3778                 break;
3779         }
3780
3781         page->max_active_luns = max_active_luns;
3782 #if 0
3783         printk("%s: total_luns = %d, max_active_luns = %d\n", __func__,
3784                page->total_luns, page->max_active_luns);
3785 #endif
3786 }
3787 #endif /* NEEDTOPORT */
3788
3789 /*
3790  * This routine could be used in the future to load default and/or saved
3791  * mode page parameters for a particuar lun.
3792  */
3793 static int
3794 ctl_init_page_index(struct ctl_lun *lun)
3795 {
3796         int i;
3797         struct ctl_page_index *page_index;
3798         struct ctl_softc *softc;
3799
3800         memcpy(&lun->mode_pages.index, page_index_template,
3801                sizeof(page_index_template));
3802
3803         softc = lun->ctl_softc;
3804
3805         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
3806
3807                 page_index = &lun->mode_pages.index[i];
3808                 /*
3809                  * If this is a disk-only mode page, there's no point in
3810                  * setting it up.  For some pages, we have to have some
3811                  * basic information about the disk in order to calculate the
3812                  * mode page data.
3813                  */
3814                 if ((lun->be_lun->lun_type != T_DIRECT)
3815                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
3816                         continue;
3817
3818                 switch (page_index->page_code & SMPH_PC_MASK) {
3819                 case SMS_FORMAT_DEVICE_PAGE: {
3820                         struct scsi_format_page *format_page;
3821
3822                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3823                                 panic("subpage is incorrect!");
3824
3825                         /*
3826                          * Sectors per track are set above.  Bytes per
3827                          * sector need to be set here on a per-LUN basis.
3828                          */
3829                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
3830                                &format_page_default,
3831                                sizeof(format_page_default));
3832                         memcpy(&lun->mode_pages.format_page[
3833                                CTL_PAGE_CHANGEABLE], &format_page_changeable,
3834                                sizeof(format_page_changeable));
3835                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
3836                                &format_page_default,
3837                                sizeof(format_page_default));
3838                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
3839                                &format_page_default,
3840                                sizeof(format_page_default));
3841
3842                         format_page = &lun->mode_pages.format_page[
3843                                 CTL_PAGE_CURRENT];
3844                         scsi_ulto2b(lun->be_lun->blocksize,
3845                                     format_page->bytes_per_sector);
3846
3847                         format_page = &lun->mode_pages.format_page[
3848                                 CTL_PAGE_DEFAULT];
3849                         scsi_ulto2b(lun->be_lun->blocksize,
3850                                     format_page->bytes_per_sector);
3851
3852                         format_page = &lun->mode_pages.format_page[
3853                                 CTL_PAGE_SAVED];
3854                         scsi_ulto2b(lun->be_lun->blocksize,
3855                                     format_page->bytes_per_sector);
3856
3857                         page_index->page_data =
3858                                 (uint8_t *)lun->mode_pages.format_page;
3859                         break;
3860                 }
3861                 case SMS_RIGID_DISK_PAGE: {
3862                         struct scsi_rigid_disk_page *rigid_disk_page;
3863                         uint32_t sectors_per_cylinder;
3864                         uint64_t cylinders;
3865 #ifndef __XSCALE__
3866                         int shift;
3867 #endif /* !__XSCALE__ */
3868
3869                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3870                                 panic("invalid subpage value %d",
3871                                       page_index->subpage);
3872
3873                         /*
3874                          * Rotation rate and sectors per track are set
3875                          * above.  We calculate the cylinders here based on
3876                          * capacity.  Due to the number of heads and
3877                          * sectors per track we're using, smaller arrays
3878                          * may turn out to have 0 cylinders.  Linux and
3879                          * FreeBSD don't pay attention to these mode pages
3880                          * to figure out capacity, but Solaris does.  It
3881                          * seems to deal with 0 cylinders just fine, and
3882                          * works out a fake geometry based on the capacity.
3883                          */
3884                         memcpy(&lun->mode_pages.rigid_disk_page[
3885                                CTL_PAGE_CURRENT], &rigid_disk_page_default,
3886                                sizeof(rigid_disk_page_default));
3887                         memcpy(&lun->mode_pages.rigid_disk_page[
3888                                CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
3889                                sizeof(rigid_disk_page_changeable));
3890                         memcpy(&lun->mode_pages.rigid_disk_page[
3891                                CTL_PAGE_DEFAULT], &rigid_disk_page_default,
3892                                sizeof(rigid_disk_page_default));
3893                         memcpy(&lun->mode_pages.rigid_disk_page[
3894                                CTL_PAGE_SAVED], &rigid_disk_page_default,
3895                                sizeof(rigid_disk_page_default));
3896
3897                         sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
3898                                 CTL_DEFAULT_HEADS;
3899
3900                         /*
3901                          * The divide method here will be more accurate,
3902                          * probably, but results in floating point being
3903                          * used in the kernel on i386 (__udivdi3()).  On the
3904                          * XScale, though, __udivdi3() is implemented in
3905                          * software.
3906                          *
3907                          * The shift method for cylinder calculation is
3908                          * accurate if sectors_per_cylinder is a power of
3909                          * 2.  Otherwise it might be slightly off -- you
3910                          * might have a bit of a truncation problem.
3911                          */
3912 #ifdef  __XSCALE__
3913                         cylinders = (lun->be_lun->maxlba + 1) /
3914                                 sectors_per_cylinder;
3915 #else
3916                         for (shift = 31; shift > 0; shift--) {
3917                                 if (sectors_per_cylinder & (1 << shift))
3918                                         break;
3919                         }
3920                         cylinders = (lun->be_lun->maxlba + 1) >> shift;
3921 #endif
3922
3923                         /*
3924                          * We've basically got 3 bytes, or 24 bits for the
3925                          * cylinder size in the mode page.  If we're over,
3926                          * just round down to 2^24.
3927                          */
3928                         if (cylinders > 0xffffff)
3929                                 cylinders = 0xffffff;
3930
3931                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
3932                                 CTL_PAGE_CURRENT];
3933                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
3934
3935                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
3936                                 CTL_PAGE_DEFAULT];
3937                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
3938
3939                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
3940                                 CTL_PAGE_SAVED];
3941                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
3942
3943                         page_index->page_data =
3944                                 (uint8_t *)lun->mode_pages.rigid_disk_page;
3945                         break;
3946                 }
3947                 case SMS_CACHING_PAGE: {
3948
3949                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3950                                 panic("invalid subpage value %d",
3951                                       page_index->subpage);
3952                         /*
3953                          * Defaults should be okay here, no calculations
3954                          * needed.
3955                          */
3956                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
3957                                &caching_page_default,
3958                                sizeof(caching_page_default));
3959                         memcpy(&lun->mode_pages.caching_page[
3960                                CTL_PAGE_CHANGEABLE], &caching_page_changeable,
3961                                sizeof(caching_page_changeable));
3962                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
3963                                &caching_page_default,
3964                                sizeof(caching_page_default));
3965                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
3966                                &caching_page_default,
3967                                sizeof(caching_page_default));
3968                         page_index->page_data =
3969                                 (uint8_t *)lun->mode_pages.caching_page;
3970                         break;
3971                 }
3972                 case SMS_CONTROL_MODE_PAGE: {
3973
3974                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3975                                 panic("invalid subpage value %d",
3976                                       page_index->subpage);
3977
3978                         /*
3979                          * Defaults should be okay here, no calculations
3980                          * needed.
3981                          */
3982                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
3983                                &control_page_default,
3984                                sizeof(control_page_default));
3985                         memcpy(&lun->mode_pages.control_page[
3986                                CTL_PAGE_CHANGEABLE], &control_page_changeable,
3987                                sizeof(control_page_changeable));
3988                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
3989                                &control_page_default,
3990                                sizeof(control_page_default));
3991                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
3992                                &control_page_default,
3993                                sizeof(control_page_default));
3994                         page_index->page_data =
3995                                 (uint8_t *)lun->mode_pages.control_page;
3996                         break;
3997
3998                 }
3999                 case SMS_VENDOR_SPECIFIC_PAGE:{
4000                         switch (page_index->subpage) {
4001                         case PWR_SUBPAGE_CODE: {
4002                                 struct copan_power_subpage *current_page,
4003                                                            *saved_page;
4004
4005                                 memcpy(&lun->mode_pages.power_subpage[
4006                                        CTL_PAGE_CURRENT],
4007                                        &power_page_default,
4008                                        sizeof(power_page_default));
4009                                 memcpy(&lun->mode_pages.power_subpage[
4010                                        CTL_PAGE_CHANGEABLE],
4011                                        &power_page_changeable,
4012                                        sizeof(power_page_changeable));
4013                                 memcpy(&lun->mode_pages.power_subpage[
4014                                        CTL_PAGE_DEFAULT],
4015                                        &power_page_default,
4016                                        sizeof(power_page_default));
4017                                 memcpy(&lun->mode_pages.power_subpage[
4018                                        CTL_PAGE_SAVED],
4019                                        &power_page_default,
4020                                        sizeof(power_page_default));
4021                                 page_index->page_data =
4022                                     (uint8_t *)lun->mode_pages.power_subpage;
4023
4024                                 current_page = (struct copan_power_subpage *)
4025                                         (page_index->page_data +
4026                                          (page_index->page_len *
4027                                           CTL_PAGE_CURRENT));
4028                                 saved_page = (struct copan_power_subpage *)
4029                                         (page_index->page_data +
4030                                          (page_index->page_len *
4031                                           CTL_PAGE_SAVED));
4032                                 break;
4033                         }
4034                         case APS_SUBPAGE_CODE: {
4035                                 struct copan_aps_subpage *current_page,
4036                                                          *saved_page;
4037
4038                                 // This gets set multiple times but
4039                                 // it should always be the same. It's
4040                                 // only done during init so who cares.
4041                                 index_to_aps_page = i;
4042
4043                                 memcpy(&lun->mode_pages.aps_subpage[
4044                                        CTL_PAGE_CURRENT],
4045                                        &aps_page_default,
4046                                        sizeof(aps_page_default));
4047                                 memcpy(&lun->mode_pages.aps_subpage[
4048                                        CTL_PAGE_CHANGEABLE],
4049                                        &aps_page_changeable,
4050                                        sizeof(aps_page_changeable));
4051                                 memcpy(&lun->mode_pages.aps_subpage[
4052                                        CTL_PAGE_DEFAULT],
4053                                        &aps_page_default,
4054                                        sizeof(aps_page_default));
4055                                 memcpy(&lun->mode_pages.aps_subpage[
4056                                        CTL_PAGE_SAVED],
4057                                        &aps_page_default,
4058                                        sizeof(aps_page_default));
4059                                 page_index->page_data =
4060                                         (uint8_t *)lun->mode_pages.aps_subpage;
4061
4062                                 current_page = (struct copan_aps_subpage *)
4063                                         (page_index->page_data +
4064                                          (page_index->page_len *
4065                                           CTL_PAGE_CURRENT));
4066                                 saved_page = (struct copan_aps_subpage *)
4067                                         (page_index->page_data +
4068                                          (page_index->page_len *
4069                                           CTL_PAGE_SAVED));
4070                                 break;
4071                         }
4072                         case DBGCNF_SUBPAGE_CODE: {
4073                                 struct copan_debugconf_subpage *current_page,
4074                                                                *saved_page;
4075
4076                                 memcpy(&lun->mode_pages.debugconf_subpage[
4077                                        CTL_PAGE_CURRENT],
4078                                        &debugconf_page_default,
4079                                        sizeof(debugconf_page_default));
4080                                 memcpy(&lun->mode_pages.debugconf_subpage[
4081                                        CTL_PAGE_CHANGEABLE],
4082                                        &debugconf_page_changeable,
4083                                        sizeof(debugconf_page_changeable));
4084                                 memcpy(&lun->mode_pages.debugconf_subpage[
4085                                        CTL_PAGE_DEFAULT],
4086                                        &debugconf_page_default,
4087                                        sizeof(debugconf_page_default));
4088                                 memcpy(&lun->mode_pages.debugconf_subpage[
4089                                        CTL_PAGE_SAVED],
4090                                        &debugconf_page_default,
4091                                        sizeof(debugconf_page_default));
4092                                 page_index->page_data =
4093                                         (uint8_t *)lun->mode_pages.debugconf_subpage;
4094
4095                                 current_page = (struct copan_debugconf_subpage *)
4096                                         (page_index->page_data +
4097                                          (page_index->page_len *
4098                                           CTL_PAGE_CURRENT));
4099                                 saved_page = (struct copan_debugconf_subpage *)
4100                                         (page_index->page_data +
4101                                          (page_index->page_len *
4102                                           CTL_PAGE_SAVED));
4103                                 break;
4104                         }
4105                         default:
4106                                 panic("invalid subpage value %d",
4107                                       page_index->subpage);
4108                                 break;
4109                         }
4110                         break;
4111                 }
4112                 default:
4113                         panic("invalid page value %d",
4114                               page_index->page_code & SMPH_PC_MASK);
4115                         break;
4116         }
4117         }
4118
4119         return (CTL_RETVAL_COMPLETE);
4120 }
4121
4122 /*
4123  * LUN allocation.
4124  *
4125  * Requirements:
4126  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4127  *   wants us to allocate the LUN and he can block.
4128  * - ctl_softc is always set
4129  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4130  *
4131  * Returns 0 for success, non-zero (errno) for failure.
4132  */
4133 static int
4134 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4135               struct ctl_be_lun *const be_lun, struct ctl_id target_id)
4136 {
4137         struct ctl_lun *nlun, *lun;
4138         struct ctl_frontend *fe;
4139         int lun_number, i;
4140
4141         if (be_lun == NULL)
4142                 return (EINVAL);
4143
4144         /*
4145          * We currently only support Direct Access or Processor LUN types.
4146          */
4147         switch (be_lun->lun_type) {
4148         case T_DIRECT:
4149                 break;
4150         case T_PROCESSOR:
4151                 break;
4152         case T_SEQUENTIAL:
4153         case T_CHANGER:
4154         default:
4155                 be_lun->lun_config_status(be_lun->be_lun,
4156                                           CTL_LUN_CONFIG_FAILURE);
4157                 break;
4158         }
4159         if (ctl_lun == NULL) {
4160                 lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4161                 if (lun == NULL) {
4162                         be_lun->lun_config_status(lun->be_lun->be_lun,
4163                                                   CTL_LUN_CONFIG_FAILURE);
4164                         return (-ENOMEM);
4165                 }
4166                 lun->flags = CTL_LUN_MALLOCED;
4167         } else
4168                 lun = ctl_lun;
4169
4170         memset(lun, 0, sizeof(*lun));
4171
4172         mtx_lock(&ctl_softc->ctl_lock);
4173         /*
4174          * See if the caller requested a particular LUN number.  If so, see
4175          * if it is available.  Otherwise, allocate the first available LUN.
4176          */
4177         if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4178                 if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4179                  || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4180                         mtx_unlock(&ctl_softc->ctl_lock);
4181                         if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4182                                 printf("ctl: requested LUN ID %d is higher "
4183                                        "than CTL_MAX_LUNS - 1 (%d)\n",
4184                                        be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4185                         } else {
4186                                 /*
4187                                  * XXX KDM return an error, or just assign
4188                                  * another LUN ID in this case??
4189                                  */
4190                                 printf("ctl: requested LUN ID %d is already "
4191                                        "in use\n", be_lun->req_lun_id);
4192                         }
4193                         if (lun->flags & CTL_LUN_MALLOCED)
4194                                 free(lun, M_CTL);
4195                         be_lun->lun_config_status(be_lun->be_lun,
4196                                                   CTL_LUN_CONFIG_FAILURE);
4197                         return (ENOSPC);
4198                 }
4199                 lun_number = be_lun->req_lun_id;
4200         } else {
4201                 lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, CTL_MAX_LUNS);
4202                 if (lun_number == -1) {
4203                         mtx_unlock(&ctl_softc->ctl_lock);
4204                         printf("ctl: can't allocate LUN on target %ju, out of "
4205                                "LUNs\n", (uintmax_t)target_id.id);
4206                         if (lun->flags & CTL_LUN_MALLOCED)
4207                                 free(lun, M_CTL);
4208                         be_lun->lun_config_status(be_lun->be_lun,
4209                                                   CTL_LUN_CONFIG_FAILURE);
4210                         return (ENOSPC);
4211                 }
4212         }
4213         ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4214
4215         lun->target = target_id;
4216         lun->lun = lun_number;
4217         lun->be_lun = be_lun;
4218         /*
4219          * The processor LUN is always enabled.  Disk LUNs come on line
4220          * disabled, and must be enabled by the backend.
4221          */
4222         lun->flags = CTL_LUN_DISABLED;
4223         lun->backend = be_lun->be;
4224         be_lun->ctl_lun = lun;
4225         be_lun->lun_id = lun_number;
4226         atomic_add_int(&be_lun->be->num_luns, 1);
4227         if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4228                 lun->flags |= CTL_LUN_STOPPED;
4229
4230         if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4231                 lun->flags |= CTL_LUN_INOPERABLE;
4232
4233         if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4234                 lun->flags |= CTL_LUN_PRIMARY_SC;
4235
4236         lun->ctl_softc = ctl_softc;
4237         TAILQ_INIT(&lun->ooa_queue);
4238         TAILQ_INIT(&lun->blocked_queue);
4239         STAILQ_INIT(&lun->error_list);
4240
4241         /*
4242          * Initialize the mode page index.
4243          */
4244         ctl_init_page_index(lun);
4245
4246         /*
4247          * Set the poweron UA for all initiators on this LUN only.
4248          */
4249         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4250                 lun->pending_sense[i].ua_pending = CTL_UA_POWERON;
4251
4252         /*
4253          * Now, before we insert this lun on the lun list, set the lun
4254          * inventory changed UA for all other luns.
4255          */
4256         STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4257                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4258                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4259                 }
4260         }
4261
4262         STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4263
4264         ctl_softc->ctl_luns[lun_number] = lun;
4265
4266         ctl_softc->num_luns++;
4267
4268         /* Setup statistics gathering */
4269         lun->stats.device_type = be_lun->lun_type;
4270         lun->stats.lun_number = lun_number;
4271         if (lun->stats.device_type == T_DIRECT)
4272                 lun->stats.blocksize = be_lun->blocksize;
4273         else
4274                 lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4275         for (i = 0;i < CTL_MAX_PORTS;i++)
4276                 lun->stats.ports[i].targ_port = i;
4277
4278         mtx_unlock(&ctl_softc->ctl_lock);
4279
4280         lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4281
4282         /*
4283          * Run through each registered FETD and bring it online if it isn't
4284          * already.  Enable the target ID if it hasn't been enabled, and
4285          * enable this particular LUN.
4286          */
4287         STAILQ_FOREACH(fe, &ctl_softc->fe_list, links) {
4288                 int retval;
4289
4290                 /*
4291                  * XXX KDM this only works for ONE TARGET ID.  We'll need
4292                  * to do things differently if we go to a multiple target
4293                  * ID scheme.
4294                  */
4295                 if ((fe->status & CTL_PORT_STATUS_TARG_ONLINE) == 0) {
4296
4297                         retval = fe->targ_enable(fe->targ_lun_arg, target_id);
4298                         if (retval != 0) {
4299                                 printf("ctl_alloc_lun: FETD %s port %d "
4300                                        "returned error %d for targ_enable on "
4301                                        "target %ju\n", fe->port_name,
4302                                        fe->targ_port, retval,
4303                                        (uintmax_t)target_id.id);
4304                         } else
4305                                 fe->status |= CTL_PORT_STATUS_TARG_ONLINE;
4306                 }
4307
4308                 retval = fe->lun_enable(fe->targ_lun_arg, target_id,lun_number);
4309                 if (retval != 0) {
4310                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4311                                "%d for lun_enable on target %ju lun %d\n",
4312                                fe->port_name, fe->targ_port, retval,
4313                                (uintmax_t)target_id.id, lun_number);
4314                 } else
4315                         fe->status |= CTL_PORT_STATUS_LUN_ONLINE;
4316         }
4317         return (0);
4318 }
4319
4320 /*
4321  * Delete a LUN.
4322  * Assumptions:
4323  * - caller holds ctl_softc->ctl_lock.
4324  * - LUN has already been marked invalid and any pending I/O has been taken
4325  *   care of.
4326  */
4327 static int
4328 ctl_free_lun(struct ctl_lun *lun)
4329 {
4330         struct ctl_softc *softc;
4331 #if 0
4332         struct ctl_frontend *fe;
4333 #endif
4334         struct ctl_lun *nlun;
4335         union ctl_io *io, *next_io;
4336         int i;
4337
4338         softc = lun->ctl_softc;
4339
4340         STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4341
4342         ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4343
4344         softc->ctl_luns[lun->lun] = NULL;
4345
4346         if (TAILQ_FIRST(&lun->ooa_queue) != NULL) {
4347                 printf("ctl_free_lun: aieee!! freeing a LUN with "
4348                        "outstanding I/O!!\n");
4349         }
4350
4351         /*
4352          * If we have anything pending on the RtR queue, remove it.
4353          */
4354         for (io = (union ctl_io *)STAILQ_FIRST(&softc->rtr_queue); io != NULL;
4355              io = next_io) {
4356                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
4357                 if ((io->io_hdr.nexus.targ_target.id == lun->target.id)
4358                  && (io->io_hdr.nexus.targ_lun == lun->lun))
4359                         STAILQ_REMOVE(&softc->rtr_queue, &io->io_hdr,
4360                                       ctl_io_hdr, links);
4361         }
4362
4363         /*
4364          * Then remove everything from the blocked queue.
4365          */
4366         for (io = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue); io != NULL;
4367              io = next_io) {
4368                 next_io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,blocked_links);
4369                 TAILQ_REMOVE(&lun->blocked_queue, &io->io_hdr, blocked_links);
4370                 io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
4371         }
4372
4373         /*
4374          * Now clear out the OOA queue, and free all the I/O.
4375          * XXX KDM should we notify the FETD here?  We probably need to
4376          * quiesce the LUN before deleting it.
4377          */
4378         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); io != NULL;
4379              io = next_io) {
4380                 next_io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr, ooa_links);
4381                 TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
4382                 ctl_free_io_internal(io, /*have_lock*/ 1);
4383         }
4384
4385         softc->num_luns--;
4386
4387         /*
4388          * XXX KDM this scheme only works for a single target/multiple LUN
4389          * setup.  It needs to be revamped for a multiple target scheme.
4390          *
4391          * XXX KDM this results in fe->lun_disable() getting called twice,
4392          * once when ctl_disable_lun() is called, and a second time here.
4393          * We really need to re-think the LUN disable semantics.  There
4394          * should probably be several steps/levels to LUN removal:
4395          *  - disable
4396          *  - invalidate
4397          *  - free
4398          *
4399          * Right now we only have a disable method when communicating to
4400          * the front end ports, at least for individual LUNs.
4401          */
4402 #if 0
4403         STAILQ_FOREACH(fe, &softc->fe_list, links) {
4404                 int retval;
4405
4406                 retval = fe->lun_disable(fe->targ_lun_arg, lun->target,
4407                                          lun->lun);
4408                 if (retval != 0) {
4409                         printf("ctl_free_lun: FETD %s port %d returned error "
4410                                "%d for lun_disable on target %ju lun %jd\n",
4411                                fe->port_name, fe->targ_port, retval,
4412                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4413                 }
4414
4415                 if (STAILQ_FIRST(&softc->lun_list) == NULL) {
4416                         fe->status &= ~CTL_PORT_STATUS_LUN_ONLINE;
4417
4418                         retval = fe->targ_disable(fe->targ_lun_arg,lun->target);
4419                         if (retval != 0) {
4420                                 printf("ctl_free_lun: FETD %s port %d "
4421                                        "returned error %d for targ_disable on "
4422                                        "target %ju\n", fe->port_name,
4423                                        fe->targ_port, retval,
4424                                        (uintmax_t)lun->target.id);
4425                         } else
4426                                 fe->status &= ~CTL_PORT_STATUS_TARG_ONLINE;
4427
4428                         if ((fe->status & CTL_PORT_STATUS_TARG_ONLINE) != 0)
4429                                 continue;
4430
4431 #if 0
4432                         fe->port_offline(fe->onoff_arg);
4433                         fe->status &= ~CTL_PORT_STATUS_ONLINE;
4434 #endif
4435                 }
4436         }
4437 #endif
4438
4439         /*
4440          * Tell the backend to free resources, if this LUN has a backend.
4441          */
4442         atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4443         lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4444
4445         if (lun->flags & CTL_LUN_MALLOCED)
4446                 free(lun, M_CTL);
4447
4448         STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4449                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4450                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4451                 }
4452         }
4453
4454         return (0);
4455 }
4456
4457 static void
4458 ctl_create_lun(struct ctl_be_lun *be_lun)
4459 {
4460         struct ctl_softc *ctl_softc;
4461
4462         ctl_softc = control_softc;
4463
4464         /*
4465          * ctl_alloc_lun() should handle all potential failure cases.
4466          */
4467         ctl_alloc_lun(ctl_softc, NULL, be_lun, ctl_softc->target);
4468 }
4469
4470 int
4471 ctl_add_lun(struct ctl_be_lun *be_lun)
4472 {
4473         struct ctl_softc *ctl_softc;
4474
4475         ctl_softc = control_softc;
4476
4477         mtx_lock(&ctl_softc->ctl_lock);
4478         STAILQ_INSERT_TAIL(&ctl_softc->pending_lun_queue, be_lun, links);
4479         mtx_unlock(&ctl_softc->ctl_lock);
4480
4481         ctl_wakeup_thread();
4482
4483         return (0);
4484 }
4485
4486 int
4487 ctl_enable_lun(struct ctl_be_lun *be_lun)
4488 {
4489         struct ctl_softc *ctl_softc;
4490         struct ctl_frontend *fe, *nfe;
4491         struct ctl_lun *lun;
4492         int retval;
4493
4494         ctl_softc = control_softc;
4495
4496         lun = (struct ctl_lun *)be_lun->ctl_lun;
4497
4498         mtx_lock(&ctl_softc->ctl_lock);
4499         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4500                 /*
4501                  * eh?  Why did we get called if the LUN is already
4502                  * enabled?
4503                  */
4504                 mtx_unlock(&ctl_softc->ctl_lock);
4505                 return (0);
4506         }
4507         lun->flags &= ~CTL_LUN_DISABLED;
4508
4509         for (fe = STAILQ_FIRST(&ctl_softc->fe_list); fe != NULL; fe = nfe) {
4510                 nfe = STAILQ_NEXT(fe, links);
4511
4512                 /*
4513                  * Drop the lock while we call the FETD's enable routine.
4514                  * This can lead to a callback into CTL (at least in the
4515                  * case of the internal initiator frontend.
4516                  */
4517                 mtx_unlock(&ctl_softc->ctl_lock);
4518                 retval = fe->lun_enable(fe->targ_lun_arg, lun->target,lun->lun);
4519                 mtx_lock(&ctl_softc->ctl_lock);
4520                 if (retval != 0) {
4521                         printf("%s: FETD %s port %d returned error "
4522                                "%d for lun_enable on target %ju lun %jd\n",
4523                                __func__, fe->port_name, fe->targ_port, retval,
4524                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4525                 }
4526 #if 0
4527                  else {
4528             /* NOTE:  TODO:  why does lun enable affect port status? */
4529                         fe->status |= CTL_PORT_STATUS_LUN_ONLINE;
4530                 }
4531 #endif
4532         }
4533
4534         mtx_unlock(&ctl_softc->ctl_lock);
4535
4536         return (0);
4537 }
4538
4539 int
4540 ctl_disable_lun(struct ctl_be_lun *be_lun)
4541 {
4542         struct ctl_softc *ctl_softc;
4543         struct ctl_frontend *fe;
4544         struct ctl_lun *lun;
4545         int retval;
4546
4547         ctl_softc = control_softc;
4548
4549         lun = (struct ctl_lun *)be_lun->ctl_lun;
4550
4551         mtx_lock(&ctl_softc->ctl_lock);
4552
4553         if (lun->flags & CTL_LUN_DISABLED) {
4554                 mtx_unlock(&ctl_softc->ctl_lock);
4555                 return (0);
4556         }
4557         lun->flags |= CTL_LUN_DISABLED;
4558
4559         STAILQ_FOREACH(fe, &ctl_softc->fe_list, links) {
4560                 mtx_unlock(&ctl_softc->ctl_lock);
4561                 /*
4562                  * Drop the lock before we call the frontend's disable
4563                  * routine, to avoid lock order reversals.
4564                  *
4565                  * XXX KDM what happens if the frontend list changes while
4566                  * we're traversing it?  It's unlikely, but should be handled.
4567                  */
4568                 retval = fe->lun_disable(fe->targ_lun_arg, lun->target,
4569                                          lun->lun);
4570                 mtx_lock(&ctl_softc->ctl_lock);
4571                 if (retval != 0) {
4572                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4573                                "%d for lun_disable on target %ju lun %jd\n",
4574                                fe->port_name, fe->targ_port, retval,
4575                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4576                 }
4577         }
4578
4579         mtx_unlock(&ctl_softc->ctl_lock);
4580
4581         return (0);
4582 }
4583
4584 int
4585 ctl_start_lun(struct ctl_be_lun *be_lun)
4586 {
4587         struct ctl_softc *ctl_softc;
4588         struct ctl_lun *lun;
4589
4590         ctl_softc = control_softc;
4591
4592         lun = (struct ctl_lun *)be_lun->ctl_lun;
4593
4594         mtx_lock(&ctl_softc->ctl_lock);
4595         lun->flags &= ~CTL_LUN_STOPPED;
4596         mtx_unlock(&ctl_softc->ctl_lock);
4597
4598         return (0);
4599 }
4600
4601 int
4602 ctl_stop_lun(struct ctl_be_lun *be_lun)
4603 {
4604         struct ctl_softc *ctl_softc;
4605         struct ctl_lun *lun;
4606
4607         ctl_softc = control_softc;
4608
4609         lun = (struct ctl_lun *)be_lun->ctl_lun;
4610
4611         mtx_lock(&ctl_softc->ctl_lock);
4612         lun->flags |= CTL_LUN_STOPPED;
4613         mtx_unlock(&ctl_softc->ctl_lock);
4614
4615         return (0);
4616 }
4617
4618 int
4619 ctl_lun_offline(struct ctl_be_lun *be_lun)
4620 {
4621         struct ctl_softc *ctl_softc;
4622         struct ctl_lun *lun;
4623
4624         ctl_softc = control_softc;
4625
4626         lun = (struct ctl_lun *)be_lun->ctl_lun;
4627
4628         mtx_lock(&ctl_softc->ctl_lock);
4629         lun->flags |= CTL_LUN_OFFLINE;
4630         mtx_unlock(&ctl_softc->ctl_lock);
4631
4632         return (0);
4633 }
4634
4635 int
4636 ctl_lun_online(struct ctl_be_lun *be_lun)
4637 {
4638         struct ctl_softc *ctl_softc;
4639         struct ctl_lun *lun;
4640
4641         ctl_softc = control_softc;
4642
4643         lun = (struct ctl_lun *)be_lun->ctl_lun;
4644
4645         mtx_lock(&ctl_softc->ctl_lock);
4646         lun->flags &= ~CTL_LUN_OFFLINE;
4647         mtx_unlock(&ctl_softc->ctl_lock);
4648
4649         return (0);
4650 }
4651
4652 int
4653 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4654 {
4655         struct ctl_softc *ctl_softc;
4656         struct ctl_lun *lun;
4657
4658         ctl_softc = control_softc;
4659
4660         lun = (struct ctl_lun *)be_lun->ctl_lun;
4661
4662         mtx_lock(&ctl_softc->ctl_lock);
4663
4664         /*
4665          * The LUN needs to be disabled before it can be marked invalid.
4666          */
4667         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4668                 mtx_unlock(&ctl_softc->ctl_lock);
4669                 return (-1);
4670         }
4671         /*
4672          * Mark the LUN invalid.
4673          */
4674         lun->flags |= CTL_LUN_INVALID;
4675
4676         /*
4677          * If there is nothing in the OOA queue, go ahead and free the LUN.
4678          * If we have something in the OOA queue, we'll free it when the
4679          * last I/O completes.
4680          */
4681         if (TAILQ_FIRST(&lun->ooa_queue) == NULL)
4682                 ctl_free_lun(lun);
4683         mtx_unlock(&ctl_softc->ctl_lock);
4684
4685         return (0);
4686 }
4687
4688 int
4689 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
4690 {
4691         struct ctl_softc *ctl_softc;
4692         struct ctl_lun *lun;
4693
4694         ctl_softc = control_softc;
4695         lun = (struct ctl_lun *)be_lun->ctl_lun;
4696
4697         mtx_lock(&ctl_softc->ctl_lock);
4698         lun->flags |= CTL_LUN_INOPERABLE;
4699         mtx_unlock(&ctl_softc->ctl_lock);
4700
4701         return (0);
4702 }
4703
4704 int
4705 ctl_lun_operable(struct ctl_be_lun *be_lun)
4706 {
4707         struct ctl_softc *ctl_softc;
4708         struct ctl_lun *lun;
4709
4710         ctl_softc = control_softc;
4711         lun = (struct ctl_lun *)be_lun->ctl_lun;
4712
4713         mtx_lock(&ctl_softc->ctl_lock);
4714         lun->flags &= ~CTL_LUN_INOPERABLE;
4715         mtx_unlock(&ctl_softc->ctl_lock);
4716
4717         return (0);
4718 }
4719
4720 int
4721 ctl_lun_power_lock(struct ctl_be_lun *be_lun, struct ctl_nexus *nexus,
4722                    int lock)
4723 {
4724         struct ctl_softc *softc;
4725         struct ctl_lun *lun;
4726         struct copan_aps_subpage *current_sp;
4727         struct ctl_page_index *page_index;
4728         int i;
4729
4730         softc = control_softc;
4731
4732         mtx_lock(&softc->ctl_lock);
4733
4734         lun = (struct ctl_lun *)be_lun->ctl_lun;
4735
4736         page_index = NULL;
4737         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4738                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
4739                      APS_PAGE_CODE)
4740                         continue;
4741
4742                 if (lun->mode_pages.index[i].subpage != APS_SUBPAGE_CODE)
4743                         continue;
4744                 page_index = &lun->mode_pages.index[i];
4745         }
4746
4747         if (page_index == NULL) {
4748                 mtx_unlock(&softc->ctl_lock);
4749                 printf("%s: APS subpage not found for lun %ju!\n", __func__,
4750                        (uintmax_t)lun->lun);
4751                 return (1);
4752         }
4753 #if 0
4754         if ((softc->aps_locked_lun != 0)
4755          && (softc->aps_locked_lun != lun->lun)) {
4756                 printf("%s: attempt to lock LUN %llu when %llu is already "
4757                        "locked\n");
4758                 mtx_unlock(&softc->ctl_lock);
4759                 return (1);
4760         }
4761 #endif
4762
4763         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
4764                 (page_index->page_len * CTL_PAGE_CURRENT));
4765
4766         if (lock != 0) {
4767                 current_sp->lock_active = APS_LOCK_ACTIVE;
4768                 softc->aps_locked_lun = lun->lun;
4769         } else {
4770                 current_sp->lock_active = 0;
4771                 softc->aps_locked_lun = 0;
4772         }
4773
4774
4775         /*
4776          * If we're in HA mode, try to send the lock message to the other
4777          * side.
4778          */
4779         if (ctl_is_single == 0) {
4780                 int isc_retval;
4781                 union ctl_ha_msg lock_msg;
4782
4783                 lock_msg.hdr.nexus = *nexus;
4784                 lock_msg.hdr.msg_type = CTL_MSG_APS_LOCK;
4785                 if (lock != 0)
4786                         lock_msg.aps.lock_flag = 1;
4787                 else
4788                         lock_msg.aps.lock_flag = 0;
4789                 isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &lock_msg,
4790                                          sizeof(lock_msg), 0);
4791                 if (isc_retval > CTL_HA_STATUS_SUCCESS) {
4792                         printf("%s: APS (lock=%d) error returned from "
4793                                "ctl_ha_msg_send: %d\n", __func__, lock, isc_retval);
4794                         mtx_unlock(&softc->ctl_lock);
4795                         return (1);
4796                 }
4797         }
4798
4799         mtx_unlock(&softc->ctl_lock);
4800
4801         return (0);
4802 }
4803
4804 void
4805 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
4806 {
4807         struct ctl_lun *lun;
4808         struct ctl_softc *softc;
4809         int i;
4810
4811         softc = control_softc;
4812
4813         mtx_lock(&softc->ctl_lock);
4814
4815         lun = (struct ctl_lun *)be_lun->ctl_lun;
4816
4817         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4818                 lun->pending_sense[i].ua_pending |= CTL_UA_CAPACITY_CHANGED;
4819
4820         mtx_unlock(&softc->ctl_lock);
4821 }
4822
4823 /*
4824  * Backend "memory move is complete" callback for requests that never
4825  * make it down to say RAIDCore's configuration code.
4826  */
4827 int
4828 ctl_config_move_done(union ctl_io *io)
4829 {
4830         int retval;
4831
4832         retval = CTL_RETVAL_COMPLETE;
4833
4834
4835         CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
4836         /*
4837          * XXX KDM this shouldn't happen, but what if it does?
4838          */
4839         if (io->io_hdr.io_type != CTL_IO_SCSI)
4840                 panic("I/O type isn't CTL_IO_SCSI!");
4841
4842         if ((io->io_hdr.port_status == 0)
4843          && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
4844          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
4845                 io->io_hdr.status = CTL_SUCCESS;
4846         else if ((io->io_hdr.port_status != 0)
4847               && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
4848               && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)){
4849                 /*
4850                  * For hardware error sense keys, the sense key
4851                  * specific value is defined to be a retry count,
4852                  * but we use it to pass back an internal FETD
4853                  * error code.  XXX KDM  Hopefully the FETD is only
4854                  * using 16 bits for an error code, since that's
4855                  * all the space we have in the sks field.
4856                  */
4857                 ctl_set_internal_failure(&io->scsiio,
4858                                          /*sks_valid*/ 1,
4859                                          /*retry_count*/
4860                                          io->io_hdr.port_status);
4861                 free(io->scsiio.kern_data_ptr, M_CTL);
4862                 ctl_done(io);
4863                 goto bailout;
4864         }
4865
4866         if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
4867          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
4868          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
4869                 /*
4870                  * XXX KDM just assuming a single pointer here, and not a
4871                  * S/G list.  If we start using S/G lists for config data,
4872                  * we'll need to know how to clean them up here as well.
4873                  */
4874                 free(io->scsiio.kern_data_ptr, M_CTL);
4875                 /* Hopefully the user has already set the status... */
4876                 ctl_done(io);
4877         } else {
4878                 /*
4879                  * XXX KDM now we need to continue data movement.  Some
4880                  * options:
4881                  * - call ctl_scsiio() again?  We don't do this for data
4882                  *   writes, because for those at least we know ahead of
4883                  *   time where the write will go and how long it is.  For
4884                  *   config writes, though, that information is largely
4885                  *   contained within the write itself, thus we need to
4886                  *   parse out the data again.
4887                  *
4888                  * - Call some other function once the data is in?
4889                  */
4890
4891                 /*
4892                  * XXX KDM call ctl_scsiio() again for now, and check flag
4893                  * bits to see whether we're allocated or not.
4894                  */
4895                 retval = ctl_scsiio(&io->scsiio);
4896         }
4897 bailout:
4898         return (retval);
4899 }
4900
4901 /*
4902  * This gets called by a backend driver when it is done with a
4903  * configuration write.
4904  */
4905 void
4906 ctl_config_write_done(union ctl_io *io)
4907 {
4908         /*
4909          * If the IO_CONT flag is set, we need to call the supplied
4910          * function to continue processing the I/O, instead of completing
4911          * the I/O just yet.
4912          *
4913          * If there is an error, though, we don't want to keep processing.
4914          * Instead, just send status back to the initiator.
4915          */
4916         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT)
4917          && (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)
4918           || ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))) {
4919                 io->scsiio.io_cont(io);
4920                 return;
4921         }
4922         /*
4923          * Since a configuration write can be done for commands that actually
4924          * have data allocated, like write buffer, and commands that have
4925          * no data, like start/stop unit, we need to check here.
4926          */
4927         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
4928                 free(io->scsiio.kern_data_ptr, M_CTL);
4929         ctl_done(io);
4930 }
4931
4932 /*
4933  * SCSI release command.
4934  */
4935 int
4936 ctl_scsi_release(struct ctl_scsiio *ctsio)
4937 {
4938         int length, longid, thirdparty_id, resv_id;
4939         struct ctl_softc *ctl_softc;
4940         struct ctl_lun *lun;
4941
4942         length = 0;
4943         resv_id = 0;
4944
4945         CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
4946
4947         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
4948         ctl_softc = control_softc;
4949
4950         switch (ctsio->cdb[0]) {
4951         case RELEASE: {
4952                 struct scsi_release *cdb;
4953
4954                 cdb = (struct scsi_release *)ctsio->cdb;
4955                 if ((cdb->byte2 & 0x1f) != 0) {
4956                         ctl_set_invalid_field(ctsio,
4957                                               /*sks_valid*/ 1,
4958                                               /*command*/ 1,
4959                                               /*field*/ 1,
4960                                               /*bit_valid*/ 0,
4961                                               /*bit*/ 0);
4962                         ctl_done((union ctl_io *)ctsio);
4963                         return (CTL_RETVAL_COMPLETE);
4964                 }
4965                 break;
4966         }
4967         case RELEASE_10: {
4968                 struct scsi_release_10 *cdb;
4969
4970                 cdb = (struct scsi_release_10 *)ctsio->cdb;
4971
4972                 if ((cdb->byte2 & SR10_EXTENT) != 0) {
4973                         ctl_set_invalid_field(ctsio,
4974                                               /*sks_valid*/ 1,
4975                                               /*command*/ 1,
4976                                               /*field*/ 1,
4977                                               /*bit_valid*/ 1,
4978                                               /*bit*/ 0);
4979                         ctl_done((union ctl_io *)ctsio);
4980                         return (CTL_RETVAL_COMPLETE);
4981
4982                 }
4983
4984                 if ((cdb->byte2 & SR10_3RDPTY) != 0) {
4985                         ctl_set_invalid_field(ctsio,
4986                                               /*sks_valid*/ 1,
4987                                               /*command*/ 1,
4988                                               /*field*/ 1,
4989                                               /*bit_valid*/ 1,
4990                                               /*bit*/ 4);
4991                         ctl_done((union ctl_io *)ctsio);
4992                         return (CTL_RETVAL_COMPLETE);
4993                 }
4994
4995                 if (cdb->byte2 & SR10_LONGID)
4996                         longid = 1;
4997                 else
4998                         thirdparty_id = cdb->thirdparty_id;
4999
5000                 resv_id = cdb->resv_id;
5001                 length = scsi_2btoul(cdb->length);
5002                 break;
5003         }
5004         }
5005
5006
5007         /*
5008          * XXX KDM right now, we only support LUN reservation.  We don't
5009          * support 3rd party reservations, or extent reservations, which
5010          * might actually need the parameter list.  If we've gotten this
5011          * far, we've got a LUN reservation.  Anything else got kicked out
5012          * above.  So, according to SPC, ignore the length.
5013          */
5014         length = 0;
5015
5016         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5017          && (length > 0)) {
5018                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5019                 if (ctsio->kern_data_ptr == NULL) {
5020                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
5021                         ctsio->io_hdr.status = SCSI_STATUS_BUSY;
5022                         ctl_done((union ctl_io *)ctsio);
5023                         return (CTL_RETVAL_COMPLETE);
5024                 }
5025                 ctsio->kern_data_len = length;
5026                 ctsio->kern_total_len = length;
5027                 ctsio->kern_data_resid = 0;
5028                 ctsio->kern_rel_offset = 0;
5029                 ctsio->kern_sg_entries = 0;
5030                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5031                 ctsio->be_move_done = ctl_config_move_done;
5032                 ctl_datamove((union ctl_io *)ctsio);
5033
5034                 return (CTL_RETVAL_COMPLETE);
5035         }
5036
5037         if (length > 0)
5038                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5039
5040         mtx_lock(&ctl_softc->ctl_lock);
5041
5042         /*
5043          * According to SPC, it is not an error for an intiator to attempt
5044          * to release a reservation on a LUN that isn't reserved, or that
5045          * is reserved by another initiator.  The reservation can only be
5046          * released, though, by the initiator who made it or by one of
5047          * several reset type events.
5048          */
5049         if (lun->flags & CTL_LUN_RESERVED) {
5050                 if ((ctsio->io_hdr.nexus.initid.id == lun->rsv_nexus.initid.id)
5051                  && (ctsio->io_hdr.nexus.targ_port == lun->rsv_nexus.targ_port)
5052                  && (ctsio->io_hdr.nexus.targ_target.id ==
5053                      lun->rsv_nexus.targ_target.id)) {
5054                         lun->flags &= ~CTL_LUN_RESERVED;
5055                 }
5056         }
5057
5058         ctsio->scsi_status = SCSI_STATUS_OK;
5059         ctsio->io_hdr.status = CTL_SUCCESS;
5060
5061         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5062                 free(ctsio->kern_data_ptr, M_CTL);
5063                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5064         }
5065
5066         mtx_unlock(&ctl_softc->ctl_lock);
5067
5068         ctl_done((union ctl_io *)ctsio);
5069         return (CTL_RETVAL_COMPLETE);
5070 }
5071
5072 int
5073 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5074 {
5075         int extent, thirdparty, longid;
5076         int resv_id, length;
5077         uint64_t thirdparty_id;
5078         struct ctl_softc *ctl_softc;
5079         struct ctl_lun *lun;
5080
5081         extent = 0;
5082         thirdparty = 0;
5083         longid = 0;
5084         resv_id = 0;
5085         length = 0;
5086         thirdparty_id = 0;
5087
5088         CTL_DEBUG_PRINT(("ctl_reserve\n"));
5089
5090         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5091         ctl_softc = control_softc;
5092
5093         switch (ctsio->cdb[0]) {
5094         case RESERVE: {
5095                 struct scsi_reserve *cdb;
5096
5097                 cdb = (struct scsi_reserve *)ctsio->cdb;
5098                 if ((cdb->byte2 & 0x1f) != 0) {
5099                         ctl_set_invalid_field(ctsio,
5100                                               /*sks_valid*/ 1,
5101                                               /*command*/ 1,
5102                                               /*field*/ 1,
5103                                               /*bit_valid*/ 0,
5104                                               /*bit*/ 0);
5105                         ctl_done((union ctl_io *)ctsio);
5106                         return (CTL_RETVAL_COMPLETE);
5107                 }
5108                 resv_id = cdb->resv_id;
5109                 length = scsi_2btoul(cdb->length);
5110                 break;
5111         }
5112         case RESERVE_10: {
5113                 struct scsi_reserve_10 *cdb;
5114
5115                 cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5116
5117                 if ((cdb->byte2 & SR10_EXTENT) != 0) {
5118                         ctl_set_invalid_field(ctsio,
5119                                               /*sks_valid*/ 1,
5120                                               /*command*/ 1,
5121                                               /*field*/ 1,
5122                                               /*bit_valid*/ 1,
5123                                               /*bit*/ 0);
5124                         ctl_done((union ctl_io *)ctsio);
5125                         return (CTL_RETVAL_COMPLETE);
5126                 }
5127                 if ((cdb->byte2 & SR10_3RDPTY) != 0) {
5128                         ctl_set_invalid_field(ctsio,
5129                                               /*sks_valid*/ 1,
5130                                               /*command*/ 1,
5131                                               /*field*/ 1,
5132                                               /*bit_valid*/ 1,
5133                                               /*bit*/ 4);
5134                         ctl_done((union ctl_io *)ctsio);
5135                         return (CTL_RETVAL_COMPLETE);
5136                 }
5137                 if (cdb->byte2 & SR10_LONGID)
5138                         longid = 1;
5139                 else
5140                         thirdparty_id = cdb->thirdparty_id;
5141
5142                 resv_id = cdb->resv_id;
5143                 length = scsi_2btoul(cdb->length);
5144                 break;
5145         }
5146         }
5147
5148         /*
5149          * XXX KDM right now, we only support LUN reservation.  We don't
5150          * support 3rd party reservations, or extent reservations, which
5151          * might actually need the parameter list.  If we've gotten this
5152          * far, we've got a LUN reservation.  Anything else got kicked out
5153          * above.  So, according to SPC, ignore the length.
5154          */
5155         length = 0;
5156
5157         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5158          && (length > 0)) {
5159                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5160                 if (ctsio->kern_data_ptr == NULL) {
5161                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
5162                         ctsio->io_hdr.status = SCSI_STATUS_BUSY;
5163                         ctl_done((union ctl_io *)ctsio);
5164                         return (CTL_RETVAL_COMPLETE);
5165                 }
5166                 ctsio->kern_data_len = length;
5167                 ctsio->kern_total_len = length;
5168                 ctsio->kern_data_resid = 0;
5169                 ctsio->kern_rel_offset = 0;
5170                 ctsio->kern_sg_entries = 0;
5171                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5172                 ctsio->be_move_done = ctl_config_move_done;
5173                 ctl_datamove((union ctl_io *)ctsio);
5174
5175                 return (CTL_RETVAL_COMPLETE);
5176         }
5177
5178         if (length > 0)
5179                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5180
5181         mtx_lock(&ctl_softc->ctl_lock);
5182         if (lun->flags & CTL_LUN_RESERVED) {
5183                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
5184                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
5185                  || (ctsio->io_hdr.nexus.targ_target.id !=
5186                      lun->rsv_nexus.targ_target.id)) {
5187                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
5188                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
5189                         goto bailout;
5190                 }
5191         }
5192
5193         lun->flags |= CTL_LUN_RESERVED;
5194         lun->rsv_nexus = ctsio->io_hdr.nexus;
5195
5196         ctsio->scsi_status = SCSI_STATUS_OK;
5197         ctsio->io_hdr.status = CTL_SUCCESS;
5198
5199 bailout:
5200         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5201                 free(ctsio->kern_data_ptr, M_CTL);
5202                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5203         }
5204
5205         mtx_unlock(&ctl_softc->ctl_lock);
5206
5207         ctl_done((union ctl_io *)ctsio);
5208         return (CTL_RETVAL_COMPLETE);
5209 }
5210
5211 int
5212 ctl_start_stop(struct ctl_scsiio *ctsio)
5213 {
5214         struct scsi_start_stop_unit *cdb;
5215         struct ctl_lun *lun;
5216         struct ctl_softc *ctl_softc;
5217         int retval;
5218
5219         CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5220
5221         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5222         ctl_softc = control_softc;
5223         retval = 0;
5224
5225         cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5226
5227         /*
5228          * XXX KDM
5229          * We don't support the immediate bit on a stop unit.  In order to
5230          * do that, we would need to code up a way to know that a stop is
5231          * pending, and hold off any new commands until it completes, one
5232          * way or another.  Then we could accept or reject those commands
5233          * depending on its status.  We would almost need to do the reverse
5234          * of what we do below for an immediate start -- return the copy of
5235          * the ctl_io to the FETD with status to send to the host (and to
5236          * free the copy!) and then free the original I/O once the stop
5237          * actually completes.  That way, the OOA queue mechanism can work
5238          * to block commands that shouldn't proceed.  Another alternative
5239          * would be to put the copy in the queue in place of the original,
5240          * and return the original back to the caller.  That could be
5241          * slightly safer..
5242          */
5243         if ((cdb->byte2 & SSS_IMMED)
5244          && ((cdb->how & SSS_START) == 0)) {
5245                 ctl_set_invalid_field(ctsio,
5246                                       /*sks_valid*/ 1,
5247                                       /*command*/ 1,
5248                                       /*field*/ 1,
5249                                       /*bit_valid*/ 1,
5250                                       /*bit*/ 0);
5251                 ctl_done((union ctl_io *)ctsio);
5252                 return (CTL_RETVAL_COMPLETE);
5253         }
5254
5255         /*
5256          * We don't support the power conditions field.  We need to check
5257          * this prior to checking the load/eject and start/stop bits.
5258          */
5259         if ((cdb->how & SSS_PC_MASK) != SSS_PC_START_VALID) {
5260                 ctl_set_invalid_field(ctsio,
5261                                       /*sks_valid*/ 1,
5262                                       /*command*/ 1,
5263                                       /*field*/ 4,
5264                                       /*bit_valid*/ 1,
5265                                       /*bit*/ 4);
5266                 ctl_done((union ctl_io *)ctsio);
5267                 return (CTL_RETVAL_COMPLETE);
5268         }
5269
5270         /*
5271          * Media isn't removable, so we can't load or eject it.
5272          */
5273         if ((cdb->how & SSS_LOEJ) != 0) {
5274                 ctl_set_invalid_field(ctsio,
5275                                       /*sks_valid*/ 1,
5276                                       /*command*/ 1,
5277                                       /*field*/ 4,
5278                                       /*bit_valid*/ 1,
5279                                       /*bit*/ 1);
5280                 ctl_done((union ctl_io *)ctsio);
5281                 return (CTL_RETVAL_COMPLETE);
5282         }
5283
5284         if ((lun->flags & CTL_LUN_PR_RESERVED)
5285          && ((cdb->how & SSS_START)==0)) {
5286                 uint32_t residx;
5287
5288                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5289                 if (!lun->per_res[residx].registered
5290                  || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5291
5292                         ctl_set_reservation_conflict(ctsio);
5293                         ctl_done((union ctl_io *)ctsio);
5294                         return (CTL_RETVAL_COMPLETE);
5295                 }
5296         }
5297
5298         /*
5299          * If there is no backend on this device, we can't start or stop
5300          * it.  In theory we shouldn't get any start/stop commands in the
5301          * first place at this level if the LUN doesn't have a backend.
5302          * That should get stopped by the command decode code.
5303          */
5304         if (lun->backend == NULL) {
5305                 ctl_set_invalid_opcode(ctsio);
5306                 ctl_done((union ctl_io *)ctsio);
5307                 return (CTL_RETVAL_COMPLETE);
5308         }
5309
5310         /*
5311          * XXX KDM Copan-specific offline behavior.
5312          * Figure out a reasonable way to port this?
5313          */
5314 #ifdef NEEDTOPORT
5315         mtx_lock(&ctl_softc->ctl_lock);
5316
5317         if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5318          && (lun->flags & CTL_LUN_OFFLINE)) {
5319                 /*
5320                  * If the LUN is offline, and the on/offline bit isn't set,
5321                  * reject the start or stop.  Otherwise, let it through.
5322                  */
5323                 mtx_unlock(&ctl_softc->ctl_lock);
5324                 ctl_set_lun_not_ready(ctsio);
5325                 ctl_done((union ctl_io *)ctsio);
5326         } else {
5327                 mtx_unlock(&ctl_softc->ctl_lock);
5328 #endif /* NEEDTOPORT */
5329                 /*
5330                  * This could be a start or a stop when we're online,
5331                  * or a stop/offline or start/online.  A start or stop when
5332                  * we're offline is covered in the case above.
5333                  */
5334                 /*
5335                  * In the non-immediate case, we send the request to
5336                  * the backend and return status to the user when
5337                  * it is done.
5338                  *
5339                  * In the immediate case, we allocate a new ctl_io
5340                  * to hold a copy of the request, and send that to
5341                  * the backend.  We then set good status on the
5342                  * user's request and return it immediately.
5343                  */
5344                 if (cdb->byte2 & SSS_IMMED) {
5345                         union ctl_io *new_io;
5346
5347                         new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5348                         if (new_io == NULL) {
5349                                 ctl_set_busy(ctsio);
5350                                 ctl_done((union ctl_io *)ctsio);
5351                         } else {
5352                                 ctl_copy_io((union ctl_io *)ctsio,
5353                                             new_io);
5354                                 retval = lun->backend->config_write(new_io);
5355                                 ctl_set_success(ctsio);
5356                                 ctl_done((union ctl_io *)ctsio);
5357                         }
5358                 } else {
5359                         retval = lun->backend->config_write(
5360                                 (union ctl_io *)ctsio);
5361                 }
5362 #ifdef NEEDTOPORT
5363         }
5364 #endif
5365         return (retval);
5366 }
5367
5368 /*
5369  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5370  * we don't really do anything with the LBA and length fields if the user
5371  * passes them in.  Instead we'll just flush out the cache for the entire
5372  * LUN.
5373  */
5374 int
5375 ctl_sync_cache(struct ctl_scsiio *ctsio)
5376 {
5377         struct ctl_lun *lun;
5378         struct ctl_softc *ctl_softc;
5379         uint64_t starting_lba;
5380         uint32_t block_count;
5381         int reladr, immed;
5382         int retval;
5383
5384         CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5385
5386         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5387         ctl_softc = control_softc;
5388         retval = 0;
5389         reladr = 0;
5390         immed = 0;
5391
5392         switch (ctsio->cdb[0]) {
5393         case SYNCHRONIZE_CACHE: {
5394                 struct scsi_sync_cache *cdb;
5395                 cdb = (struct scsi_sync_cache *)ctsio->cdb;
5396
5397                 if (cdb->byte2 & SSC_RELADR)
5398                         reladr = 1;
5399
5400                 if (cdb->byte2 & SSC_IMMED)
5401                         immed = 1;
5402
5403                 starting_lba = scsi_4btoul(cdb->begin_lba);
5404                 block_count = scsi_2btoul(cdb->lb_count);
5405                 break;
5406         }
5407         case SYNCHRONIZE_CACHE_16: {
5408                 struct scsi_sync_cache_16 *cdb;
5409                 cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5410
5411                 if (cdb->byte2 & SSC_RELADR)
5412                         reladr = 1;
5413
5414                 if (cdb->byte2 & SSC_IMMED)
5415                         immed = 1;
5416
5417                 starting_lba = scsi_8btou64(cdb->begin_lba);
5418                 block_count = scsi_4btoul(cdb->lb_count);
5419                 break;
5420         }
5421         default:
5422                 ctl_set_invalid_opcode(ctsio);
5423                 ctl_done((union ctl_io *)ctsio);
5424                 goto bailout;
5425                 break; /* NOTREACHED */
5426         }
5427
5428         if (immed) {
5429                 /*
5430                  * We don't support the immediate bit.  Since it's in the
5431                  * same place for the 10 and 16 byte SYNCHRONIZE CACHE
5432                  * commands, we can just return the same error in either
5433                  * case.
5434                  */
5435                 ctl_set_invalid_field(ctsio,
5436                                       /*sks_valid*/ 1,
5437                                       /*command*/ 1,
5438                                       /*field*/ 1,
5439                                       /*bit_valid*/ 1,
5440                                       /*bit*/ 1);
5441                 ctl_done((union ctl_io *)ctsio);
5442                 goto bailout;
5443         }
5444
5445         if (reladr) {
5446                 /*
5447                  * We don't support the reladr bit either.  It can only be
5448                  * used with linked commands, and we don't support linked
5449                  * commands.  Since the bit is in the same place for the
5450                  * 10 and 16 byte SYNCHRONIZE CACHE * commands, we can
5451                  * just return the same error in either case.
5452                  */
5453                 ctl_set_invalid_field(ctsio,
5454                                       /*sks_valid*/ 1,
5455                                       /*command*/ 1,
5456                                       /*field*/ 1,
5457                                       /*bit_valid*/ 1,
5458                                       /*bit*/ 0);
5459                 ctl_done((union ctl_io *)ctsio);
5460                 goto bailout;
5461         }
5462
5463         /*
5464          * We check the LBA and length, but don't do anything with them.
5465          * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5466          * get flushed.  This check will just help satisfy anyone who wants
5467          * to see an error for an out of range LBA.
5468          */
5469         if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5470                 ctl_set_lba_out_of_range(ctsio);
5471                 ctl_done((union ctl_io *)ctsio);
5472                 goto bailout;
5473         }
5474
5475         /*
5476          * If this LUN has no backend, we can't flush the cache anyway.
5477          */
5478         if (lun->backend == NULL) {
5479                 ctl_set_invalid_opcode(ctsio);
5480                 ctl_done((union ctl_io *)ctsio);
5481                 goto bailout;
5482         }
5483
5484         /*
5485          * Check to see whether we're configured to send the SYNCHRONIZE
5486          * CACHE command directly to the back end.
5487          */
5488         mtx_lock(&ctl_softc->ctl_lock);
5489         if ((ctl_softc->flags & CTL_FLAG_REAL_SYNC)
5490          && (++(lun->sync_count) >= lun->sync_interval)) {
5491                 lun->sync_count = 0;
5492                 mtx_unlock(&ctl_softc->ctl_lock);
5493                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5494         } else {
5495                 mtx_unlock(&ctl_softc->ctl_lock);
5496                 ctl_set_success(ctsio);
5497                 ctl_done((union ctl_io *)ctsio);
5498         }
5499
5500 bailout:
5501
5502         return (retval);
5503 }
5504
5505 int
5506 ctl_format(struct ctl_scsiio *ctsio)
5507 {
5508         struct scsi_format *cdb;
5509         struct ctl_lun *lun;
5510         struct ctl_softc *ctl_softc;
5511         int length, defect_list_len;
5512
5513         CTL_DEBUG_PRINT(("ctl_format\n"));
5514
5515         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5516         ctl_softc = control_softc;
5517
5518         cdb = (struct scsi_format *)ctsio->cdb;
5519
5520         length = 0;
5521         if (cdb->byte2 & SF_FMTDATA) {
5522                 if (cdb->byte2 & SF_LONGLIST)
5523                         length = sizeof(struct scsi_format_header_long);
5524                 else
5525                         length = sizeof(struct scsi_format_header_short);
5526         }
5527
5528         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5529          && (length > 0)) {
5530                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5531                 if (ctsio->kern_data_ptr == NULL) {
5532                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
5533                         ctsio->io_hdr.status = SCSI_STATUS_BUSY;
5534                         ctl_done((union ctl_io *)ctsio);
5535                         return (CTL_RETVAL_COMPLETE);
5536                 }
5537                 ctsio->kern_data_len = length;
5538                 ctsio->kern_total_len = length;
5539                 ctsio->kern_data_resid = 0;
5540                 ctsio->kern_rel_offset = 0;
5541                 ctsio->kern_sg_entries = 0;
5542                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5543                 ctsio->be_move_done = ctl_config_move_done;
5544                 ctl_datamove((union ctl_io *)ctsio);
5545
5546                 return (CTL_RETVAL_COMPLETE);
5547         }
5548
5549         defect_list_len = 0;
5550
5551         if (cdb->byte2 & SF_FMTDATA) {
5552                 if (cdb->byte2 & SF_LONGLIST) {
5553                         struct scsi_format_header_long *header;
5554
5555                         header = (struct scsi_format_header_long *)
5556                                 ctsio->kern_data_ptr;
5557
5558                         defect_list_len = scsi_4btoul(header->defect_list_len);
5559                         if (defect_list_len != 0) {
5560                                 ctl_set_invalid_field(ctsio,
5561                                                       /*sks_valid*/ 1,
5562                                                       /*command*/ 0,
5563                                                       /*field*/ 2,
5564                                                       /*bit_valid*/ 0,
5565                                                       /*bit*/ 0);
5566                                 goto bailout;
5567                         }
5568                 } else {
5569                         struct scsi_format_header_short *header;
5570
5571                         header = (struct scsi_format_header_short *)
5572                                 ctsio->kern_data_ptr;
5573
5574                         defect_list_len = scsi_2btoul(header->defect_list_len);
5575                         if (defect_list_len != 0) {
5576                                 ctl_set_invalid_field(ctsio,
5577                                                       /*sks_valid*/ 1,
5578                                                       /*command*/ 0,
5579                                                       /*field*/ 2,
5580                                                       /*bit_valid*/ 0,
5581                                                       /*bit*/ 0);
5582                                 goto bailout;
5583                         }
5584                 }
5585         }
5586
5587         /*
5588          * The format command will clear out the "Medium format corrupted"
5589          * status if set by the configuration code.  That status is really
5590          * just a way to notify the host that we have lost the media, and
5591          * get them to issue a command that will basically make them think
5592          * they're blowing away the media.
5593          */
5594         mtx_lock(&ctl_softc->ctl_lock);
5595         lun->flags &= ~CTL_LUN_INOPERABLE;
5596         mtx_unlock(&ctl_softc->ctl_lock);
5597
5598         ctsio->scsi_status = SCSI_STATUS_OK;
5599         ctsio->io_hdr.status = CTL_SUCCESS;
5600 bailout:
5601
5602         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5603                 free(ctsio->kern_data_ptr, M_CTL);
5604                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5605         }
5606
5607         ctl_done((union ctl_io *)ctsio);
5608         return (CTL_RETVAL_COMPLETE);
5609 }
5610
5611 int
5612 ctl_write_buffer(struct ctl_scsiio *ctsio)
5613 {
5614         struct scsi_write_buffer *cdb;
5615         struct copan_page_header *header;
5616         struct ctl_lun *lun;
5617         struct ctl_softc *ctl_softc;
5618         int buffer_offset, len;
5619         int retval;
5620
5621         header = NULL;
5622
5623         retval = CTL_RETVAL_COMPLETE;
5624
5625         CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5626
5627         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5628         ctl_softc = control_softc;
5629         cdb = (struct scsi_write_buffer *)ctsio->cdb;
5630
5631         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5632                 ctl_set_invalid_field(ctsio,
5633                                       /*sks_valid*/ 1,
5634                                       /*command*/ 1,
5635                                       /*field*/ 1,
5636                                       /*bit_valid*/ 1,
5637                                       /*bit*/ 4);
5638                 ctl_done((union ctl_io *)ctsio);
5639                 return (CTL_RETVAL_COMPLETE);
5640         }
5641         if (cdb->buffer_id != 0) {
5642                 ctl_set_invalid_field(ctsio,
5643                                       /*sks_valid*/ 1,
5644                                       /*command*/ 1,
5645                                       /*field*/ 2,
5646                                       /*bit_valid*/ 0,
5647                                       /*bit*/ 0);
5648                 ctl_done((union ctl_io *)ctsio);
5649                 return (CTL_RETVAL_COMPLETE);
5650         }
5651
5652         len = scsi_3btoul(cdb->length);
5653         buffer_offset = scsi_3btoul(cdb->offset);
5654
5655         if (len > sizeof(lun->write_buffer)) {
5656                 ctl_set_invalid_field(ctsio,
5657                                       /*sks_valid*/ 1,
5658                                       /*command*/ 1,
5659                                       /*field*/ 6,
5660                                       /*bit_valid*/ 0,
5661                                       /*bit*/ 0);
5662                 ctl_done((union ctl_io *)ctsio);
5663                 return (CTL_RETVAL_COMPLETE);
5664         }
5665
5666         if (buffer_offset != 0) {
5667                 ctl_set_invalid_field(ctsio,
5668                                       /*sks_valid*/ 1,
5669                                       /*command*/ 1,
5670                                       /*field*/ 3,
5671                                       /*bit_valid*/ 0,
5672                                       /*bit*/ 0);
5673                 ctl_done((union ctl_io *)ctsio);
5674                 return (CTL_RETVAL_COMPLETE);
5675         }
5676
5677         /*
5678          * If we've got a kernel request that hasn't been malloced yet,
5679          * malloc it and tell the caller the data buffer is here.
5680          */
5681         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5682                 ctsio->kern_data_ptr = lun->write_buffer;
5683                 ctsio->kern_data_len = len;
5684                 ctsio->kern_total_len = len;
5685                 ctsio->kern_data_resid = 0;
5686                 ctsio->kern_rel_offset = 0;
5687                 ctsio->kern_sg_entries = 0;
5688                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5689                 ctsio->be_move_done = ctl_config_move_done;
5690                 ctl_datamove((union ctl_io *)ctsio);
5691
5692                 return (CTL_RETVAL_COMPLETE);
5693         }
5694
5695         ctl_done((union ctl_io *)ctsio);
5696
5697         return (CTL_RETVAL_COMPLETE);
5698 }
5699
5700 /*
5701  * Note that this function currently doesn't actually do anything inside
5702  * CTL to enforce things if the DQue bit is turned on.
5703  *
5704  * Also note that this function can't be used in the default case, because
5705  * the DQue bit isn't set in the changeable mask for the control mode page
5706  * anyway.  This is just here as an example for how to implement a page
5707  * handler, and a placeholder in case we want to allow the user to turn
5708  * tagged queueing on and off.
5709  *
5710  * The D_SENSE bit handling is functional, however, and will turn
5711  * descriptor sense on and off for a given LUN.
5712  */
5713 int
5714 ctl_control_page_handler(struct ctl_scsiio *ctsio,
5715                          struct ctl_page_index *page_index, uint8_t *page_ptr)
5716 {
5717         struct scsi_control_page *current_cp, *saved_cp, *user_cp;
5718         struct ctl_lun *lun;
5719         struct ctl_softc *softc;
5720         int set_ua;
5721         uint32_t initidx;
5722
5723         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5724         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5725         set_ua = 0;
5726
5727         user_cp = (struct scsi_control_page *)page_ptr;
5728         current_cp = (struct scsi_control_page *)
5729                 (page_index->page_data + (page_index->page_len *
5730                 CTL_PAGE_CURRENT));
5731         saved_cp = (struct scsi_control_page *)
5732                 (page_index->page_data + (page_index->page_len *
5733                 CTL_PAGE_SAVED));
5734
5735         softc = control_softc;
5736
5737         mtx_lock(&softc->ctl_lock);
5738         if (((current_cp->rlec & SCP_DSENSE) == 0)
5739          && ((user_cp->rlec & SCP_DSENSE) != 0)) {
5740                 /*
5741                  * Descriptor sense is currently turned off and the user
5742                  * wants to turn it on.
5743                  */
5744                 current_cp->rlec |= SCP_DSENSE;
5745                 saved_cp->rlec |= SCP_DSENSE;
5746                 lun->flags |= CTL_LUN_SENSE_DESC;
5747                 set_ua = 1;
5748         } else if (((current_cp->rlec & SCP_DSENSE) != 0)
5749                 && ((user_cp->rlec & SCP_DSENSE) == 0)) {
5750                 /*
5751                  * Descriptor sense is currently turned on, and the user
5752                  * wants to turn it off.
5753                  */
5754                 current_cp->rlec &= ~SCP_DSENSE;
5755                 saved_cp->rlec &= ~SCP_DSENSE;
5756                 lun->flags &= ~CTL_LUN_SENSE_DESC;
5757                 set_ua = 1;
5758         }
5759         if (current_cp->queue_flags & SCP_QUEUE_DQUE) {
5760                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
5761 #ifdef NEEDTOPORT
5762                         csevent_log(CSC_CTL | CSC_SHELF_SW |
5763                                     CTL_UNTAG_TO_UNTAG,
5764                                     csevent_LogType_Trace,
5765                                     csevent_Severity_Information,
5766                                     csevent_AlertLevel_Green,
5767                                     csevent_FRU_Firmware,
5768                                     csevent_FRU_Unknown,
5769                                     "Received untagged to untagged transition");
5770 #endif /* NEEDTOPORT */
5771                 } else {
5772 #ifdef NEEDTOPORT
5773                         csevent_log(CSC_CTL | CSC_SHELF_SW |
5774                                     CTL_UNTAG_TO_TAG,
5775                                     csevent_LogType_ConfigChange,
5776                                     csevent_Severity_Information,
5777                                     csevent_AlertLevel_Green,
5778                                     csevent_FRU_Firmware,
5779                                     csevent_FRU_Unknown,
5780                                     "Received untagged to tagged "
5781                                     "queueing transition");
5782 #endif /* NEEDTOPORT */
5783
5784                         current_cp->queue_flags &= ~SCP_QUEUE_DQUE;
5785                         saved_cp->queue_flags &= ~SCP_QUEUE_DQUE;
5786                         set_ua = 1;
5787                 }
5788         } else {
5789                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
5790 #ifdef NEEDTOPORT
5791                         csevent_log(CSC_CTL | CSC_SHELF_SW |
5792                                     CTL_TAG_TO_UNTAG,
5793                                     csevent_LogType_ConfigChange,
5794                                     csevent_Severity_Warning,
5795                                     csevent_AlertLevel_Yellow,
5796                                     csevent_FRU_Firmware,
5797                                     csevent_FRU_Unknown,
5798                                     "Received tagged queueing to untagged "
5799                                     "transition");
5800 #endif /* NEEDTOPORT */
5801
5802                         current_cp->queue_flags |= SCP_QUEUE_DQUE;
5803                         saved_cp->queue_flags |= SCP_QUEUE_DQUE;
5804                         set_ua = 1;
5805                 } else {
5806 #ifdef NEEDTOPORT
5807                         csevent_log(CSC_CTL | CSC_SHELF_SW |
5808                                     CTL_TAG_TO_TAG,
5809                                     csevent_LogType_Trace,
5810                                     csevent_Severity_Information,
5811                                     csevent_AlertLevel_Green,
5812                                     csevent_FRU_Firmware,
5813                                     csevent_FRU_Unknown,
5814                                     "Received tagged queueing to tagged "
5815                                     "queueing transition");
5816 #endif /* NEEDTOPORT */
5817                 }
5818         }
5819         if (set_ua != 0) {
5820                 int i;
5821                 /*
5822                  * Let other initiators know that the mode
5823                  * parameters for this LUN have changed.
5824                  */
5825                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
5826                         if (i == initidx)
5827                                 continue;
5828
5829                         lun->pending_sense[i].ua_pending |=
5830                                 CTL_UA_MODE_CHANGE;
5831                 }
5832         }
5833         mtx_unlock(&softc->ctl_lock);
5834
5835         return (0);
5836 }
5837
5838 int
5839 ctl_power_sp_handler(struct ctl_scsiio *ctsio,
5840                      struct ctl_page_index *page_index, uint8_t *page_ptr)
5841 {
5842         return (0);
5843 }
5844
5845 int
5846 ctl_power_sp_sense_handler(struct ctl_scsiio *ctsio,
5847                            struct ctl_page_index *page_index, int pc)
5848 {
5849         struct copan_power_subpage *page;
5850
5851         page = (struct copan_power_subpage *)page_index->page_data +
5852                 (page_index->page_len * pc);
5853
5854         switch (pc) {
5855         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
5856                 /*
5857                  * We don't update the changable bits for this page.
5858                  */
5859                 break;
5860         case SMS_PAGE_CTRL_CURRENT >> 6:
5861         case SMS_PAGE_CTRL_DEFAULT >> 6:
5862         case SMS_PAGE_CTRL_SAVED >> 6:
5863 #ifdef NEEDTOPORT
5864                 ctl_update_power_subpage(page);
5865 #endif
5866                 break;
5867         default:
5868 #ifdef NEEDTOPORT
5869                 EPRINT(0, "Invalid PC %d!!", pc);
5870 #endif
5871                 break;
5872         }
5873         return (0);
5874 }
5875
5876
5877 int
5878 ctl_aps_sp_handler(struct ctl_scsiio *ctsio,
5879                    struct ctl_page_index *page_index, uint8_t *page_ptr)
5880 {
5881         struct copan_aps_subpage *user_sp;
5882         struct copan_aps_subpage *current_sp;
5883         union ctl_modepage_info *modepage_info;
5884         struct ctl_softc *softc;
5885         struct ctl_lun *lun;
5886         int retval;
5887
5888         retval = CTL_RETVAL_COMPLETE;
5889         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
5890                      (page_index->page_len * CTL_PAGE_CURRENT));
5891         softc = control_softc;
5892         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5893
5894         user_sp = (struct copan_aps_subpage *)page_ptr;
5895
5896         modepage_info = (union ctl_modepage_info *)
5897                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
5898
5899         modepage_info->header.page_code = page_index->page_code & SMPH_PC_MASK;
5900         modepage_info->header.subpage = page_index->subpage;
5901         modepage_info->aps.lock_active = user_sp->lock_active;
5902
5903         mtx_lock(&softc->ctl_lock);
5904
5905         /*
5906          * If there is a request to lock the LUN and another LUN is locked
5907          * this is an error. If the requested LUN is already locked ignore
5908          * the request. If no LUN is locked attempt to lock it.
5909          * if there is a request to unlock the LUN and the LUN is currently
5910          * locked attempt to unlock it. Otherwise ignore the request. i.e.
5911          * if another LUN is locked or no LUN is locked.
5912          */
5913         if (user_sp->lock_active & APS_LOCK_ACTIVE) {
5914                 if (softc->aps_locked_lun == lun->lun) {
5915                         /*
5916                          * This LUN is already locked, so we're done.
5917                          */
5918                         retval = CTL_RETVAL_COMPLETE;
5919                 } else if (softc->aps_locked_lun == 0) {
5920                         /*
5921                          * No one has the lock, pass the request to the
5922                          * backend.
5923                          */
5924                         retval = lun->backend->config_write(
5925                                 (union ctl_io *)ctsio);
5926                 } else {
5927                         /*
5928                          * Someone else has the lock, throw out the request.
5929                          */
5930                         ctl_set_already_locked(ctsio);
5931                         free(ctsio->kern_data_ptr, M_CTL);
5932                         ctl_done((union ctl_io *)ctsio);
5933
5934                         /*
5935                          * Set the return value so that ctl_do_mode_select()
5936                          * won't try to complete the command.  We already
5937                          * completed it here.
5938                          */
5939                         retval = CTL_RETVAL_ERROR;
5940                 }
5941         } else if (softc->aps_locked_lun == lun->lun) {
5942                 /*
5943                  * This LUN is locked, so pass the unlock request to the
5944                  * backend.
5945                  */
5946                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5947         }
5948         mtx_unlock(&softc->ctl_lock);
5949
5950         return (retval);
5951 }
5952
5953 int
5954 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
5955                                 struct ctl_page_index *page_index,
5956                                 uint8_t *page_ptr)
5957 {
5958         uint8_t *c;
5959         int i;
5960
5961         c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
5962         ctl_time_io_secs =
5963                 (c[0] << 8) |
5964                 (c[1] << 0) |
5965                 0;
5966         CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
5967         printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
5968         printf("page data:");
5969         for (i=0; i<8; i++)
5970                 printf(" %.2x",page_ptr[i]);
5971         printf("\n");
5972         return (0);
5973 }
5974
5975 int
5976 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
5977                                struct ctl_page_index *page_index,
5978                                int pc)
5979 {
5980         struct copan_debugconf_subpage *page;
5981
5982         page = (struct copan_debugconf_subpage *)page_index->page_data +
5983                 (page_index->page_len * pc);
5984
5985         switch (pc) {
5986         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
5987         case SMS_PAGE_CTRL_DEFAULT >> 6:
5988         case SMS_PAGE_CTRL_SAVED >> 6:
5989                 /*
5990                  * We don't update the changable or default bits for this page.
5991                  */
5992                 break;
5993         case SMS_PAGE_CTRL_CURRENT >> 6:
5994                 page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
5995                 page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
5996                 break;
5997         default:
5998 #ifdef NEEDTOPORT
5999                 EPRINT(0, "Invalid PC %d!!", pc);
6000 #endif /* NEEDTOPORT */
6001                 break;
6002         }
6003         return (0);
6004 }
6005
6006
6007 static int
6008 ctl_do_mode_select(union ctl_io *io)
6009 {
6010         struct scsi_mode_page_header *page_header;
6011         struct ctl_page_index *page_index;
6012         struct ctl_scsiio *ctsio;
6013         int control_dev, page_len;
6014         int page_len_offset, page_len_size;
6015         union ctl_modepage_info *modepage_info;
6016         struct ctl_lun *lun;
6017         int *len_left, *len_used;
6018         int retval, i;
6019
6020         ctsio = &io->scsiio;
6021         page_index = NULL;
6022         page_len = 0;
6023         retval = CTL_RETVAL_COMPLETE;
6024
6025         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6026
6027         if (lun->be_lun->lun_type != T_DIRECT)
6028                 control_dev = 1;
6029         else
6030                 control_dev = 0;
6031
6032         modepage_info = (union ctl_modepage_info *)
6033                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6034         len_left = &modepage_info->header.len_left;
6035         len_used = &modepage_info->header.len_used;
6036
6037 do_next_page:
6038
6039         page_header = (struct scsi_mode_page_header *)
6040                 (ctsio->kern_data_ptr + *len_used);
6041
6042         if (*len_left == 0) {
6043                 free(ctsio->kern_data_ptr, M_CTL);
6044                 ctl_set_success(ctsio);
6045                 ctl_done((union ctl_io *)ctsio);
6046                 return (CTL_RETVAL_COMPLETE);
6047         } else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6048
6049                 free(ctsio->kern_data_ptr, M_CTL);
6050                 ctl_set_param_len_error(ctsio);
6051                 ctl_done((union ctl_io *)ctsio);
6052                 return (CTL_RETVAL_COMPLETE);
6053
6054         } else if ((page_header->page_code & SMPH_SPF)
6055                 && (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6056
6057                 free(ctsio->kern_data_ptr, M_CTL);
6058                 ctl_set_param_len_error(ctsio);
6059                 ctl_done((union ctl_io *)ctsio);
6060                 return (CTL_RETVAL_COMPLETE);
6061         }
6062
6063
6064         /*
6065          * XXX KDM should we do something with the block descriptor?
6066          */
6067         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6068
6069                 if ((control_dev != 0)
6070                  && (lun->mode_pages.index[i].page_flags &
6071                      CTL_PAGE_FLAG_DISK_ONLY))
6072                         continue;
6073
6074                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6075                     (page_header->page_code & SMPH_PC_MASK))
6076                         continue;
6077
6078                 /*
6079                  * If neither page has a subpage code, then we've got a
6080                  * match.
6081                  */
6082                 if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6083                  && ((page_header->page_code & SMPH_SPF) == 0)) {
6084                         page_index = &lun->mode_pages.index[i];
6085                         page_len = page_header->page_length;
6086                         break;
6087                 }
6088
6089                 /*
6090                  * If both pages have subpages, then the subpage numbers
6091                  * have to match.
6092                  */
6093                 if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6094                   && (page_header->page_code & SMPH_SPF)) {
6095                         struct scsi_mode_page_header_sp *sph;
6096
6097                         sph = (struct scsi_mode_page_header_sp *)page_header;
6098
6099                         if (lun->mode_pages.index[i].subpage ==
6100                             sph->subpage) {
6101                                 page_index = &lun->mode_pages.index[i];
6102                                 page_len = scsi_2btoul(sph->page_length);
6103                                 break;
6104                         }
6105                 }
6106         }
6107
6108         /*
6109          * If we couldn't find the page, or if we don't have a mode select
6110          * handler for it, send back an error to the user.
6111          */
6112         if ((page_index == NULL)
6113          || (page_index->select_handler == NULL)) {
6114                 ctl_set_invalid_field(ctsio,
6115                                       /*sks_valid*/ 1,
6116                                       /*command*/ 0,
6117                                       /*field*/ *len_used,
6118                                       /*bit_valid*/ 0,
6119                                       /*bit*/ 0);
6120                 free(ctsio->kern_data_ptr, M_CTL);
6121                 ctl_done((union ctl_io *)ctsio);
6122                 return (CTL_RETVAL_COMPLETE);
6123         }
6124
6125         if (page_index->page_code & SMPH_SPF) {
6126                 page_len_offset = 2;
6127                 page_len_size = 2;
6128         } else {
6129                 page_len_size = 1;
6130                 page_len_offset = 1;
6131         }
6132
6133         /*
6134          * If the length the initiator gives us isn't the one we specify in
6135          * the mode page header, or if they didn't specify enough data in
6136          * the CDB to avoid truncating this page, kick out the request.
6137          */
6138         if ((page_len != (page_index->page_len - page_len_offset -
6139                           page_len_size))
6140          || (*len_left < page_index->page_len)) {
6141
6142
6143                 ctl_set_invalid_field(ctsio,
6144                                       /*sks_valid*/ 1,
6145                                       /*command*/ 0,
6146                                       /*field*/ *len_used + page_len_offset,
6147                                       /*bit_valid*/ 0,
6148                                       /*bit*/ 0);
6149                 free(ctsio->kern_data_ptr, M_CTL);
6150                 ctl_done((union ctl_io *)ctsio);
6151                 return (CTL_RETVAL_COMPLETE);
6152         }
6153
6154         /*
6155          * Run through the mode page, checking to make sure that the bits
6156          * the user changed are actually legal for him to change.
6157          */
6158         for (i = 0; i < page_index->page_len; i++) {
6159                 uint8_t *user_byte, *change_mask, *current_byte;
6160                 int bad_bit;
6161                 int j;
6162
6163                 user_byte = (uint8_t *)page_header + i;
6164                 change_mask = page_index->page_data +
6165                               (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6166                 current_byte = page_index->page_data +
6167                                (page_index->page_len * CTL_PAGE_CURRENT) + i;
6168
6169                 /*
6170                  * Check to see whether the user set any bits in this byte
6171                  * that he is not allowed to set.
6172                  */
6173                 if ((*user_byte & ~(*change_mask)) ==
6174                     (*current_byte & ~(*change_mask)))
6175                         continue;
6176
6177                 /*
6178                  * Go through bit by bit to determine which one is illegal.
6179                  */
6180                 bad_bit = 0;
6181                 for (j = 7; j >= 0; j--) {
6182                         if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6183                             (((1 << i) & ~(*change_mask)) & *current_byte)) {
6184                                 bad_bit = i;
6185                                 break;
6186                         }
6187                 }
6188                 ctl_set_invalid_field(ctsio,
6189                                       /*sks_valid*/ 1,
6190                                       /*command*/ 0,
6191                                       /*field*/ *len_used + i,
6192                                       /*bit_valid*/ 1,
6193                                       /*bit*/ bad_bit);
6194                 free(ctsio->kern_data_ptr, M_CTL);
6195                 ctl_done((union ctl_io *)ctsio);
6196                 return (CTL_RETVAL_COMPLETE);
6197         }
6198
6199         /*
6200          * Decrement these before we call the page handler, since we may
6201          * end up getting called back one way or another before the handler
6202          * returns to this context.
6203          */
6204         *len_left -= page_index->page_len;
6205         *len_used += page_index->page_len;
6206
6207         retval = page_index->select_handler(ctsio, page_index,
6208                                             (uint8_t *)page_header);
6209
6210         /*
6211          * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6212          * wait until this queued command completes to finish processing
6213          * the mode page.  If it returns anything other than
6214          * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6215          * already set the sense information, freed the data pointer, and
6216          * completed the io for us.
6217          */
6218         if (retval != CTL_RETVAL_COMPLETE)
6219                 goto bailout_no_done;
6220
6221         /*
6222          * If the initiator sent us more than one page, parse the next one.
6223          */
6224         if (*len_left > 0)
6225                 goto do_next_page;
6226
6227         ctl_set_success(ctsio);
6228         free(ctsio->kern_data_ptr, M_CTL);
6229         ctl_done((union ctl_io *)ctsio);
6230
6231 bailout_no_done:
6232
6233         return (CTL_RETVAL_COMPLETE);
6234
6235 }
6236
6237 int
6238 ctl_mode_select(struct ctl_scsiio *ctsio)
6239 {
6240         int param_len, pf, sp;
6241         int header_size, bd_len;
6242         int len_left, len_used;
6243         struct ctl_page_index *page_index;
6244         struct ctl_lun *lun;
6245         int control_dev, page_len;
6246         union ctl_modepage_info *modepage_info;
6247         int retval;
6248
6249         pf = 0;
6250         sp = 0;
6251         page_len = 0;
6252         len_used = 0;
6253         len_left = 0;
6254         retval = 0;
6255         bd_len = 0;
6256         page_index = NULL;
6257
6258         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6259
6260         if (lun->be_lun->lun_type != T_DIRECT)
6261                 control_dev = 1;
6262         else
6263                 control_dev = 0;
6264
6265         switch (ctsio->cdb[0]) {
6266         case MODE_SELECT_6: {
6267                 struct scsi_mode_select_6 *cdb;
6268
6269                 cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6270
6271                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6272                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6273
6274                 param_len = cdb->length;
6275                 header_size = sizeof(struct scsi_mode_header_6);
6276                 break;
6277         }
6278         case MODE_SELECT_10: {
6279                 struct scsi_mode_select_10 *cdb;
6280
6281                 cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6282
6283                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6284                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6285
6286                 param_len = scsi_2btoul(cdb->length);
6287                 header_size = sizeof(struct scsi_mode_header_10);
6288                 break;
6289         }
6290         default:
6291                 ctl_set_invalid_opcode(ctsio);
6292                 ctl_done((union ctl_io *)ctsio);
6293                 return (CTL_RETVAL_COMPLETE);
6294                 break; /* NOTREACHED */
6295         }
6296
6297         /*
6298          * From SPC-3:
6299          * "A parameter list length of zero indicates that the Data-Out Buffer
6300          * shall be empty. This condition shall not be considered as an error."
6301          */
6302         if (param_len == 0) {
6303                 ctl_set_success(ctsio);
6304                 ctl_done((union ctl_io *)ctsio);
6305                 return (CTL_RETVAL_COMPLETE);
6306         }
6307
6308         /*
6309          * Since we'll hit this the first time through, prior to
6310          * allocation, we don't need to free a data buffer here.
6311          */
6312         if (param_len < header_size) {
6313                 ctl_set_param_len_error(ctsio);
6314                 ctl_done((union ctl_io *)ctsio);
6315                 return (CTL_RETVAL_COMPLETE);
6316         }
6317
6318         /*
6319          * Allocate the data buffer and grab the user's data.  In theory,
6320          * we shouldn't have to sanity check the parameter list length here
6321          * because the maximum size is 64K.  We should be able to malloc
6322          * that much without too many problems.
6323          */
6324         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6325                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6326                 if (ctsio->kern_data_ptr == NULL) {
6327                         ctl_set_busy(ctsio);
6328                         ctl_done((union ctl_io *)ctsio);
6329                         return (CTL_RETVAL_COMPLETE);
6330                 }
6331                 ctsio->kern_data_len = param_len;
6332                 ctsio->kern_total_len = param_len;
6333                 ctsio->kern_data_resid = 0;
6334                 ctsio->kern_rel_offset = 0;
6335                 ctsio->kern_sg_entries = 0;
6336                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6337                 ctsio->be_move_done = ctl_config_move_done;
6338                 ctl_datamove((union ctl_io *)ctsio);
6339
6340                 return (CTL_RETVAL_COMPLETE);
6341         }
6342
6343         switch (ctsio->cdb[0]) {
6344         case MODE_SELECT_6: {
6345                 struct scsi_mode_header_6 *mh6;
6346
6347                 mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6348                 bd_len = mh6->blk_desc_len;
6349                 break;
6350         }
6351         case MODE_SELECT_10: {
6352                 struct scsi_mode_header_10 *mh10;
6353
6354                 mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6355                 bd_len = scsi_2btoul(mh10->blk_desc_len);
6356                 break;
6357         }
6358         default:
6359                 panic("Invalid CDB type %#x", ctsio->cdb[0]);
6360                 break;
6361         }
6362
6363         if (param_len < (header_size + bd_len)) {
6364                 free(ctsio->kern_data_ptr, M_CTL);
6365                 ctl_set_param_len_error(ctsio);
6366                 ctl_done((union ctl_io *)ctsio);
6367                 return (CTL_RETVAL_COMPLETE);
6368         }
6369
6370         /*
6371          * Set the IO_CONT flag, so that if this I/O gets passed to
6372          * ctl_config_write_done(), it'll get passed back to
6373          * ctl_do_mode_select() for further processing, or completion if
6374          * we're all done.
6375          */
6376         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6377         ctsio->io_cont = ctl_do_mode_select;
6378
6379         modepage_info = (union ctl_modepage_info *)
6380                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6381
6382         memset(modepage_info, 0, sizeof(*modepage_info));
6383
6384         len_left = param_len - header_size - bd_len;
6385         len_used = header_size + bd_len;
6386
6387         modepage_info->header.len_left = len_left;
6388         modepage_info->header.len_used = len_used;
6389
6390         return (ctl_do_mode_select((union ctl_io *)ctsio));
6391 }
6392
6393 int
6394 ctl_mode_sense(struct ctl_scsiio *ctsio)
6395 {
6396         struct ctl_lun *lun;
6397         int pc, page_code, dbd, llba, subpage;
6398         int alloc_len, page_len, header_len, total_len;
6399         struct scsi_mode_block_descr *block_desc;
6400         struct ctl_page_index *page_index;
6401         int control_dev;
6402
6403         dbd = 0;
6404         llba = 0;
6405         block_desc = NULL;
6406         page_index = NULL;
6407
6408         CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6409
6410         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6411
6412         if (lun->be_lun->lun_type != T_DIRECT)
6413                 control_dev = 1;
6414         else
6415                 control_dev = 0;
6416
6417         switch (ctsio->cdb[0]) {
6418         case MODE_SENSE_6: {
6419                 struct scsi_mode_sense_6 *cdb;
6420
6421                 cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6422
6423                 header_len = sizeof(struct scsi_mode_hdr_6);
6424                 if (cdb->byte2 & SMS_DBD)
6425                         dbd = 1;
6426                 else
6427                         header_len += sizeof(struct scsi_mode_block_descr);
6428
6429                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6430                 page_code = cdb->page & SMS_PAGE_CODE;
6431                 subpage = cdb->subpage;
6432                 alloc_len = cdb->length;
6433                 break;
6434         }
6435         case MODE_SENSE_10: {
6436                 struct scsi_mode_sense_10 *cdb;
6437
6438                 cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6439
6440                 header_len = sizeof(struct scsi_mode_hdr_10);
6441
6442                 if (cdb->byte2 & SMS_DBD)
6443                         dbd = 1;
6444                 else
6445                         header_len += sizeof(struct scsi_mode_block_descr);
6446                 if (cdb->byte2 & SMS10_LLBAA)
6447                         llba = 1;
6448                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6449                 page_code = cdb->page & SMS_PAGE_CODE;
6450                 subpage = cdb->subpage;
6451                 alloc_len = scsi_2btoul(cdb->length);
6452                 break;
6453         }
6454         default:
6455                 ctl_set_invalid_opcode(ctsio);
6456                 ctl_done((union ctl_io *)ctsio);
6457                 return (CTL_RETVAL_COMPLETE);
6458                 break; /* NOTREACHED */
6459         }
6460
6461         /*
6462          * We have to make a first pass through to calculate the size of
6463          * the pages that match the user's query.  Then we allocate enough
6464          * memory to hold it, and actually copy the data into the buffer.
6465          */
6466         switch (page_code) {
6467         case SMS_ALL_PAGES_PAGE: {
6468                 int i;
6469
6470                 page_len = 0;
6471
6472                 /*
6473                  * At the moment, values other than 0 and 0xff here are
6474                  * reserved according to SPC-3.
6475                  */
6476                 if ((subpage != SMS_SUBPAGE_PAGE_0)
6477                  && (subpage != SMS_SUBPAGE_ALL)) {
6478                         ctl_set_invalid_field(ctsio,
6479                                               /*sks_valid*/ 1,
6480                                               /*command*/ 1,
6481                                               /*field*/ 3,
6482                                               /*bit_valid*/ 0,
6483                                               /*bit*/ 0);
6484                         ctl_done((union ctl_io *)ctsio);
6485                         return (CTL_RETVAL_COMPLETE);
6486                 }
6487
6488                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6489                         if ((control_dev != 0)
6490                          && (lun->mode_pages.index[i].page_flags &
6491                              CTL_PAGE_FLAG_DISK_ONLY))
6492                                 continue;
6493
6494                         /*
6495                          * We don't use this subpage if the user didn't
6496                          * request all subpages.
6497                          */
6498                         if ((lun->mode_pages.index[i].subpage != 0)
6499                          && (subpage == SMS_SUBPAGE_PAGE_0))
6500                                 continue;
6501
6502 #if 0
6503                         printf("found page %#x len %d\n",
6504                                lun->mode_pages.index[i].page_code &
6505                                SMPH_PC_MASK,
6506                                lun->mode_pages.index[i].page_len);
6507 #endif
6508                         page_len += lun->mode_pages.index[i].page_len;
6509                 }
6510                 break;
6511         }
6512         default: {
6513                 int i;
6514
6515                 page_len = 0;
6516
6517                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6518                         /* Look for the right page code */
6519                         if ((lun->mode_pages.index[i].page_code &
6520                              SMPH_PC_MASK) != page_code)
6521                                 continue;
6522
6523                         /* Look for the right subpage or the subpage wildcard*/
6524                         if ((lun->mode_pages.index[i].subpage != subpage)
6525                          && (subpage != SMS_SUBPAGE_ALL))
6526                                 continue;
6527
6528                         /* Make sure the page is supported for this dev type */
6529                         if ((control_dev != 0)
6530                          && (lun->mode_pages.index[i].page_flags &
6531                              CTL_PAGE_FLAG_DISK_ONLY))
6532                                 continue;
6533
6534 #if 0
6535                         printf("found page %#x len %d\n",
6536                                lun->mode_pages.index[i].page_code &
6537                                SMPH_PC_MASK,
6538                                lun->mode_pages.index[i].page_len);
6539 #endif
6540
6541                         page_len += lun->mode_pages.index[i].page_len;
6542                 }
6543
6544                 if (page_len == 0) {
6545                         ctl_set_invalid_field(ctsio,
6546                                               /*sks_valid*/ 1,
6547                                               /*command*/ 1,
6548                                               /*field*/ 2,
6549                                               /*bit_valid*/ 1,
6550                                               /*bit*/ 5);
6551                         ctl_done((union ctl_io *)ctsio);
6552                         return (CTL_RETVAL_COMPLETE);
6553                 }
6554                 break;
6555         }
6556         }
6557
6558         total_len = header_len + page_len;
6559 #if 0
6560         printf("header_len = %d, page_len = %d, total_len = %d\n",
6561                header_len, page_len, total_len);
6562 #endif
6563
6564         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK);
6565         if (ctsio->kern_data_ptr == NULL) {
6566                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
6567                 ctsio->scsi_status = SCSI_STATUS_BUSY;
6568                 ctl_done((union ctl_io *)ctsio);
6569                 return (CTL_RETVAL_COMPLETE);
6570         }
6571         ctsio->kern_sg_entries = 0;
6572         ctsio->kern_data_resid = 0;
6573         ctsio->kern_rel_offset = 0;
6574         if (total_len < alloc_len) {
6575                 ctsio->residual = alloc_len - total_len;
6576                 ctsio->kern_data_len = total_len;
6577                 ctsio->kern_total_len = total_len;
6578         } else {
6579                 ctsio->residual = 0;
6580                 ctsio->kern_data_len = alloc_len;
6581                 ctsio->kern_total_len = alloc_len;
6582         }
6583         memset(ctsio->kern_data_ptr, 0, total_len);
6584
6585         switch (ctsio->cdb[0]) {
6586         case MODE_SENSE_6: {
6587                 struct scsi_mode_hdr_6 *header;
6588
6589                 header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6590
6591                 header->datalen = ctl_min(total_len - 1, 254);
6592
6593                 if (dbd)
6594                         header->block_descr_len = 0;
6595                 else
6596                         header->block_descr_len =
6597                                 sizeof(struct scsi_mode_block_descr);
6598                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6599                 break;
6600         }
6601         case MODE_SENSE_10: {
6602                 struct scsi_mode_hdr_10 *header;
6603                 int datalen;
6604
6605                 header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6606
6607                 datalen = ctl_min(total_len - 2, 65533);
6608                 scsi_ulto2b(datalen, header->datalen);
6609                 if (dbd)
6610                         scsi_ulto2b(0, header->block_descr_len);
6611                 else
6612                         scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
6613                                     header->block_descr_len);
6614                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6615                 break;
6616         }
6617         default:
6618                 panic("invalid CDB type %#x", ctsio->cdb[0]);
6619                 break; /* NOTREACHED */
6620         }
6621
6622         /*
6623          * If we've got a disk, use its blocksize in the block
6624          * descriptor.  Otherwise, just set it to 0.
6625          */
6626         if (dbd == 0) {
6627                 if (control_dev != 0)
6628                         scsi_ulto3b(lun->be_lun->blocksize,
6629                                     block_desc->block_len);
6630                 else
6631                         scsi_ulto3b(0, block_desc->block_len);
6632         }
6633
6634         switch (page_code) {
6635         case SMS_ALL_PAGES_PAGE: {
6636                 int i, data_used;
6637
6638                 data_used = header_len;
6639                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6640                         struct ctl_page_index *page_index;
6641
6642                         page_index = &lun->mode_pages.index[i];
6643
6644                         if ((control_dev != 0)
6645                          && (page_index->page_flags &
6646                             CTL_PAGE_FLAG_DISK_ONLY))
6647                                 continue;
6648
6649                         /*
6650                          * We don't use this subpage if the user didn't
6651                          * request all subpages.  We already checked (above)
6652                          * to make sure the user only specified a subpage
6653                          * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
6654                          */
6655                         if ((page_index->subpage != 0)
6656                          && (subpage == SMS_SUBPAGE_PAGE_0))
6657                                 continue;
6658
6659                         /*
6660                          * Call the handler, if it exists, to update the
6661                          * page to the latest values.
6662                          */
6663                         if (page_index->sense_handler != NULL)
6664                                 page_index->sense_handler(ctsio, page_index,pc);
6665
6666                         memcpy(ctsio->kern_data_ptr + data_used,
6667                                page_index->page_data +
6668                                (page_index->page_len * pc),
6669                                page_index->page_len);
6670                         data_used += page_index->page_len;
6671                 }
6672                 break;
6673         }
6674         default: {
6675                 int i, data_used;
6676
6677                 data_used = header_len;
6678
6679                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6680                         struct ctl_page_index *page_index;
6681
6682                         page_index = &lun->mode_pages.index[i];
6683
6684                         /* Look for the right page code */
6685                         if ((page_index->page_code & SMPH_PC_MASK) != page_code)
6686                                 continue;
6687
6688                         /* Look for the right subpage or the subpage wildcard*/
6689                         if ((page_index->subpage != subpage)
6690                          && (subpage != SMS_SUBPAGE_ALL))
6691                                 continue;
6692
6693                         /* Make sure the page is supported for this dev type */
6694                         if ((control_dev != 0)
6695                          && (page_index->page_flags &
6696                              CTL_PAGE_FLAG_DISK_ONLY))
6697                                 continue;
6698
6699                         /*
6700                          * Call the handler, if it exists, to update the
6701                          * page to the latest values.
6702                          */
6703                         if (page_index->sense_handler != NULL)
6704                                 page_index->sense_handler(ctsio, page_index,pc);
6705
6706                         memcpy(ctsio->kern_data_ptr + data_used,
6707                                page_index->page_data +
6708                                (page_index->page_len * pc),
6709                                page_index->page_len);
6710                         data_used += page_index->page_len;
6711                 }
6712                 break;
6713         }
6714         }
6715
6716         ctsio->scsi_status = SCSI_STATUS_OK;
6717
6718         ctsio->be_move_done = ctl_config_move_done;
6719         ctl_datamove((union ctl_io *)ctsio);
6720
6721         return (CTL_RETVAL_COMPLETE);
6722 }
6723
6724 int
6725 ctl_read_capacity(struct ctl_scsiio *ctsio)
6726 {
6727         struct scsi_read_capacity *cdb;
6728         struct scsi_read_capacity_data *data;
6729         struct ctl_lun *lun;
6730         uint32_t lba;
6731
6732         CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
6733
6734         cdb = (struct scsi_read_capacity *)ctsio->cdb;
6735
6736         lba = scsi_4btoul(cdb->addr);
6737         if (((cdb->pmi & SRC_PMI) == 0)
6738          && (lba != 0)) {
6739                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6740                                       /*sks_valid*/ 1,
6741                                       /*command*/ 1,
6742                                       /*field*/ 2,
6743                                       /*bit_valid*/ 0,
6744                                       /*bit*/ 0);
6745                 ctl_done((union ctl_io *)ctsio);
6746                 return (CTL_RETVAL_COMPLETE);
6747         }
6748
6749         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6750
6751         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK);
6752         if (ctsio->kern_data_ptr == NULL) {
6753                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
6754                 ctsio->scsi_status = SCSI_STATUS_BUSY;
6755                 ctl_done((union ctl_io *)ctsio);
6756                 return (CTL_RETVAL_COMPLETE);
6757         }
6758         data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
6759         ctsio->residual = 0;
6760         ctsio->kern_data_len = sizeof(*data);
6761         ctsio->kern_total_len = sizeof(*data);
6762         ctsio->kern_data_resid = 0;
6763         ctsio->kern_rel_offset = 0;
6764         ctsio->kern_sg_entries = 0;
6765
6766         memset(data, 0, sizeof(*data));
6767
6768         /*
6769          * If the maximum LBA is greater than 0xfffffffe, the user must
6770          * issue a SERVICE ACTION IN (16) command, with the read capacity
6771          * serivce action set.
6772          */
6773         if (lun->be_lun->maxlba > 0xfffffffe)
6774                 scsi_ulto4b(0xffffffff, data->addr);
6775         else
6776                 scsi_ulto4b(lun->be_lun->maxlba, data->addr);
6777
6778         /*
6779          * XXX KDM this may not be 512 bytes...
6780          */
6781         scsi_ulto4b(lun->be_lun->blocksize, data->length);
6782
6783         ctsio->scsi_status = SCSI_STATUS_OK;
6784
6785         ctsio->be_move_done = ctl_config_move_done;
6786         ctl_datamove((union ctl_io *)ctsio);
6787
6788         return (CTL_RETVAL_COMPLETE);
6789 }
6790
6791 static int
6792 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
6793 {
6794         struct scsi_read_capacity_16 *cdb;
6795         struct scsi_read_capacity_data_long *data;
6796         struct ctl_lun *lun;
6797         uint64_t lba;
6798         uint32_t alloc_len;
6799
6800         CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
6801
6802         cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
6803
6804         alloc_len = scsi_4btoul(cdb->alloc_len);
6805         lba = scsi_8btou64(cdb->addr);
6806
6807         if ((cdb->reladr & SRC16_PMI)
6808          && (lba != 0)) {
6809                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6810                                       /*sks_valid*/ 1,
6811                                       /*command*/ 1,
6812                                       /*field*/ 2,
6813                                       /*bit_valid*/ 0,
6814                                       /*bit*/ 0);
6815                 ctl_done((union ctl_io *)ctsio);
6816                 return (CTL_RETVAL_COMPLETE);
6817         }
6818
6819         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6820
6821         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK);
6822         if (ctsio->kern_data_ptr == NULL) {
6823                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
6824                 ctsio->scsi_status = SCSI_STATUS_BUSY;
6825                 ctl_done((union ctl_io *)ctsio);
6826                 return (CTL_RETVAL_COMPLETE);
6827         }
6828         data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
6829
6830         if (sizeof(*data) < alloc_len) {
6831                 ctsio->residual = alloc_len - sizeof(*data);
6832                 ctsio->kern_data_len = sizeof(*data);
6833                 ctsio->kern_total_len = sizeof(*data);
6834         } else {
6835                 ctsio->residual = 0;
6836                 ctsio->kern_data_len = alloc_len;
6837                 ctsio->kern_total_len = alloc_len;
6838         }
6839         ctsio->kern_data_resid = 0;
6840         ctsio->kern_rel_offset = 0;
6841         ctsio->kern_sg_entries = 0;
6842
6843         memset(data, 0, sizeof(*data));
6844
6845         scsi_u64to8b(lun->be_lun->maxlba, data->addr);
6846         /* XXX KDM this may not be 512 bytes... */
6847         scsi_ulto4b(lun->be_lun->blocksize, data->length);
6848
6849         ctsio->scsi_status = SCSI_STATUS_OK;
6850
6851         ctsio->be_move_done = ctl_config_move_done;
6852         ctl_datamove((union ctl_io *)ctsio);
6853
6854         return (CTL_RETVAL_COMPLETE);
6855 }
6856
6857 int
6858 ctl_service_action_in(struct ctl_scsiio *ctsio)
6859 {
6860         struct scsi_service_action_in *cdb;
6861         int retval;
6862
6863         CTL_DEBUG_PRINT(("ctl_service_action_in\n"));
6864
6865         cdb = (struct scsi_service_action_in *)ctsio->cdb;
6866
6867         retval = CTL_RETVAL_COMPLETE;
6868
6869         switch (cdb->service_action) {
6870         case SRC16_SERVICE_ACTION:
6871                 retval = ctl_read_capacity_16(ctsio);
6872                 break;
6873         default:
6874                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6875                                       /*sks_valid*/ 1,
6876                                       /*command*/ 1,
6877                                       /*field*/ 1,
6878                                       /*bit_valid*/ 1,
6879                                       /*bit*/ 4);
6880                 ctl_done((union ctl_io *)ctsio);
6881                 break;
6882         }
6883
6884         return (retval);
6885 }
6886
6887 int
6888 ctl_maintenance_in(struct ctl_scsiio *ctsio)
6889 {
6890         struct scsi_maintenance_in *cdb;
6891         int retval;
6892         int alloc_len, total_len = 0;
6893         int num_target_port_groups;
6894         struct ctl_lun *lun;
6895         struct ctl_softc *softc;
6896         struct scsi_target_group_data *rtg_ptr;
6897         struct scsi_target_port_group_descriptor *tpg_desc_ptr1, *tpg_desc_ptr2;
6898         struct scsi_target_port_descriptor  *tp_desc_ptr1_1, *tp_desc_ptr1_2,
6899                                             *tp_desc_ptr2_1, *tp_desc_ptr2_2;
6900
6901         CTL_DEBUG_PRINT(("ctl_maintenance_in\n"));
6902
6903         cdb = (struct scsi_maintenance_in *)ctsio->cdb;
6904         softc = control_softc;
6905         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6906
6907         retval = CTL_RETVAL_COMPLETE;
6908         mtx_lock(&softc->ctl_lock);
6909
6910         if ((cdb->byte2 & SERVICE_ACTION_MASK) != SA_RPRT_TRGT_GRP) {
6911                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6912                                       /*sks_valid*/ 1,
6913                                       /*command*/ 1,
6914                                       /*field*/ 1,
6915                                       /*bit_valid*/ 1,
6916                                       /*bit*/ 4);
6917                 ctl_done((union ctl_io *)ctsio);
6918                 return(retval);
6919         }
6920
6921         if (ctl_is_single)
6922                 num_target_port_groups = NUM_TARGET_PORT_GROUPS - 1;
6923         else
6924                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
6925
6926         total_len = sizeof(struct scsi_target_group_data) +
6927                 sizeof(struct scsi_target_port_group_descriptor) *
6928                 num_target_port_groups +
6929                 sizeof(struct scsi_target_port_descriptor) *
6930                 NUM_PORTS_PER_GRP * num_target_port_groups;
6931
6932         alloc_len = scsi_4btoul(cdb->length);
6933
6934         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK);
6935         if (ctsio->kern_data_ptr == NULL) {
6936                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
6937                 ctsio->scsi_status = SCSI_STATUS_BUSY;
6938                 ctl_done((union ctl_io *)ctsio);
6939                 return (CTL_RETVAL_COMPLETE);
6940         }
6941         memset(ctsio->kern_data_ptr, 0, total_len);
6942
6943         ctsio->kern_sg_entries = 0;
6944
6945         if (total_len < alloc_len) {
6946                 ctsio->residual = alloc_len - total_len;
6947                 ctsio->kern_data_len = total_len;
6948                 ctsio->kern_total_len = total_len;
6949         } else {
6950                 ctsio->residual = 0;
6951                 ctsio->kern_data_len = alloc_len;
6952                 ctsio->kern_total_len = alloc_len;
6953         }
6954         ctsio->kern_data_resid = 0;
6955         ctsio->kern_rel_offset = 0;
6956
6957         rtg_ptr = (struct scsi_target_group_data *)ctsio->kern_data_ptr;
6958
6959         tpg_desc_ptr1 = &rtg_ptr->groups[0];
6960         tp_desc_ptr1_1 = &tpg_desc_ptr1->descriptors[0];
6961         tp_desc_ptr1_2 = (struct scsi_target_port_descriptor *)
6962                 &tp_desc_ptr1_1->desc_list[0];
6963
6964
6965
6966         if (ctl_is_single == 0) {
6967                 tpg_desc_ptr2 = (struct scsi_target_port_group_descriptor *)
6968                         &tp_desc_ptr1_2->desc_list[0];
6969                 tp_desc_ptr2_1 = &tpg_desc_ptr2->descriptors[0];
6970                 tp_desc_ptr2_2 = (struct scsi_target_port_descriptor *)
6971                         &tp_desc_ptr2_1->desc_list[0];
6972         } else {
6973                 tpg_desc_ptr2 = NULL;
6974                 tp_desc_ptr2_1 = NULL;
6975                 tp_desc_ptr2_2 = NULL;
6976         }
6977
6978         scsi_ulto4b(total_len - 4, rtg_ptr->length);
6979         if (ctl_is_single == 0) {
6980                 if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
6981                         if (lun->flags & CTL_LUN_PRIMARY_SC) {
6982                                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
6983                                 tpg_desc_ptr2->pref_state =
6984                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
6985                         } else {
6986                                 tpg_desc_ptr1->pref_state =
6987                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
6988                                 tpg_desc_ptr2->pref_state = TPG_PRIMARY;
6989                         }
6990                 } else {
6991                         if (lun->flags & CTL_LUN_PRIMARY_SC) {
6992                                 tpg_desc_ptr1->pref_state =
6993                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
6994                                 tpg_desc_ptr2->pref_state = TPG_PRIMARY;
6995                         } else {
6996                                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
6997                                 tpg_desc_ptr2->pref_state =
6998                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
6999                         }
7000                 }
7001         } else {
7002                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7003         }
7004         tpg_desc_ptr1->support = 0;
7005         tpg_desc_ptr1->target_port_group[1] = 1;
7006         tpg_desc_ptr1->status = TPG_IMPLICIT;
7007         tpg_desc_ptr1->target_port_count= NUM_PORTS_PER_GRP;
7008
7009         if (ctl_is_single == 0) {
7010                 tpg_desc_ptr2->support = 0;
7011                 tpg_desc_ptr2->target_port_group[1] = 2;
7012                 tpg_desc_ptr2->status = TPG_IMPLICIT;
7013                 tpg_desc_ptr2->target_port_count = NUM_PORTS_PER_GRP;
7014
7015                 tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7016                 tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7017
7018                 tp_desc_ptr2_1->relative_target_port_identifier[1] = 9;
7019                 tp_desc_ptr2_2->relative_target_port_identifier[1] = 10;
7020         } else {
7021                 if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7022                         tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7023                         tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7024                 } else {
7025                         tp_desc_ptr1_1->relative_target_port_identifier[1] = 9;
7026                         tp_desc_ptr1_2->relative_target_port_identifier[1] = 10;
7027                 }
7028         }
7029
7030         mtx_unlock(&softc->ctl_lock);
7031
7032         ctsio->be_move_done = ctl_config_move_done;
7033
7034         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7035                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7036                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7037                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7038                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7039
7040         ctl_datamove((union ctl_io *)ctsio);
7041         return(retval);
7042 }
7043
7044 int
7045 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7046 {
7047         struct scsi_per_res_in *cdb;
7048         int alloc_len, total_len = 0;
7049         /* struct scsi_per_res_in_rsrv in_data; */
7050         struct ctl_lun *lun;
7051         struct ctl_softc *softc;
7052
7053         CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7054
7055         softc = control_softc;
7056
7057         cdb = (struct scsi_per_res_in *)ctsio->cdb;
7058
7059         alloc_len = scsi_2btoul(cdb->length);
7060
7061         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7062
7063 retry:
7064         mtx_lock(&softc->ctl_lock);
7065         switch (cdb->action) {
7066         case SPRI_RK: /* read keys */
7067                 total_len = sizeof(struct scsi_per_res_in_keys) +
7068                         lun->pr_key_count *
7069                         sizeof(struct scsi_per_res_key);
7070                 break;
7071         case SPRI_RR: /* read reservation */
7072                 if (lun->flags & CTL_LUN_PR_RESERVED)
7073                         total_len = sizeof(struct scsi_per_res_in_rsrv);
7074                 else
7075                         total_len = sizeof(struct scsi_per_res_in_header);
7076                 break;
7077         case SPRI_RC: /* report capabilities */
7078                 total_len = sizeof(struct scsi_per_res_cap);
7079                 break;
7080         case SPRI_RS: /* read full status */
7081         default:
7082                 mtx_unlock(&softc->ctl_lock);
7083                 ctl_set_invalid_field(ctsio,
7084                                       /*sks_valid*/ 1,
7085                                       /*command*/ 1,
7086                                       /*field*/ 1,
7087                                       /*bit_valid*/ 1,
7088                                       /*bit*/ 0);
7089                 ctl_done((union ctl_io *)ctsio);
7090                 return (CTL_RETVAL_COMPLETE);
7091                 break; /* NOTREACHED */
7092         }
7093         mtx_unlock(&softc->ctl_lock);
7094
7095         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK);
7096         if (ctsio->kern_data_ptr == NULL) {
7097                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
7098                 ctsio->scsi_status = SCSI_STATUS_BUSY;
7099                 ctl_done((union ctl_io *)ctsio);
7100                 return (CTL_RETVAL_COMPLETE);
7101         }
7102
7103         if (total_len < alloc_len) {
7104                 ctsio->residual = alloc_len - total_len;
7105                 ctsio->kern_data_len = total_len;
7106                 ctsio->kern_total_len = total_len;
7107         } else {
7108                 ctsio->residual = 0;
7109                 ctsio->kern_data_len = alloc_len;
7110                 ctsio->kern_total_len = alloc_len;
7111         }
7112
7113         ctsio->kern_data_resid = 0;
7114         ctsio->kern_rel_offset = 0;
7115         ctsio->kern_sg_entries = 0;
7116
7117         memset(ctsio->kern_data_ptr, 0, total_len);
7118
7119         mtx_lock(&softc->ctl_lock);
7120         switch (cdb->action) {
7121         case SPRI_RK: { // read keys
7122         struct scsi_per_res_in_keys *res_keys;
7123                 int i, key_count;
7124
7125                 res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7126
7127                 /*
7128                  * We had to drop the lock to allocate our buffer, which
7129                  * leaves time for someone to come in with another
7130                  * persistent reservation.  (That is unlikely, though,
7131                  * since this should be the only persistent reservation
7132                  * command active right now.)
7133                  */
7134                 if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7135                     (lun->pr_key_count *
7136                      sizeof(struct scsi_per_res_key)))){
7137                         mtx_unlock(&softc->ctl_lock);
7138                         free(ctsio->kern_data_ptr, M_CTL);
7139                         printf("%s: reservation length changed, retrying\n",
7140                                __func__);
7141                         goto retry;
7142                 }
7143
7144                 scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7145
7146                 scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7147                              lun->pr_key_count, res_keys->header.length);
7148
7149                 for (i = 0, key_count = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7150                         if (!lun->per_res[i].registered)
7151                                 continue;
7152
7153                         /*
7154                          * We used lun->pr_key_count to calculate the
7155                          * size to allocate.  If it turns out the number of
7156                          * initiators with the registered flag set is
7157                          * larger than that (i.e. they haven't been kept in
7158                          * sync), we've got a problem.
7159                          */
7160                         if (key_count >= lun->pr_key_count) {
7161 #ifdef NEEDTOPORT
7162                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
7163                                             CTL_PR_ERROR,
7164                                             csevent_LogType_Fault,
7165                                             csevent_AlertLevel_Yellow,
7166                                             csevent_FRU_ShelfController,
7167                                             csevent_FRU_Firmware,
7168                                         csevent_FRU_Unknown,
7169                                             "registered keys %d >= key "
7170                                             "count %d", key_count,
7171                                             lun->pr_key_count);
7172 #endif
7173                                 key_count++;
7174                                 continue;
7175                         }
7176                         memcpy(res_keys->keys[key_count].key,
7177                                lun->per_res[i].res_key.key,
7178                                ctl_min(sizeof(res_keys->keys[key_count].key),
7179                                sizeof(lun->per_res[i].res_key)));
7180                         key_count++;
7181                 }
7182                 break;
7183         }
7184         case SPRI_RR: { // read reservation
7185                 struct scsi_per_res_in_rsrv *res;
7186                 int tmp_len, header_only;
7187
7188                 res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7189
7190                 scsi_ulto4b(lun->PRGeneration, res->header.generation);
7191
7192                 if (lun->flags & CTL_LUN_PR_RESERVED)
7193                 {
7194                         tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7195                         scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7196                                     res->header.length);
7197                         header_only = 0;
7198                 } else {
7199                         tmp_len = sizeof(struct scsi_per_res_in_header);
7200                         scsi_ulto4b(0, res->header.length);
7201                         header_only = 1;
7202                 }
7203
7204                 /*
7205                  * We had to drop the lock to allocate our buffer, which
7206                  * leaves time for someone to come in with another
7207                  * persistent reservation.  (That is unlikely, though,
7208                  * since this should be the only persistent reservation
7209                  * command active right now.)
7210                  */
7211                 if (tmp_len != total_len) {
7212                         mtx_unlock(&softc->ctl_lock);
7213                         free(ctsio->kern_data_ptr, M_CTL);
7214                         printf("%s: reservation status changed, retrying\n",
7215                                __func__);
7216                         goto retry;
7217                 }
7218
7219                 /*
7220                  * No reservation held, so we're done.
7221                  */
7222                 if (header_only != 0)
7223                         break;
7224
7225                 /*
7226                  * If the registration is an All Registrants type, the key
7227                  * is 0, since it doesn't really matter.
7228                  */
7229                 if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7230                         memcpy(res->data.reservation,
7231                                &lun->per_res[lun->pr_res_idx].res_key,
7232                                sizeof(struct scsi_per_res_key));
7233                 }
7234                 res->data.scopetype = lun->res_type;
7235                 break;
7236         }
7237         case SPRI_RC:     //report capabilities
7238         {
7239                 struct scsi_per_res_cap *res_cap;
7240                 uint16_t type_mask;
7241
7242                 res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7243                 scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7244                 res_cap->flags2 |= SPRI_TMV;
7245                 type_mask = SPRI_TM_WR_EX_AR |
7246                             SPRI_TM_EX_AC_RO |
7247                             SPRI_TM_WR_EX_RO |
7248                             SPRI_TM_EX_AC |
7249                             SPRI_TM_WR_EX |
7250                             SPRI_TM_EX_AC_AR;
7251                 scsi_ulto2b(type_mask, res_cap->type_mask);
7252                 break;
7253         }
7254         case SPRI_RS: //read full status
7255         default:
7256                 /*
7257                  * This is a bug, because we just checked for this above,
7258                  * and should have returned an error.
7259                  */
7260                 panic("Invalid PR type %x", cdb->action);
7261                 break; /* NOTREACHED */
7262         }
7263         mtx_unlock(&softc->ctl_lock);
7264
7265         ctsio->be_move_done = ctl_config_move_done;
7266
7267         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7268                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7269                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7270                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7271                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7272
7273         ctl_datamove((union ctl_io *)ctsio);
7274
7275         return (CTL_RETVAL_COMPLETE);
7276 }
7277
7278 /*
7279  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
7280  * it should return.
7281  */
7282 static int
7283 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
7284                 uint64_t sa_res_key, uint8_t type, uint32_t residx,
7285                 struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
7286                 struct scsi_per_res_out_parms* param)
7287 {
7288         union ctl_ha_msg persis_io;
7289         int retval, i;
7290         int isc_retval;
7291
7292         retval = 0;
7293
7294         if (sa_res_key == 0) {
7295                 mtx_lock(&softc->ctl_lock);
7296                 if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7297                         /* validate scope and type */
7298                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7299                              SPR_LU_SCOPE) {
7300                                 mtx_unlock(&softc->ctl_lock);
7301                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7302                                                       /*sks_valid*/ 1,
7303                                                       /*command*/ 1,
7304                                                       /*field*/ 2,
7305                                                       /*bit_valid*/ 1,
7306                                                       /*bit*/ 4);
7307                                 ctl_done((union ctl_io *)ctsio);
7308                                 return (1);
7309                         }
7310
7311                         if (type>8 || type==2 || type==4 || type==0) {
7312                                 mtx_unlock(&softc->ctl_lock);
7313                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7314                                                       /*sks_valid*/ 1,
7315                                                       /*command*/ 1,
7316                                                       /*field*/ 2,
7317                                                       /*bit_valid*/ 1,
7318                                                       /*bit*/ 0);
7319                                 ctl_done((union ctl_io *)ctsio);
7320                                 return (1);
7321                         }
7322
7323                         /* temporarily unregister this nexus */
7324                         lun->per_res[residx].registered = 0;
7325
7326                         /*
7327                          * Unregister everybody else and build UA for
7328                          * them
7329                          */
7330                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7331                                 if (lun->per_res[i].registered == 0)
7332                                         continue;
7333
7334                                 if (!persis_offset
7335                                  && i <CTL_MAX_INITIATORS)
7336                                         lun->pending_sense[i].ua_pending |=
7337                                                 CTL_UA_REG_PREEMPT;
7338                                 else if (persis_offset
7339                                       && i >= persis_offset)
7340                                         lun->pending_sense[i-persis_offset
7341                                                 ].ua_pending |=
7342                                                 CTL_UA_REG_PREEMPT;
7343                                 lun->per_res[i].registered = 0;
7344                                 memset(&lun->per_res[i].res_key, 0,
7345                                        sizeof(struct scsi_per_res_key));
7346                         }
7347                         lun->per_res[residx].registered = 1;
7348                         lun->pr_key_count = 1;
7349                         lun->res_type = type;
7350                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7351                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7352                                 lun->pr_res_idx = residx;
7353
7354                         mtx_unlock(&softc->ctl_lock);
7355                         /* send msg to other side */
7356                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7357                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7358                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7359                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7360                         persis_io.pr.pr_info.res_type = type;
7361                         memcpy(persis_io.pr.pr_info.sa_res_key,
7362                                param->serv_act_res_key,
7363                                sizeof(param->serv_act_res_key));
7364                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7365                              &persis_io, sizeof(persis_io), 0)) >
7366                              CTL_HA_STATUS_SUCCESS) {
7367                                 printf("CTL:Persis Out error returned "
7368                                        "from ctl_ha_msg_send %d\n",
7369                                        isc_retval);
7370                         }
7371                 } else {
7372                         /* not all registrants */
7373                         mtx_unlock(&softc->ctl_lock);
7374                         free(ctsio->kern_data_ptr, M_CTL);
7375                         ctl_set_invalid_field(ctsio,
7376                                               /*sks_valid*/ 1,
7377                                               /*command*/ 0,
7378                                               /*field*/ 8,
7379                                               /*bit_valid*/ 0,
7380                                               /*bit*/ 0);
7381                         ctl_done((union ctl_io *)ctsio);
7382                         return (1);
7383                 }
7384         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7385                 || !(lun->flags & CTL_LUN_PR_RESERVED)) {
7386                 int found = 0;
7387
7388                 mtx_lock(&softc->ctl_lock);
7389                 if (res_key == sa_res_key) {
7390                         /* special case */
7391                         /*
7392                          * The spec implies this is not good but doesn't
7393                          * say what to do. There are two choices either
7394                          * generate a res conflict or check condition
7395                          * with illegal field in parameter data. Since
7396                          * that is what is done when the sa_res_key is
7397                          * zero I'll take that approach since this has
7398                          * to do with the sa_res_key.
7399                          */
7400                         mtx_unlock(&softc->ctl_lock);
7401                         free(ctsio->kern_data_ptr, M_CTL);
7402                         ctl_set_invalid_field(ctsio,
7403                                               /*sks_valid*/ 1,
7404                                               /*command*/ 0,
7405                                               /*field*/ 8,
7406                                               /*bit_valid*/ 0,
7407                                               /*bit*/ 0);
7408                         ctl_done((union ctl_io *)ctsio);
7409                         return (1);
7410                 }
7411
7412                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7413                         if (lun->per_res[i].registered
7414                          && memcmp(param->serv_act_res_key,
7415                             lun->per_res[i].res_key.key,
7416                             sizeof(struct scsi_per_res_key)) != 0)
7417                                 continue;
7418
7419                         found = 1;
7420                         lun->per_res[i].registered = 0;
7421                         memset(&lun->per_res[i].res_key, 0,
7422                                sizeof(struct scsi_per_res_key));
7423                         lun->pr_key_count--;
7424
7425                         if (!persis_offset
7426                          && i < CTL_MAX_INITIATORS)
7427                                 lun->pending_sense[i].ua_pending |=
7428                                         CTL_UA_REG_PREEMPT;
7429                         else if (persis_offset
7430                               && i >= persis_offset)
7431                                 lun->pending_sense[i-persis_offset].ua_pending|=
7432                                         CTL_UA_REG_PREEMPT;
7433                 }
7434                 mtx_unlock(&softc->ctl_lock);
7435                 if (!found) {
7436                         free(ctsio->kern_data_ptr, M_CTL);
7437                         ctl_set_reservation_conflict(ctsio);
7438                         ctl_done((union ctl_io *)ctsio);
7439                         return (CTL_RETVAL_COMPLETE);
7440                 }
7441                 /* send msg to other side */
7442                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7443                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7444                 persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7445                 persis_io.pr.pr_info.residx = lun->pr_res_idx;
7446                 persis_io.pr.pr_info.res_type = type;
7447                 memcpy(persis_io.pr.pr_info.sa_res_key,
7448                        param->serv_act_res_key,
7449                        sizeof(param->serv_act_res_key));
7450                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7451                      &persis_io, sizeof(persis_io), 0)) >
7452                      CTL_HA_STATUS_SUCCESS) {
7453                         printf("CTL:Persis Out error returned from "
7454                                "ctl_ha_msg_send %d\n", isc_retval);
7455                 }
7456         } else {
7457                 /* Reserved but not all registrants */
7458                 /* sa_res_key is res holder */
7459                 if (memcmp(param->serv_act_res_key,
7460                    lun->per_res[lun->pr_res_idx].res_key.key,
7461                    sizeof(struct scsi_per_res_key)) == 0) {
7462                         /* validate scope and type */
7463                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7464                              SPR_LU_SCOPE) {
7465                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7466                                                       /*sks_valid*/ 1,
7467                                                       /*command*/ 1,
7468                                                       /*field*/ 2,
7469                                                       /*bit_valid*/ 1,
7470                                                       /*bit*/ 4);
7471                                 ctl_done((union ctl_io *)ctsio);
7472                                 return (1);
7473                         }
7474
7475                         if (type>8 || type==2 || type==4 || type==0) {
7476                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7477                                                       /*sks_valid*/ 1,
7478                                                       /*command*/ 1,
7479                                                       /*field*/ 2,
7480                                                       /*bit_valid*/ 1,
7481                                                       /*bit*/ 0);
7482                                 ctl_done((union ctl_io *)ctsio);
7483                                 return (1);
7484                         }
7485
7486                         /*
7487                          * Do the following:
7488                          * if sa_res_key != res_key remove all
7489                          * registrants w/sa_res_key and generate UA
7490                          * for these registrants(Registrations
7491                          * Preempted) if it wasn't an exclusive
7492                          * reservation generate UA(Reservations
7493                          * Preempted) for all other registered nexuses
7494                          * if the type has changed. Establish the new
7495                          * reservation and holder. If res_key and
7496                          * sa_res_key are the same do the above
7497                          * except don't unregister the res holder.
7498                          */
7499
7500                         /*
7501                          * Temporarily unregister so it won't get
7502                          * removed or UA generated
7503                          */
7504                         lun->per_res[residx].registered = 0;
7505                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7506                                 if (lun->per_res[i].registered == 0)
7507                                         continue;
7508
7509                                 if (memcmp(param->serv_act_res_key,
7510                                     lun->per_res[i].res_key.key,
7511                                     sizeof(struct scsi_per_res_key)) == 0) {
7512                                         lun->per_res[i].registered = 0;
7513                                         memset(&lun->per_res[i].res_key,
7514                                                0,
7515                                                sizeof(struct scsi_per_res_key));
7516                                         lun->pr_key_count--;
7517
7518                                         if (!persis_offset
7519                                          && i < CTL_MAX_INITIATORS)
7520                                                 lun->pending_sense[i
7521                                                         ].ua_pending |=
7522                                                         CTL_UA_REG_PREEMPT;
7523                                         else if (persis_offset
7524                                               && i >= persis_offset)
7525                                                 lun->pending_sense[
7526                                                   i-persis_offset].ua_pending |=
7527                                                   CTL_UA_REG_PREEMPT;
7528                                 } else if (type != lun->res_type
7529                                         && (lun->res_type == SPR_TYPE_WR_EX_RO
7530                                          || lun->res_type ==SPR_TYPE_EX_AC_RO)){
7531                                                 if (!persis_offset
7532                                                  && i < CTL_MAX_INITIATORS)
7533                                                         lun->pending_sense[i
7534                                                         ].ua_pending |=
7535                                                         CTL_UA_RES_RELEASE;
7536                                                 else if (persis_offset
7537                                                       && i >= persis_offset)
7538                                                         lun->pending_sense[
7539                                                         i-persis_offset
7540                                                         ].ua_pending |=
7541                                                         CTL_UA_RES_RELEASE;
7542                                 }
7543                         }
7544                         lun->per_res[residx].registered = 1;
7545                         lun->res_type = type;
7546                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7547                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7548                                 lun->pr_res_idx = residx;
7549                         else
7550                                 lun->pr_res_idx =
7551                                         CTL_PR_ALL_REGISTRANTS;
7552
7553                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7554                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7555                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7556                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7557                         persis_io.pr.pr_info.res_type = type;
7558                         memcpy(persis_io.pr.pr_info.sa_res_key,
7559                                param->serv_act_res_key,
7560                                sizeof(param->serv_act_res_key));
7561                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7562                              &persis_io, sizeof(persis_io), 0)) >
7563                              CTL_HA_STATUS_SUCCESS) {
7564                                 printf("CTL:Persis Out error returned "
7565                                        "from ctl_ha_msg_send %d\n",
7566                                        isc_retval);
7567                         }
7568                 } else {
7569                         /*
7570                          * sa_res_key is not the res holder just
7571                          * remove registrants
7572                          */
7573                         int found=0;
7574                         mtx_lock(&softc->ctl_lock);
7575
7576                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7577                                 if (memcmp(param->serv_act_res_key,
7578                                     lun->per_res[i].res_key.key,
7579                                     sizeof(struct scsi_per_res_key)) != 0)
7580                                         continue;
7581
7582                                 found = 1;
7583                                 lun->per_res[i].registered = 0;
7584                                 memset(&lun->per_res[i].res_key, 0,
7585                                        sizeof(struct scsi_per_res_key));
7586                                 lun->pr_key_count--;
7587
7588                                 if (!persis_offset
7589                                  && i < CTL_MAX_INITIATORS)
7590                                         lun->pending_sense[i].ua_pending |=
7591                                                 CTL_UA_REG_PREEMPT;
7592                                 else if (persis_offset
7593                                       && i >= persis_offset)
7594                                         lun->pending_sense[
7595                                                 i-persis_offset].ua_pending |=
7596                                                 CTL_UA_REG_PREEMPT;
7597                         }
7598
7599                         if (!found) {
7600                                 mtx_unlock(&softc->ctl_lock);
7601                                 free(ctsio->kern_data_ptr, M_CTL);
7602                                 ctl_set_reservation_conflict(ctsio);
7603                                 ctl_done((union ctl_io *)ctsio);
7604                                 return (1);
7605                         }
7606                         mtx_unlock(&softc->ctl_lock);
7607                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7608                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7609                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7610                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7611                         persis_io.pr.pr_info.res_type = type;
7612                         memcpy(persis_io.pr.pr_info.sa_res_key,
7613                                param->serv_act_res_key,
7614                                sizeof(param->serv_act_res_key));
7615                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7616                              &persis_io, sizeof(persis_io), 0)) >
7617                              CTL_HA_STATUS_SUCCESS) {
7618                                 printf("CTL:Persis Out error returned "
7619                                        "from ctl_ha_msg_send %d\n",
7620                                 isc_retval);
7621                         }
7622                 }
7623         }
7624
7625         lun->PRGeneration++;
7626
7627         return (retval);
7628 }
7629
7630 static void
7631 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
7632 {
7633         int i;
7634
7635         if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7636          || lun->pr_res_idx == CTL_PR_NO_RESERVATION
7637          || memcmp(&lun->per_res[lun->pr_res_idx].res_key,
7638                    msg->pr.pr_info.sa_res_key,
7639                    sizeof(struct scsi_per_res_key)) != 0) {
7640                 uint64_t sa_res_key;
7641                 sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
7642
7643                 if (sa_res_key == 0) {
7644                         /* temporarily unregister this nexus */
7645                         lun->per_res[msg->pr.pr_info.residx].registered = 0;
7646
7647                         /*
7648                          * Unregister everybody else and build UA for
7649                          * them
7650                          */
7651                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7652                                 if (lun->per_res[i].registered == 0)
7653                                         continue;
7654
7655                                 if (!persis_offset
7656                                  && i < CTL_MAX_INITIATORS)
7657                                         lun->pending_sense[i].ua_pending |=
7658                                                 CTL_UA_REG_PREEMPT;
7659                                 else if (persis_offset && i >= persis_offset)
7660                                         lun->pending_sense[i -
7661                                                 persis_offset].ua_pending |=
7662                                                 CTL_UA_REG_PREEMPT;
7663                                 lun->per_res[i].registered = 0;
7664                                 memset(&lun->per_res[i].res_key, 0,
7665                                        sizeof(struct scsi_per_res_key));
7666                         }
7667
7668                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
7669                         lun->pr_key_count = 1;
7670                         lun->res_type = msg->pr.pr_info.res_type;
7671                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7672                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7673                                 lun->pr_res_idx = msg->pr.pr_info.residx;
7674                 } else {
7675                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7676                                 if (memcmp(msg->pr.pr_info.sa_res_key,
7677                                    lun->per_res[i].res_key.key,
7678                                    sizeof(struct scsi_per_res_key)) != 0)
7679                                         continue;
7680
7681                                 lun->per_res[i].registered = 0;
7682                                 memset(&lun->per_res[i].res_key, 0,
7683                                        sizeof(struct scsi_per_res_key));
7684                                 lun->pr_key_count--;
7685
7686                                 if (!persis_offset
7687                                  && i < persis_offset)
7688                                         lun->pending_sense[i].ua_pending |=
7689                                                 CTL_UA_REG_PREEMPT;
7690                                 else if (persis_offset
7691                                       && i >= persis_offset)
7692                                         lun->pending_sense[i -
7693                                                 persis_offset].ua_pending |=
7694                                                 CTL_UA_REG_PREEMPT;
7695                         }
7696                 }
7697         } else {
7698                 /*
7699                  * Temporarily unregister so it won't get removed
7700                  * or UA generated
7701                  */
7702                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
7703                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7704                         if (lun->per_res[i].registered == 0)
7705                                 continue;
7706
7707                         if (memcmp(msg->pr.pr_info.sa_res_key,
7708                            lun->per_res[i].res_key.key,
7709                            sizeof(struct scsi_per_res_key)) == 0) {
7710                                 lun->per_res[i].registered = 0;
7711                                 memset(&lun->per_res[i].res_key, 0,
7712                                        sizeof(struct scsi_per_res_key));
7713                                 lun->pr_key_count--;
7714                                 if (!persis_offset
7715                                  && i < CTL_MAX_INITIATORS)
7716                                         lun->pending_sense[i].ua_pending |=
7717                                                 CTL_UA_REG_PREEMPT;
7718                                 else if (persis_offset
7719                                       && i >= persis_offset)
7720                                         lun->pending_sense[i -
7721                                                 persis_offset].ua_pending |=
7722                                                 CTL_UA_REG_PREEMPT;
7723                         } else if (msg->pr.pr_info.res_type != lun->res_type
7724                                 && (lun->res_type == SPR_TYPE_WR_EX_RO
7725                                  || lun->res_type == SPR_TYPE_EX_AC_RO)) {
7726                                         if (!persis_offset
7727                                          && i < persis_offset)
7728                                                 lun->pending_sense[i
7729                                                         ].ua_pending |=
7730                                                         CTL_UA_RES_RELEASE;
7731                                         else if (persis_offset
7732                                               && i >= persis_offset)
7733                                         lun->pending_sense[i -
7734                                                 persis_offset].ua_pending |=
7735                                                 CTL_UA_RES_RELEASE;
7736                         }
7737                 }
7738                 lun->per_res[msg->pr.pr_info.residx].registered = 1;
7739                 lun->res_type = msg->pr.pr_info.res_type;
7740                 if (lun->res_type != SPR_TYPE_WR_EX_AR
7741                  && lun->res_type != SPR_TYPE_EX_AC_AR)
7742                         lun->pr_res_idx = msg->pr.pr_info.residx;
7743                 else
7744                         lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
7745         }
7746         lun->PRGeneration++;
7747
7748 }
7749
7750
7751 int
7752 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
7753 {
7754         int retval;
7755         int isc_retval;
7756         u_int32_t param_len;
7757         struct scsi_per_res_out *cdb;
7758         struct ctl_lun *lun;
7759         struct scsi_per_res_out_parms* param;
7760         struct ctl_softc *softc;
7761         uint32_t residx;
7762         uint64_t res_key, sa_res_key;
7763         uint8_t type;
7764         union ctl_ha_msg persis_io;
7765         int    i;
7766
7767         CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
7768
7769         retval = CTL_RETVAL_COMPLETE;
7770
7771         softc = control_softc;
7772
7773         cdb = (struct scsi_per_res_out *)ctsio->cdb;
7774         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7775
7776         /*
7777          * We only support whole-LUN scope.  The scope & type are ignored for
7778          * register, register and ignore existing key and clear.
7779          * We sometimes ignore scope and type on preempts too!!
7780          * Verify reservation type here as well.
7781          */
7782         type = cdb->scope_type & SPR_TYPE_MASK;
7783         if ((cdb->action == SPRO_RESERVE)
7784          || (cdb->action == SPRO_RELEASE)) {
7785                 if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
7786                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7787                                               /*sks_valid*/ 1,
7788                                               /*command*/ 1,
7789                                               /*field*/ 2,
7790                                               /*bit_valid*/ 1,
7791                                               /*bit*/ 4);
7792                         ctl_done((union ctl_io *)ctsio);
7793                         return (CTL_RETVAL_COMPLETE);
7794                 }
7795
7796                 if (type>8 || type==2 || type==4 || type==0) {
7797                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7798                                               /*sks_valid*/ 1,
7799                                               /*command*/ 1,
7800                                               /*field*/ 2,
7801                                               /*bit_valid*/ 1,
7802                                               /*bit*/ 0);
7803                         ctl_done((union ctl_io *)ctsio);
7804                         return (CTL_RETVAL_COMPLETE);
7805                 }
7806         }
7807
7808         switch (cdb->action & SPRO_ACTION_MASK) {
7809         case SPRO_REGISTER:
7810         case SPRO_RESERVE:
7811         case SPRO_RELEASE:
7812         case SPRO_CLEAR:
7813         case SPRO_PREEMPT:
7814         case SPRO_REG_IGNO:
7815                 break;
7816         case SPRO_REG_MOVE:
7817         case SPRO_PRE_ABO:
7818         default:
7819                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7820                                       /*sks_valid*/ 1,
7821                                       /*command*/ 1,
7822                                       /*field*/ 1,
7823                                       /*bit_valid*/ 1,
7824                                       /*bit*/ 0);
7825                 ctl_done((union ctl_io *)ctsio);
7826                 return (CTL_RETVAL_COMPLETE);
7827                 break; /* NOTREACHED */
7828         }
7829
7830         param_len = scsi_4btoul(cdb->length);
7831
7832         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
7833                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
7834                 if (ctsio->kern_data_ptr == NULL) {
7835                         ctl_set_busy(ctsio);
7836                         ctl_done((union ctl_io *)ctsio);
7837                         return (CTL_RETVAL_COMPLETE);
7838                 }
7839                 ctsio->kern_data_len = param_len;
7840                 ctsio->kern_total_len = param_len;
7841                 ctsio->kern_data_resid = 0;
7842                 ctsio->kern_rel_offset = 0;
7843                 ctsio->kern_sg_entries = 0;
7844                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7845                 ctsio->be_move_done = ctl_config_move_done;
7846                 ctl_datamove((union ctl_io *)ctsio);
7847
7848                 return (CTL_RETVAL_COMPLETE);
7849         }
7850
7851         param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
7852
7853         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
7854         res_key = scsi_8btou64(param->res_key.key);
7855         sa_res_key = scsi_8btou64(param->serv_act_res_key);
7856
7857         /*
7858          * Validate the reservation key here except for SPRO_REG_IGNO
7859          * This must be done for all other service actions
7860          */
7861         if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
7862                 mtx_lock(&softc->ctl_lock);
7863                 if (lun->per_res[residx].registered) {
7864                     if (memcmp(param->res_key.key,
7865                                lun->per_res[residx].res_key.key,
7866                                ctl_min(sizeof(param->res_key),
7867                                sizeof(lun->per_res[residx].res_key))) != 0) {
7868                                 /*
7869                                  * The current key passed in doesn't match
7870                                  * the one the initiator previously
7871                                  * registered.
7872                                  */
7873                                 mtx_unlock(&softc->ctl_lock);
7874                                 free(ctsio->kern_data_ptr, M_CTL);
7875                                 ctl_set_reservation_conflict(ctsio);
7876                                 ctl_done((union ctl_io *)ctsio);
7877                                 return (CTL_RETVAL_COMPLETE);
7878                         }
7879                 } else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
7880                     /*
7881                          * We are not registered
7882                          */
7883                         mtx_unlock(&softc->ctl_lock);
7884                         free(ctsio->kern_data_ptr, M_CTL);
7885                         ctl_set_reservation_conflict(ctsio);
7886                         ctl_done((union ctl_io *)ctsio);
7887                         return (CTL_RETVAL_COMPLETE);
7888                 } else if (res_key != 0) {
7889                         /*
7890                          * We are not registered and trying to register but
7891                          * the register key isn't zero.
7892                          */
7893                         mtx_unlock(&softc->ctl_lock);
7894                         free(ctsio->kern_data_ptr, M_CTL);
7895                         ctl_set_reservation_conflict(ctsio);
7896                         ctl_done((union ctl_io *)ctsio);
7897                         return (CTL_RETVAL_COMPLETE);
7898                 }
7899                 mtx_unlock(&softc->ctl_lock);
7900         }
7901
7902         switch (cdb->action & SPRO_ACTION_MASK) {
7903         case SPRO_REGISTER:
7904         case SPRO_REG_IGNO: {
7905
7906 #if 0
7907                 printf("Registration received\n");
7908 #endif
7909
7910                 /*
7911                  * We don't support any of these options, as we report in
7912                  * the read capabilities request (see
7913                  * ctl_persistent_reserve_in(), above).
7914                  */
7915                 if ((param->flags & SPR_SPEC_I_PT)
7916                  || (param->flags & SPR_ALL_TG_PT)
7917                  || (param->flags & SPR_APTPL)) {
7918                         int bit_ptr;
7919
7920                         if (param->flags & SPR_APTPL)
7921                                 bit_ptr = 0;
7922                         else if (param->flags & SPR_ALL_TG_PT)
7923                                 bit_ptr = 2;
7924                         else /* SPR_SPEC_I_PT */
7925                                 bit_ptr = 3;
7926
7927                         free(ctsio->kern_data_ptr, M_CTL);
7928                         ctl_set_invalid_field(ctsio,
7929                                               /*sks_valid*/ 1,
7930                                               /*command*/ 0,
7931                                               /*field*/ 20,
7932                                               /*bit_valid*/ 1,
7933                                               /*bit*/ bit_ptr);
7934                         ctl_done((union ctl_io *)ctsio);
7935                         return (CTL_RETVAL_COMPLETE);
7936                 }
7937
7938                 mtx_lock(&softc->ctl_lock);
7939
7940                 /*
7941                  * The initiator wants to clear the
7942                  * key/unregister.
7943                  */
7944                 if (sa_res_key == 0) {
7945                         if ((res_key == 0
7946                           && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
7947                          || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
7948                           && !lun->per_res[residx].registered)) {
7949                                 mtx_unlock(&softc->ctl_lock);
7950                                 goto done;
7951                         }
7952
7953                         lun->per_res[residx].registered = 0;
7954                         memset(&lun->per_res[residx].res_key,
7955                                0, sizeof(lun->per_res[residx].res_key));
7956                         lun->pr_key_count--;
7957
7958                         if (residx == lun->pr_res_idx) {
7959                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
7960                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
7961
7962                                 if ((lun->res_type == SPR_TYPE_WR_EX_RO
7963                                   || lun->res_type == SPR_TYPE_EX_AC_RO)
7964                                  && lun->pr_key_count) {
7965                                         /*
7966                                          * If the reservation is a registrants
7967                                          * only type we need to generate a UA
7968                                          * for other registered inits.  The
7969                                          * sense code should be RESERVATIONS
7970                                          * RELEASED
7971                                          */
7972
7973                                         for (i = 0; i < CTL_MAX_INITIATORS;i++){
7974                                                 if (lun->per_res[
7975                                                     i+persis_offset].registered
7976                                                     == 0)
7977                                                         continue;
7978                                                 lun->pending_sense[i
7979                                                         ].ua_pending |=
7980                                                         CTL_UA_RES_RELEASE;
7981                                         }
7982                                 }
7983                                 lun->res_type = 0;
7984                         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7985                                 if (lun->pr_key_count==0) {
7986                                         lun->flags &= ~CTL_LUN_PR_RESERVED;
7987                                         lun->res_type = 0;
7988                                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
7989                                 }
7990                         }
7991                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7992                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7993                         persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
7994                         persis_io.pr.pr_info.residx = residx;
7995                         if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7996                              &persis_io, sizeof(persis_io), 0 )) >
7997                              CTL_HA_STATUS_SUCCESS) {
7998                                 printf("CTL:Persis Out error returned from "
7999                                        "ctl_ha_msg_send %d\n", isc_retval);
8000                         }
8001                         mtx_unlock(&softc->ctl_lock);
8002                 } else /* sa_res_key != 0 */ {
8003
8004                         /*
8005                          * If we aren't registered currently then increment
8006                          * the key count and set the registered flag.
8007                          */
8008                         if (!lun->per_res[residx].registered) {
8009                                 lun->pr_key_count++;
8010                                 lun->per_res[residx].registered = 1;
8011                         }
8012
8013                         memcpy(&lun->per_res[residx].res_key,
8014                                param->serv_act_res_key,
8015                                ctl_min(sizeof(param->serv_act_res_key),
8016                                sizeof(lun->per_res[residx].res_key)));
8017
8018                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8019                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8020                         persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8021                         persis_io.pr.pr_info.residx = residx;
8022                         memcpy(persis_io.pr.pr_info.sa_res_key,
8023                                param->serv_act_res_key,
8024                                sizeof(param->serv_act_res_key));
8025                         mtx_unlock(&softc->ctl_lock);
8026                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8027                              &persis_io, sizeof(persis_io), 0)) >
8028                              CTL_HA_STATUS_SUCCESS) {
8029                                 printf("CTL:Persis Out error returned from "
8030                                        "ctl_ha_msg_send %d\n", isc_retval);
8031                         }
8032                 }
8033                 lun->PRGeneration++;
8034
8035                 break;
8036         }
8037         case SPRO_RESERVE:
8038 #if 0
8039                 printf("Reserve executed type %d\n", type);
8040 #endif
8041                 mtx_lock(&softc->ctl_lock);
8042                 if (lun->flags & CTL_LUN_PR_RESERVED) {
8043                         /*
8044                          * if this isn't the reservation holder and it's
8045                          * not a "all registrants" type or if the type is
8046                          * different then we have a conflict
8047                          */
8048                         if ((lun->pr_res_idx != residx
8049                           && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8050                          || lun->res_type != type) {
8051                                 mtx_unlock(&softc->ctl_lock);
8052                                 free(ctsio->kern_data_ptr, M_CTL);
8053                                 ctl_set_reservation_conflict(ctsio);
8054                                 ctl_done((union ctl_io *)ctsio);
8055                                 return (CTL_RETVAL_COMPLETE);
8056                         }
8057                 } else /* create a reservation */ {
8058                         /*
8059                          * If it's not an "all registrants" type record
8060                          * reservation holder
8061                          */
8062                         if (type != SPR_TYPE_WR_EX_AR
8063                          && type != SPR_TYPE_EX_AC_AR)
8064                                 lun->pr_res_idx = residx; /* Res holder */
8065                         else
8066                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8067
8068                         lun->flags |= CTL_LUN_PR_RESERVED;
8069                         lun->res_type = type;
8070
8071                         mtx_unlock(&softc->ctl_lock);
8072
8073                         /* send msg to other side */
8074                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8075                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8076                         persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8077                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8078                         persis_io.pr.pr_info.res_type = type;
8079                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8080                              &persis_io, sizeof(persis_io), 0)) >
8081                              CTL_HA_STATUS_SUCCESS) {
8082                                 printf("CTL:Persis Out error returned from "
8083                                        "ctl_ha_msg_send %d\n", isc_retval);
8084                         }
8085                 }
8086                 break;
8087
8088         case SPRO_RELEASE:
8089                 mtx_lock(&softc->ctl_lock);
8090                 if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8091                         /* No reservation exists return good status */
8092                         mtx_unlock(&softc->ctl_lock);
8093                         goto done;
8094                 }
8095                 /*
8096                  * Is this nexus a reservation holder?
8097                  */
8098                 if (lun->pr_res_idx != residx
8099                  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8100                         /*
8101                          * not a res holder return good status but
8102                          * do nothing
8103                          */
8104                         mtx_unlock(&softc->ctl_lock);
8105                         goto done;
8106                 }
8107
8108                 if (lun->res_type != type) {
8109                         mtx_unlock(&softc->ctl_lock);
8110                         free(ctsio->kern_data_ptr, M_CTL);
8111                         ctl_set_illegal_pr_release(ctsio);
8112                         ctl_done((union ctl_io *)ctsio);
8113                         return (CTL_RETVAL_COMPLETE);
8114                 }
8115
8116                 /* okay to release */
8117                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8118                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8119                 lun->res_type = 0;
8120
8121                 /*
8122                  * if this isn't an exclusive access
8123                  * res generate UA for all other
8124                  * registrants.
8125                  */
8126                 if (type != SPR_TYPE_EX_AC
8127                  && type != SPR_TYPE_WR_EX) {
8128                         /*
8129                          * temporarily unregister so we don't generate UA
8130                          */
8131                         lun->per_res[residx].registered = 0;
8132
8133                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8134                                 if (lun->per_res[i+persis_offset].registered
8135                                     == 0)
8136                                         continue;
8137                                 lun->pending_sense[i].ua_pending |=
8138                                         CTL_UA_RES_RELEASE;
8139                         }
8140
8141                         lun->per_res[residx].registered = 1;
8142                 }
8143                 mtx_unlock(&softc->ctl_lock);
8144                 /* Send msg to other side */
8145                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8146                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8147                 persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8148                 if ((isc_retval=ctl_ha_msg_send( CTL_HA_CHAN_CTL, &persis_io,
8149                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8150                         printf("CTL:Persis Out error returned from "
8151                                "ctl_ha_msg_send %d\n", isc_retval);
8152                 }
8153                 break;
8154
8155         case SPRO_CLEAR:
8156                 /* send msg to other side */
8157
8158                 mtx_lock(&softc->ctl_lock);
8159                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8160                 lun->res_type = 0;
8161                 lun->pr_key_count = 0;
8162                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8163
8164
8165                 memset(&lun->per_res[residx].res_key,
8166                        0, sizeof(lun->per_res[residx].res_key));
8167                 lun->per_res[residx].registered = 0;
8168
8169                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++)
8170                         if (lun->per_res[i].registered) {
8171                                 if (!persis_offset && i < CTL_MAX_INITIATORS)
8172                                         lun->pending_sense[i].ua_pending |=
8173                                                 CTL_UA_RES_PREEMPT;
8174                                 else if (persis_offset && i >= persis_offset)
8175                                         lun->pending_sense[i-persis_offset
8176                                             ].ua_pending |= CTL_UA_RES_PREEMPT;
8177
8178                                 memset(&lun->per_res[i].res_key,
8179                                        0, sizeof(struct scsi_per_res_key));
8180                                 lun->per_res[i].registered = 0;
8181                         }
8182                 lun->PRGeneration++;
8183                 mtx_unlock(&softc->ctl_lock);
8184                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8185                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8186                 persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8187                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8188                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8189                         printf("CTL:Persis Out error returned from "
8190                                "ctl_ha_msg_send %d\n", isc_retval);
8191                 }
8192                 break;
8193
8194         case SPRO_PREEMPT: {
8195                 int nretval;
8196
8197                 nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8198                                           residx, ctsio, cdb, param);
8199                 if (nretval != 0)
8200                         return (CTL_RETVAL_COMPLETE);
8201                 break;
8202         }
8203         case SPRO_REG_MOVE:
8204         case SPRO_PRE_ABO:
8205         default:
8206                 free(ctsio->kern_data_ptr, M_CTL);
8207                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8208                                       /*sks_valid*/ 1,
8209                                       /*command*/ 1,
8210                                       /*field*/ 1,
8211                                       /*bit_valid*/ 1,
8212                                       /*bit*/ 0);
8213                 ctl_done((union ctl_io *)ctsio);
8214                 return (CTL_RETVAL_COMPLETE);
8215                 break; /* NOTREACHED */
8216         }
8217
8218 done:
8219         free(ctsio->kern_data_ptr, M_CTL);
8220         ctl_set_success(ctsio);
8221         ctl_done((union ctl_io *)ctsio);
8222
8223         return (retval);
8224 }
8225
8226 /*
8227  * This routine is for handling a message from the other SC pertaining to
8228  * persistent reserve out. All the error checking will have been done
8229  * so only perorming the action need be done here to keep the two
8230  * in sync.
8231  */
8232 static void
8233 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8234 {
8235         struct ctl_lun *lun;
8236         struct ctl_softc *softc;
8237         int i;
8238
8239         softc = control_softc;
8240
8241         mtx_lock(&softc->ctl_lock);
8242
8243         lun = softc->ctl_luns[msg->hdr.nexus.targ_lun];
8244         switch(msg->pr.pr_info.action) {
8245         case CTL_PR_REG_KEY:
8246                 if (!lun->per_res[msg->pr.pr_info.residx].registered) {
8247                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
8248                         lun->pr_key_count++;
8249                 }
8250                 lun->PRGeneration++;
8251                 memcpy(&lun->per_res[msg->pr.pr_info.residx].res_key,
8252                        msg->pr.pr_info.sa_res_key,
8253                        sizeof(struct scsi_per_res_key));
8254                 break;
8255
8256         case CTL_PR_UNREG_KEY:
8257                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
8258                 memset(&lun->per_res[msg->pr.pr_info.residx].res_key,
8259                        0, sizeof(struct scsi_per_res_key));
8260                 lun->pr_key_count--;
8261
8262                 /* XXX Need to see if the reservation has been released */
8263                 /* if so do we need to generate UA? */
8264                 if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8265                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8266                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8267
8268                         if ((lun->res_type == SPR_TYPE_WR_EX_RO
8269                           || lun->res_type == SPR_TYPE_EX_AC_RO)
8270                          && lun->pr_key_count) {
8271                                 /*
8272                                  * If the reservation is a registrants
8273                                  * only type we need to generate a UA
8274                                  * for other registered inits.  The
8275                                  * sense code should be RESERVATIONS
8276                                  * RELEASED
8277                                  */
8278
8279                                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8280                                         if (lun->per_res[i+
8281                                             persis_offset].registered == 0)
8282                                                 continue;
8283
8284                                         lun->pending_sense[i
8285                                                 ].ua_pending |=
8286                                                 CTL_UA_RES_RELEASE;
8287                                 }
8288                         }
8289                         lun->res_type = 0;
8290                 } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8291                         if (lun->pr_key_count==0) {
8292                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8293                                 lun->res_type = 0;
8294                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8295                         }
8296                 }
8297                 lun->PRGeneration++;
8298                 break;
8299
8300         case CTL_PR_RESERVE:
8301                 lun->flags |= CTL_LUN_PR_RESERVED;
8302                 lun->res_type = msg->pr.pr_info.res_type;
8303                 lun->pr_res_idx = msg->pr.pr_info.residx;
8304
8305                 break;
8306
8307         case CTL_PR_RELEASE:
8308                 /*
8309                  * if this isn't an exclusive access res generate UA for all
8310                  * other registrants.
8311                  */
8312                 if (lun->res_type != SPR_TYPE_EX_AC
8313                  && lun->res_type != SPR_TYPE_WR_EX) {
8314                         for (i = 0; i < CTL_MAX_INITIATORS; i++)
8315                                 if (lun->per_res[i+persis_offset].registered)
8316                                         lun->pending_sense[i].ua_pending |=
8317                                                 CTL_UA_RES_RELEASE;
8318                 }
8319
8320                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8321                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8322                 lun->res_type = 0;
8323                 break;
8324
8325         case CTL_PR_PREEMPT:
8326                 ctl_pro_preempt_other(lun, msg);
8327                 break;
8328         case CTL_PR_CLEAR:
8329                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8330                 lun->res_type = 0;
8331                 lun->pr_key_count = 0;
8332                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8333
8334                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8335                         if (lun->per_res[i].registered == 0)
8336                                 continue;
8337                         if (!persis_offset
8338                          && i < CTL_MAX_INITIATORS)
8339                                 lun->pending_sense[i].ua_pending |=
8340                                         CTL_UA_RES_PREEMPT;
8341                         else if (persis_offset
8342                               && i >= persis_offset)
8343                                 lun->pending_sense[i-persis_offset].ua_pending|=
8344                                         CTL_UA_RES_PREEMPT;
8345                         memset(&lun->per_res[i].res_key, 0,
8346                                sizeof(struct scsi_per_res_key));
8347                         lun->per_res[i].registered = 0;
8348                 }
8349                 lun->PRGeneration++;
8350                 break;
8351         }
8352
8353         mtx_unlock(&softc->ctl_lock);
8354 }
8355
8356 int
8357 ctl_read_write(struct ctl_scsiio *ctsio)
8358 {
8359         struct ctl_lun *lun;
8360         struct ctl_lba_len lbalen;
8361         uint64_t lba;
8362         uint32_t num_blocks;
8363         int reladdr, fua, dpo, ebp;
8364         int retval;
8365         int isread;
8366
8367         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8368
8369         CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8370
8371         reladdr = 0;
8372         fua = 0;
8373         dpo = 0;
8374         ebp = 0;
8375
8376         retval = CTL_RETVAL_COMPLETE;
8377
8378         isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8379               || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8380         if (lun->flags & CTL_LUN_PR_RESERVED && isread) {
8381                 uint32_t residx;
8382
8383                 /*
8384                  * XXX KDM need a lock here.
8385                  */
8386                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8387                 if ((lun->res_type == SPR_TYPE_EX_AC
8388                   && residx != lun->pr_res_idx)
8389                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
8390                    || lun->res_type == SPR_TYPE_EX_AC_AR)
8391                   && !lun->per_res[residx].registered)) {
8392                         ctl_set_reservation_conflict(ctsio);
8393                         ctl_done((union ctl_io *)ctsio);
8394                         return (CTL_RETVAL_COMPLETE);
8395                 }
8396         }
8397
8398         switch (ctsio->cdb[0]) {
8399         case READ_6:
8400         case WRITE_6: {
8401                 struct scsi_rw_6 *cdb;
8402
8403                 cdb = (struct scsi_rw_6 *)ctsio->cdb;
8404
8405                 lba = scsi_3btoul(cdb->addr);
8406                 /* only 5 bits are valid in the most significant address byte */
8407                 lba &= 0x1fffff;
8408                 num_blocks = cdb->length;
8409                 /*
8410                  * This is correct according to SBC-2.
8411                  */
8412                 if (num_blocks == 0)
8413                         num_blocks = 256;
8414                 break;
8415         }
8416         case READ_10:
8417         case WRITE_10: {
8418                 struct scsi_rw_10 *cdb;
8419
8420                 cdb = (struct scsi_rw_10 *)ctsio->cdb;
8421
8422                 if (cdb->byte2 & SRW10_RELADDR)
8423                         reladdr = 1;
8424                 if (cdb->byte2 & SRW10_FUA)
8425                         fua = 1;
8426                 if (cdb->byte2 & SRW10_DPO)
8427                         dpo = 1;
8428
8429                 if ((cdb->opcode == WRITE_10)
8430                  && (cdb->byte2 & SRW10_EBP))
8431                         ebp = 1;
8432
8433                 lba = scsi_4btoul(cdb->addr);
8434                 num_blocks = scsi_2btoul(cdb->length);
8435                 break;
8436         }
8437         case WRITE_VERIFY_10: {
8438                 struct scsi_write_verify_10 *cdb;
8439
8440                 cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
8441
8442                 /*
8443                  * XXX KDM we should do actual write verify support at some
8444                  * point.  This is obviously fake, we're just translating
8445                  * things to a write.  So we don't even bother checking the
8446                  * BYTCHK field, since we don't do any verification.  If
8447                  * the user asks for it, we'll just pretend we did it.
8448                  */
8449                 if (cdb->byte2 & SWV_DPO)
8450                         dpo = 1;
8451
8452                 lba = scsi_4btoul(cdb->addr);
8453                 num_blocks = scsi_2btoul(cdb->length);
8454                 break;
8455         }
8456         case READ_12:
8457         case WRITE_12: {
8458                 struct scsi_rw_12 *cdb;
8459
8460                 cdb = (struct scsi_rw_12 *)ctsio->cdb;
8461
8462                 if (cdb->byte2 & SRW12_RELADDR)
8463                         reladdr = 1;
8464                 if (cdb->byte2 & SRW12_FUA)
8465                         fua = 1;
8466                 if (cdb->byte2 & SRW12_DPO)
8467                         dpo = 1;
8468                 lba = scsi_4btoul(cdb->addr);
8469                 num_blocks = scsi_4btoul(cdb->length);
8470                 break;
8471         }
8472         case WRITE_VERIFY_12: {
8473                 struct scsi_write_verify_12 *cdb;
8474
8475                 cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
8476
8477                 if (cdb->byte2 & SWV_DPO)
8478                         dpo = 1;
8479
8480                 lba = scsi_4btoul(cdb->addr);
8481                 num_blocks = scsi_4btoul(cdb->length);
8482
8483                 break;
8484         }
8485         case READ_16:
8486         case WRITE_16: {
8487                 struct scsi_rw_16 *cdb;
8488
8489                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
8490
8491                 if (cdb->byte2 & SRW12_RELADDR)
8492                         reladdr = 1;
8493                 if (cdb->byte2 & SRW12_FUA)
8494                         fua = 1;
8495                 if (cdb->byte2 & SRW12_DPO)
8496                         dpo = 1;
8497
8498                 lba = scsi_8btou64(cdb->addr);
8499                 num_blocks = scsi_4btoul(cdb->length);
8500                 break;
8501         }
8502         case WRITE_VERIFY_16: {
8503                 struct scsi_write_verify_16 *cdb;
8504
8505                 cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
8506
8507                 if (cdb->byte2 & SWV_DPO)
8508                         dpo = 1;
8509
8510                 lba = scsi_8btou64(cdb->addr);
8511                 num_blocks = scsi_4btoul(cdb->length);
8512                 break;
8513         }
8514         default:
8515                 /*
8516                  * We got a command we don't support.  This shouldn't
8517                  * happen, commands should be filtered out above us.
8518                  */
8519                 ctl_set_invalid_opcode(ctsio);
8520                 ctl_done((union ctl_io *)ctsio);
8521
8522                 return (CTL_RETVAL_COMPLETE);
8523                 break; /* NOTREACHED */
8524         }
8525
8526         /*
8527          * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
8528          * interesting for us, but if RAIDCore is in write-back mode,
8529          * getting it to do write-through for a particular transaction may
8530          * not be possible.
8531          */
8532         /*
8533          * We don't support relative addressing.  That also requires
8534          * supporting linked commands, which we don't do.
8535          */
8536         if (reladdr != 0) {
8537                 ctl_set_invalid_field(ctsio,
8538                                       /*sks_valid*/ 1,
8539                                       /*command*/ 1,
8540                                       /*field*/ 1,
8541                                       /*bit_valid*/ 1,
8542                                       /*bit*/ 0);
8543                 ctl_done((union ctl_io *)ctsio);
8544                 return (CTL_RETVAL_COMPLETE);
8545         }
8546
8547         /*
8548          * The first check is to make sure we're in bounds, the second
8549          * check is to catch wrap-around problems.  If the lba + num blocks
8550          * is less than the lba, then we've wrapped around and the block
8551          * range is invalid anyway.
8552          */
8553         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
8554          || ((lba + num_blocks) < lba)) {
8555                 ctl_set_lba_out_of_range(ctsio);
8556                 ctl_done((union ctl_io *)ctsio);
8557                 return (CTL_RETVAL_COMPLETE);
8558         }
8559
8560         /*
8561          * According to SBC-3, a transfer length of 0 is not an error.
8562          * Note that this cannot happen with WRITE(6) or READ(6), since 0
8563          * translates to 256 blocks for those commands.
8564          */
8565         if (num_blocks == 0) {
8566                 ctl_set_success(ctsio);
8567                 ctl_done((union ctl_io *)ctsio);
8568                 return (CTL_RETVAL_COMPLETE);
8569         }
8570
8571         lbalen.lba = lba;
8572         lbalen.len = num_blocks;
8573         memcpy(ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes, &lbalen,
8574                sizeof(lbalen));
8575
8576         CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
8577
8578         retval = lun->backend->data_submit((union ctl_io *)ctsio);
8579
8580         return (retval);
8581 }
8582
8583 int
8584 ctl_report_luns(struct ctl_scsiio *ctsio)
8585 {
8586         struct scsi_report_luns *cdb;
8587         struct scsi_report_luns_data *lun_data;
8588         struct ctl_lun *lun, *request_lun;
8589         int num_luns, retval;
8590         uint32_t alloc_len, lun_datalen;
8591         int num_filled, well_known;
8592         uint32_t initidx;
8593
8594         retval = CTL_RETVAL_COMPLETE;
8595         well_known = 0;
8596
8597         cdb = (struct scsi_report_luns *)ctsio->cdb;
8598
8599         CTL_DEBUG_PRINT(("ctl_report_luns\n"));
8600
8601         mtx_lock(&control_softc->ctl_lock);
8602         num_luns = control_softc->num_luns;
8603         mtx_unlock(&control_softc->ctl_lock);
8604
8605         switch (cdb->select_report) {
8606         case RPL_REPORT_DEFAULT:
8607         case RPL_REPORT_ALL:
8608                 break;
8609         case RPL_REPORT_WELLKNOWN:
8610                 well_known = 1;
8611                 num_luns = 0;
8612                 break;
8613         default:
8614                 ctl_set_invalid_field(ctsio,
8615                                       /*sks_valid*/ 1,
8616                                       /*command*/ 1,
8617                                       /*field*/ 2,
8618                                       /*bit_valid*/ 0,
8619                                       /*bit*/ 0);
8620                 ctl_done((union ctl_io *)ctsio);
8621                 return (retval);
8622                 break; /* NOTREACHED */
8623         }
8624
8625         alloc_len = scsi_4btoul(cdb->length);
8626         /*
8627          * The initiator has to allocate at least 16 bytes for this request,
8628          * so he can at least get the header and the first LUN.  Otherwise
8629          * we reject the request (per SPC-3 rev 14, section 6.21).
8630          */
8631         if (alloc_len < (sizeof(struct scsi_report_luns_data) +
8632             sizeof(struct scsi_report_luns_lundata))) {
8633                 ctl_set_invalid_field(ctsio,
8634                                       /*sks_valid*/ 1,
8635                                       /*command*/ 1,
8636                                       /*field*/ 6,
8637                                       /*bit_valid*/ 0,
8638                                       /*bit*/ 0);
8639                 ctl_done((union ctl_io *)ctsio);
8640                 return (retval);
8641         }
8642
8643         request_lun = (struct ctl_lun *)
8644                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8645
8646         lun_datalen = sizeof(*lun_data) +
8647                 (num_luns * sizeof(struct scsi_report_luns_lundata));
8648
8649         ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK);
8650         if (ctsio->kern_data_ptr == NULL) {
8651                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
8652                 ctsio->scsi_status = SCSI_STATUS_BUSY;
8653                 ctl_done((union ctl_io *)ctsio);
8654                 return (CTL_RETVAL_COMPLETE);
8655         }
8656
8657         lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
8658         ctsio->kern_sg_entries = 0;
8659
8660         if (lun_datalen < alloc_len) {
8661                 ctsio->residual = alloc_len - lun_datalen;
8662                 ctsio->kern_data_len = lun_datalen;
8663                 ctsio->kern_total_len = lun_datalen;
8664         } else {
8665                 ctsio->residual = 0;
8666                 ctsio->kern_data_len = alloc_len;
8667                 ctsio->kern_total_len = alloc_len;
8668         }
8669         ctsio->kern_data_resid = 0;
8670         ctsio->kern_rel_offset = 0;
8671         ctsio->kern_sg_entries = 0;
8672
8673         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
8674
8675         memset(lun_data, 0, lun_datalen);
8676
8677         /*
8678          * We set this to the actual data length, regardless of how much
8679          * space we actually have to return results.  If the user looks at
8680          * this value, he'll know whether or not he allocated enough space
8681          * and reissue the command if necessary.  We don't support well
8682          * known logical units, so if the user asks for that, return none.
8683          */
8684         scsi_ulto4b(lun_datalen - 8, lun_data->length);
8685
8686         mtx_lock(&control_softc->ctl_lock);
8687         for (num_filled = 0, lun = STAILQ_FIRST(&control_softc->lun_list);
8688              (lun != NULL) && (num_filled < num_luns);
8689              lun = STAILQ_NEXT(lun, links)) {
8690
8691                 if (lun->lun <= 0xff) {
8692                         /*
8693                          * Peripheral addressing method, bus number 0.
8694                          */
8695                         lun_data->luns[num_filled].lundata[0] =
8696                                 RPL_LUNDATA_ATYP_PERIPH;
8697                         lun_data->luns[num_filled].lundata[1] = lun->lun;
8698                         num_filled++;
8699                 } else if (lun->lun <= 0x3fff) {
8700                         /*
8701                          * Flat addressing method.
8702                          */
8703                         lun_data->luns[num_filled].lundata[0] =
8704                                 RPL_LUNDATA_ATYP_FLAT |
8705                                 (lun->lun & RPL_LUNDATA_FLAT_LUN_MASK);
8706 #ifdef OLDCTLHEADERS
8707                                 (SRLD_ADDR_FLAT << SRLD_ADDR_SHIFT) |
8708                                 (lun->lun & SRLD_BUS_LUN_MASK);
8709 #endif
8710                         lun_data->luns[num_filled].lundata[1] =
8711 #ifdef OLDCTLHEADERS
8712                                 lun->lun >> SRLD_BUS_LUN_BITS;
8713 #endif
8714                                 lun->lun >> RPL_LUNDATA_FLAT_LUN_BITS;
8715                         num_filled++;
8716                 } else {
8717                         printf("ctl_report_luns: bogus LUN number %jd, "
8718                                "skipping\n", (intmax_t)lun->lun);
8719                 }
8720                 /*
8721                  * According to SPC-3, rev 14 section 6.21:
8722                  *
8723                  * "The execution of a REPORT LUNS command to any valid and
8724                  * installed logical unit shall clear the REPORTED LUNS DATA
8725                  * HAS CHANGED unit attention condition for all logical
8726                  * units of that target with respect to the requesting
8727                  * initiator. A valid and installed logical unit is one
8728                  * having a PERIPHERAL QUALIFIER of 000b in the standard
8729                  * INQUIRY data (see 6.4.2)."
8730                  *
8731                  * If request_lun is NULL, the LUN this report luns command
8732                  * was issued to is either disabled or doesn't exist. In that
8733                  * case, we shouldn't clear any pending lun change unit
8734                  * attention.
8735                  */
8736                 if (request_lun != NULL)
8737                         lun->pending_sense[initidx].ua_pending &=
8738                                 ~CTL_UA_LUN_CHANGE;
8739         }
8740         mtx_unlock(&control_softc->ctl_lock);
8741
8742         /*
8743          * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
8744          * this request.
8745          */
8746         ctsio->scsi_status = SCSI_STATUS_OK;
8747
8748         ctsio->be_move_done = ctl_config_move_done;
8749         ctl_datamove((union ctl_io *)ctsio);
8750
8751         return (retval);
8752 }
8753
8754 int
8755 ctl_request_sense(struct ctl_scsiio *ctsio)
8756 {
8757         struct scsi_request_sense *cdb;
8758         struct scsi_sense_data *sense_ptr;
8759         struct ctl_lun *lun;
8760         uint32_t initidx;
8761         int have_error;
8762         scsi_sense_data_type sense_format;
8763
8764         cdb = (struct scsi_request_sense *)ctsio->cdb;
8765
8766         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8767
8768         CTL_DEBUG_PRINT(("ctl_request_sense\n"));
8769
8770         /*
8771          * Determine which sense format the user wants.
8772          */
8773         if (cdb->byte2 & SRS_DESC)
8774                 sense_format = SSD_TYPE_DESC;
8775         else
8776                 sense_format = SSD_TYPE_FIXED;
8777
8778         ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
8779         if (ctsio->kern_data_ptr == NULL) {
8780                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
8781                 ctsio->scsi_status = SCSI_STATUS_BUSY;
8782                 ctl_done((union ctl_io *)ctsio);
8783                 return (CTL_RETVAL_COMPLETE);
8784         }
8785         sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
8786         ctsio->kern_sg_entries = 0;
8787
8788         /*
8789          * struct scsi_sense_data, which is currently set to 256 bytes, is
8790          * larger than the largest allowed value for the length field in the
8791          * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
8792          */
8793         ctsio->residual = 0;
8794         ctsio->kern_data_len = cdb->length;
8795         ctsio->kern_total_len = cdb->length;
8796
8797         ctsio->kern_data_resid = 0;
8798         ctsio->kern_rel_offset = 0;
8799         ctsio->kern_sg_entries = 0;
8800
8801         /*
8802          * If we don't have a LUN, we don't have any pending sense.
8803          */
8804         if (lun == NULL)
8805                 goto no_sense;
8806
8807         have_error = 0;
8808         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
8809         /*
8810          * Check for pending sense, and then for pending unit attentions.
8811          * Pending sense gets returned first, then pending unit attentions.
8812          */
8813         mtx_lock(&lun->ctl_softc->ctl_lock);
8814         if (ctl_is_set(lun->have_ca, initidx)) {
8815                 scsi_sense_data_type stored_format;
8816
8817                 /*
8818                  * Check to see which sense format was used for the stored
8819                  * sense data.
8820                  */
8821                 stored_format = scsi_sense_type(
8822                     &lun->pending_sense[initidx].sense);
8823
8824                 /*
8825                  * If the user requested a different sense format than the
8826                  * one we stored, then we need to convert it to the other
8827                  * format.  If we're going from descriptor to fixed format
8828                  * sense data, we may lose things in translation, depending
8829                  * on what options were used.
8830                  *
8831                  * If the stored format is SSD_TYPE_NONE (i.e. invalid),
8832                  * for some reason we'll just copy it out as-is.
8833                  */
8834                 if ((stored_format == SSD_TYPE_FIXED)
8835                  && (sense_format == SSD_TYPE_DESC))
8836                         ctl_sense_to_desc((struct scsi_sense_data_fixed *)
8837                             &lun->pending_sense[initidx].sense,
8838                             (struct scsi_sense_data_desc *)sense_ptr);
8839                 else if ((stored_format == SSD_TYPE_DESC)
8840                       && (sense_format == SSD_TYPE_FIXED))
8841                         ctl_sense_to_fixed((struct scsi_sense_data_desc *)
8842                             &lun->pending_sense[initidx].sense,
8843                             (struct scsi_sense_data_fixed *)sense_ptr);
8844                 else
8845                         memcpy(sense_ptr, &lun->pending_sense[initidx].sense,
8846                                ctl_min(sizeof(*sense_ptr),
8847                                sizeof(lun->pending_sense[initidx].sense)));
8848
8849                 ctl_clear_mask(lun->have_ca, initidx);
8850                 have_error = 1;
8851         } else if (lun->pending_sense[initidx].ua_pending != CTL_UA_NONE) {
8852                 ctl_ua_type ua_type;
8853
8854                 ua_type = ctl_build_ua(lun->pending_sense[initidx].ua_pending,
8855                                        sense_ptr, sense_format);
8856                 if (ua_type != CTL_UA_NONE) {
8857                         have_error = 1;
8858                         /* We're reporting this UA, so clear it */
8859                         lun->pending_sense[initidx].ua_pending &= ~ua_type;
8860                 }
8861         }
8862         mtx_unlock(&lun->ctl_softc->ctl_lock);
8863
8864         /*
8865          * We already have a pending error, return it.
8866          */
8867         if (have_error != 0) {
8868                 /*
8869                  * We report the SCSI status as OK, since the status of the
8870                  * request sense command itself is OK.
8871                  */
8872                 ctsio->scsi_status = SCSI_STATUS_OK;
8873
8874                 /*
8875                  * We report 0 for the sense length, because we aren't doing
8876                  * autosense in this case.  We're reporting sense as
8877                  * parameter data.
8878                  */
8879                 ctsio->sense_len = 0;
8880
8881                 ctsio->be_move_done = ctl_config_move_done;
8882                 ctl_datamove((union ctl_io *)ctsio);
8883
8884                 return (CTL_RETVAL_COMPLETE);
8885         }
8886
8887 no_sense:
8888
8889         /*
8890          * No sense information to report, so we report that everything is
8891          * okay.
8892          */
8893         ctl_set_sense_data(sense_ptr,
8894                            lun,
8895                            sense_format,
8896                            /*current_error*/ 1,
8897                            /*sense_key*/ SSD_KEY_NO_SENSE,
8898                            /*asc*/ 0x00,
8899                            /*ascq*/ 0x00,
8900                            SSD_ELEM_NONE);
8901
8902         ctsio->scsi_status = SCSI_STATUS_OK;
8903
8904         /*
8905          * We report 0 for the sense length, because we aren't doing
8906          * autosense in this case.  We're reporting sense as parameter data.
8907          */
8908         ctsio->sense_len = 0;
8909         ctsio->be_move_done = ctl_config_move_done;
8910         ctl_datamove((union ctl_io *)ctsio);
8911
8912         return (CTL_RETVAL_COMPLETE);
8913 }
8914
8915 int
8916 ctl_tur(struct ctl_scsiio *ctsio)
8917 {
8918         struct ctl_lun *lun;
8919
8920         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8921
8922         CTL_DEBUG_PRINT(("ctl_tur\n"));
8923
8924         if (lun == NULL)
8925                 return (-EINVAL);
8926
8927         ctsio->scsi_status = SCSI_STATUS_OK;
8928         ctsio->io_hdr.status = CTL_SUCCESS;
8929
8930         ctl_done((union ctl_io *)ctsio);
8931
8932         return (CTL_RETVAL_COMPLETE);
8933 }
8934
8935 #ifdef notyet
8936 static int
8937 ctl_cmddt_inquiry(struct ctl_scsiio *ctsio)
8938 {
8939
8940 }
8941 #endif
8942
8943 static int
8944 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
8945 {
8946         struct scsi_vpd_supported_pages *pages;
8947         int sup_page_size;
8948         struct ctl_lun *lun;
8949
8950         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8951
8952         sup_page_size = sizeof(struct scsi_vpd_supported_pages) +
8953                 SCSI_EVPD_NUM_SUPPORTED_PAGES;
8954         /*
8955          * XXX KDM GFP_???  We probably don't want to wait here,
8956          * unless we end up having a process/thread context.
8957          */
8958         ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK);
8959         if (ctsio->kern_data_ptr == NULL) {
8960                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
8961                 ctsio->scsi_status = SCSI_STATUS_BUSY;
8962                 ctl_done((union ctl_io *)ctsio);
8963                 return (CTL_RETVAL_COMPLETE);
8964         }
8965         pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
8966         ctsio->kern_sg_entries = 0;
8967
8968         if (sup_page_size < alloc_len) {
8969                 ctsio->residual = alloc_len - sup_page_size;
8970                 ctsio->kern_data_len = sup_page_size;
8971                 ctsio->kern_total_len = sup_page_size;
8972         } else {
8973                 ctsio->residual = 0;
8974                 ctsio->kern_data_len = alloc_len;
8975                 ctsio->kern_total_len = alloc_len;
8976         }
8977         ctsio->kern_data_resid = 0;
8978         ctsio->kern_rel_offset = 0;
8979         ctsio->kern_sg_entries = 0;
8980
8981         memset(pages, 0, sup_page_size);
8982
8983         /*
8984          * The control device is always connected.  The disk device, on the
8985          * other hand, may not be online all the time.  Need to change this
8986          * to figure out whether the disk device is actually online or not.
8987          */
8988         if (lun != NULL)
8989                 pages->device = (SID_QUAL_LU_CONNECTED << 5) |
8990                                 lun->be_lun->lun_type;
8991         else
8992                 pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
8993
8994         pages->length = SCSI_EVPD_NUM_SUPPORTED_PAGES;
8995         /* Supported VPD pages */
8996         pages->page_list[0] = SVPD_SUPPORTED_PAGES;
8997         /* Serial Number */
8998         pages->page_list[1] = SVPD_UNIT_SERIAL_NUMBER;
8999         /* Device Identification */
9000         pages->page_list[2] = SVPD_DEVICE_ID;
9001
9002         ctsio->scsi_status = SCSI_STATUS_OK;
9003
9004         ctsio->be_move_done = ctl_config_move_done;
9005         ctl_datamove((union ctl_io *)ctsio);
9006
9007         return (CTL_RETVAL_COMPLETE);
9008 }
9009
9010 static int
9011 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9012 {
9013         struct scsi_vpd_unit_serial_number *sn_ptr;
9014         struct ctl_lun *lun;
9015 #ifndef CTL_USE_BACKEND_SN
9016         char tmpstr[32];
9017 #endif
9018
9019         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9020
9021         /* XXX KDM which malloc flags here?? */
9022         ctsio->kern_data_ptr = malloc(sizeof(*sn_ptr), M_CTL, M_WAITOK);
9023         if (ctsio->kern_data_ptr == NULL) {
9024                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
9025                 ctsio->scsi_status = SCSI_STATUS_BUSY;
9026                 ctl_done((union ctl_io *)ctsio);
9027                 return (CTL_RETVAL_COMPLETE);
9028         }
9029         sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9030         ctsio->kern_sg_entries = 0;
9031
9032         if (sizeof(*sn_ptr) < alloc_len) {
9033                 ctsio->residual = alloc_len - sizeof(*sn_ptr);
9034                 ctsio->kern_data_len = sizeof(*sn_ptr);
9035                 ctsio->kern_total_len = sizeof(*sn_ptr);
9036         } else {
9037                 ctsio->residual = 0;
9038                 ctsio->kern_data_len = alloc_len;
9039                 ctsio->kern_total_len = alloc_len;
9040         }
9041         ctsio->kern_data_resid = 0;
9042         ctsio->kern_rel_offset = 0;
9043         ctsio->kern_sg_entries = 0;
9044
9045         memset(sn_ptr, 0, sizeof(*sn_ptr));
9046
9047         /*
9048          * The control device is always connected.  The disk device, on the
9049          * other hand, may not be online all the time.  Need to change this
9050          * to figure out whether the disk device is actually online or not.
9051          */
9052         if (lun != NULL)
9053                 sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9054                                   lun->be_lun->lun_type;
9055         else
9056                 sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9057
9058         sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9059         sn_ptr->length = ctl_min(sizeof(*sn_ptr) - 4, CTL_SN_LEN);
9060 #ifdef CTL_USE_BACKEND_SN
9061         /*
9062          * If we don't have a LUN, we just leave the serial number as
9063          * all spaces.
9064          */
9065         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9066         if (lun != NULL) {
9067                 strncpy((char *)sn_ptr->serial_num,
9068                         (char *)lun->be_lun->serial_num, CTL_SN_LEN);
9069         }
9070 #else
9071         /*
9072          * Note that we're using a non-unique serial number here,
9073          */
9074         snprintf(tmpstr, sizeof(tmpstr), "MYSERIALNUMIS000");
9075         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9076         strncpy(sn_ptr->serial_num, tmpstr, ctl_min(CTL_SN_LEN,
9077                 ctl_min(sizeof(tmpstr), sizeof(*sn_ptr) - 4)));
9078 #endif
9079         ctsio->scsi_status = SCSI_STATUS_OK;
9080
9081         ctsio->be_move_done = ctl_config_move_done;
9082         ctl_datamove((union ctl_io *)ctsio);
9083
9084         return (CTL_RETVAL_COMPLETE);
9085 }
9086
9087
9088 static int
9089 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9090 {
9091         struct scsi_vpd_device_id *devid_ptr;
9092         struct scsi_vpd_id_descriptor *desc, *desc1;
9093         struct scsi_vpd_id_descriptor *desc2, *desc3; /* for types 4h and 5h */
9094         struct scsi_vpd_id_t10 *t10id;
9095         struct ctl_softc *ctl_softc;
9096         struct ctl_lun *lun;
9097         struct ctl_frontend *fe;
9098 #ifndef CTL_USE_BACKEND_SN
9099         char tmpstr[32];
9100 #endif /* CTL_USE_BACKEND_SN */
9101         int devid_len;
9102
9103         ctl_softc = control_softc;
9104         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9105
9106         devid_len = sizeof(struct scsi_vpd_device_id) +
9107                 sizeof(struct scsi_vpd_id_descriptor) +
9108                 sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN +
9109                 sizeof(struct scsi_vpd_id_descriptor) + CTL_WWPN_LEN +
9110                 sizeof(struct scsi_vpd_id_descriptor) +
9111                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9112                 sizeof(struct scsi_vpd_id_descriptor) +
9113                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9114
9115         /* XXX KDM which malloc flags here ?? */
9116         ctsio->kern_data_ptr = malloc(devid_len, M_CTL, M_WAITOK);
9117         if (ctsio->kern_data_ptr == NULL) {
9118                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
9119                 ctsio->scsi_status = SCSI_STATUS_BUSY;
9120                 ctl_done((union ctl_io *)ctsio);
9121                 return (CTL_RETVAL_COMPLETE);
9122         }
9123         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9124         ctsio->kern_sg_entries = 0;
9125
9126         if (devid_len < alloc_len) {
9127                 ctsio->residual = alloc_len - devid_len;
9128                 ctsio->kern_data_len = devid_len;
9129                 ctsio->kern_total_len = devid_len;
9130         } else {
9131                 ctsio->residual = 0;
9132                 ctsio->kern_data_len = alloc_len;
9133                 ctsio->kern_total_len = alloc_len;
9134         }
9135         ctsio->kern_data_resid = 0;
9136         ctsio->kern_rel_offset = 0;
9137         ctsio->kern_sg_entries = 0;
9138
9139         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
9140         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
9141         desc1 = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9142                 sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN);
9143         desc2 = (struct scsi_vpd_id_descriptor *)(&desc1->identifier[0] +
9144                   CTL_WWPN_LEN);
9145         desc3 = (struct scsi_vpd_id_descriptor *)(&desc2->identifier[0] +
9146                  sizeof(struct scsi_vpd_id_rel_trgt_port_id));
9147         memset(devid_ptr, 0, devid_len);
9148
9149         /*
9150          * The control device is always connected.  The disk device, on the
9151          * other hand, may not be online all the time.
9152          */
9153         if (lun != NULL)
9154                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9155                                      lun->be_lun->lun_type;
9156         else
9157                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9158
9159         devid_ptr->page_code = SVPD_DEVICE_ID;
9160
9161         scsi_ulto2b(devid_len - 4, devid_ptr->length);
9162
9163         mtx_lock(&ctl_softc->ctl_lock);
9164
9165         fe = ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)];
9166
9167         /*
9168          * For Fibre channel,
9169          */
9170         if (fe->port_type == CTL_PORT_FC)
9171         {
9172                 desc->proto_codeset = (SCSI_PROTO_FC << 4) |
9173                                       SVPD_ID_CODESET_ASCII;
9174                 desc1->proto_codeset = (SCSI_PROTO_FC << 4) |
9175                               SVPD_ID_CODESET_BINARY;
9176         }
9177         else
9178         {
9179                 desc->proto_codeset = (SCSI_PROTO_SPI << 4) |
9180                                       SVPD_ID_CODESET_ASCII;
9181                 desc1->proto_codeset = (SCSI_PROTO_SPI << 4) |
9182                               SVPD_ID_CODESET_BINARY;
9183         }
9184         desc2->proto_codeset = desc3->proto_codeset = desc1->proto_codeset;
9185         mtx_unlock(&ctl_softc->ctl_lock);
9186
9187         /*
9188          * We're using a LUN association here.  i.e., this device ID is a
9189          * per-LUN identifier.
9190          */
9191         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
9192         desc->length = sizeof(*t10id) + CTL_DEVID_LEN;
9193         strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
9194
9195         /*
9196          * desc1 is for the WWPN which is a port asscociation.
9197          */
9198         desc1->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT | SVPD_ID_TYPE_NAA;
9199         desc1->length = CTL_WWPN_LEN;
9200         /* XXX Call Reggie's get_WWNN func here then add port # to the end */
9201         /* For testing just create the WWPN */
9202 #if 0
9203         ddb_GetWWNN((char *)desc1->identifier);
9204
9205         /* NOTE: if the port is 0 or 8 we don't want to subtract 1 */
9206         /* This is so Copancontrol will return something sane */
9207         if (ctsio->io_hdr.nexus.targ_port!=0 &&
9208             ctsio->io_hdr.nexus.targ_port!=8)
9209                 desc1->identifier[7] += ctsio->io_hdr.nexus.targ_port-1;
9210         else
9211                 desc1->identifier[7] += ctsio->io_hdr.nexus.targ_port;
9212 #endif
9213
9214         be64enc(desc1->identifier, fe->wwpn);
9215
9216         /*
9217          * desc2 is for the Relative Target Port(type 4h) identifier
9218          */
9219         desc2->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT
9220                          | SVPD_ID_TYPE_RELTARG;
9221         desc2->length = 4;
9222 //#if 0
9223         /* NOTE: if the port is 0 or 8 we don't want to subtract 1 */
9224         /* This is so Copancontrol will return something sane */
9225         if (ctsio->io_hdr.nexus.targ_port!=0 &&
9226             ctsio->io_hdr.nexus.targ_port!=8)
9227                 desc2->identifier[3] = ctsio->io_hdr.nexus.targ_port - 1;
9228         else
9229                 desc2->identifier[3] = ctsio->io_hdr.nexus.targ_port;
9230 //#endif
9231
9232         /*
9233          * desc3 is for the Target Port Group(type 5h) identifier
9234          */
9235         desc3->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT
9236                          | SVPD_ID_TYPE_TPORTGRP;
9237         desc3->length = 4;
9238         if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS || ctl_is_single)
9239                 desc3->identifier[3] = 1;
9240         else
9241                 desc3->identifier[3] = 2;
9242
9243 #ifdef CTL_USE_BACKEND_SN
9244         /*
9245          * If we've actually got a backend, copy the device id from the
9246          * per-LUN data.  Otherwise, set it to all spaces.
9247          */
9248         if (lun != NULL) {
9249                 /*
9250                  * Copy the backend's LUN ID.
9251                  */
9252                 strncpy((char *)t10id->vendor_spec_id,
9253                         (char *)lun->be_lun->device_id, CTL_DEVID_LEN);
9254         } else {
9255                 /*
9256                  * No backend, set this to spaces.
9257                  */
9258                 memset(t10id->vendor_spec_id, 0x20, CTL_DEVID_LEN);
9259         }
9260 #else
9261         snprintf(tmpstr, sizeof(tmpstr), "MYDEVICEIDIS%4d",
9262                  (lun != NULL) ?  (int)lun->lun : 0);
9263         strncpy(t10id->vendor_spec_id, tmpstr, ctl_min(CTL_DEVID_LEN,
9264                 sizeof(tmpstr)));
9265 #endif
9266
9267         ctsio->scsi_status = SCSI_STATUS_OK;
9268
9269         ctsio->be_move_done = ctl_config_move_done;
9270         ctl_datamove((union ctl_io *)ctsio);
9271
9272         return (CTL_RETVAL_COMPLETE);
9273 }
9274
9275 static int
9276 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
9277 {
9278         struct scsi_inquiry *cdb;
9279         int alloc_len, retval;
9280
9281         cdb = (struct scsi_inquiry *)ctsio->cdb;
9282
9283         retval = CTL_RETVAL_COMPLETE;
9284
9285         alloc_len = scsi_2btoul(cdb->length);
9286
9287         switch (cdb->page_code) {
9288         case SVPD_SUPPORTED_PAGES:
9289                 retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
9290                 break;
9291         case SVPD_UNIT_SERIAL_NUMBER:
9292                 retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
9293                 break;
9294         case SVPD_DEVICE_ID:
9295                 retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
9296                 break;
9297         default:
9298                 ctl_set_invalid_field(ctsio,
9299                                       /*sks_valid*/ 1,
9300                                       /*command*/ 1,
9301                                       /*field*/ 2,
9302                                       /*bit_valid*/ 0,
9303                                       /*bit*/ 0);
9304                 ctl_done((union ctl_io *)ctsio);
9305                 retval = CTL_RETVAL_COMPLETE;
9306                 break;
9307         }
9308
9309         return (retval);
9310 }
9311
9312 static int
9313 ctl_inquiry_std(struct ctl_scsiio *ctsio)
9314 {
9315         struct scsi_inquiry_data *inq_ptr;
9316         struct scsi_inquiry *cdb;
9317         struct ctl_softc *ctl_softc;
9318         struct ctl_lun *lun;
9319         uint32_t alloc_len;
9320         int is_fc;
9321
9322         ctl_softc = control_softc;
9323
9324         /*
9325          * Figure out whether we're talking to a Fibre Channel port or not.
9326          * We treat the ioctl front end, and any SCSI adapters, as packetized
9327          * SCSI front ends.
9328          */
9329         mtx_lock(&ctl_softc->ctl_lock);
9330         if (ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)]->port_type !=
9331             CTL_PORT_FC)
9332                 is_fc = 0;
9333         else
9334                 is_fc = 1;
9335         mtx_unlock(&ctl_softc->ctl_lock);
9336
9337         lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9338         cdb = (struct scsi_inquiry *)ctsio->cdb;
9339         alloc_len = scsi_2btoul(cdb->length);
9340
9341         /*
9342          * We malloc the full inquiry data size here and fill it
9343          * in.  If the user only asks for less, we'll give him
9344          * that much.
9345          */
9346         /* XXX KDM what malloc flags should we use here?? */
9347         ctsio->kern_data_ptr = malloc(sizeof(*inq_ptr), M_CTL, M_WAITOK);
9348         if (ctsio->kern_data_ptr == NULL) {
9349                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
9350                 ctsio->scsi_status = SCSI_STATUS_BUSY;
9351                 ctl_done((union ctl_io *)ctsio);
9352                 return (CTL_RETVAL_COMPLETE);
9353         }
9354         inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
9355         ctsio->kern_sg_entries = 0;
9356         ctsio->kern_data_resid = 0;
9357         ctsio->kern_rel_offset = 0;
9358
9359         if (sizeof(*inq_ptr) < alloc_len) {
9360                 ctsio->residual = alloc_len - sizeof(*inq_ptr);
9361                 ctsio->kern_data_len = sizeof(*inq_ptr);
9362                 ctsio->kern_total_len = sizeof(*inq_ptr);
9363         } else {
9364                 ctsio->residual = 0;
9365                 ctsio->kern_data_len = alloc_len;
9366                 ctsio->kern_total_len = alloc_len;
9367         }
9368
9369         memset(inq_ptr, 0, sizeof(*inq_ptr));
9370
9371         /*
9372          * The control device is always connected.  The disk device, on the
9373          * other hand, may not be online all the time.  If we don't have a
9374          * LUN mapping, we'll just say it's offline.
9375          */
9376         if (lun != NULL)
9377                 inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9378                                   lun->be_lun->lun_type;
9379         else
9380                 inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9381
9382         /* RMB in byte 2 is 0 */
9383         inq_ptr->version = SCSI_REV_SPC3;
9384
9385         /*
9386          * According to SAM-3, even if a device only supports a single
9387          * level of LUN addressing, it should still set the HISUP bit:
9388          *
9389          * 4.9.1 Logical unit numbers overview
9390          *
9391          * All logical unit number formats described in this standard are
9392          * hierarchical in structure even when only a single level in that
9393          * hierarchy is used. The HISUP bit shall be set to one in the
9394          * standard INQUIRY data (see SPC-2) when any logical unit number
9395          * format described in this standard is used.  Non-hierarchical
9396          * formats are outside the scope of this standard.
9397          *
9398          * Therefore we set the HiSup bit here.
9399          *
9400          * The reponse format is 2, per SPC-3.
9401          */
9402         inq_ptr->response_format = SID_HiSup | 2;
9403
9404         inq_ptr->additional_length = sizeof(*inq_ptr) - 4;
9405         CTL_DEBUG_PRINT(("additional_length = %d\n",
9406                          inq_ptr->additional_length));
9407
9408         inq_ptr->spc3_flags = SPC3_SID_TPGS_IMPLICIT;
9409         /* 16 bit addressing */
9410         if (is_fc == 0)
9411                 inq_ptr->spc2_flags = SPC2_SID_ADDR16;
9412         /* XXX set the SID_MultiP bit here if we're actually going to
9413            respond on multiple ports */
9414         inq_ptr->spc2_flags |= SPC2_SID_MultiP;
9415
9416         /* 16 bit data bus, synchronous transfers */
9417         /* XXX these flags don't apply for FC */
9418         if (is_fc == 0)
9419                 inq_ptr->flags = SID_WBus16 | SID_Sync;
9420         /*
9421          * XXX KDM do we want to support tagged queueing on the control
9422          * device at all?
9423          */
9424         if ((lun == NULL)
9425          || (lun->be_lun->lun_type != T_PROCESSOR))
9426                 inq_ptr->flags |= SID_CmdQue;
9427         /*
9428          * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
9429          * We have 8 bytes for the vendor name, and 16 bytes for the device
9430          * name and 4 bytes for the revision.
9431          */
9432         strncpy(inq_ptr->vendor, CTL_VENDOR, sizeof(inq_ptr->vendor));
9433         if (lun == NULL) {
9434                 strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
9435         } else {
9436                 switch (lun->be_lun->lun_type) {
9437                 case T_DIRECT:
9438                         strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
9439                         break;
9440                 case T_PROCESSOR:
9441                         strcpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT);
9442                         break;
9443                 default:
9444                         strcpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT);
9445                         break;
9446                 }
9447         }
9448
9449         /*
9450          * XXX make this a macro somewhere so it automatically gets
9451          * incremented when we make changes.
9452          */
9453         strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
9454
9455         /*
9456          * For parallel SCSI, we support double transition and single
9457          * transition clocking.  We also support QAS (Quick Arbitration
9458          * and Selection) and Information Unit transfers on both the
9459          * control and array devices.
9460          */
9461         if (is_fc == 0)
9462                 inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
9463                                     SID_SPI_IUS;
9464
9465         /* SAM-3 */
9466         scsi_ulto2b(0x0060, inq_ptr->version1);
9467         /* SPC-3 (no version claimed) XXX should we claim a version? */
9468         scsi_ulto2b(0x0300, inq_ptr->version2);
9469         if (is_fc) {
9470                 /* FCP-2 ANSI INCITS.350:2003 */
9471                 scsi_ulto2b(0x0917, inq_ptr->version3);
9472         } else {
9473                 /* SPI-4 ANSI INCITS.362:200x */
9474                 scsi_ulto2b(0x0B56, inq_ptr->version3);
9475         }
9476
9477         if (lun == NULL) {
9478                 /* SBC-2 (no version claimed) XXX should we claim a version? */
9479                 scsi_ulto2b(0x0320, inq_ptr->version4);
9480         } else {
9481                 switch (lun->be_lun->lun_type) {
9482                 case T_DIRECT:
9483                         /*
9484                          * SBC-2 (no version claimed) XXX should we claim a
9485                          * version?
9486                          */
9487                         scsi_ulto2b(0x0320, inq_ptr->version4);
9488                         break;
9489                 case T_PROCESSOR:
9490                 default:
9491                         break;
9492                 }
9493         }
9494         sprintf((char *)inq_ptr->vendor_specific1, "Copyright (C) 2004, COPAN "
9495                 "Systems, Inc.  All Rights Reserved.");
9496
9497         ctsio->scsi_status = SCSI_STATUS_OK;
9498         if (ctsio->kern_data_len > 0) {
9499                 ctsio->be_move_done = ctl_config_move_done;
9500                 ctl_datamove((union ctl_io *)ctsio);
9501         } else {
9502                 ctsio->io_hdr.status = CTL_SUCCESS;
9503                 ctl_done((union ctl_io *)ctsio);
9504         }
9505
9506         return (CTL_RETVAL_COMPLETE);
9507 }
9508
9509 int
9510 ctl_inquiry(struct ctl_scsiio *ctsio)
9511 {
9512         struct scsi_inquiry *cdb;
9513         int retval;
9514
9515         cdb = (struct scsi_inquiry *)ctsio->cdb;
9516
9517         retval = 0;
9518
9519         CTL_DEBUG_PRINT(("ctl_inquiry\n"));
9520
9521         /*
9522          * Right now, we don't support the CmdDt inquiry information.
9523          * This would be nice to support in the future.  When we do
9524          * support it, we should change this test so that it checks to make
9525          * sure SI_EVPD and SI_CMDDT aren't both set at the same time.
9526          */
9527 #ifdef notyet
9528         if (((cdb->byte2 & SI_EVPD)
9529          && (cdb->byte2 & SI_CMDDT)))
9530 #endif
9531         if (cdb->byte2 & SI_CMDDT) {
9532                 /*
9533                  * Point to the SI_CMDDT bit.  We might change this
9534                  * when we support SI_CMDDT, but since both bits would be
9535                  * "wrong", this should probably just stay as-is then.
9536                  */
9537                 ctl_set_invalid_field(ctsio,
9538                                       /*sks_valid*/ 1,
9539                                       /*command*/ 1,
9540                                       /*field*/ 1,
9541                                       /*bit_valid*/ 1,
9542                                       /*bit*/ 1);
9543                 ctl_done((union ctl_io *)ctsio);
9544                 return (CTL_RETVAL_COMPLETE);
9545         }
9546         if (cdb->byte2 & SI_EVPD)
9547                 retval = ctl_inquiry_evpd(ctsio);
9548 #ifdef notyet
9549         else if (cdb->byte2 & SI_CMDDT)
9550                 retval = ctl_inquiry_cmddt(ctsio);
9551 #endif
9552         else
9553                 retval = ctl_inquiry_std(ctsio);
9554
9555         return (retval);
9556 }
9557
9558 /*
9559  * For known CDB types, parse the LBA and length.
9560  */
9561 static int
9562 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len)
9563 {
9564         if (io->io_hdr.io_type != CTL_IO_SCSI)
9565                 return (1);
9566
9567         switch (io->scsiio.cdb[0]) {
9568         case READ_6:
9569         case WRITE_6: {
9570                 struct scsi_rw_6 *cdb;
9571
9572                 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
9573
9574                 *lba = scsi_3btoul(cdb->addr);
9575                 /* only 5 bits are valid in the most significant address byte */
9576                 *lba &= 0x1fffff;
9577                 *len = cdb->length;
9578                 break;
9579         }
9580         case READ_10:
9581         case WRITE_10: {
9582                 struct scsi_rw_10 *cdb;
9583
9584                 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
9585
9586                 *lba = scsi_4btoul(cdb->addr);
9587                 *len = scsi_2btoul(cdb->length);
9588                 break;
9589         }
9590         case WRITE_VERIFY_10: {
9591                 struct scsi_write_verify_10 *cdb;
9592
9593                 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
9594
9595                 *lba = scsi_4btoul(cdb->addr);
9596                 *len = scsi_2btoul(cdb->length);
9597                 break;
9598         }
9599         case READ_12:
9600         case WRITE_12: {
9601                 struct scsi_rw_12 *cdb;
9602
9603                 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
9604
9605                 *lba = scsi_4btoul(cdb->addr);
9606                 *len = scsi_4btoul(cdb->length);
9607                 break;
9608         }
9609         case WRITE_VERIFY_12: {
9610                 struct scsi_write_verify_12 *cdb;
9611
9612                 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
9613
9614                 *lba = scsi_4btoul(cdb->addr);
9615                 *len = scsi_4btoul(cdb->length);
9616                 break;
9617         }
9618         case READ_16:
9619         case WRITE_16: {
9620                 struct scsi_rw_16 *cdb;
9621
9622                 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
9623
9624                 *lba = scsi_8btou64(cdb->addr);
9625                 *len = scsi_4btoul(cdb->length);
9626                 break;
9627         }
9628         case WRITE_VERIFY_16: {
9629                 struct scsi_write_verify_16 *cdb;
9630
9631                 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
9632
9633
9634                 *lba = scsi_8btou64(cdb->addr);
9635                 *len = scsi_4btoul(cdb->length);
9636                 break;
9637         }
9638         default:
9639                 return (1);
9640                 break; /* NOTREACHED */
9641         }
9642
9643         return (0);
9644 }
9645
9646 static ctl_action
9647 ctl_extent_check_lba(uint64_t lba1, uint32_t len1, uint64_t lba2, uint32_t len2)
9648 {
9649         uint64_t endlba1, endlba2;
9650
9651         endlba1 = lba1 + len1 - 1;
9652         endlba2 = lba2 + len2 - 1;
9653
9654         if ((endlba1 < lba2)
9655          || (endlba2 < lba1))
9656                 return (CTL_ACTION_PASS);
9657         else
9658                 return (CTL_ACTION_BLOCK);
9659 }
9660
9661 static ctl_action
9662 ctl_extent_check(union ctl_io *io1, union ctl_io *io2)
9663 {
9664         uint64_t lba1, lba2;
9665         uint32_t len1, len2;
9666         int retval;
9667
9668         retval = ctl_get_lba_len(io1, &lba1, &len1);
9669         if (retval != 0)
9670                 return (CTL_ACTION_ERROR);
9671
9672         retval = ctl_get_lba_len(io2, &lba2, &len2);
9673         if (retval != 0)
9674                 return (CTL_ACTION_ERROR);
9675
9676         return (ctl_extent_check_lba(lba1, len1, lba2, len2));
9677 }
9678
9679 static ctl_action
9680 ctl_check_for_blockage(union ctl_io *pending_io, union ctl_io *ooa_io)
9681 {
9682         struct ctl_cmd_entry *pending_entry, *ooa_entry;
9683         ctl_serialize_action *serialize_row;
9684
9685         /*
9686          * The initiator attempted multiple untagged commands at the same
9687          * time.  Can't do that.
9688          */
9689         if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
9690          && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
9691          && ((pending_io->io_hdr.nexus.targ_port ==
9692               ooa_io->io_hdr.nexus.targ_port)
9693           && (pending_io->io_hdr.nexus.initid.id ==
9694               ooa_io->io_hdr.nexus.initid.id))
9695          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
9696                 return (CTL_ACTION_OVERLAP);
9697
9698         /*
9699          * The initiator attempted to send multiple tagged commands with
9700          * the same ID.  (It's fine if different initiators have the same
9701          * tag ID.)
9702          *
9703          * Even if all of those conditions are true, we don't kill the I/O
9704          * if the command ahead of us has been aborted.  We won't end up
9705          * sending it to the FETD, and it's perfectly legal to resend a
9706          * command with the same tag number as long as the previous
9707          * instance of this tag number has been aborted somehow.
9708          */
9709         if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
9710          && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
9711          && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
9712          && ((pending_io->io_hdr.nexus.targ_port ==
9713               ooa_io->io_hdr.nexus.targ_port)
9714           && (pending_io->io_hdr.nexus.initid.id ==
9715               ooa_io->io_hdr.nexus.initid.id))
9716          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
9717                 return (CTL_ACTION_OVERLAP_TAG);
9718
9719         /*
9720          * If we get a head of queue tag, SAM-3 says that we should
9721          * immediately execute it.
9722          *
9723          * What happens if this command would normally block for some other
9724          * reason?  e.g. a request sense with a head of queue tag
9725          * immediately after a write.  Normally that would block, but this
9726          * will result in its getting executed immediately...
9727          *
9728          * We currently return "pass" instead of "skip", so we'll end up
9729          * going through the rest of the queue to check for overlapped tags.
9730          *
9731          * XXX KDM check for other types of blockage first??
9732          */
9733         if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
9734                 return (CTL_ACTION_PASS);
9735
9736         /*
9737          * Ordered tags have to block until all items ahead of them
9738          * have completed.  If we get called with an ordered tag, we always
9739          * block, if something else is ahead of us in the queue.
9740          */
9741         if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
9742                 return (CTL_ACTION_BLOCK);
9743
9744         /*
9745          * Simple tags get blocked until all head of queue and ordered tags
9746          * ahead of them have completed.  I'm lumping untagged commands in
9747          * with simple tags here.  XXX KDM is that the right thing to do?
9748          */
9749         if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
9750           || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
9751          && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
9752           || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
9753                 return (CTL_ACTION_BLOCK);
9754
9755         pending_entry = &ctl_cmd_table[pending_io->scsiio.cdb[0]];
9756         ooa_entry = &ctl_cmd_table[ooa_io->scsiio.cdb[0]];
9757
9758         serialize_row = ctl_serialize_table[ooa_entry->seridx];
9759
9760         switch (serialize_row[pending_entry->seridx]) {
9761         case CTL_SER_BLOCK:
9762                 return (CTL_ACTION_BLOCK);
9763                 break; /* NOTREACHED */
9764         case CTL_SER_EXTENT:
9765                 return (ctl_extent_check(pending_io, ooa_io));
9766                 break; /* NOTREACHED */
9767         case CTL_SER_PASS:
9768                 return (CTL_ACTION_PASS);
9769                 break; /* NOTREACHED */
9770         case CTL_SER_SKIP:
9771                 return (CTL_ACTION_SKIP);
9772                 break;
9773         default:
9774                 panic("invalid serialization value %d",
9775                       serialize_row[pending_entry->seridx]);
9776                 break; /* NOTREACHED */
9777         }
9778
9779         return (CTL_ACTION_ERROR);
9780 }
9781
9782 /*
9783  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
9784  * Assumptions:
9785  * - caller holds ctl_lock
9786  * - pending_io is generally either incoming, or on the blocked queue
9787  * - starting I/O is the I/O we want to start the check with.
9788  */
9789 static ctl_action
9790 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
9791               union ctl_io *starting_io)
9792 {
9793         union ctl_io *ooa_io;
9794         ctl_action action;
9795
9796         /*
9797          * Run back along the OOA queue, starting with the current
9798          * blocked I/O and going through every I/O before it on the
9799          * queue.  If starting_io is NULL, we'll just end up returning
9800          * CTL_ACTION_PASS.
9801          */
9802         for (ooa_io = starting_io; ooa_io != NULL;
9803              ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
9804              ooa_links)){
9805
9806                 /*
9807                  * This routine just checks to see whether
9808                  * cur_blocked is blocked by ooa_io, which is ahead
9809                  * of it in the queue.  It doesn't queue/dequeue
9810                  * cur_blocked.
9811                  */
9812                 action = ctl_check_for_blockage(pending_io, ooa_io);
9813                 switch (action) {
9814                 case CTL_ACTION_BLOCK:
9815                 case CTL_ACTION_OVERLAP:
9816                 case CTL_ACTION_OVERLAP_TAG:
9817                 case CTL_ACTION_SKIP:
9818                 case CTL_ACTION_ERROR:
9819                         return (action);
9820                         break; /* NOTREACHED */
9821                 case CTL_ACTION_PASS:
9822                         break;
9823                 default:
9824                         panic("invalid action %d", action);
9825                         break;  /* NOTREACHED */
9826                 }
9827         }
9828
9829         return (CTL_ACTION_PASS);
9830 }
9831
9832 /*
9833  * Assumptions:
9834  * - An I/O has just completed, and has been removed from the per-LUN OOA
9835  *   queue, so some items on the blocked queue may now be unblocked.
9836  * - The caller holds ctl_softc->ctl_lock
9837  */
9838 static int
9839 ctl_check_blocked(struct ctl_lun *lun)
9840 {
9841         union ctl_io *cur_blocked, *next_blocked;
9842
9843         /*
9844          * Run forward from the head of the blocked queue, checking each
9845          * entry against the I/Os prior to it on the OOA queue to see if
9846          * there is still any blockage.
9847          *
9848          * We cannot use the TAILQ_FOREACH() macro, because it can't deal
9849          * with our removing a variable on it while it is traversing the
9850          * list.
9851          */
9852         for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
9853              cur_blocked != NULL; cur_blocked = next_blocked) {
9854                 union ctl_io *prev_ooa;
9855                 ctl_action action;
9856
9857                 next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
9858                                                           blocked_links);
9859
9860                 prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
9861                                                       ctl_ooaq, ooa_links);
9862
9863                 /*
9864                  * If cur_blocked happens to be the first item in the OOA
9865                  * queue now, prev_ooa will be NULL, and the action
9866                  * returned will just be CTL_ACTION_PASS.
9867                  */
9868                 action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
9869
9870                 switch (action) {
9871                 case CTL_ACTION_BLOCK:
9872                         /* Nothing to do here, still blocked */
9873                         break;
9874                 case CTL_ACTION_OVERLAP:
9875                 case CTL_ACTION_OVERLAP_TAG:
9876                         /*
9877                          * This shouldn't happen!  In theory we've already
9878                          * checked this command for overlap...
9879                          */
9880                         break;
9881                 case CTL_ACTION_PASS:
9882                 case CTL_ACTION_SKIP: {
9883                         struct ctl_softc *softc;
9884                         struct ctl_cmd_entry *entry;
9885                         uint32_t initidx;
9886                         uint8_t opcode;
9887                         int isc_retval;
9888
9889                         /*
9890                          * The skip case shouldn't happen, this transaction
9891                          * should have never made it onto the blocked queue.
9892                          */
9893                         /*
9894                          * This I/O is no longer blocked, we can remove it
9895                          * from the blocked queue.  Since this is a TAILQ
9896                          * (doubly linked list), we can do O(1) removals
9897                          * from any place on the list.
9898                          */
9899                         TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
9900                                      blocked_links);
9901                         cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
9902
9903                         if (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC){
9904                                 /*
9905                                  * Need to send IO back to original side to
9906                                  * run
9907                                  */
9908                                 union ctl_ha_msg msg_info;
9909
9910                                 msg_info.hdr.original_sc =
9911                                         cur_blocked->io_hdr.original_sc;
9912                                 msg_info.hdr.serializing_sc = cur_blocked;
9913                                 msg_info.hdr.msg_type = CTL_MSG_R2R;
9914                                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
9915                                      &msg_info, sizeof(msg_info), 0)) >
9916                                      CTL_HA_STATUS_SUCCESS) {
9917                                         printf("CTL:Check Blocked error from "
9918                                                "ctl_ha_msg_send %d\n",
9919                                                isc_retval);
9920                                 }
9921                                 break;
9922                         }
9923                         opcode = cur_blocked->scsiio.cdb[0];
9924                         entry = &ctl_cmd_table[opcode];
9925                         softc = control_softc;
9926
9927                         initidx = ctl_get_initindex(&cur_blocked->io_hdr.nexus);
9928
9929                         /*
9930                          * Check this I/O for LUN state changes that may
9931                          * have happened while this command was blocked.
9932                          * The LUN state may have been changed by a command
9933                          * ahead of us in the queue, so we need to re-check
9934                          * for any states that can be caused by SCSI
9935                          * commands.
9936                          */
9937                         if (ctl_scsiio_lun_check(softc, lun, entry,
9938                                                  &cur_blocked->scsiio) == 0) {
9939                                 cur_blocked->io_hdr.flags |=
9940                                                       CTL_FLAG_IS_WAS_ON_RTR;
9941                                 STAILQ_INSERT_TAIL(&lun->ctl_softc->rtr_queue,
9942                                                    &cur_blocked->io_hdr, links);
9943                                 /*
9944                                  * In the non CTL_DONE_THREAD case, we need
9945                                  * to wake up the work thread here.  When
9946                                  * we're processing completed requests from
9947                                  * the work thread context, we'll pop back
9948                                  * around and end up pulling things off the
9949                                  * RtR queue.  When we aren't processing
9950                                  * things from the work thread context,
9951                                  * though, we won't ever check the RtR queue.
9952                                  * So we need to wake up the thread to clear
9953                                  * things off the queue.  Otherwise this
9954                                  * transaction will just sit on the RtR queue
9955                                  * until a new I/O comes in.  (Which may or
9956                                  * may not happen...)
9957                                  */
9958 #ifndef CTL_DONE_THREAD
9959                                 ctl_wakeup_thread();
9960 #endif
9961                         } else
9962                                 ctl_done_lock(cur_blocked, /*have_lock*/ 1);
9963                         break;
9964                 }
9965                 default:
9966                         /*
9967                          * This probably shouldn't happen -- we shouldn't
9968                          * get CTL_ACTION_ERROR, or anything else.
9969                          */
9970                         break;
9971                 }
9972         }
9973
9974         return (CTL_RETVAL_COMPLETE);
9975 }
9976
9977 /*
9978  * This routine (with one exception) checks LUN flags that can be set by
9979  * commands ahead of us in the OOA queue.  These flags have to be checked
9980  * when a command initially comes in, and when we pull a command off the
9981  * blocked queue and are preparing to execute it.  The reason we have to
9982  * check these flags for commands on the blocked queue is that the LUN
9983  * state may have been changed by a command ahead of us while we're on the
9984  * blocked queue.
9985  *
9986  * Ordering is somewhat important with these checks, so please pay
9987  * careful attention to the placement of any new checks.
9988  */
9989 static int
9990 ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
9991                      struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
9992 {
9993         int retval;
9994
9995         retval = 0;
9996
9997         /*
9998          * If this shelf is a secondary shelf controller, we have to reject
9999          * any media access commands.
10000          */
10001 #if 0
10002         /* No longer needed for HA */
10003         if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
10004          && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
10005                 ctl_set_lun_standby(ctsio);
10006                 retval = 1;
10007                 goto bailout;
10008         }
10009 #endif
10010
10011         /*
10012          * Check for a reservation conflict.  If this command isn't allowed
10013          * even on reserved LUNs, and if this initiator isn't the one who
10014          * reserved us, reject the command with a reservation conflict.
10015          */
10016         if ((lun->flags & CTL_LUN_RESERVED)
10017          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
10018                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
10019                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
10020                  || (ctsio->io_hdr.nexus.targ_target.id !=
10021                      lun->rsv_nexus.targ_target.id)) {
10022                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10023                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
10024                         retval = 1;
10025                         goto bailout;
10026                 }
10027         }
10028
10029         if ( (lun->flags & CTL_LUN_PR_RESERVED)
10030          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV) == 0)) {
10031                 uint32_t residx;
10032
10033                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
10034                 /*
10035                  * if we aren't registered or it's a res holder type
10036                  * reservation and this isn't the res holder then set a
10037                  * conflict.
10038                  * NOTE: Commands which might be allowed on write exclusive
10039                  * type reservations are checked in the particular command
10040                  * for a conflict. Read and SSU are the only ones.
10041                  */
10042                 if (!lun->per_res[residx].registered
10043                  || (residx != lun->pr_res_idx && lun->res_type < 4)) {
10044                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10045                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
10046                         retval = 1;
10047                         goto bailout;
10048                 }
10049
10050         }
10051
10052         if ((lun->flags & CTL_LUN_OFFLINE)
10053          && ((entry->flags & CTL_CMD_FLAG_OK_ON_OFFLINE) == 0)) {
10054                 ctl_set_lun_not_ready(ctsio);
10055                 retval = 1;
10056                 goto bailout;
10057         }
10058
10059         /*
10060          * If the LUN is stopped, see if this particular command is allowed
10061          * for a stopped lun.  Otherwise, reject it with 0x04,0x02.
10062          */
10063         if ((lun->flags & CTL_LUN_STOPPED)
10064          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
10065                 /* "Logical unit not ready, initializing cmd. required" */
10066                 ctl_set_lun_stopped(ctsio);
10067                 retval = 1;
10068                 goto bailout;
10069         }
10070
10071         if ((lun->flags & CTL_LUN_INOPERABLE)
10072          && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
10073                 /* "Medium format corrupted" */
10074                 ctl_set_medium_format_corrupted(ctsio);
10075                 retval = 1;
10076                 goto bailout;
10077         }
10078
10079 bailout:
10080         return (retval);
10081
10082 }
10083
10084 static void
10085 ctl_failover_io(union ctl_io *io, int have_lock)
10086 {
10087         ctl_set_busy(&io->scsiio);
10088         ctl_done_lock(io, have_lock);
10089 }
10090
10091 static void
10092 ctl_failover(void)
10093 {
10094         struct ctl_lun *lun;
10095         struct ctl_softc *ctl_softc;
10096         union ctl_io *next_io, *pending_io;
10097         union ctl_io *io;
10098         int lun_idx;
10099         int i;
10100
10101         ctl_softc = control_softc;
10102
10103         mtx_lock(&ctl_softc->ctl_lock);
10104         /*
10105          * Remove any cmds from the other SC from the rtr queue.  These
10106          * will obviously only be for LUNs for which we're the primary.
10107          * We can't send status or get/send data for these commands.
10108          * Since they haven't been executed yet, we can just remove them.
10109          * We'll either abort them or delete them below, depending on
10110          * which HA mode we're in.
10111          */
10112         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->rtr_queue);
10113              io != NULL; io = next_io) {
10114                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
10115                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10116                         STAILQ_REMOVE(&ctl_softc->rtr_queue, &io->io_hdr,
10117                                       ctl_io_hdr, links);
10118         }
10119
10120         for (lun_idx=0; lun_idx < ctl_softc->num_luns; lun_idx++) {
10121                 lun = ctl_softc->ctl_luns[lun_idx];
10122                 if (lun==NULL)
10123                         continue;
10124
10125                 /*
10126                  * Processor LUNs are primary on both sides.
10127                  * XXX will this always be true?
10128                  */
10129                 if (lun->be_lun->lun_type == T_PROCESSOR)
10130                         continue;
10131
10132                 if ((lun->flags & CTL_LUN_PRIMARY_SC)
10133                  && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10134                         printf("FAILOVER: primary lun %d\n", lun_idx);
10135                         /*
10136                          * Remove all commands from the other SC. First from the
10137                          * blocked queue then from the ooa queue. Once we have
10138                          * removed them. Call ctl_check_blocked to see if there
10139                          * is anything that can run.
10140                          */
10141                         for (io = (union ctl_io *)TAILQ_FIRST(
10142                              &lun->blocked_queue); io != NULL; io = next_io) {
10143
10144                                 next_io = (union ctl_io *)TAILQ_NEXT(
10145                                     &io->io_hdr, blocked_links);
10146
10147                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10148                                         TAILQ_REMOVE(&lun->blocked_queue,
10149                                                      &io->io_hdr,blocked_links);
10150                                         io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10151                                         TAILQ_REMOVE(&lun->ooa_queue,
10152                                                      &io->io_hdr, ooa_links);
10153
10154                                         ctl_free_io_internal(io, 1);
10155                                 }
10156                         }
10157
10158                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10159                              io != NULL; io = next_io) {
10160
10161                                 next_io = (union ctl_io *)TAILQ_NEXT(
10162                                     &io->io_hdr, ooa_links);
10163
10164                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10165
10166                                         TAILQ_REMOVE(&lun->ooa_queue,
10167                                                 &io->io_hdr,
10168                                                 ooa_links);
10169
10170                                         ctl_free_io_internal(io, 1);
10171                                 }
10172                         }
10173                         ctl_check_blocked(lun);
10174                 } else if ((lun->flags & CTL_LUN_PRIMARY_SC)
10175                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10176
10177                         printf("FAILOVER: primary lun %d\n", lun_idx);
10178                         /*
10179                          * Abort all commands from the other SC.  We can't
10180                          * send status back for them now.  These should get
10181                          * cleaned up when they are completed or come out
10182                          * for a datamove operation.
10183                          */
10184                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10185                              io != NULL; io = next_io) {
10186                                 next_io = (union ctl_io *)TAILQ_NEXT(
10187                                         &io->io_hdr, ooa_links);
10188
10189                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10190                                         io->io_hdr.flags |= CTL_FLAG_ABORT;
10191                         }
10192                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10193                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10194
10195                         printf("FAILOVER: secondary lun %d\n", lun_idx);
10196
10197                         lun->flags |= CTL_LUN_PRIMARY_SC;
10198
10199                         /*
10200                          * We send all I/O that was sent to this controller
10201                          * and redirected to the other side back with
10202                          * busy status, and have the initiator retry it.
10203                          * Figuring out how much data has been transferred,
10204                          * etc. and picking up where we left off would be
10205                          * very tricky.
10206                          *
10207                          * XXX KDM need to remove I/O from the blocked
10208                          * queue as well!
10209                          */
10210                         for (pending_io = (union ctl_io *)TAILQ_FIRST(
10211                              &lun->ooa_queue); pending_io != NULL;
10212                              pending_io = next_io) {
10213
10214                                 next_io =  (union ctl_io *)TAILQ_NEXT(
10215                                         &pending_io->io_hdr, ooa_links);
10216
10217                                 pending_io->io_hdr.flags &=
10218                                         ~CTL_FLAG_SENT_2OTHER_SC;
10219
10220                                 if (pending_io->io_hdr.flags &
10221                                     CTL_FLAG_IO_ACTIVE) {
10222                                         pending_io->io_hdr.flags |=
10223                                                 CTL_FLAG_FAILOVER;
10224                                 } else {
10225                                         ctl_set_busy(&pending_io->scsiio);
10226                                         ctl_done_lock(pending_io,
10227                                                       /*have_lock*/1);
10228                                 }
10229                         }
10230
10231                         /*
10232                          * Build Unit Attention
10233                          */
10234                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10235                                 lun->pending_sense[i].ua_pending |=
10236                                                      CTL_UA_ASYM_ACC_CHANGE;
10237                         }
10238                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10239                         && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10240                         printf("FAILOVER: secondary lun %d\n", lun_idx);
10241                         /*
10242                          * if the first io on the OOA is not on the RtR queue
10243                          * add it.
10244                          */
10245                         lun->flags |= CTL_LUN_PRIMARY_SC;
10246
10247                         pending_io = (union ctl_io *)TAILQ_FIRST(
10248                             &lun->ooa_queue);
10249                         if (pending_io==NULL) {
10250                                 printf("Nothing on OOA queue\n");
10251                                 continue;
10252                         }
10253
10254                         pending_io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
10255                         if ((pending_io->io_hdr.flags &
10256                              CTL_FLAG_IS_WAS_ON_RTR) == 0) {
10257                                 pending_io->io_hdr.flags |=
10258                                     CTL_FLAG_IS_WAS_ON_RTR;
10259                                 STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
10260                                                    &pending_io->io_hdr, links);
10261                         }
10262 #if 0
10263                         else
10264                         {
10265                                 printf("Tag 0x%04x is running\n",
10266                                       pending_io->scsiio.tag_num);
10267                         }
10268 #endif
10269
10270                         next_io = (union ctl_io *)TAILQ_NEXT(
10271                             &pending_io->io_hdr, ooa_links);
10272                         for (pending_io=next_io; pending_io != NULL;
10273                              pending_io = next_io) {
10274                                 pending_io->io_hdr.flags &=
10275                                     ~CTL_FLAG_SENT_2OTHER_SC;
10276                                 next_io = (union ctl_io *)TAILQ_NEXT(
10277                                         &pending_io->io_hdr, ooa_links);
10278                                 if (pending_io->io_hdr.flags &
10279                                     CTL_FLAG_IS_WAS_ON_RTR) {
10280 #if 0
10281                                         printf("Tag 0x%04x is running\n",
10282                                                 pending_io->scsiio.tag_num);
10283 #endif
10284                                         continue;
10285                                 }
10286
10287                                 switch (ctl_check_ooa(lun, pending_io,
10288                                     (union ctl_io *)TAILQ_PREV(
10289                                     &pending_io->io_hdr, ctl_ooaq,
10290                                     ooa_links))) {
10291
10292                                 case CTL_ACTION_BLOCK:
10293                                         TAILQ_INSERT_TAIL(&lun->blocked_queue,
10294                                                           &pending_io->io_hdr,
10295                                                           blocked_links);
10296                                         pending_io->io_hdr.flags |=
10297                                             CTL_FLAG_BLOCKED;
10298                                         break;
10299                                 case CTL_ACTION_PASS:
10300                                 case CTL_ACTION_SKIP:
10301                                         pending_io->io_hdr.flags |=
10302                                             CTL_FLAG_IS_WAS_ON_RTR;
10303                                         STAILQ_INSERT_TAIL(
10304                                             &ctl_softc->rtr_queue,
10305                                             &pending_io->io_hdr, links);
10306                                         break;
10307                                 case CTL_ACTION_OVERLAP:
10308                                         ctl_set_overlapped_cmd(
10309                                             (struct ctl_scsiio *)pending_io);
10310                                         ctl_done_lock(pending_io,
10311                                                       /*have_lock*/ 1);
10312                                         break;
10313                                 case CTL_ACTION_OVERLAP_TAG:
10314                                         ctl_set_overlapped_tag(
10315                                             (struct ctl_scsiio *)pending_io,
10316                                             pending_io->scsiio.tag_num & 0xff);
10317                                         ctl_done_lock(pending_io,
10318                                                       /*have_lock*/ 1);
10319                                         break;
10320                                 case CTL_ACTION_ERROR:
10321                                 default:
10322                                         ctl_set_internal_failure(
10323                                                 (struct ctl_scsiio *)pending_io,
10324                                                 0,  // sks_valid
10325                                                 0); //retry count
10326                                         ctl_done_lock(pending_io,
10327                                                       /*have_lock*/ 1);
10328                                         break;
10329                                 }
10330                         }
10331
10332                         /*
10333                          * Build Unit Attention
10334                          */
10335                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10336                                 lun->pending_sense[i].ua_pending |=
10337                                                      CTL_UA_ASYM_ACC_CHANGE;
10338                         }
10339                 } else {
10340                         panic("Unhandled HA mode failover, LUN flags = %#x, "
10341                               "ha_mode = #%x", lun->flags, ctl_softc->ha_mode);
10342                 }
10343         }
10344         ctl_pause_rtr = 0;
10345         mtx_unlock(&ctl_softc->ctl_lock);
10346 }
10347
10348 static int
10349 ctl_scsiio_precheck(struct ctl_softc *ctl_softc, struct ctl_scsiio *ctsio)
10350 {
10351         struct ctl_lun *lun;
10352         struct ctl_cmd_entry *entry;
10353         uint8_t opcode;
10354         uint32_t initidx;
10355         int retval;
10356
10357         retval = 0;
10358
10359         lun = NULL;
10360
10361         opcode = ctsio->cdb[0];
10362
10363         mtx_lock(&ctl_softc->ctl_lock);
10364
10365         if ((ctsio->io_hdr.nexus.targ_lun < CTL_MAX_LUNS)
10366          && (ctl_softc->ctl_luns[ctsio->io_hdr.nexus.targ_lun] != NULL)) {
10367                 lun = ctl_softc->ctl_luns[ctsio->io_hdr.nexus.targ_lun];
10368                 /*
10369                  * If the LUN is invalid, pretend that it doesn't exist.
10370                  * It will go away as soon as all pending I/O has been
10371                  * completed.
10372                  */
10373                 if (lun->flags & CTL_LUN_DISABLED) {
10374                         lun = NULL;
10375                 } else {
10376                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
10377                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
10378                                 lun->be_lun;
10379                         if (lun->be_lun->lun_type == T_PROCESSOR) {
10380                                 ctsio->io_hdr.flags |= CTL_FLAG_CONTROL_DEV;
10381                         }
10382                 }
10383         } else {
10384                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
10385                 ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
10386         }
10387
10388         entry = &ctl_cmd_table[opcode];
10389
10390         ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
10391         ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
10392
10393         /*
10394          * Check to see whether we can send this command to LUNs that don't
10395          * exist.  This should pretty much only be the case for inquiry
10396          * and request sense.  Further checks, below, really require having
10397          * a LUN, so we can't really check the command anymore.  Just put
10398          * it on the rtr queue.
10399          */
10400         if (lun == NULL) {
10401                 if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10402                         goto queue_rtr;
10403
10404                 ctl_set_unsupported_lun(ctsio);
10405                 mtx_unlock(&ctl_softc->ctl_lock);
10406                 ctl_done((union ctl_io *)ctsio);
10407                 goto bailout;
10408         } else {
10409                 /*
10410                  * Every I/O goes into the OOA queue for a particular LUN, and
10411                  * stays there until completion.
10412                  */
10413                 TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
10414
10415                 /*
10416                  * Make sure we support this particular command on this LUN.
10417                  * e.g., we don't support writes to the control LUN.
10418                  */
10419                 switch (lun->be_lun->lun_type) {
10420                 case T_PROCESSOR:
10421                         if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0)
10422                          && ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10423                               == 0)) {
10424                                 ctl_set_invalid_opcode(ctsio);
10425                                 mtx_unlock(&ctl_softc->ctl_lock);
10426                                 ctl_done((union ctl_io *)ctsio);
10427                                 goto bailout;
10428                         }
10429                         break;
10430                 case T_DIRECT:
10431                         if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0)
10432                          && ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10433                               == 0)){
10434                                 ctl_set_invalid_opcode(ctsio);
10435                                 mtx_unlock(&ctl_softc->ctl_lock);
10436                                 ctl_done((union ctl_io *)ctsio);
10437                                 goto bailout;
10438                         }
10439                         break;
10440                 default:
10441                         printf("Unsupported CTL LUN type %d\n",
10442                                lun->be_lun->lun_type);
10443                         panic("Unsupported CTL LUN type %d\n",
10444                               lun->be_lun->lun_type);
10445                         break; /* NOTREACHED */
10446                 }
10447         }
10448
10449         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
10450
10451         /*
10452          * If we've got a request sense, it'll clear the contingent
10453          * allegiance condition.  Otherwise, if we have a CA condition for
10454          * this initiator, clear it, because it sent down a command other
10455          * than request sense.
10456          */
10457         if ((opcode != REQUEST_SENSE)
10458          && (ctl_is_set(lun->have_ca, initidx)))
10459                 ctl_clear_mask(lun->have_ca, initidx);
10460
10461         /*
10462          * If the command has this flag set, it handles its own unit
10463          * attention reporting, we shouldn't do anything.  Otherwise we
10464          * check for any pending unit attentions, and send them back to the
10465          * initiator.  We only do this when a command initially comes in,
10466          * not when we pull it off the blocked queue.
10467          *
10468          * According to SAM-3, section 5.3.2, the order that things get
10469          * presented back to the host is basically unit attentions caused
10470          * by some sort of reset event, busy status, reservation conflicts
10471          * or task set full, and finally any other status.
10472          *
10473          * One issue here is that some of the unit attentions we report
10474          * don't fall into the "reset" category (e.g. "reported luns data
10475          * has changed").  So reporting it here, before the reservation
10476          * check, may be technically wrong.  I guess the only thing to do
10477          * would be to check for and report the reset events here, and then
10478          * check for the other unit attention types after we check for a
10479          * reservation conflict.
10480          *
10481          * XXX KDM need to fix this
10482          */
10483         if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
10484                 ctl_ua_type ua_type;
10485
10486                 ua_type = lun->pending_sense[initidx].ua_pending;
10487                 if (ua_type != CTL_UA_NONE) {
10488                         scsi_sense_data_type sense_format;
10489
10490                         if (lun != NULL)
10491                                 sense_format = (lun->flags &
10492                                     CTL_LUN_SENSE_DESC) ? SSD_TYPE_DESC :
10493                                     SSD_TYPE_FIXED;
10494                         else
10495                                 sense_format = SSD_TYPE_FIXED;
10496
10497                         ua_type = ctl_build_ua(ua_type, &ctsio->sense_data,
10498                                                sense_format);
10499                         if (ua_type != CTL_UA_NONE) {
10500                                 ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
10501                                 ctsio->io_hdr.status = CTL_SCSI_ERROR |
10502                                                        CTL_AUTOSENSE;
10503                                 ctsio->sense_len = SSD_FULL_SIZE;
10504                                 lun->pending_sense[initidx].ua_pending &=
10505                                         ~ua_type;
10506                                 mtx_unlock(&ctl_softc->ctl_lock);
10507                                 ctl_done((union ctl_io *)ctsio);
10508                                 goto bailout;
10509                         }
10510                 }
10511         }
10512
10513
10514         if (ctl_scsiio_lun_check(ctl_softc, lun, entry, ctsio) != 0) {
10515                 mtx_unlock(&ctl_softc->ctl_lock);
10516                 ctl_done((union ctl_io *)ctsio);
10517                 goto bailout;
10518         }
10519
10520         /*
10521          * XXX CHD this is where we want to send IO to other side if
10522          * this LUN is secondary on this SC. We will need to make a copy
10523          * of the IO and flag the IO on this side as SENT_2OTHER and the flag
10524          * the copy we send as FROM_OTHER.
10525          * We also need to stuff the address of the original IO so we can
10526          * find it easily. Something similar will need be done on the other
10527          * side so when we are done we can find the copy.
10528          */
10529         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
10530                 union ctl_ha_msg msg_info;
10531                 int isc_retval;
10532
10533                 ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
10534
10535                 msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
10536                 msg_info.hdr.original_sc = (union ctl_io *)ctsio;
10537 #if 0
10538                 printf("1. ctsio %p\n", ctsio);
10539 #endif
10540                 msg_info.hdr.serializing_sc = NULL;
10541                 msg_info.hdr.nexus = ctsio->io_hdr.nexus;
10542                 msg_info.scsi.tag_num = ctsio->tag_num;
10543                 msg_info.scsi.tag_type = ctsio->tag_type;
10544                 memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
10545
10546                 ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
10547
10548                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10549                     (void *)&msg_info, sizeof(msg_info), 0)) >
10550                     CTL_HA_STATUS_SUCCESS) {
10551                         printf("CTL:precheck, ctl_ha_msg_send returned %d\n",
10552                                isc_retval);
10553                         printf("CTL:opcode is %x\n",opcode);
10554                 } else {
10555 #if 0
10556                         printf("CTL:Precheck sent msg, opcode is %x\n",opcode);
10557 #endif
10558                 }
10559
10560                 /*
10561                  * XXX KDM this I/O is off the incoming queue, but hasn't
10562                  * been inserted on any other queue.  We may need to come
10563                  * up with a holding queue while we wait for serialization
10564                  * so that we have an idea of what we're waiting for from
10565                  * the other side.
10566                  */
10567                 goto bailout_unlock;
10568         }
10569
10570         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
10571                               (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
10572                               ctl_ooaq, ooa_links))) {
10573         case CTL_ACTION_BLOCK:
10574                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
10575                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
10576                                   blocked_links);
10577                 goto bailout_unlock;
10578                 break; /* NOTREACHED */
10579         case CTL_ACTION_PASS:
10580         case CTL_ACTION_SKIP:
10581                 goto queue_rtr;
10582                 break; /* NOTREACHED */
10583         case CTL_ACTION_OVERLAP:
10584                 ctl_set_overlapped_cmd(ctsio);
10585                 mtx_unlock(&ctl_softc->ctl_lock);
10586                 ctl_done((union ctl_io *)ctsio);
10587                 goto bailout;
10588                 break; /* NOTREACHED */
10589         case CTL_ACTION_OVERLAP_TAG:
10590                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
10591                 mtx_unlock(&ctl_softc->ctl_lock);
10592                 ctl_done((union ctl_io *)ctsio);
10593                 goto bailout;
10594                 break; /* NOTREACHED */
10595         case CTL_ACTION_ERROR:
10596         default:
10597                 ctl_set_internal_failure(ctsio,
10598                                          /*sks_valid*/ 0,
10599                                          /*retry_count*/ 0);
10600                 mtx_unlock(&ctl_softc->ctl_lock);
10601                 ctl_done((union ctl_io *)ctsio);
10602                 goto bailout;
10603                 break; /* NOTREACHED */
10604         }
10605
10606         goto bailout_unlock;
10607
10608 queue_rtr:
10609         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
10610         STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue, &ctsio->io_hdr, links);
10611
10612 bailout_unlock:
10613         mtx_unlock(&ctl_softc->ctl_lock);
10614
10615 bailout:
10616         return (retval);
10617 }
10618
10619 static int
10620 ctl_scsiio(struct ctl_scsiio *ctsio)
10621 {
10622         int retval;
10623         struct ctl_cmd_entry *entry;
10624
10625         retval = CTL_RETVAL_COMPLETE;
10626
10627         CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
10628
10629         entry = &ctl_cmd_table[ctsio->cdb[0]];
10630
10631         /*
10632          * If this I/O has been aborted, just send it straight to
10633          * ctl_done() without executing it.
10634          */
10635         if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
10636                 ctl_done((union ctl_io *)ctsio);
10637                 goto bailout;
10638         }
10639
10640         /*
10641          * All the checks should have been handled by ctl_scsiio_precheck().
10642          * We should be clear now to just execute the I/O.
10643          */
10644         retval = entry->execute(ctsio);
10645
10646 bailout:
10647         return (retval);
10648 }
10649
10650 /*
10651  * Since we only implement one target right now, a bus reset simply resets
10652  * our single target.
10653  */
10654 static int
10655 ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io)
10656 {
10657         return(ctl_target_reset(ctl_softc, io, CTL_UA_BUS_RESET));
10658 }
10659
10660 static int
10661 ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
10662                  ctl_ua_type ua_type)
10663 {
10664         struct ctl_lun *lun;
10665         int retval;
10666
10667         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
10668                 union ctl_ha_msg msg_info;
10669
10670                 io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
10671                 msg_info.hdr.nexus = io->io_hdr.nexus;
10672                 if (ua_type==CTL_UA_TARG_RESET)
10673                         msg_info.task.task_action = CTL_TASK_TARGET_RESET;
10674                 else
10675                         msg_info.task.task_action = CTL_TASK_BUS_RESET;
10676                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
10677                 msg_info.hdr.original_sc = NULL;
10678                 msg_info.hdr.serializing_sc = NULL;
10679                 if (CTL_HA_STATUS_SUCCESS != ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10680                     (void *)&msg_info, sizeof(msg_info), 0)) {
10681                 }
10682         }
10683         retval = 0;
10684
10685         STAILQ_FOREACH(lun, &ctl_softc->lun_list, links)
10686                 retval += ctl_lun_reset(lun, io, ua_type);
10687
10688         return (retval);
10689 }
10690
10691 /*
10692  * The LUN should always be set.  The I/O is optional, and is used to
10693  * distinguish between I/Os sent by this initiator, and by other
10694  * initiators.  We set unit attention for initiators other than this one.
10695  * SAM-3 is vague on this point.  It does say that a unit attention should
10696  * be established for other initiators when a LUN is reset (see section
10697  * 5.7.3), but it doesn't specifically say that the unit attention should
10698  * be established for this particular initiator when a LUN is reset.  Here
10699  * is the relevant text, from SAM-3 rev 8:
10700  *
10701  * 5.7.2 When a SCSI initiator port aborts its own tasks
10702  *
10703  * When a SCSI initiator port causes its own task(s) to be aborted, no
10704  * notification that the task(s) have been aborted shall be returned to
10705  * the SCSI initiator port other than the completion response for the
10706  * command or task management function action that caused the task(s) to
10707  * be aborted and notification(s) associated with related effects of the
10708  * action (e.g., a reset unit attention condition).
10709  *
10710  * XXX KDM for now, we're setting unit attention for all initiators.
10711  */
10712 static int
10713 ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
10714 {
10715         union ctl_io *xio;
10716 #if 0
10717         uint32_t initindex;
10718 #endif
10719         int i;
10720
10721         /*
10722          * Run through the OOA queue and abort each I/O.
10723          */
10724 #if 0
10725         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
10726 #endif
10727         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
10728              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
10729                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
10730         }
10731
10732         /*
10733          * This version sets unit attention for every
10734          */
10735 #if 0
10736         initindex = ctl_get_initindex(&io->io_hdr.nexus);
10737         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10738                 if (initindex == i)
10739                         continue;
10740                 lun->pending_sense[i].ua_pending |= ua_type;
10741         }
10742 #endif
10743
10744         /*
10745          * A reset (any kind, really) clears reservations established with
10746          * RESERVE/RELEASE.  It does not clear reservations established
10747          * with PERSISTENT RESERVE OUT, but we don't support that at the
10748          * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
10749          * reservations made with the RESERVE/RELEASE commands, because
10750          * those commands are obsolete in SPC-3.
10751          */
10752         lun->flags &= ~CTL_LUN_RESERVED;
10753
10754         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10755                 ctl_clear_mask(lun->have_ca, i);
10756                 lun->pending_sense[i].ua_pending |= ua_type;
10757         }
10758
10759         return (0);
10760 }
10761
10762 static int
10763 ctl_abort_task(union ctl_io *io)
10764 {
10765         union ctl_io *xio;
10766         struct ctl_lun *lun;
10767         struct ctl_softc *ctl_softc;
10768 #if 0
10769         struct sbuf sb;
10770         char printbuf[128];
10771 #endif
10772         int found;
10773
10774         ctl_softc = control_softc;
10775         found = 0;
10776
10777         /*
10778          * Look up the LUN.
10779          */
10780         if ((io->io_hdr.nexus.targ_lun < CTL_MAX_LUNS)
10781          && (ctl_softc->ctl_luns[io->io_hdr.nexus.targ_lun] != NULL))
10782                 lun = ctl_softc->ctl_luns[io->io_hdr.nexus.targ_lun];
10783         else
10784                 goto bailout;
10785
10786 #if 0
10787         printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
10788                lun->lun, io->taskio.tag_num, io->taskio.tag_type);
10789 #endif
10790
10791         /*
10792          * Run through the OOA queue and attempt to find the given I/O.
10793          * The target port, initiator ID, tag type and tag number have to
10794          * match the values that we got from the initiator.  If we have an
10795          * untagged command to abort, simply abort the first untagged command
10796          * we come to.  We only allow one untagged command at a time of course.
10797          */
10798 #if 0
10799         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
10800 #endif
10801         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
10802              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
10803 #if 0
10804                 sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
10805
10806                 sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
10807                             lun->lun, xio->scsiio.tag_num,
10808                             xio->scsiio.tag_type,
10809                             (xio->io_hdr.blocked_links.tqe_prev
10810                             == NULL) ? "" : " BLOCKED",
10811                             (xio->io_hdr.flags &
10812                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
10813                             (xio->io_hdr.flags &
10814                             CTL_FLAG_ABORT) ? " ABORT" : ""),
10815                             (xio->io_hdr.flags &
10816                             CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : "");
10817                 ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
10818                 sbuf_finish(&sb);
10819                 printf("%s\n", sbuf_data(&sb));
10820 #endif
10821
10822                 if ((xio->io_hdr.nexus.targ_port == io->io_hdr.nexus.targ_port)
10823                  && (xio->io_hdr.nexus.initid.id ==
10824                      io->io_hdr.nexus.initid.id)) {
10825                         /*
10826                          * If the abort says that the task is untagged, the
10827                          * task in the queue must be untagged.  Otherwise,
10828                          * we just check to see whether the tag numbers
10829                          * match.  This is because the QLogic firmware
10830                          * doesn't pass back the tag type in an abort
10831                          * request.
10832                          */
10833 #if 0
10834                         if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
10835                           && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
10836                          || (xio->scsiio.tag_num == io->taskio.tag_num)) {
10837 #endif
10838                         /*
10839                          * XXX KDM we've got problems with FC, because it
10840                          * doesn't send down a tag type with aborts.  So we
10841                          * can only really go by the tag number...
10842                          * This may cause problems with parallel SCSI.
10843                          * Need to figure that out!!
10844                          */
10845                         if (xio->scsiio.tag_num == io->taskio.tag_num) {
10846                                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
10847                                 found = 1;
10848                                 if ((io->io_hdr.flags &
10849                                      CTL_FLAG_FROM_OTHER_SC) == 0 &&
10850                                     !(lun->flags & CTL_LUN_PRIMARY_SC)) {
10851                                         union ctl_ha_msg msg_info;
10852
10853                                         io->io_hdr.flags |=
10854                                                         CTL_FLAG_SENT_2OTHER_SC;
10855                                         msg_info.hdr.nexus = io->io_hdr.nexus;
10856                                         msg_info.task.task_action =
10857                                                 CTL_TASK_ABORT_TASK;
10858                                         msg_info.task.tag_num =
10859                                                 io->taskio.tag_num;
10860                                         msg_info.task.tag_type =
10861                                                 io->taskio.tag_type;
10862                                         msg_info.hdr.msg_type =
10863                                                 CTL_MSG_MANAGE_TASKS;
10864                                         msg_info.hdr.original_sc = NULL;
10865                                         msg_info.hdr.serializing_sc = NULL;
10866 #if 0
10867                                         printf("Sent Abort to other side\n");
10868 #endif
10869                                         if (CTL_HA_STATUS_SUCCESS !=
10870                                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10871                                                 (void *)&msg_info,
10872                                                 sizeof(msg_info), 0)) {
10873                                         }
10874                                 }
10875 #if 0
10876                                 printf("ctl_abort_task: found I/O to abort\n");
10877 #endif
10878                                 break;
10879                         }
10880                 }
10881         }
10882
10883 bailout:
10884
10885         if (found == 0) {
10886                 /*
10887                  * This isn't really an error.  It's entirely possible for
10888                  * the abort and command completion to cross on the wire.
10889                  * This is more of an informative/diagnostic error.
10890                  */
10891 #if 0
10892                 printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
10893                        "%d:%d:%d:%d tag %d type %d\n",
10894                        io->io_hdr.nexus.initid.id,
10895                        io->io_hdr.nexus.targ_port,
10896                        io->io_hdr.nexus.targ_target.id,
10897                        io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
10898                        io->taskio.tag_type);
10899 #endif
10900                 return (1);
10901         } else
10902                 return (0);
10903 }
10904
10905 /*
10906  * Assumptions:  caller holds ctl_softc->ctl_lock
10907  *
10908  * This routine cannot block!  It must be callable from an interrupt
10909  * handler as well as from the work thread.
10910  */
10911 static void
10912 ctl_run_task_queue(struct ctl_softc *ctl_softc)
10913 {
10914         union ctl_io *io, *next_io;
10915
10916         CTL_DEBUG_PRINT(("ctl_run_task_queue\n"));
10917
10918         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->task_queue);
10919              io != NULL; io = next_io) {
10920                 int retval;
10921                 const char *task_desc;
10922
10923                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
10924
10925                 retval = 0;
10926
10927                 switch (io->io_hdr.io_type) {
10928                 case CTL_IO_TASK: {
10929                         task_desc = ctl_scsi_task_string(&io->taskio);
10930                         if (task_desc != NULL) {
10931 #ifdef NEEDTOPORT
10932                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
10933                                             CTL_TASK_REPORT,
10934                                             csevent_LogType_Trace,
10935                                             csevent_Severity_Information,
10936                                             csevent_AlertLevel_Green,
10937                                             csevent_FRU_Firmware,
10938                                             csevent_FRU_Unknown,
10939                                             "CTL: received task: %s",task_desc);
10940 #endif
10941                         } else {
10942 #ifdef NEEDTOPORT
10943                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
10944                                             CTL_TASK_REPORT,
10945                                             csevent_LogType_Trace,
10946                                             csevent_Severity_Information,
10947                                             csevent_AlertLevel_Green,
10948                                             csevent_FRU_Firmware,
10949                                             csevent_FRU_Unknown,
10950                                             "CTL: received unknown task "
10951                                             "type: %d (%#x)",
10952                                             io->taskio.task_action,
10953                                             io->taskio.task_action);
10954 #endif
10955                         }
10956                         switch (io->taskio.task_action) {
10957                         case CTL_TASK_ABORT_TASK:
10958                                 retval = ctl_abort_task(io);
10959                                 break;
10960                         case CTL_TASK_ABORT_TASK_SET:
10961                                 break;
10962                         case CTL_TASK_CLEAR_ACA:
10963                                 break;
10964                         case CTL_TASK_CLEAR_TASK_SET:
10965                                 break;
10966                         case CTL_TASK_LUN_RESET: {
10967                                 struct ctl_lun *lun;
10968                                 uint32_t targ_lun;
10969                                 int retval;
10970
10971                                 targ_lun = io->io_hdr.nexus.targ_lun;
10972
10973                                 if ((targ_lun < CTL_MAX_LUNS)
10974                                  && (ctl_softc->ctl_luns[targ_lun] != NULL))
10975                                         lun = ctl_softc->ctl_luns[targ_lun];
10976                                 else {
10977                                         retval = 1;
10978                                         break;
10979                                 }
10980
10981                                 if (!(io->io_hdr.flags &
10982                                     CTL_FLAG_FROM_OTHER_SC)) {
10983                                         union ctl_ha_msg msg_info;
10984
10985                                         io->io_hdr.flags |=
10986                                                 CTL_FLAG_SENT_2OTHER_SC;
10987                                         msg_info.hdr.msg_type =
10988                                                 CTL_MSG_MANAGE_TASKS;
10989                                         msg_info.hdr.nexus = io->io_hdr.nexus;
10990                                         msg_info.task.task_action =
10991                                                 CTL_TASK_LUN_RESET;
10992                                         msg_info.hdr.original_sc = NULL;
10993                                         msg_info.hdr.serializing_sc = NULL;
10994                                         if (CTL_HA_STATUS_SUCCESS !=
10995                                             ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10996                                             (void *)&msg_info,
10997                                             sizeof(msg_info), 0)) {
10998                                         }
10999                                 }
11000
11001                                 retval = ctl_lun_reset(lun, io,
11002                                                        CTL_UA_LUN_RESET);
11003                                 break;
11004                         }
11005                         case CTL_TASK_TARGET_RESET:
11006                                 retval = ctl_target_reset(ctl_softc, io,
11007                                                           CTL_UA_TARG_RESET);
11008                                 break;
11009                         case CTL_TASK_BUS_RESET:
11010                                 retval = ctl_bus_reset(ctl_softc, io);
11011                                 break;
11012                         case CTL_TASK_PORT_LOGIN:
11013                                 break;
11014                         case CTL_TASK_PORT_LOGOUT:
11015                                 break;
11016                         default:
11017                                 printf("ctl_run_task_queue: got unknown task "
11018                                        "management event %d\n",
11019                                        io->taskio.task_action);
11020                                 break;
11021                         }
11022                         if (retval == 0)
11023                                 io->io_hdr.status = CTL_SUCCESS;
11024                         else
11025                                 io->io_hdr.status = CTL_ERROR;
11026
11027                         STAILQ_REMOVE(&ctl_softc->task_queue, &io->io_hdr,
11028                                       ctl_io_hdr, links);
11029                         /*
11030                          * This will queue this I/O to the done queue, but the
11031                          * work thread won't be able to process it until we
11032                          * return and the lock is released.
11033                          */
11034                         ctl_done_lock(io, /*have_lock*/ 1);
11035                         break;
11036                 }
11037                 default: {
11038
11039                         printf("%s: invalid I/O type %d msg %d cdb %x"
11040                                " iptl: %ju:%d:%ju:%d tag 0x%04x\n",
11041                                __func__, io->io_hdr.io_type,
11042                                io->io_hdr.msg_type, io->scsiio.cdb[0],
11043                                (uintmax_t)io->io_hdr.nexus.initid.id,
11044                                io->io_hdr.nexus.targ_port,
11045                                (uintmax_t)io->io_hdr.nexus.targ_target.id,
11046                                io->io_hdr.nexus.targ_lun,
11047                                (io->io_hdr.io_type == CTL_IO_TASK) ?
11048                                io->taskio.tag_num : io->scsiio.tag_num);
11049                         STAILQ_REMOVE(&ctl_softc->task_queue, &io->io_hdr,
11050                                       ctl_io_hdr, links);
11051                         ctl_free_io_internal(io, 1);
11052                         break;
11053                 }
11054                 }
11055         }
11056
11057         ctl_softc->flags &= ~CTL_FLAG_TASK_PENDING;
11058 }
11059
11060 /*
11061  * For HA operation.  Handle commands that come in from the other
11062  * controller.
11063  */
11064 static void
11065 ctl_handle_isc(union ctl_io *io)
11066 {
11067         int free_io;
11068         struct ctl_lun *lun;
11069         struct ctl_softc *ctl_softc;
11070
11071         ctl_softc = control_softc;
11072
11073         lun = ctl_softc->ctl_luns[io->io_hdr.nexus.targ_lun];
11074
11075         switch (io->io_hdr.msg_type) {
11076         case CTL_MSG_SERIALIZE:
11077                 free_io = ctl_serialize_other_sc_cmd(&io->scsiio,
11078                                                      /*have_lock*/ 0);
11079                 break;
11080         case CTL_MSG_R2R: {
11081                 uint8_t opcode;
11082                 struct ctl_cmd_entry *entry;
11083
11084                 /*
11085                  * This is only used in SER_ONLY mode.
11086                  */
11087                 free_io = 0;
11088                 opcode = io->scsiio.cdb[0];
11089                 entry = &ctl_cmd_table[opcode];
11090                 mtx_lock(&ctl_softc->ctl_lock);
11091                 if (ctl_scsiio_lun_check(ctl_softc, lun,
11092                     entry, (struct ctl_scsiio *)io) != 0) {
11093                         ctl_done_lock(io, /*have_lock*/ 1);
11094                         mtx_unlock(&ctl_softc->ctl_lock);
11095                         break;
11096                 }
11097                 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11098                 STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
11099                                    &io->io_hdr, links);
11100                 mtx_unlock(&ctl_softc->ctl_lock);
11101                 break;
11102         }
11103         case CTL_MSG_FINISH_IO:
11104                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
11105                         free_io = 0;
11106                         ctl_done_lock(io, /*have_lock*/ 0);
11107                 } else {
11108                         free_io = 1;
11109                         mtx_lock(&ctl_softc->ctl_lock);
11110                         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
11111                                      ooa_links);
11112                         STAILQ_REMOVE(&ctl_softc->task_queue,
11113                                       &io->io_hdr, ctl_io_hdr, links);
11114                         ctl_check_blocked(lun);
11115                         mtx_unlock(&ctl_softc->ctl_lock);
11116                 }
11117                 break;
11118         case CTL_MSG_PERS_ACTION:
11119                 ctl_hndl_per_res_out_on_other_sc(
11120                         (union ctl_ha_msg *)&io->presio.pr_msg);
11121                 free_io = 1;
11122                 break;
11123         case CTL_MSG_BAD_JUJU:
11124                 free_io = 0;
11125                 ctl_done_lock(io, /*have_lock*/ 0);
11126                 break;
11127         case CTL_MSG_DATAMOVE:
11128                 /* Only used in XFER mode */
11129                 free_io = 0;
11130                 ctl_datamove_remote(io);
11131                 break;
11132         case CTL_MSG_DATAMOVE_DONE:
11133                 /* Only used in XFER mode */
11134                 free_io = 0;
11135                 io->scsiio.be_move_done(io);
11136                 break;
11137         default:
11138                 free_io = 1;
11139                 printf("%s: Invalid message type %d\n",
11140                        __func__, io->io_hdr.msg_type);
11141                 break;
11142         }
11143         if (free_io)
11144                 ctl_free_io_internal(io, 0);
11145
11146 }
11147
11148
11149 /*
11150  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
11151  * there is no match.
11152  */
11153 static ctl_lun_error_pattern
11154 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
11155 {
11156         struct ctl_cmd_entry *entry;
11157         ctl_lun_error_pattern filtered_pattern, pattern;
11158         uint8_t opcode;
11159
11160         pattern = desc->error_pattern;
11161
11162         /*
11163          * XXX KDM we need more data passed into this function to match a
11164          * custom pattern, and we actually need to implement custom pattern
11165          * matching.
11166          */
11167         if (pattern & CTL_LUN_PAT_CMD)
11168                 return (CTL_LUN_PAT_CMD);
11169
11170         if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
11171                 return (CTL_LUN_PAT_ANY);
11172
11173         opcode = ctsio->cdb[0];
11174         entry = &ctl_cmd_table[opcode];
11175
11176         filtered_pattern = entry->pattern & pattern;
11177
11178         /*
11179          * If the user requested specific flags in the pattern (e.g.
11180          * CTL_LUN_PAT_RANGE), make sure the command supports all of those
11181          * flags.
11182          *
11183          * If the user did not specify any flags, it doesn't matter whether
11184          * or not the command supports the flags.
11185          */
11186         if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
11187              (pattern & ~CTL_LUN_PAT_MASK))
11188                 return (CTL_LUN_PAT_NONE);
11189
11190         /*
11191          * If the user asked for a range check, see if the requested LBA
11192          * range overlaps with this command's LBA range.
11193          */
11194         if (filtered_pattern & CTL_LUN_PAT_RANGE) {
11195                 uint64_t lba1;
11196                 uint32_t len1;
11197                 ctl_action action;
11198                 int retval;
11199
11200                 retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
11201                 if (retval != 0)
11202                         return (CTL_LUN_PAT_NONE);
11203
11204                 action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
11205                                               desc->lba_range.len);
11206                 /*
11207                  * A "pass" means that the LBA ranges don't overlap, so
11208                  * this doesn't match the user's range criteria.
11209                  */
11210                 if (action == CTL_ACTION_PASS)
11211                         return (CTL_LUN_PAT_NONE);
11212         }
11213
11214         return (filtered_pattern);
11215 }
11216
11217 /*
11218  * Called with the CTL lock held.
11219  */
11220 static void
11221 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
11222 {
11223         struct ctl_error_desc *desc, *desc2;
11224
11225         STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
11226                 ctl_lun_error_pattern pattern;
11227                 /*
11228                  * Check to see whether this particular command matches
11229                  * the pattern in the descriptor.
11230                  */
11231                 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
11232                 if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
11233                         continue;
11234
11235                 switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
11236                 case CTL_LUN_INJ_ABORTED:
11237                         ctl_set_aborted(&io->scsiio);
11238                         break;
11239                 case CTL_LUN_INJ_MEDIUM_ERR:
11240                         ctl_set_medium_error(&io->scsiio);
11241                         break;
11242                 case CTL_LUN_INJ_UA:
11243                         /* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
11244                          * OCCURRED */
11245                         ctl_set_ua(&io->scsiio, 0x29, 0x00);
11246                         break;
11247                 case CTL_LUN_INJ_CUSTOM:
11248                         /*
11249                          * We're assuming the user knows what he is doing.
11250                          * Just copy the sense information without doing
11251                          * checks.
11252                          */
11253                         bcopy(&desc->custom_sense, &io->scsiio.sense_data,
11254                               ctl_min(sizeof(desc->custom_sense),
11255                                       sizeof(io->scsiio.sense_data)));
11256                         io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
11257                         io->scsiio.sense_len = SSD_FULL_SIZE;
11258                         io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
11259                         break;
11260                 case CTL_LUN_INJ_NONE:
11261                 default:
11262                         /*
11263                          * If this is an error injection type we don't know
11264                          * about, clear the continuous flag (if it is set)
11265                          * so it will get deleted below.
11266                          */
11267                         desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
11268                         break;
11269                 }
11270                 /*
11271                  * By default, each error injection action is a one-shot
11272                  */
11273                 if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
11274                         continue;
11275
11276                 STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
11277
11278                 free(desc, M_CTL);
11279         }
11280 }
11281
11282 #ifdef CTL_IO_DELAY
11283 static void
11284 ctl_datamove_timer_wakeup(void *arg)
11285 {
11286         union ctl_io *io;
11287
11288         io = (union ctl_io *)arg;
11289
11290         ctl_datamove(io);
11291 }
11292 #endif /* CTL_IO_DELAY */
11293
11294 /*
11295  * Assumption:  caller does NOT hold ctl_lock
11296  */
11297 void
11298 ctl_datamove(union ctl_io *io)
11299 {
11300         void (*fe_datamove)(union ctl_io *io);
11301
11302         CTL_DEBUG_PRINT(("ctl_datamove\n"));
11303
11304 #ifdef CTL_TIME_IO
11305         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
11306                 char str[256];
11307                 char path_str[64];
11308                 struct sbuf sb;
11309
11310                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
11311                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
11312
11313                 sbuf_cat(&sb, path_str);
11314                 switch (io->io_hdr.io_type) {
11315                 case CTL_IO_SCSI:
11316                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
11317                         sbuf_printf(&sb, "\n");
11318                         sbuf_cat(&sb, path_str);
11319                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
11320                                     io->scsiio.tag_num, io->scsiio.tag_type);
11321                         break;
11322                 case CTL_IO_TASK:
11323                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
11324                                     "Tag Type: %d\n", io->taskio.task_action,
11325                                     io->taskio.tag_num, io->taskio.tag_type);
11326                         break;
11327                 default:
11328                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
11329                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
11330                         break;
11331                 }
11332                 sbuf_cat(&sb, path_str);
11333                 sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
11334                             (intmax_t)time_uptime - io->io_hdr.start_time);
11335                 sbuf_finish(&sb);
11336                 printf("%s", sbuf_data(&sb));
11337         }
11338 #endif /* CTL_TIME_IO */
11339
11340         mtx_lock(&control_softc->ctl_lock);
11341 #ifdef CTL_IO_DELAY
11342         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
11343                 struct ctl_lun *lun;
11344
11345                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
11346
11347                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
11348         } else {
11349                 struct ctl_lun *lun;
11350
11351                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
11352                 if ((lun != NULL)
11353                  && (lun->delay_info.datamove_delay > 0)) {
11354                         struct callout *callout;
11355
11356                         callout = (struct callout *)&io->io_hdr.timer_bytes;
11357                         callout_init(callout, /*mpsafe*/ 1);
11358                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
11359                         callout_reset(callout,
11360                                       lun->delay_info.datamove_delay * hz,
11361                                       ctl_datamove_timer_wakeup, io);
11362                         if (lun->delay_info.datamove_type ==
11363                             CTL_DELAY_TYPE_ONESHOT)
11364                                 lun->delay_info.datamove_delay = 0;
11365                         mtx_unlock(&control_softc->ctl_lock);
11366                         return;
11367                 }
11368         }
11369 #endif
11370         /*
11371          * If we have any pending task management commands, process them
11372          * first.  This is necessary to eliminate a race condition with the
11373          * FETD:
11374          *
11375          * - FETD submits a task management command, like an abort.
11376          * - Back end calls fe_datamove() to move the data for the aborted
11377          *   command.  The FETD can't really accept it, but if it did, it
11378          *   would end up transmitting data for a command that the initiator
11379          *   told us to abort.
11380          *
11381          * We close the race by processing all pending task management
11382          * commands here (we can't block!), and then check this I/O to see
11383          * if it has been aborted.  If so, return it to the back end with
11384          * bad status, so the back end can say return an error to the back end
11385          * and then when the back end returns an error, we can return the
11386          * aborted command to the FETD, so it can clean up its resources.
11387          */
11388         if (control_softc->flags & CTL_FLAG_TASK_PENDING)
11389                 ctl_run_task_queue(control_softc);
11390
11391         /*
11392          * This command has been aborted.  Set the port status, so we fail
11393          * the data move.
11394          */
11395         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
11396                 printf("ctl_datamove: tag 0x%04x on (%ju:%d:%ju:%d) aborted\n",
11397                        io->scsiio.tag_num,(uintmax_t)io->io_hdr.nexus.initid.id,
11398                        io->io_hdr.nexus.targ_port,
11399                        (uintmax_t)io->io_hdr.nexus.targ_target.id,
11400                        io->io_hdr.nexus.targ_lun);
11401                 io->io_hdr.status = CTL_CMD_ABORTED;
11402                 io->io_hdr.port_status = 31337;
11403                 mtx_unlock(&control_softc->ctl_lock);
11404                 /*
11405                  * Note that the backend, in this case, will get the
11406                  * callback in its context.  In other cases it may get
11407                  * called in the frontend's interrupt thread context.
11408                  */
11409                 io->scsiio.be_move_done(io);
11410                 return;
11411         }
11412
11413         /*
11414          * If we're in XFER mode and this I/O is from the other shelf
11415          * controller, we need to send the DMA to the other side to
11416          * actually transfer the data to/from the host.  In serialize only
11417          * mode the transfer happens below CTL and ctl_datamove() is only
11418          * called on the machine that originally received the I/O.
11419          */
11420         if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
11421          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11422                 union ctl_ha_msg msg;
11423                 uint32_t sg_entries_sent;
11424                 int do_sg_copy;
11425                 int i;
11426
11427                 memset(&msg, 0, sizeof(msg));
11428                 msg.hdr.msg_type = CTL_MSG_DATAMOVE;
11429                 msg.hdr.original_sc = io->io_hdr.original_sc;
11430                 msg.hdr.serializing_sc = io;
11431                 msg.hdr.nexus = io->io_hdr.nexus;
11432                 msg.dt.flags = io->io_hdr.flags;
11433                 /*
11434                  * We convert everything into a S/G list here.  We can't
11435                  * pass by reference, only by value between controllers.
11436                  * So we can't pass a pointer to the S/G list, only as many
11437                  * S/G entries as we can fit in here.  If it's possible for
11438                  * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
11439                  * then we need to break this up into multiple transfers.
11440                  */
11441                 if (io->scsiio.kern_sg_entries == 0) {
11442                         msg.dt.kern_sg_entries = 1;
11443                         /*
11444                          * If this is in cached memory, flush the cache
11445                          * before we send the DMA request to the other
11446                          * controller.  We want to do this in either the
11447                          * read or the write case.  The read case is
11448                          * straightforward.  In the write case, we want to
11449                          * make sure nothing is in the local cache that
11450                          * could overwrite the DMAed data.
11451                          */
11452                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
11453                                 /*
11454                                  * XXX KDM use bus_dmamap_sync() here.
11455                                  */
11456                         }
11457
11458                         /*
11459                          * Convert to a physical address if this is a
11460                          * virtual address.
11461                          */
11462                         if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
11463                                 msg.dt.sg_list[0].addr =
11464                                         io->scsiio.kern_data_ptr;
11465                         } else {
11466                                 /*
11467                                  * XXX KDM use busdma here!
11468                                  */
11469 #if 0
11470                                 msg.dt.sg_list[0].addr = (void *)
11471                                         vtophys(io->scsiio.kern_data_ptr);
11472 #endif
11473                         }
11474
11475                         msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
11476                         do_sg_copy = 0;
11477                 } else {
11478                         struct ctl_sg_entry *sgl;
11479
11480                         do_sg_copy = 1;
11481                         msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
11482                         sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
11483                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
11484                                 /*
11485                                  * XXX KDM use bus_dmamap_sync() here.
11486                                  */
11487                         }
11488                 }
11489
11490                 msg.dt.kern_data_len = io->scsiio.kern_data_len;
11491                 msg.dt.kern_total_len = io->scsiio.kern_total_len;
11492                 msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
11493                 msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
11494                 msg.dt.sg_sequence = 0;
11495
11496                 /*
11497                  * Loop until we've sent all of the S/G entries.  On the
11498                  * other end, we'll recompose these S/G entries into one
11499                  * contiguous list before passing it to the
11500                  */
11501                 for (sg_entries_sent = 0; sg_entries_sent <
11502                      msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
11503                         msg.dt.cur_sg_entries = ctl_min((sizeof(msg.dt.sg_list)/
11504                                 sizeof(msg.dt.sg_list[0])),
11505                                 msg.dt.kern_sg_entries - sg_entries_sent);
11506
11507                         if (do_sg_copy != 0) {
11508                                 struct ctl_sg_entry *sgl;
11509                                 int j;
11510
11511                                 sgl = (struct ctl_sg_entry *)
11512                                         io->scsiio.kern_data_ptr;
11513                                 /*
11514                                  * If this is in cached memory, flush the cache
11515                                  * before we send the DMA request to the other
11516                                  * controller.  We want to do this in either
11517                                  * the * read or the write case.  The read
11518                                  * case is straightforward.  In the write
11519                                  * case, we want to make sure nothing is
11520                                  * in the local cache that could overwrite
11521                                  * the DMAed data.
11522                                  */
11523
11524                                 for (i = sg_entries_sent, j = 0;
11525                                      i < msg.dt.cur_sg_entries; i++, j++) {
11526                                         if ((io->io_hdr.flags &
11527                                              CTL_FLAG_NO_DATASYNC) == 0) {
11528                                                 /*
11529                                                  * XXX KDM use bus_dmamap_sync()
11530                                                  */
11531                                         }
11532                                         if ((io->io_hdr.flags &
11533                                              CTL_FLAG_BUS_ADDR) == 0) {
11534                                                 /*
11535                                                  * XXX KDM use busdma.
11536                                                  */
11537 #if 0
11538                                                 msg.dt.sg_list[j].addr =(void *)
11539                                                        vtophys(sgl[i].addr);
11540 #endif
11541                                         } else {
11542                                                 msg.dt.sg_list[j].addr =
11543                                                         sgl[i].addr;
11544                                         }
11545                                         msg.dt.sg_list[j].len = sgl[i].len;
11546                                 }
11547                         }
11548
11549                         sg_entries_sent += msg.dt.cur_sg_entries;
11550                         if (sg_entries_sent >= msg.dt.kern_sg_entries)
11551                                 msg.dt.sg_last = 1;
11552                         else
11553                                 msg.dt.sg_last = 0;
11554
11555                         /*
11556                          * XXX KDM drop and reacquire the lock here?
11557                          */
11558                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
11559                             sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
11560                                 /*
11561                                  * XXX do something here.
11562                                  */
11563                         }
11564
11565                         msg.dt.sent_sg_entries = sg_entries_sent;
11566                 }
11567                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11568                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER)
11569                         ctl_failover_io(io, /*have_lock*/ 1);
11570
11571         } else {
11572
11573                 /*
11574                  * Lookup the fe_datamove() function for this particular
11575                  * front end.
11576                  */
11577                 fe_datamove =
11578                     control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
11579                 mtx_unlock(&control_softc->ctl_lock);
11580
11581                 fe_datamove(io);
11582         }
11583 }
11584
11585 static void
11586 ctl_send_datamove_done(union ctl_io *io, int have_lock)
11587 {
11588         union ctl_ha_msg msg;
11589         int isc_status;
11590
11591         memset(&msg, 0, sizeof(msg));
11592
11593         msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
11594         msg.hdr.original_sc = io;
11595         msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
11596         msg.hdr.nexus = io->io_hdr.nexus;
11597         msg.hdr.status = io->io_hdr.status;
11598         msg.scsi.tag_num = io->scsiio.tag_num;
11599         msg.scsi.tag_type = io->scsiio.tag_type;
11600         msg.scsi.scsi_status = io->scsiio.scsi_status;
11601         memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
11602                sizeof(io->scsiio.sense_data));
11603         msg.scsi.sense_len = io->scsiio.sense_len;
11604         msg.scsi.sense_residual = io->scsiio.sense_residual;
11605         msg.scsi.fetd_status = io->io_hdr.port_status;
11606         msg.scsi.residual = io->scsiio.residual;
11607         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11608
11609         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
11610                 ctl_failover_io(io, /*have_lock*/ have_lock);
11611                 return;
11612         }
11613
11614         isc_status = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0);
11615         if (isc_status > CTL_HA_STATUS_SUCCESS) {
11616                 /* XXX do something if this fails */
11617         }
11618
11619 }
11620
11621 /*
11622  * The DMA to the remote side is done, now we need to tell the other side
11623  * we're done so it can continue with its data movement.
11624  */
11625 static void
11626 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
11627 {
11628         union ctl_io *io;
11629
11630         io = rq->context;
11631
11632         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
11633                 printf("%s: ISC DMA write failed with error %d", __func__,
11634                        rq->ret);
11635                 ctl_set_internal_failure(&io->scsiio,
11636                                          /*sks_valid*/ 1,
11637                                          /*retry_count*/ rq->ret);
11638         }
11639
11640         ctl_dt_req_free(rq);
11641
11642         /*
11643          * In this case, we had to malloc the memory locally.  Free it.
11644          */
11645         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
11646                 int i;
11647                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
11648                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
11649         }
11650         /*
11651          * The data is in local and remote memory, so now we need to send
11652          * status (good or back) back to the other side.
11653          */
11654         ctl_send_datamove_done(io, /*have_lock*/ 0);
11655 }
11656
11657 /*
11658  * We've moved the data from the host/controller into local memory.  Now we
11659  * need to push it over to the remote controller's memory.
11660  */
11661 static int
11662 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
11663 {
11664         int retval;
11665
11666         retval = 0;
11667
11668         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
11669                                           ctl_datamove_remote_write_cb);
11670
11671         return (retval);
11672 }
11673
11674 static void
11675 ctl_datamove_remote_write(union ctl_io *io)
11676 {
11677         int retval;
11678         void (*fe_datamove)(union ctl_io *io);
11679
11680         /*
11681          * - Get the data from the host/HBA into local memory.
11682          * - DMA memory from the local controller to the remote controller.
11683          * - Send status back to the remote controller.
11684          */
11685
11686         retval = ctl_datamove_remote_sgl_setup(io);
11687         if (retval != 0)
11688                 return;
11689
11690         /* Switch the pointer over so the FETD knows what to do */
11691         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
11692
11693         /*
11694          * Use a custom move done callback, since we need to send completion
11695          * back to the other controller, not to the backend on this side.
11696          */
11697         io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
11698
11699         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
11700
11701         fe_datamove(io);
11702
11703         return;
11704
11705 }
11706
11707 static int
11708 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
11709 {
11710 #if 0
11711         char str[256];
11712         char path_str[64];
11713         struct sbuf sb;
11714 #endif
11715
11716         /*
11717          * In this case, we had to malloc the memory locally.  Free it.
11718          */
11719         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
11720                 int i;
11721                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
11722                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
11723         }
11724
11725 #if 0
11726         scsi_path_string(io, path_str, sizeof(path_str));
11727         sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
11728         sbuf_cat(&sb, path_str);
11729         scsi_command_string(&io->scsiio, NULL, &sb);
11730         sbuf_printf(&sb, "\n");
11731         sbuf_cat(&sb, path_str);
11732         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
11733                     io->scsiio.tag_num, io->scsiio.tag_type);
11734         sbuf_cat(&sb, path_str);
11735         sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
11736                     io->io_hdr.flags, io->io_hdr.status);
11737         sbuf_finish(&sb);
11738         printk("%s", sbuf_data(&sb));
11739 #endif
11740
11741
11742         /*
11743          * The read is done, now we need to send status (good or bad) back
11744          * to the other side.
11745          */
11746         ctl_send_datamove_done(io, /*have_lock*/ 0);
11747
11748         return (0);
11749 }
11750
11751 static void
11752 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
11753 {
11754         union ctl_io *io;
11755         void (*fe_datamove)(union ctl_io *io);
11756
11757         io = rq->context;
11758
11759         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
11760                 printf("%s: ISC DMA read failed with error %d", __func__,
11761                        rq->ret);
11762                 ctl_set_internal_failure(&io->scsiio,
11763                                          /*sks_valid*/ 1,
11764                                          /*retry_count*/ rq->ret);
11765         }
11766
11767         ctl_dt_req_free(rq);
11768
11769         /* Switch the pointer over so the FETD knows what to do */
11770         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
11771
11772         /*
11773          * Use a custom move done callback, since we need to send completion
11774          * back to the other controller, not to the backend on this side.
11775          */
11776         io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
11777
11778         /* XXX KDM add checks like the ones in ctl_datamove? */
11779
11780         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
11781
11782         fe_datamove(io);
11783 }
11784
11785 static int
11786 ctl_datamove_remote_sgl_setup(union ctl_io *io)
11787 {
11788         struct ctl_sg_entry *local_sglist, *remote_sglist;
11789         struct ctl_sg_entry *local_dma_sglist, *remote_dma_sglist;
11790         struct ctl_softc *softc;
11791         int retval;
11792         int i;
11793
11794         retval = 0;
11795         softc = control_softc;
11796
11797         local_sglist = io->io_hdr.local_sglist;
11798         local_dma_sglist = io->io_hdr.local_dma_sglist;
11799         remote_sglist = io->io_hdr.remote_sglist;
11800         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
11801
11802         if (io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) {
11803                 for (i = 0; i < io->scsiio.kern_sg_entries; i++) {
11804                         local_sglist[i].len = remote_sglist[i].len;
11805
11806                         /*
11807                          * XXX Detect the situation where the RS-level I/O
11808                          * redirector on the other side has already read the
11809                          * data off of the AOR RS on this side, and
11810                          * transferred it to remote (mirror) memory on the
11811                          * other side.  Since we already have the data in
11812                          * memory here, we just need to use it.
11813                          *
11814                          * XXX KDM this can probably be removed once we
11815                          * get the cache device code in and take the
11816                          * current AOR implementation out.
11817                          */
11818 #ifdef NEEDTOPORT
11819                         if ((remote_sglist[i].addr >=
11820                              (void *)vtophys(softc->mirr->addr))
11821                          && (remote_sglist[i].addr <
11822                              ((void *)vtophys(softc->mirr->addr) +
11823                              CacheMirrorOffset))) {
11824                                 local_sglist[i].addr = remote_sglist[i].addr -
11825                                         CacheMirrorOffset;
11826                                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
11827                                      CTL_FLAG_DATA_IN)
11828                                         io->io_hdr.flags |= CTL_FLAG_REDIR_DONE;
11829                         } else {
11830                                 local_sglist[i].addr = remote_sglist[i].addr +
11831                                         CacheMirrorOffset;
11832                         }
11833 #endif
11834 #if 0
11835                         printf("%s: local %p, remote %p, len %d\n",
11836                                __func__, local_sglist[i].addr,
11837                                remote_sglist[i].addr, local_sglist[i].len);
11838 #endif
11839                 }
11840         } else {
11841                 uint32_t len_to_go;
11842
11843                 /*
11844                  * In this case, we don't have automatically allocated
11845                  * memory for this I/O on this controller.  This typically
11846                  * happens with internal CTL I/O -- e.g. inquiry, mode
11847                  * sense, etc.  Anything coming from RAIDCore will have
11848                  * a mirror area available.
11849                  */
11850                 len_to_go = io->scsiio.kern_data_len;
11851
11852                 /*
11853                  * Clear the no datasync flag, we have to use malloced
11854                  * buffers.
11855                  */
11856                 io->io_hdr.flags &= ~CTL_FLAG_NO_DATASYNC;
11857
11858                 /*
11859                  * The difficult thing here is that the size of the various
11860                  * S/G segments may be different than the size from the
11861                  * remote controller.  That'll make it harder when DMAing
11862                  * the data back to the other side.
11863                  */
11864                 for (i = 0; (i < sizeof(io->io_hdr.remote_sglist) /
11865                      sizeof(io->io_hdr.remote_sglist[0])) &&
11866                      (len_to_go > 0); i++) {
11867                         local_sglist[i].len = ctl_min(len_to_go, 131072);
11868                         CTL_SIZE_8B(local_dma_sglist[i].len,
11869                                     local_sglist[i].len);
11870                         local_sglist[i].addr =
11871                                 malloc(local_dma_sglist[i].len, M_CTL,M_WAITOK);
11872
11873                         local_dma_sglist[i].addr = local_sglist[i].addr;
11874
11875                         if (local_sglist[i].addr == NULL) {
11876                                 int j;
11877
11878                                 printf("malloc failed for %zd bytes!",
11879                                        local_dma_sglist[i].len);
11880                                 for (j = 0; j < i; j++) {
11881                                         free(local_sglist[j].addr, M_CTL);
11882                                 }
11883                                 ctl_set_internal_failure(&io->scsiio,
11884                                                          /*sks_valid*/ 1,
11885                                                          /*retry_count*/ 4857);
11886                                 retval = 1;
11887                                 goto bailout_error;
11888
11889                         }
11890                         /* XXX KDM do we need a sync here? */
11891
11892                         len_to_go -= local_sglist[i].len;
11893                 }
11894                 /*
11895                  * Reset the number of S/G entries accordingly.  The
11896                  * original number of S/G entries is available in
11897                  * rem_sg_entries.
11898                  */
11899                 io->scsiio.kern_sg_entries = i;
11900
11901 #if 0
11902                 printf("%s: kern_sg_entries = %d\n", __func__,
11903                        io->scsiio.kern_sg_entries);
11904                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
11905                         printf("%s: sg[%d] = %p, %d (DMA: %d)\n", __func__, i,
11906                                local_sglist[i].addr, local_sglist[i].len,
11907                                local_dma_sglist[i].len);
11908 #endif
11909         }
11910
11911
11912         return (retval);
11913
11914 bailout_error:
11915
11916         ctl_send_datamove_done(io, /*have_lock*/ 0);
11917
11918         return (retval);
11919 }
11920
11921 static int
11922 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
11923                          ctl_ha_dt_cb callback)
11924 {
11925         struct ctl_ha_dt_req *rq;
11926         struct ctl_sg_entry *remote_sglist, *local_sglist;
11927         struct ctl_sg_entry *remote_dma_sglist, *local_dma_sglist;
11928         uint32_t local_used, remote_used, total_used;
11929         int retval;
11930         int i, j;
11931
11932         retval = 0;
11933
11934         rq = ctl_dt_req_alloc();
11935
11936         /*
11937          * If we failed to allocate the request, and if the DMA didn't fail
11938          * anyway, set busy status.  This is just a resource allocation
11939          * failure.
11940          */
11941         if ((rq == NULL)
11942          && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
11943                 ctl_set_busy(&io->scsiio);
11944
11945         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
11946
11947                 if (rq != NULL)
11948                         ctl_dt_req_free(rq);
11949
11950                 /*
11951                  * The data move failed.  We need to return status back
11952                  * to the other controller.  No point in trying to DMA
11953                  * data to the remote controller.
11954                  */
11955
11956                 ctl_send_datamove_done(io, /*have_lock*/ 0);
11957
11958                 retval = 1;
11959
11960                 goto bailout;
11961         }
11962
11963         local_sglist = io->io_hdr.local_sglist;
11964         local_dma_sglist = io->io_hdr.local_dma_sglist;
11965         remote_sglist = io->io_hdr.remote_sglist;
11966         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
11967         local_used = 0;
11968         remote_used = 0;
11969         total_used = 0;
11970
11971         if (io->io_hdr.flags & CTL_FLAG_REDIR_DONE) {
11972                 rq->ret = CTL_HA_STATUS_SUCCESS;
11973                 rq->context = io;
11974                 callback(rq);
11975                 goto bailout;
11976         }
11977
11978         /*
11979          * Pull/push the data over the wire from/to the other controller.
11980          * This takes into account the possibility that the local and
11981          * remote sglists may not be identical in terms of the size of
11982          * the elements and the number of elements.
11983          *
11984          * One fundamental assumption here is that the length allocated for
11985          * both the local and remote sglists is identical.  Otherwise, we've
11986          * essentially got a coding error of some sort.
11987          */
11988         for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
11989                 int isc_ret;
11990                 uint32_t cur_len, dma_length;
11991                 uint8_t *tmp_ptr;
11992
11993                 rq->id = CTL_HA_DATA_CTL;
11994                 rq->command = command;
11995                 rq->context = io;
11996
11997                 /*
11998                  * Both pointers should be aligned.  But it is possible
11999                  * that the allocation length is not.  They should both
12000                  * also have enough slack left over at the end, though,
12001                  * to round up to the next 8 byte boundary.
12002                  */
12003                 cur_len = ctl_min(local_sglist[i].len - local_used,
12004                                   remote_sglist[j].len - remote_used);
12005
12006                 /*
12007                  * In this case, we have a size issue and need to decrease
12008                  * the size, except in the case where we actually have less
12009                  * than 8 bytes left.  In that case, we need to increase
12010                  * the DMA length to get the last bit.
12011                  */
12012                 if ((cur_len & 0x7) != 0) {
12013                         if (cur_len > 0x7) {
12014                                 cur_len = cur_len - (cur_len & 0x7);
12015                                 dma_length = cur_len;
12016                         } else {
12017                                 CTL_SIZE_8B(dma_length, cur_len);
12018                         }
12019
12020                 } else
12021                         dma_length = cur_len;
12022
12023                 /*
12024                  * If we had to allocate memory for this I/O, instead of using
12025                  * the non-cached mirror memory, we'll need to flush the cache
12026                  * before trying to DMA to the other controller.
12027                  *
12028                  * We could end up doing this multiple times for the same
12029                  * segment if we have a larger local segment than remote
12030                  * segment.  That shouldn't be an issue.
12031                  */
12032                 if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12033                         /*
12034                          * XXX KDM use bus_dmamap_sync() here.
12035                          */
12036                 }
12037
12038                 rq->size = dma_length;
12039
12040                 tmp_ptr = (uint8_t *)local_sglist[i].addr;
12041                 tmp_ptr += local_used;
12042
12043                 /* Use physical addresses when talking to ISC hardware */
12044                 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
12045                         /* XXX KDM use busdma */
12046 #if 0
12047                         rq->local = vtophys(tmp_ptr);
12048 #endif
12049                 } else
12050                         rq->local = tmp_ptr;
12051
12052                 tmp_ptr = (uint8_t *)remote_sglist[j].addr;
12053                 tmp_ptr += remote_used;
12054                 rq->remote = tmp_ptr;
12055
12056                 rq->callback = NULL;
12057
12058                 local_used += cur_len;
12059                 if (local_used >= local_sglist[i].len) {
12060                         i++;
12061                         local_used = 0;
12062                 }
12063
12064                 remote_used += cur_len;
12065                 if (remote_used >= remote_sglist[j].len) {
12066                         j++;
12067                         remote_used = 0;
12068                 }
12069                 total_used += cur_len;
12070
12071                 if (total_used >= io->scsiio.kern_data_len)
12072                         rq->callback = callback;
12073
12074                 if ((rq->size & 0x7) != 0) {
12075                         printf("%s: warning: size %d is not on 8b boundary\n",
12076                                __func__, rq->size);
12077                 }
12078                 if (((uintptr_t)rq->local & 0x7) != 0) {
12079                         printf("%s: warning: local %p not on 8b boundary\n",
12080                                __func__, rq->local);
12081                 }
12082                 if (((uintptr_t)rq->remote & 0x7) != 0) {
12083                         printf("%s: warning: remote %p not on 8b boundary\n",
12084                                __func__, rq->local);
12085                 }
12086 #if 0
12087                 printf("%s: %s: local %#x remote %#x size %d\n", __func__,
12088                        (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
12089                        rq->local, rq->remote, rq->size);
12090 #endif
12091
12092                 isc_ret = ctl_dt_single(rq);
12093                 if (isc_ret == CTL_HA_STATUS_WAIT)
12094                         continue;
12095
12096                 if (isc_ret == CTL_HA_STATUS_DISCONNECT) {
12097                         rq->ret = CTL_HA_STATUS_SUCCESS;
12098                 } else {
12099                         rq->ret = isc_ret;
12100                 }
12101                 callback(rq);
12102                 goto bailout;
12103         }
12104
12105 bailout:
12106         return (retval);
12107
12108 }
12109
12110 static void
12111 ctl_datamove_remote_read(union ctl_io *io)
12112 {
12113         int retval;
12114         int i;
12115
12116         /*
12117          * This will send an error to the other controller in the case of a
12118          * failure.
12119          */
12120         retval = ctl_datamove_remote_sgl_setup(io);
12121         if (retval != 0)
12122                 return;
12123
12124         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
12125                                           ctl_datamove_remote_read_cb);
12126         if ((retval != 0)
12127          && ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0)) {
12128                 /*
12129                  * Make sure we free memory if there was an error..  The
12130                  * ctl_datamove_remote_xfer() function will send the
12131                  * datamove done message, or call the callback with an
12132                  * error if there is a problem.
12133                  */
12134                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12135                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12136         }
12137
12138         return;
12139 }
12140
12141 /*
12142  * Process a datamove request from the other controller.  This is used for
12143  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
12144  * first.  Once that is complete, the data gets DMAed into the remote
12145  * controller's memory.  For reads, we DMA from the remote controller's
12146  * memory into our memory first, and then move it out to the FETD.
12147  *
12148  * Should be called without the ctl_lock held.
12149  */
12150 static void
12151 ctl_datamove_remote(union ctl_io *io)
12152 {
12153         struct ctl_softc *softc;
12154
12155         softc = control_softc;
12156
12157         /*
12158          * Note that we look for an aborted I/O here, but don't do some of
12159          * the other checks that ctl_datamove() normally does.  We don't
12160          * need to run the task queue, because this I/O is on the ISC
12161          * queue, which is executed by the work thread after the task queue.
12162          * We don't need to run the datamove delay code, since that should
12163          * have been done if need be on the other controller.
12164          */
12165         mtx_lock(&softc->ctl_lock);
12166
12167         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12168
12169                 printf("%s: tag 0x%04x on (%d:%d:%d:%d) aborted\n", __func__,
12170                        io->scsiio.tag_num, io->io_hdr.nexus.initid.id,
12171                        io->io_hdr.nexus.targ_port,
12172                        io->io_hdr.nexus.targ_target.id,
12173                        io->io_hdr.nexus.targ_lun);
12174                 io->io_hdr.status = CTL_CMD_ABORTED;
12175                 io->io_hdr.port_status = 31338;
12176
12177                 mtx_unlock(&softc->ctl_lock);
12178
12179                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12180
12181                 return;
12182         }
12183
12184         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT) {
12185                 mtx_unlock(&softc->ctl_lock);
12186                 ctl_datamove_remote_write(io);
12187         } else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN){
12188                 mtx_unlock(&softc->ctl_lock);
12189                 ctl_datamove_remote_read(io);
12190         } else {
12191                 union ctl_ha_msg msg;
12192                 struct scsi_sense_data *sense;
12193                 uint8_t sks[3];
12194                 int retry_count;
12195
12196                 memset(&msg, 0, sizeof(msg));
12197
12198                 msg.hdr.msg_type = CTL_MSG_BAD_JUJU;
12199                 msg.hdr.status = CTL_SCSI_ERROR;
12200                 msg.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
12201
12202                 retry_count = 4243;
12203
12204                 sense = &msg.scsi.sense_data;
12205                 sks[0] = SSD_SCS_VALID;
12206                 sks[1] = (retry_count >> 8) & 0xff;
12207                 sks[2] = retry_count & 0xff;
12208
12209                 /* "Internal target failure" */
12210                 scsi_set_sense_data(sense,
12211                                     /*sense_format*/ SSD_TYPE_NONE,
12212                                     /*current_error*/ 1,
12213                                     /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
12214                                     /*asc*/ 0x44,
12215                                     /*ascq*/ 0x00,
12216                                     /*type*/ SSD_ELEM_SKS,
12217                                     /*size*/ sizeof(sks),
12218                                     /*data*/ sks,
12219                                     SSD_ELEM_NONE);
12220
12221                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12222                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12223                         ctl_failover_io(io, /*have_lock*/ 1);
12224                         mtx_unlock(&softc->ctl_lock);
12225                         return;
12226                 }
12227
12228                 mtx_unlock(&softc->ctl_lock);
12229
12230                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0) >
12231                     CTL_HA_STATUS_SUCCESS) {
12232                         /* XXX KDM what to do if this fails? */
12233                 }
12234                 return;
12235         }
12236
12237 }
12238
12239 static int
12240 ctl_process_done(union ctl_io *io, int have_lock)
12241 {
12242         struct ctl_lun *lun;
12243         struct ctl_softc *ctl_softc;
12244         void (*fe_done)(union ctl_io *io);
12245         uint32_t targ_port = ctl_port_idx(io->io_hdr.nexus.targ_port);
12246
12247         CTL_DEBUG_PRINT(("ctl_process_done\n"));
12248
12249         fe_done =
12250             control_softc->ctl_ports[targ_port]->fe_done;
12251
12252 #ifdef CTL_TIME_IO
12253         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12254                 char str[256];
12255                 char path_str[64];
12256                 struct sbuf sb;
12257
12258                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12259                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12260
12261                 sbuf_cat(&sb, path_str);
12262                 switch (io->io_hdr.io_type) {
12263                 case CTL_IO_SCSI:
12264                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12265                         sbuf_printf(&sb, "\n");
12266                         sbuf_cat(&sb, path_str);
12267                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12268                                     io->scsiio.tag_num, io->scsiio.tag_type);
12269                         break;
12270                 case CTL_IO_TASK:
12271                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12272                                     "Tag Type: %d\n", io->taskio.task_action,
12273                                     io->taskio.tag_num, io->taskio.tag_type);
12274                         break;
12275                 default:
12276                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12277                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12278                         break;
12279                 }
12280                 sbuf_cat(&sb, path_str);
12281                 sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
12282                             (intmax_t)time_uptime - io->io_hdr.start_time);
12283                 sbuf_finish(&sb);
12284                 printf("%s", sbuf_data(&sb));
12285         }
12286 #endif /* CTL_TIME_IO */
12287
12288         switch (io->io_hdr.io_type) {
12289         case CTL_IO_SCSI:
12290                 break;
12291         case CTL_IO_TASK:
12292                 ctl_io_error_print(io, NULL);
12293                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
12294                         ctl_free_io_internal(io, /*have_lock*/ 0);
12295                 else
12296                         fe_done(io);
12297                 return (CTL_RETVAL_COMPLETE);
12298                 break;
12299         default:
12300                 printf("ctl_process_done: invalid io type %d\n",
12301                        io->io_hdr.io_type);
12302                 panic("ctl_process_done: invalid io type %d\n",
12303                       io->io_hdr.io_type);
12304                 break; /* NOTREACHED */
12305         }
12306
12307         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12308         if (lun == NULL) {
12309                 CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
12310                                  io->io_hdr.nexus.targ_lun));
12311                 fe_done(io);
12312                 goto bailout;
12313         }
12314         ctl_softc = lun->ctl_softc;
12315
12316         /*
12317          * Remove this from the OOA queue.
12318          */
12319         if (have_lock == 0)
12320                 mtx_lock(&ctl_softc->ctl_lock);
12321
12322         /*
12323          * Check to see if we have any errors to inject here.  We only
12324          * inject errors for commands that don't already have errors set.
12325          */
12326         if ((STAILQ_FIRST(&lun->error_list) != NULL)
12327          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))
12328                 ctl_inject_error(lun, io);
12329
12330         /*
12331          * XXX KDM how do we treat commands that aren't completed
12332          * successfully?
12333          *
12334          * XXX KDM should we also track I/O latency?
12335          */
12336         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
12337                 uint32_t blocksize;
12338 #ifdef CTL_TIME_IO
12339                 struct bintime cur_bt;
12340 #endif
12341
12342                 if ((lun->be_lun != NULL)
12343                  && (lun->be_lun->blocksize != 0))
12344                         blocksize = lun->be_lun->blocksize;
12345                 else
12346                         blocksize = 512;
12347
12348                 switch (io->io_hdr.io_type) {
12349                 case CTL_IO_SCSI: {
12350                         int isread;
12351                         struct ctl_lba_len lbalen;
12352
12353                         isread = 0;
12354                         switch (io->scsiio.cdb[0]) {
12355                         case READ_6:
12356                         case READ_10:
12357                         case READ_12:
12358                         case READ_16:
12359                                 isread = 1;
12360                                 /* FALLTHROUGH */
12361                         case WRITE_6:
12362                         case WRITE_10:
12363                         case WRITE_12:
12364                         case WRITE_16:
12365                         case WRITE_VERIFY_10:
12366                         case WRITE_VERIFY_12:
12367                         case WRITE_VERIFY_16:
12368                                 memcpy(&lbalen, io->io_hdr.ctl_private[
12369                                        CTL_PRIV_LBA_LEN].bytes, sizeof(lbalen));
12370
12371                                 if (isread) {
12372                                         lun->stats.ports[targ_port].bytes[CTL_STATS_READ] +=
12373                                                 lbalen.len * blocksize;
12374                                         lun->stats.ports[targ_port].operations[CTL_STATS_READ]++;
12375
12376 #ifdef CTL_TIME_IO
12377                                         bintime_add(
12378                                            &lun->stats.ports[targ_port].dma_time[CTL_STATS_READ],
12379                                            &io->io_hdr.dma_bt);
12380                                         lun->stats.ports[targ_port].num_dmas[CTL_STATS_READ] +=
12381                                                 io->io_hdr.num_dmas;
12382                                         getbintime(&cur_bt);
12383                                         bintime_sub(&cur_bt,
12384                                                     &io->io_hdr.start_bt);
12385
12386                                         bintime_add(
12387                                             &lun->stats.ports[targ_port].time[CTL_STATS_READ],
12388                                             &cur_bt);
12389
12390 #if 0
12391                                         cs_prof_gettime(&cur_ticks);
12392                                         lun->stats.time[CTL_STATS_READ] +=
12393                                                 cur_ticks -
12394                                                 io->io_hdr.start_ticks;
12395 #endif
12396 #if 0
12397                                         lun->stats.time[CTL_STATS_READ] +=
12398                                                 jiffies - io->io_hdr.start_time;
12399 #endif
12400 #endif /* CTL_TIME_IO */
12401                                 } else {
12402                                         lun->stats.ports[targ_port].bytes[CTL_STATS_WRITE] +=
12403                                                 lbalen.len * blocksize;
12404                                         lun->stats.ports[targ_port].operations[
12405                                                 CTL_STATS_WRITE]++;
12406
12407 #ifdef CTL_TIME_IO
12408                                         bintime_add(
12409                                           &lun->stats.ports[targ_port].dma_time[CTL_STATS_WRITE],
12410                                           &io->io_hdr.dma_bt);
12411                                         lun->stats.ports[targ_port].num_dmas[CTL_STATS_WRITE] +=
12412                                                 io->io_hdr.num_dmas;
12413                                         getbintime(&cur_bt);
12414                                         bintime_sub(&cur_bt,
12415                                                     &io->io_hdr.start_bt);
12416
12417                                         bintime_add(
12418                                             &lun->stats.ports[targ_port].time[CTL_STATS_WRITE],
12419                                             &cur_bt);
12420 #if 0
12421                                         cs_prof_gettime(&cur_ticks);
12422                                         lun->stats.ports[targ_port].time[CTL_STATS_WRITE] +=
12423                                                 cur_ticks -
12424                                                 io->io_hdr.start_ticks;
12425                                         lun->stats.ports[targ_port].time[CTL_STATS_WRITE] +=
12426                                                 jiffies - io->io_hdr.start_time;
12427 #endif
12428 #endif /* CTL_TIME_IO */
12429                                 }
12430                                 break;
12431                         default:
12432                                 lun->stats.ports[targ_port].operations[CTL_STATS_NO_IO]++;
12433
12434 #ifdef CTL_TIME_IO
12435                                 bintime_add(
12436                                   &lun->stats.ports[targ_port].dma_time[CTL_STATS_NO_IO],
12437                                   &io->io_hdr.dma_bt);
12438                                 lun->stats.ports[targ_port].num_dmas[CTL_STATS_NO_IO] +=
12439                                         io->io_hdr.num_dmas;
12440                                 getbintime(&cur_bt);
12441                                 bintime_sub(&cur_bt, &io->io_hdr.start_bt);
12442
12443                                 bintime_add(&lun->stats.ports[targ_port].time[CTL_STATS_NO_IO],
12444                                             &cur_bt);
12445
12446 #if 0
12447                                 cs_prof_gettime(&cur_ticks);
12448                                 lun->stats.ports[targ_port].time[CTL_STATS_NO_IO] +=
12449                                         cur_ticks -
12450                                         io->io_hdr.start_ticks;
12451                                 lun->stats.ports[targ_port].time[CTL_STATS_NO_IO] +=
12452                                         jiffies - io->io_hdr.start_time;
12453 #endif
12454 #endif /* CTL_TIME_IO */
12455                                 break;
12456                         }
12457                         break;
12458                 }
12459                 default:
12460                         break;
12461                 }
12462         }
12463
12464         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
12465
12466         /*
12467          * Run through the blocked queue on this LUN and see if anything
12468          * has become unblocked, now that this transaction is done.
12469          */
12470         ctl_check_blocked(lun);
12471
12472         /*
12473          * If the LUN has been invalidated, free it if there is nothing
12474          * left on its OOA queue.
12475          */
12476         if ((lun->flags & CTL_LUN_INVALID)
12477          && (TAILQ_FIRST(&lun->ooa_queue) == NULL))
12478                 ctl_free_lun(lun);
12479
12480         /*
12481          * If this command has been aborted, make sure we set the status
12482          * properly.  The FETD is responsible for freeing the I/O and doing
12483          * whatever it needs to do to clean up its state.
12484          */
12485         if (io->io_hdr.flags & CTL_FLAG_ABORT)
12486                 io->io_hdr.status = CTL_CMD_ABORTED;
12487
12488         /*
12489          * We print out status for every task management command.  For SCSI
12490          * commands, we filter out any unit attention errors; they happen
12491          * on every boot, and would clutter up the log.  Note:  task
12492          * management commands aren't printed here, they are printed above,
12493          * since they should never even make it down here.
12494          */
12495         switch (io->io_hdr.io_type) {
12496         case CTL_IO_SCSI: {
12497                 int error_code, sense_key, asc, ascq;
12498
12499                 sense_key = 0;
12500
12501                 if (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR)
12502                  && (io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)) {
12503                         /*
12504                          * Since this is just for printing, no need to
12505                          * show errors here.
12506                          */
12507                         scsi_extract_sense_len(&io->scsiio.sense_data,
12508                                                io->scsiio.sense_len,
12509                                                &error_code,
12510                                                &sense_key,
12511                                                &asc,
12512                                                &ascq,
12513                                                /*show_errors*/ 0);
12514                 }
12515
12516                 if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
12517                  && (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SCSI_ERROR)
12518                   || (io->scsiio.scsi_status != SCSI_STATUS_CHECK_COND)
12519                   || (sense_key != SSD_KEY_UNIT_ATTENTION))) {
12520
12521                         if ((time_uptime - ctl_softc->last_print_jiffies) <= 0){
12522                                 ctl_softc->skipped_prints++;
12523                                 if (have_lock == 0)
12524                                         mtx_unlock(&ctl_softc->ctl_lock);
12525                         } else {
12526                                 uint32_t skipped_prints;
12527
12528                                 skipped_prints = ctl_softc->skipped_prints;
12529
12530                                 ctl_softc->skipped_prints = 0;
12531                                 ctl_softc->last_print_jiffies = time_uptime;
12532
12533                                 if (have_lock == 0)
12534                                         mtx_unlock(&ctl_softc->ctl_lock);
12535                                 if (skipped_prints > 0) {
12536 #ifdef NEEDTOPORT
12537                                         csevent_log(CSC_CTL | CSC_SHELF_SW |
12538                                             CTL_ERROR_REPORT,
12539                                             csevent_LogType_Trace,
12540                                             csevent_Severity_Information,
12541                                             csevent_AlertLevel_Green,
12542                                             csevent_FRU_Firmware,
12543                                             csevent_FRU_Unknown,
12544                                             "High CTL error volume, %d prints "
12545                                             "skipped", skipped_prints);
12546 #endif
12547                                 }
12548                                 ctl_io_error_print(io, NULL);
12549                         }
12550                 } else {
12551                         if (have_lock == 0)
12552                                 mtx_unlock(&ctl_softc->ctl_lock);
12553                 }
12554                 break;
12555         }
12556         case CTL_IO_TASK:
12557                 if (have_lock == 0)
12558                         mtx_unlock(&ctl_softc->ctl_lock);
12559                 ctl_io_error_print(io, NULL);
12560                 break;
12561         default:
12562                 if (have_lock == 0)
12563                         mtx_unlock(&ctl_softc->ctl_lock);
12564                 break;
12565         }
12566
12567         /*
12568          * Tell the FETD or the other shelf controller we're done with this
12569          * command.  Note that only SCSI commands get to this point.  Task
12570          * management commands are completed above.
12571          *
12572          * We only send status to the other controller if we're in XFER
12573          * mode.  In SER_ONLY mode, the I/O is done on the controller that
12574          * received the I/O (from CTL's perspective), and so the status is
12575          * generated there.
12576          *
12577          * XXX KDM if we hold the lock here, we could cause a deadlock
12578          * if the frontend comes back in in this context to queue
12579          * something.
12580          */
12581         if ((ctl_softc->ha_mode == CTL_HA_MODE_XFER)
12582          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12583                 union ctl_ha_msg msg;
12584
12585                 memset(&msg, 0, sizeof(msg));
12586                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
12587                 msg.hdr.original_sc = io->io_hdr.original_sc;
12588                 msg.hdr.nexus = io->io_hdr.nexus;
12589                 msg.hdr.status = io->io_hdr.status;
12590                 msg.scsi.scsi_status = io->scsiio.scsi_status;
12591                 msg.scsi.tag_num = io->scsiio.tag_num;
12592                 msg.scsi.tag_type = io->scsiio.tag_type;
12593                 msg.scsi.sense_len = io->scsiio.sense_len;
12594                 msg.scsi.sense_residual = io->scsiio.sense_residual;
12595                 msg.scsi.residual = io->scsiio.residual;
12596                 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12597                        sizeof(io->scsiio.sense_data));
12598                 /*
12599                  * We copy this whether or not this is an I/O-related
12600                  * command.  Otherwise, we'd have to go and check to see
12601                  * whether it's a read/write command, and it really isn't
12602                  * worth it.
12603                  */
12604                 memcpy(&msg.scsi.lbalen,
12605                        &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
12606                        sizeof(msg.scsi.lbalen));;
12607
12608                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12609                                 sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
12610                         /* XXX do something here */
12611                 }
12612
12613                 ctl_free_io_internal(io, /*have_lock*/ 0);
12614         } else
12615                 fe_done(io);
12616
12617 bailout:
12618
12619         return (CTL_RETVAL_COMPLETE);
12620 }
12621
12622 /*
12623  * Front end should call this if it doesn't do autosense.  When the request
12624  * sense comes back in from the initiator, we'll dequeue this and send it.
12625  */
12626 int
12627 ctl_queue_sense(union ctl_io *io)
12628 {
12629         struct ctl_lun *lun;
12630         struct ctl_softc *ctl_softc;
12631         uint32_t initidx;
12632
12633         ctl_softc = control_softc;
12634
12635         CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
12636
12637         /*
12638          * LUN lookup will likely move to the ctl_work_thread() once we
12639          * have our new queueing infrastructure (that doesn't put things on
12640          * a per-LUN queue initially).  That is so that we can handle
12641          * things like an INQUIRY to a LUN that we don't have enabled.  We
12642          * can't deal with that right now.
12643          */
12644         mtx_lock(&ctl_softc->ctl_lock);
12645
12646         /*
12647          * If we don't have a LUN for this, just toss the sense
12648          * information.
12649          */
12650         if ((io->io_hdr.nexus.targ_lun < CTL_MAX_LUNS)
12651          && (ctl_softc->ctl_luns[io->io_hdr.nexus.targ_lun] != NULL))
12652                 lun = ctl_softc->ctl_luns[io->io_hdr.nexus.targ_lun];
12653         else
12654                 goto bailout;
12655
12656         initidx = ctl_get_initindex(&io->io_hdr.nexus);
12657
12658         /*
12659          * Already have CA set for this LUN...toss the sense information.
12660          */
12661         if (ctl_is_set(lun->have_ca, initidx))
12662                 goto bailout;
12663
12664         memcpy(&lun->pending_sense[initidx].sense, &io->scsiio.sense_data,
12665                ctl_min(sizeof(lun->pending_sense[initidx].sense),
12666                sizeof(io->scsiio.sense_data)));
12667         ctl_set_mask(lun->have_ca, initidx);
12668
12669 bailout:
12670         mtx_unlock(&ctl_softc->ctl_lock);
12671
12672         ctl_free_io(io);
12673
12674         return (CTL_RETVAL_COMPLETE);
12675 }
12676
12677 /*
12678  * Primary command inlet from frontend ports.  All SCSI and task I/O
12679  * requests must go through this function.
12680  */
12681 int
12682 ctl_queue(union ctl_io *io)
12683 {
12684         struct ctl_softc *ctl_softc;
12685
12686         CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
12687
12688         ctl_softc = control_softc;
12689
12690 #ifdef CTL_TIME_IO
12691         io->io_hdr.start_time = time_uptime;
12692         getbintime(&io->io_hdr.start_bt);
12693 #endif /* CTL_TIME_IO */
12694
12695         mtx_lock(&ctl_softc->ctl_lock);
12696
12697         switch (io->io_hdr.io_type) {
12698         case CTL_IO_SCSI:
12699                 STAILQ_INSERT_TAIL(&ctl_softc->incoming_queue, &io->io_hdr,
12700                                    links);
12701                 break;
12702         case CTL_IO_TASK:
12703                 STAILQ_INSERT_TAIL(&ctl_softc->task_queue, &io->io_hdr, links);
12704                 /*
12705                  * Set the task pending flag.  This is necessary to close a
12706                  * race condition with the FETD:
12707                  *
12708                  * - FETD submits a task management command, like an abort.
12709                  * - Back end calls fe_datamove() to move the data for the
12710                  *   aborted command.  The FETD can't really accept it, but
12711                  *   if it did, it would end up transmitting data for a
12712                  *   command that the initiator told us to abort.
12713                  *
12714                  * We close the race condition by setting the flag here,
12715                  * and checking it in ctl_datamove(), before calling the
12716                  * FETD's fe_datamove routine.  If we've got a task
12717                  * pending, we run the task queue and then check to see
12718                  * whether our particular I/O has been aborted.
12719                  */
12720                 ctl_softc->flags |= CTL_FLAG_TASK_PENDING;
12721                 break;
12722         default:
12723                 mtx_unlock(&ctl_softc->ctl_lock);
12724                 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
12725                 return (-EINVAL);
12726                 break; /* NOTREACHED */
12727         }
12728         mtx_unlock(&ctl_softc->ctl_lock);
12729
12730         ctl_wakeup_thread();
12731
12732         return (CTL_RETVAL_COMPLETE);
12733 }
12734
12735 #ifdef CTL_IO_DELAY
12736 static void
12737 ctl_done_timer_wakeup(void *arg)
12738 {
12739         union ctl_io *io;
12740
12741         io = (union ctl_io *)arg;
12742         ctl_done_lock(io, /*have_lock*/ 0);
12743 }
12744 #endif /* CTL_IO_DELAY */
12745
12746 void
12747 ctl_done_lock(union ctl_io *io, int have_lock)
12748 {
12749         struct ctl_softc *ctl_softc;
12750 #ifndef CTL_DONE_THREAD
12751         union ctl_io *xio;
12752 #endif /* !CTL_DONE_THREAD */
12753
12754         ctl_softc = control_softc;
12755
12756         if (have_lock == 0)
12757                 mtx_lock(&ctl_softc->ctl_lock);
12758
12759         /*
12760          * Enable this to catch duplicate completion issues.
12761          */
12762 #if 0
12763         if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
12764                 printf("%s: type %d msg %d cdb %x iptl: "
12765                        "%d:%d:%d:%d tag 0x%04x "
12766                        "flag %#x status %x\n",
12767                         __func__,
12768                         io->io_hdr.io_type,
12769                         io->io_hdr.msg_type,
12770                         io->scsiio.cdb[0],
12771                         io->io_hdr.nexus.initid.id,
12772                         io->io_hdr.nexus.targ_port,
12773                         io->io_hdr.nexus.targ_target.id,
12774                         io->io_hdr.nexus.targ_lun,
12775                         (io->io_hdr.io_type ==
12776                         CTL_IO_TASK) ?
12777                         io->taskio.tag_num :
12778                         io->scsiio.tag_num,
12779                         io->io_hdr.flags,
12780                         io->io_hdr.status);
12781         } else
12782                 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
12783 #endif
12784
12785         /*
12786          * This is an internal copy of an I/O, and should not go through
12787          * the normal done processing logic.
12788          */
12789         if (io->io_hdr.flags & CTL_FLAG_INT_COPY) {
12790                 if (have_lock == 0)
12791                         mtx_unlock(&ctl_softc->ctl_lock);
12792                 return;
12793         }
12794
12795         /*
12796          * We need to send a msg to the serializing shelf to finish the IO
12797          * as well.  We don't send a finish message to the other shelf if
12798          * this is a task management command.  Task management commands
12799          * aren't serialized in the OOA queue, but rather just executed on
12800          * both shelf controllers for commands that originated on that
12801          * controller.
12802          */
12803         if ((io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)
12804          && (io->io_hdr.io_type != CTL_IO_TASK)) {
12805                 union ctl_ha_msg msg_io;
12806
12807                 msg_io.hdr.msg_type = CTL_MSG_FINISH_IO;
12808                 msg_io.hdr.serializing_sc = io->io_hdr.serializing_sc;
12809                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_io,
12810                     sizeof(msg_io), 0 ) != CTL_HA_STATUS_SUCCESS) {
12811                 }
12812                 /* continue on to finish IO */
12813         }
12814 #ifdef CTL_IO_DELAY
12815         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12816                 struct ctl_lun *lun;
12817
12818                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12819
12820                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12821         } else {
12822                 struct ctl_lun *lun;
12823
12824                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12825
12826                 if ((lun != NULL)
12827                  && (lun->delay_info.done_delay > 0)) {
12828                         struct callout *callout;
12829
12830                         callout = (struct callout *)&io->io_hdr.timer_bytes;
12831                         callout_init(callout, /*mpsafe*/ 1);
12832                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12833                         callout_reset(callout,
12834                                       lun->delay_info.done_delay * hz,
12835                                       ctl_done_timer_wakeup, io);
12836                         if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
12837                                 lun->delay_info.done_delay = 0;
12838                         if (have_lock == 0)
12839                                 mtx_unlock(&ctl_softc->ctl_lock);
12840                         return;
12841                 }
12842         }
12843 #endif /* CTL_IO_DELAY */
12844
12845         STAILQ_INSERT_TAIL(&ctl_softc->done_queue, &io->io_hdr, links);
12846
12847 #ifdef CTL_DONE_THREAD
12848         if (have_lock == 0)
12849                 mtx_unlock(&ctl_softc->ctl_lock);
12850
12851         ctl_wakeup_thread();
12852 #else /* CTL_DONE_THREAD */
12853         for (xio = (union ctl_io *)STAILQ_FIRST(&ctl_softc->done_queue);
12854              xio != NULL;
12855              xio =(union ctl_io *)STAILQ_FIRST(&ctl_softc->done_queue)) {
12856
12857                 STAILQ_REMOVE_HEAD(&ctl_softc->done_queue, links);
12858
12859                 ctl_process_done(xio, /*have_lock*/ 1);
12860         }
12861         if (have_lock == 0)
12862                 mtx_unlock(&ctl_softc->ctl_lock);
12863 #endif /* CTL_DONE_THREAD */
12864 }
12865
12866 void
12867 ctl_done(union ctl_io *io)
12868 {
12869         ctl_done_lock(io, /*have_lock*/ 0);
12870 }
12871
12872 int
12873 ctl_isc(struct ctl_scsiio *ctsio)
12874 {
12875         struct ctl_lun *lun;
12876         int retval;
12877
12878         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12879
12880         CTL_DEBUG_PRINT(("ctl_isc: command: %02x\n", ctsio->cdb[0]));
12881
12882         CTL_DEBUG_PRINT(("ctl_isc: calling data_submit()\n"));
12883
12884         retval = lun->backend->data_submit((union ctl_io *)ctsio);
12885
12886         return (retval);
12887 }
12888
12889
12890 static void
12891 ctl_work_thread(void *arg)
12892 {
12893         struct ctl_softc *softc;
12894         union ctl_io *io;
12895         struct ctl_be_lun *be_lun;
12896         int retval;
12897
12898         CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
12899
12900         softc = (struct ctl_softc *)arg;
12901         if (softc == NULL)
12902                 return;
12903
12904         mtx_lock(&softc->ctl_lock);
12905         for (;;) {
12906                 retval = 0;
12907
12908                 /*
12909                  * We handle the queues in this order:
12910                  * - task management
12911                  * - ISC
12912                  * - done queue (to free up resources, unblock other commands)
12913                  * - RtR queue
12914                  * - incoming queue
12915                  *
12916                  * If those queues are empty, we break out of the loop and
12917                  * go to sleep.
12918                  */
12919                 io = (union ctl_io *)STAILQ_FIRST(&softc->task_queue);
12920                 if (io != NULL) {
12921                         ctl_run_task_queue(softc);
12922                         continue;
12923                 }
12924                 io = (union ctl_io *)STAILQ_FIRST(&softc->isc_queue);
12925                 if (io != NULL) {
12926                         STAILQ_REMOVE_HEAD(&softc->isc_queue, links);
12927                         ctl_handle_isc(io);
12928                         continue;
12929                 }
12930                 io = (union ctl_io *)STAILQ_FIRST(&softc->done_queue);
12931                 if (io != NULL) {
12932                         STAILQ_REMOVE_HEAD(&softc->done_queue, links);
12933                         /* clear any blocked commands, call fe_done */
12934                         mtx_unlock(&softc->ctl_lock);
12935                         /*
12936                          * XXX KDM
12937                          * Call this without a lock for now.  This will
12938                          * depend on whether there is any way the FETD can
12939                          * sleep or deadlock if called with the CTL lock
12940                          * held.
12941                          */
12942                         retval = ctl_process_done(io, /*have_lock*/ 0);
12943                         mtx_lock(&softc->ctl_lock);
12944                         continue;
12945                 }
12946                 if (!ctl_pause_rtr) {
12947                         io = (union ctl_io *)STAILQ_FIRST(&softc->rtr_queue);
12948                         if (io != NULL) {
12949                                 STAILQ_REMOVE_HEAD(&softc->rtr_queue, links);
12950                                 mtx_unlock(&softc->ctl_lock);
12951                                 goto execute;
12952                         }
12953                 }
12954                 io = (union ctl_io *)STAILQ_FIRST(&softc->incoming_queue);
12955                 if (io != NULL) {
12956                         STAILQ_REMOVE_HEAD(&softc->incoming_queue, links);
12957                         mtx_unlock(&softc->ctl_lock);
12958                         ctl_scsiio_precheck(softc, &io->scsiio);
12959                         mtx_lock(&softc->ctl_lock);
12960                         continue;
12961                 }
12962                 /*
12963                  * We might want to move this to a separate thread, so that
12964                  * configuration requests (in this case LUN creations)
12965                  * won't impact the I/O path.
12966                  */
12967                 be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
12968                 if (be_lun != NULL) {
12969                         STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
12970                         mtx_unlock(&softc->ctl_lock);
12971                         ctl_create_lun(be_lun);
12972                         mtx_lock(&softc->ctl_lock);
12973                         continue;
12974                 }
12975
12976                 /* XXX KDM use the PDROP flag?? */
12977                 /* Sleep until we have something to do. */
12978                 mtx_sleep(softc, &softc->ctl_lock, PRIBIO, "ctl_work", 0);
12979
12980                 /* Back to the top of the loop to see what woke us up. */
12981                 continue;
12982
12983 execute:
12984                 retval = ctl_scsiio(&io->scsiio);
12985                 switch (retval) {
12986                 case CTL_RETVAL_COMPLETE:
12987                         break;
12988                 default:
12989                         /*
12990                          * Probably need to make sure this doesn't happen.
12991                          */
12992                         break;
12993                 }
12994                 mtx_lock(&softc->ctl_lock);
12995         }
12996 }
12997
12998 void
12999 ctl_wakeup_thread()
13000 {
13001         struct ctl_softc *softc;
13002
13003         softc = control_softc;
13004
13005         wakeup(softc);
13006 }
13007
13008 /* Initialization and failover */
13009
13010 void
13011 ctl_init_isc_msg(void)
13012 {
13013         printf("CTL: Still calling this thing\n");
13014 }
13015
13016 /*
13017  * Init component
13018  *      Initializes component into configuration defined by bootMode
13019  *      (see hasc-sv.c)
13020  *      returns hasc_Status:
13021  *              OK
13022  *              ERROR - fatal error
13023  */
13024 static ctl_ha_comp_status
13025 ctl_isc_init(struct ctl_ha_component *c)
13026 {
13027         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13028
13029         c->status = ret;
13030         return ret;
13031 }
13032
13033 /* Start component
13034  *      Starts component in state requested. If component starts successfully,
13035  *      it must set its own state to the requestrd state
13036  *      When requested state is HASC_STATE_HA, the component may refine it
13037  *      by adding _SLAVE or _MASTER flags.
13038  *      Currently allowed state transitions are:
13039  *      UNKNOWN->HA             - initial startup
13040  *      UNKNOWN->SINGLE - initial startup when no parter detected
13041  *      HA->SINGLE              - failover
13042  * returns ctl_ha_comp_status:
13043  *              OK      - component successfully started in requested state
13044  *              FAILED  - could not start the requested state, failover may
13045  *                        be possible
13046  *              ERROR   - fatal error detected, no future startup possible
13047  */
13048 static ctl_ha_comp_status
13049 ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
13050 {
13051         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13052
13053         // UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
13054         if (c->state == CTL_HA_STATE_UNKNOWN ) {
13055                 ctl_is_single = 0;
13056                 if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
13057                     != CTL_HA_STATUS_SUCCESS) {
13058                         printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
13059                         ret = CTL_HA_COMP_STATUS_ERROR;
13060                 }
13061         } else if (CTL_HA_STATE_IS_HA(c->state)
13062                 && CTL_HA_STATE_IS_SINGLE(state)){
13063                 // HA->SINGLE transition
13064                 ctl_failover();
13065                 ctl_is_single = 1;
13066         } else {
13067                 printf("ctl_isc_start:Invalid state transition %X->%X\n",
13068                        c->state, state);
13069                 ret = CTL_HA_COMP_STATUS_ERROR;
13070         }
13071         if (CTL_HA_STATE_IS_SINGLE(state))
13072                 ctl_is_single = 1;
13073
13074         c->state = state;
13075         c->status = ret;
13076         return ret;
13077 }
13078
13079 /*
13080  * Quiesce component
13081  * The component must clear any error conditions (set status to OK) and
13082  * prepare itself to another Start call
13083  * returns ctl_ha_comp_status:
13084  *      OK
13085  *      ERROR
13086  */
13087 static ctl_ha_comp_status
13088 ctl_isc_quiesce(struct ctl_ha_component *c)
13089 {
13090         int ret = CTL_HA_COMP_STATUS_OK;
13091
13092         ctl_pause_rtr = 1;
13093         c->status = ret;
13094         return ret;
13095 }
13096
13097 struct ctl_ha_component ctl_ha_component_ctlisc =
13098 {
13099         .name = "CTL ISC",
13100         .state = CTL_HA_STATE_UNKNOWN,
13101         .init = ctl_isc_init,
13102         .start = ctl_isc_start,
13103         .quiesce = ctl_isc_quiesce
13104 };
13105
13106 /*
13107  *  vim: ts=8
13108  */