sys/cam/ctl/ctl.c

   1 /*-
   2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
   3  * Copyright (c) 2012 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Edward Tomasz Napierala
   7  * under sponsorship from the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  16  *    substantially similar to the "NO WARRANTY" disclaimer below
  17  *    ("Disclaimer") and any redistribution must be conditioned upon
  18  *    including a substantially similar Disclaimer requirement for further
  19  *    binary redistribution.
  20  *
  21  * NO WARRANTY
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  30  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  31  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGES.
  33  *
  34  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl.c#8 $
  35  */
  36 /*
  37  * CAM Target Layer, a SCSI device emulation subsystem.
  38  *
  39  * Author: Ken Merry <ken@FreeBSD.org>
  40  */
  41
  42 #define _CTL_C
  43
  44 #include <sys/cdefs.h>
  45 __FBSDID("$FreeBSD$");
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/types.h>
  51 #include <sys/kthread.h>
  52 #include <sys/bio.h>
  53 #include <sys/fcntl.h>
  54 #include <sys/lock.h>
  55 #include <sys/module.h>
  56 #include <sys/mutex.h>
  57 #include <sys/condvar.h>
  58 #include <sys/malloc.h>
  59 #include <sys/conf.h>
  60 #include <sys/ioccom.h>
  61 #include <sys/queue.h>
  62 #include <sys/sbuf.h>
  63 #include <sys/smp.h>
  64 #include <sys/endian.h>
  65 #include <sys/sysctl.h>
  66
  67 #include <cam/cam.h>
  68 #include <cam/scsi/scsi_all.h>
  69 #include <cam/scsi/scsi_da.h>
  70 #include <cam/ctl/ctl_io.h>
  71 #include <cam/ctl/ctl.h>
  72 #include <cam/ctl/ctl_frontend.h>
  73 #include <cam/ctl/ctl_frontend_internal.h>
  74 #include <cam/ctl/ctl_util.h>
  75 #include <cam/ctl/ctl_backend.h>
  76 #include <cam/ctl/ctl_ioctl.h>
  77 #include <cam/ctl/ctl_ha.h>
  78 #include <cam/ctl/ctl_private.h>
  79 #include <cam/ctl/ctl_debug.h>
  80 #include <cam/ctl/ctl_scsi_all.h>
  81 #include <cam/ctl/ctl_error.h>
  82
  83 struct ctl_softc *control_softc = NULL;
  84
  85 /*
  86  * Size and alignment macros needed for Copan-specific HA hardware.  These
  87  * can go away when the HA code is re-written, and uses busdma for any
  88  * hardware.
  89  */
  90 #define CTL_ALIGN_8B(target, source, type)                              \
  91         if (((uint32_t)source & 0x7) != 0)                              \
  92                 target = (type)(source + (0x8 - ((uint32_t)source & 0x7)));\
  93         else                                                            \
  94                 target = (type)source;
  95
  96 #define CTL_SIZE_8B(target, size)                                       \
  97         if ((size & 0x7) != 0)                                          \
  98                 target = size + (0x8 - (size & 0x7));                   \
  99         else                                                            \
 100                 target = size;
 101
 102 #define CTL_ALIGN_8B_MARGIN     16
 103
 104 /*
 105  * Template mode pages.
 106  */
 107
 108 /*
 109  * Note that these are default values only.  The actual values will be
 110  * filled in when the user does a mode sense.
 111  */
 112 static struct copan_power_subpage power_page_default = {
 113         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 114         /*subpage*/ PWR_SUBPAGE_CODE,
 115         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 116                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 117         /*page_version*/ PWR_VERSION,
 118         /* total_luns */ 26,
 119         /* max_active_luns*/ PWR_DFLT_MAX_LUNS,
 120         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 121                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 122                       0, 0, 0, 0, 0, 0}
 123 };
 124
 125 static struct copan_power_subpage power_page_changeable = {
 126         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 127         /*subpage*/ PWR_SUBPAGE_CODE,
 128         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 129                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 130         /*page_version*/ 0,
 131         /* total_luns */ 0,
 132         /* max_active_luns*/ 0,
 133         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 134                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 135                       0, 0, 0, 0, 0, 0}
 136 };
 137
 138 static struct copan_aps_subpage aps_page_default = {
 139         APS_PAGE_CODE | SMPH_SPF, //page_code
 140         APS_SUBPAGE_CODE, //subpage
 141         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 142          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 143         APS_VERSION, //page_version
 144         0, //lock_active
 145         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 146         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 147         0, 0, 0, 0, 0} //reserved
 148 };
 149
 150 static struct copan_aps_subpage aps_page_changeable = {
 151         APS_PAGE_CODE | SMPH_SPF, //page_code
 152         APS_SUBPAGE_CODE, //subpage
 153         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 154          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 155         0, //page_version
 156         0, //lock_active
 157         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 158         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 159         0, 0, 0, 0, 0} //reserved
 160 };
 161
 162 static struct copan_debugconf_subpage debugconf_page_default = {
 163         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 164         DBGCNF_SUBPAGE_CODE,            /* subpage */
 165         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 166          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 167         DBGCNF_VERSION,                 /* page_version */
 168         {CTL_TIME_IO_DEFAULT_SECS>>8,
 169          CTL_TIME_IO_DEFAULT_SECS>>0},  /* ctl_time_io_secs */
 170 };
 171
 172 static struct copan_debugconf_subpage debugconf_page_changeable = {
 173         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 174         DBGCNF_SUBPAGE_CODE,            /* subpage */
 175         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 176          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 177         0,                              /* page_version */
 178         {0xff,0xff},                    /* ctl_time_io_secs */
 179 };
 180
 181 static struct scsi_format_page format_page_default = {
 182         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 183         /*page_length*/sizeof(struct scsi_format_page) - 2,
 184         /*tracks_per_zone*/ {0, 0},
 185         /*alt_sectors_per_zone*/ {0, 0},
 186         /*alt_tracks_per_zone*/ {0, 0},
 187         /*alt_tracks_per_lun*/ {0, 0},
 188         /*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
 189                                 CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
 190         /*bytes_per_sector*/ {0, 0},
 191         /*interleave*/ {0, 0},
 192         /*track_skew*/ {0, 0},
 193         /*cylinder_skew*/ {0, 0},
 194         /*flags*/ SFP_HSEC,
 195         /*reserved*/ {0, 0, 0}
 196 };
 197
 198 static struct scsi_format_page format_page_changeable = {
 199         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 200         /*page_length*/sizeof(struct scsi_format_page) - 2,
 201         /*tracks_per_zone*/ {0, 0},
 202         /*alt_sectors_per_zone*/ {0, 0},
 203         /*alt_tracks_per_zone*/ {0, 0},
 204         /*alt_tracks_per_lun*/ {0, 0},
 205         /*sectors_per_track*/ {0, 0},
 206         /*bytes_per_sector*/ {0, 0},
 207         /*interleave*/ {0, 0},
 208         /*track_skew*/ {0, 0},
 209         /*cylinder_skew*/ {0, 0},
 210         /*flags*/ 0,
 211         /*reserved*/ {0, 0, 0}
 212 };
 213
 214 static struct scsi_rigid_disk_page rigid_disk_page_default = {
 215         /*page_code*/SMS_RIGID_DISK_PAGE,
 216         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 217         /*cylinders*/ {0, 0, 0},
 218         /*heads*/ CTL_DEFAULT_HEADS,
 219         /*start_write_precomp*/ {0, 0, 0},
 220         /*start_reduced_current*/ {0, 0, 0},
 221         /*step_rate*/ {0, 0},
 222         /*landing_zone_cylinder*/ {0, 0, 0},
 223         /*rpl*/ SRDP_RPL_DISABLED,
 224         /*rotational_offset*/ 0,
 225         /*reserved1*/ 0,
 226         /*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
 227                            CTL_DEFAULT_ROTATION_RATE & 0xff},
 228         /*reserved2*/ {0, 0}
 229 };
 230
 231 static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
 232         /*page_code*/SMS_RIGID_DISK_PAGE,
 233         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 234         /*cylinders*/ {0, 0, 0},
 235         /*heads*/ 0,
 236         /*start_write_precomp*/ {0, 0, 0},
 237         /*start_reduced_current*/ {0, 0, 0},
 238         /*step_rate*/ {0, 0},
 239         /*landing_zone_cylinder*/ {0, 0, 0},
 240         /*rpl*/ 0,
 241         /*rotational_offset*/ 0,
 242         /*reserved1*/ 0,
 243         /*rotation_rate*/ {0, 0},
 244         /*reserved2*/ {0, 0}
 245 };
 246
 247 static struct scsi_caching_page caching_page_default = {
 248         /*page_code*/SMS_CACHING_PAGE,
 249         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 250         /*flags1*/ SCP_DISC | SCP_WCE,
 251         /*ret_priority*/ 0,
 252         /*disable_pf_transfer_len*/ {0xff, 0xff},
 253         /*min_prefetch*/ {0, 0},
 254         /*max_prefetch*/ {0xff, 0xff},
 255         /*max_pf_ceiling*/ {0xff, 0xff},
 256         /*flags2*/ 0,
 257         /*cache_segments*/ 0,
 258         /*cache_seg_size*/ {0, 0},
 259         /*reserved*/ 0,
 260         /*non_cache_seg_size*/ {0, 0, 0}
 261 };
 262
 263 static struct scsi_caching_page caching_page_changeable = {
 264         /*page_code*/SMS_CACHING_PAGE,
 265         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 266         /*flags1*/ 0,
 267         /*ret_priority*/ 0,
 268         /*disable_pf_transfer_len*/ {0, 0},
 269         /*min_prefetch*/ {0, 0},
 270         /*max_prefetch*/ {0, 0},
 271         /*max_pf_ceiling*/ {0, 0},
 272         /*flags2*/ 0,
 273         /*cache_segments*/ 0,
 274         /*cache_seg_size*/ {0, 0},
 275         /*reserved*/ 0,
 276         /*non_cache_seg_size*/ {0, 0, 0}
 277 };
 278
 279 static struct scsi_control_page control_page_default = {
 280         /*page_code*/SMS_CONTROL_MODE_PAGE,
 281         /*page_length*/sizeof(struct scsi_control_page) - 2,
 282         /*rlec*/0,
 283         /*queue_flags*/0,
 284         /*eca_and_aen*/0,
 285         /*reserved*/0,
 286         /*aen_holdoff_period*/{0, 0}
 287 };
 288
 289 static struct scsi_control_page control_page_changeable = {
 290         /*page_code*/SMS_CONTROL_MODE_PAGE,
 291         /*page_length*/sizeof(struct scsi_control_page) - 2,
 292         /*rlec*/SCP_DSENSE,
 293         /*queue_flags*/0,
 294         /*eca_and_aen*/0,
 295         /*reserved*/0,
 296         /*aen_holdoff_period*/{0, 0}
 297 };
 298
 299
 300 /*
 301  * XXX KDM move these into the softc.
 302  */
 303 static int rcv_sync_msg;
 304 static int persis_offset;
 305 static uint8_t ctl_pause_rtr;
 306 static int     ctl_is_single = 1;
 307 static int     index_to_aps_page;
 308
 309 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 310 static int worker_threads = -1;
 311 TUNABLE_INT("kern.cam.ctl.worker_threads", &worker_threads);
 312 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
 313     &worker_threads, 1, "Number of worker threads");
 314 static int verbose = 0;
 315 TUNABLE_INT("kern.cam.ctl.verbose", &verbose);
 316 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, verbose, CTLFLAG_RWTUN,
 317     &verbose, 0, "Show SCSI errors returned to initiator");
 318
 319 /*
 320  * Serial number (0x80), device id (0x83), supported pages (0x00),
 321  * Block limits (0xB0) and Logical Block Provisioning (0xB2)
 322  */
 323 #define SCSI_EVPD_NUM_SUPPORTED_PAGES   5
 324
 325 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 326                                   int param);
 327 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 328 static int ctl_init(void);
 329 void ctl_shutdown(void);
 330 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 331 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 332 static void ctl_ioctl_online(void *arg);
 333 static void ctl_ioctl_offline(void *arg);
 334 static int ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id);
 335 static int ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id);
 336 static int ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio);
 337 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
 338 static int ctl_ioctl_submit_wait(union ctl_io *io);
 339 static void ctl_ioctl_datamove(union ctl_io *io);
 340 static void ctl_ioctl_done(union ctl_io *io);
 341 static void ctl_ioctl_hard_startstop_callback(void *arg,
 342                                               struct cfi_metatask *metatask);
 343 static void ctl_ioctl_bbrread_callback(void *arg,struct cfi_metatask *metatask);
 344 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 345                               struct ctl_ooa *ooa_hdr,
 346                               struct ctl_ooa_entry *kern_entries);
 347 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 348                      struct thread *td);
 349 uint32_t ctl_get_resindex(struct ctl_nexus *nexus);
 350 uint32_t ctl_port_idx(int port_num);
 351 #ifdef unused
 352 static union ctl_io *ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port,
 353                                    uint32_t targ_target, uint32_t targ_lun,
 354                                    int can_wait);
 355 static void ctl_kfree_io(union ctl_io *io);
 356 #endif /* unused */
 357 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
 358                          struct ctl_be_lun *be_lun, struct ctl_id target_id);
 359 static int ctl_free_lun(struct ctl_lun *lun);
 360 static void ctl_create_lun(struct ctl_be_lun *be_lun);
 361 /**
 362 static void ctl_failover_change_pages(struct ctl_softc *softc,
 363                                       struct ctl_scsiio *ctsio, int master);
 364 **/
 365
 366 static int ctl_do_mode_select(union ctl_io *io);
 367 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
 368                            uint64_t res_key, uint64_t sa_res_key,
 369                            uint8_t type, uint32_t residx,
 370                            struct ctl_scsiio *ctsio,
 371                            struct scsi_per_res_out *cdb,
 372                            struct scsi_per_res_out_parms* param);
 373 static void ctl_pro_preempt_other(struct ctl_lun *lun,
 374                                   union ctl_ha_msg *msg);
 375 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
 376 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
 377 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
 378 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
 379 static int ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio,
 380                                          int alloc_len);
 381 static int ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len);
 382 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
 383 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
 384 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len);
 385 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2);
 386 static ctl_action ctl_check_for_blockage(union ctl_io *pending_io,
 387                                          union ctl_io *ooa_io);
 388 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
 389                                 union ctl_io *starting_io);
 390 static int ctl_check_blocked(struct ctl_lun *lun);
 391 static int ctl_scsiio_lun_check(struct ctl_softc *ctl_softc,
 392                                 struct ctl_lun *lun,
 393                                 const struct ctl_cmd_entry *entry,
 394                                 struct ctl_scsiio *ctsio);
 395 //static int ctl_check_rtr(union ctl_io *pending_io, struct ctl_softc *softc);
 396 static void ctl_failover(void);
 397 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 398                                struct ctl_scsiio *ctsio);
 399 static int ctl_scsiio(struct ctl_scsiio *ctsio);
 400
 401 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 402 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 403                             ctl_ua_type ua_type);
 404 static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 405                          ctl_ua_type ua_type);
 406 static int ctl_abort_task(union ctl_io *io);
 407 static void ctl_run_task(union ctl_io *io);
 408 #ifdef CTL_IO_DELAY
 409 static void ctl_datamove_timer_wakeup(void *arg);
 410 static void ctl_done_timer_wakeup(void *arg);
 411 #endif /* CTL_IO_DELAY */
 412
 413 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 414 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
 415 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
 416 static void ctl_datamove_remote_write(union ctl_io *io);
 417 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
 418 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 419 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 420 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 421                                     ctl_ha_dt_cb callback);
 422 static void ctl_datamove_remote_read(union ctl_io *io);
 423 static void ctl_datamove_remote(union ctl_io *io);
 424 static int ctl_process_done(union ctl_io *io);
 425 static void ctl_lun_thread(void *arg);
 426 static void ctl_work_thread(void *arg);
 427 static void ctl_enqueue_incoming(union ctl_io *io);
 428 static void ctl_enqueue_rtr(union ctl_io *io);
 429 static void ctl_enqueue_done(union ctl_io *io);
 430 static void ctl_enqueue_isc(union ctl_io *io);
 431 static const struct ctl_cmd_entry *
 432     ctl_get_cmd_entry(struct ctl_scsiio *ctsio);
 433 static const struct ctl_cmd_entry *
 434     ctl_validate_command(struct ctl_scsiio *ctsio);
 435 static int ctl_cmd_applicable(uint8_t lun_type,
 436     const struct ctl_cmd_entry *entry);
 437
 438 /*
 439  * Load the serialization table.  This isn't very pretty, but is probably
 440  * the easiest way to do it.
 441  */
 442 #include "ctl_ser_table.c"
 443
 444 /*
 445  * We only need to define open, close and ioctl routines for this driver.
 446  */
 447 static struct cdevsw ctl_cdevsw = {
 448         .d_version =    D_VERSION,
 449         .d_flags =      0,
 450         .d_open =       ctl_open,
 451         .d_close =      ctl_close,
 452         .d_ioctl =      ctl_ioctl,
 453         .d_name =       "ctl",
 454 };
 455
 456
 457 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 458 MALLOC_DEFINE(M_CTLIO, "ctlio", "Memory used for CTL requests");
 459
 460 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 461
 462 static moduledata_t ctl_moduledata = {
 463         "ctl",
 464         ctl_module_event_handler,
 465         NULL
 466 };
 467
 468 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
 469 MODULE_VERSION(ctl, 1);
 470
 471 static struct ctl_frontend ioctl_frontend =
 472 {
 473         .name = "ioctl",
 474 };
 475
 476 static void
 477 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 478                             union ctl_ha_msg *msg_info)
 479 {
 480         struct ctl_scsiio *ctsio;
 481
 482         if (msg_info->hdr.original_sc == NULL) {
 483                 printf("%s: original_sc == NULL!\n", __func__);
 484                 /* XXX KDM now what? */
 485                 return;
 486         }
 487
 488         ctsio = &msg_info->hdr.original_sc->scsiio;
 489         ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 490         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 491         ctsio->io_hdr.status = msg_info->hdr.status;
 492         ctsio->scsi_status = msg_info->scsi.scsi_status;
 493         ctsio->sense_len = msg_info->scsi.sense_len;
 494         ctsio->sense_residual = msg_info->scsi.sense_residual;
 495         ctsio->residual = msg_info->scsi.residual;
 496         memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 497                sizeof(ctsio->sense_data));
 498         memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
 499                &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));
 500         ctl_enqueue_isc((union ctl_io *)ctsio);
 501 }
 502
 503 static void
 504 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
 505                                 union ctl_ha_msg *msg_info)
 506 {
 507         struct ctl_scsiio *ctsio;
 508
 509         if (msg_info->hdr.serializing_sc == NULL) {
 510                 printf("%s: serializing_sc == NULL!\n", __func__);
 511                 /* XXX KDM now what? */
 512                 return;
 513         }
 514
 515         ctsio = &msg_info->hdr.serializing_sc->scsiio;
 516 #if 0
 517         /*
 518          * Attempt to catch the situation where an I/O has
 519          * been freed, and we're using it again.
 520          */
 521         if (ctsio->io_hdr.io_type == 0xff) {
 522                 union ctl_io *tmp_io;
 523                 tmp_io = (union ctl_io *)ctsio;
 524                 printf("%s: %p use after free!\n", __func__,
 525                        ctsio);
 526                 printf("%s: type %d msg %d cdb %x iptl: "
 527                        "%d:%d:%d:%d tag 0x%04x "
 528                        "flag %#x status %x\n",
 529                         __func__,
 530                         tmp_io->io_hdr.io_type,
 531                         tmp_io->io_hdr.msg_type,
 532                         tmp_io->scsiio.cdb[0],
 533                         tmp_io->io_hdr.nexus.initid.id,
 534                         tmp_io->io_hdr.nexus.targ_port,
 535                         tmp_io->io_hdr.nexus.targ_target.id,
 536                         tmp_io->io_hdr.nexus.targ_lun,
 537                         (tmp_io->io_hdr.io_type ==
 538                         CTL_IO_TASK) ?
 539                         tmp_io->taskio.tag_num :
 540                         tmp_io->scsiio.tag_num,
 541                         tmp_io->io_hdr.flags,
 542                         tmp_io->io_hdr.status);
 543         }
 544 #endif
 545         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 546         ctl_enqueue_isc((union ctl_io *)ctsio);
 547 }
 548
 549 /*
 550  * ISC (Inter Shelf Communication) event handler.  Events from the HA
 551  * subsystem come in here.
 552  */
 553 static void
 554 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 555 {
 556         struct ctl_softc *ctl_softc;
 557         union ctl_io *io;
 558         struct ctl_prio *presio;
 559         ctl_ha_status isc_status;
 560
 561         ctl_softc = control_softc;
 562         io = NULL;
 563
 564
 565 #if 0
 566         printf("CTL: Isc Msg event %d\n", event);
 567 #endif
 568         if (event == CTL_HA_EVT_MSG_RECV) {
 569                 union ctl_ha_msg msg_info;
 570
 571                 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
 572                                              sizeof(msg_info), /*wait*/ 0);
 573 #if 0
 574                 printf("CTL: msg_type %d\n", msg_info.msg_type);
 575 #endif
 576                 if (isc_status != 0) {
 577                         printf("Error receiving message, status = %d\n",
 578                                isc_status);
 579                         return;
 580                 }
 581
 582                 switch (msg_info.hdr.msg_type) {
 583                 case CTL_MSG_SERIALIZE:
 584 #if 0
 585                         printf("Serialize\n");
 586 #endif
 587                         io = ctl_alloc_io((void *)ctl_softc->othersc_pool);
 588                         if (io == NULL) {
 589                                 printf("ctl_isc_event_handler: can't allocate "
 590                                        "ctl_io!\n");
 591                                 /* Bad Juju */
 592                                 /* Need to set busy and send msg back */
 593                                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
 594                                 msg_info.hdr.status = CTL_SCSI_ERROR;
 595                                 msg_info.scsi.scsi_status = SCSI_STATUS_BUSY;
 596                                 msg_info.scsi.sense_len = 0;
 597                                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
 598                                     sizeof(msg_info), 0) > CTL_HA_STATUS_SUCCESS){
 599                                 }
 600                                 goto bailout;
 601                         }
 602                         ctl_zero_io(io);
 603                         // populate ctsio from msg_info
 604                         io->io_hdr.io_type = CTL_IO_SCSI;
 605                         io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
 606                         io->io_hdr.original_sc = msg_info.hdr.original_sc;
 607 #if 0
 608                         printf("pOrig %x\n", (int)msg_info.original_sc);
 609 #endif
 610                         io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
 611                                             CTL_FLAG_IO_ACTIVE;
 612                         /*
 613                          * If we're in serialization-only mode, we don't
 614                          * want to go through full done processing.  Thus
 615                          * the COPY flag.
 616                          *
 617                          * XXX KDM add another flag that is more specific.
 618                          */
 619                         if (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)
 620                                 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
 621                         io->io_hdr.nexus = msg_info.hdr.nexus;
 622 #if 0
 623                         printf("targ %d, port %d, iid %d, lun %d\n",
 624                                io->io_hdr.nexus.targ_target.id,
 625                                io->io_hdr.nexus.targ_port,
 626                                io->io_hdr.nexus.initid.id,
 627                                io->io_hdr.nexus.targ_lun);
 628 #endif
 629                         io->scsiio.tag_num = msg_info.scsi.tag_num;
 630                         io->scsiio.tag_type = msg_info.scsi.tag_type;
 631                         memcpy(io->scsiio.cdb, msg_info.scsi.cdb,
 632                                CTL_MAX_CDBLEN);
 633                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
 634                                 const struct ctl_cmd_entry *entry;
 635
 636                                 entry = ctl_get_cmd_entry(&io->scsiio);
 637                                 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
 638                                 io->io_hdr.flags |=
 639                                         entry->flags & CTL_FLAG_DATA_MASK;
 640                         }
 641                         ctl_enqueue_isc(io);
 642                         break;
 643
 644                 /* Performed on the Originating SC, XFER mode only */
 645                 case CTL_MSG_DATAMOVE: {
 646                         struct ctl_sg_entry *sgl;
 647                         int i, j;
 648
 649                         io = msg_info.hdr.original_sc;
 650                         if (io == NULL) {
 651                                 printf("%s: original_sc == NULL!\n", __func__);
 652                                 /* XXX KDM do something here */
 653                                 break;
 654                         }
 655                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
 656                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 657                         /*
 658                          * Keep track of this, we need to send it back over
 659                          * when the datamove is complete.
 660                          */
 661                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 662
 663                         if (msg_info.dt.sg_sequence == 0) {
 664                                 /*
 665                                  * XXX KDM we use the preallocated S/G list
 666                                  * here, but we'll need to change this to
 667                                  * dynamic allocation if we need larger S/G
 668                                  * lists.
 669                                  */
 670                                 if (msg_info.dt.kern_sg_entries >
 671                                     sizeof(io->io_hdr.remote_sglist) /
 672                                     sizeof(io->io_hdr.remote_sglist[0])) {
 673                                         printf("%s: number of S/G entries "
 674                                             "needed %u > allocated num %zd\n",
 675                                             __func__,
 676                                             msg_info.dt.kern_sg_entries,
 677                                             sizeof(io->io_hdr.remote_sglist)/
 678                                             sizeof(io->io_hdr.remote_sglist[0]));
 679
 680                                         /*
 681                                          * XXX KDM send a message back to
 682                                          * the other side to shut down the
 683                                          * DMA.  The error will come back
 684                                          * through via the normal channel.
 685                                          */
 686                                         break;
 687                                 }
 688                                 sgl = io->io_hdr.remote_sglist;
 689                                 memset(sgl, 0,
 690                                        sizeof(io->io_hdr.remote_sglist));
 691
 692                                 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
 693
 694                                 io->scsiio.kern_sg_entries =
 695                                         msg_info.dt.kern_sg_entries;
 696                                 io->scsiio.rem_sg_entries =
 697                                         msg_info.dt.kern_sg_entries;
 698                                 io->scsiio.kern_data_len =
 699                                         msg_info.dt.kern_data_len;
 700                                 io->scsiio.kern_total_len =
 701                                         msg_info.dt.kern_total_len;
 702                                 io->scsiio.kern_data_resid =
 703                                         msg_info.dt.kern_data_resid;
 704                                 io->scsiio.kern_rel_offset =
 705                                         msg_info.dt.kern_rel_offset;
 706                                 /*
 707                                  * Clear out per-DMA flags.
 708                                  */
 709                                 io->io_hdr.flags &= ~CTL_FLAG_RDMA_MASK;
 710                                 /*
 711                                  * Add per-DMA flags that are set for this
 712                                  * particular DMA request.
 713                                  */
 714                                 io->io_hdr.flags |= msg_info.dt.flags &
 715                                                     CTL_FLAG_RDMA_MASK;
 716                         } else
 717                                 sgl = (struct ctl_sg_entry *)
 718                                         io->scsiio.kern_data_ptr;
 719
 720                         for (i = msg_info.dt.sent_sg_entries, j = 0;
 721                              i < (msg_info.dt.sent_sg_entries +
 722                              msg_info.dt.cur_sg_entries); i++, j++) {
 723                                 sgl[i].addr = msg_info.dt.sg_list[j].addr;
 724                                 sgl[i].len = msg_info.dt.sg_list[j].len;
 725
 726 #if 0
 727                                 printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
 728                                        __func__,
 729                                        msg_info.dt.sg_list[j].addr,
 730                                        msg_info.dt.sg_list[j].len,
 731                                        sgl[i].addr, sgl[i].len, j, i);
 732 #endif
 733                         }
 734 #if 0
 735                         memcpy(&sgl[msg_info.dt.sent_sg_entries],
 736                                msg_info.dt.sg_list,
 737                                sizeof(*sgl) * msg_info.dt.cur_sg_entries);
 738 #endif
 739
 740                         /*
 741                          * If this is the last piece of the I/O, we've got
 742                          * the full S/G list.  Queue processing in the thread.
 743                          * Otherwise wait for the next piece.
 744                          */
 745                         if (msg_info.dt.sg_last != 0)
 746                                 ctl_enqueue_isc(io);
 747                         break;
 748                 }
 749                 /* Performed on the Serializing (primary) SC, XFER mode only */
 750                 case CTL_MSG_DATAMOVE_DONE: {
 751                         if (msg_info.hdr.serializing_sc == NULL) {
 752                                 printf("%s: serializing_sc == NULL!\n",
 753                                        __func__);
 754                                 /* XXX KDM now what? */
 755                                 break;
 756                         }
 757                         /*
 758                          * We grab the sense information here in case
 759                          * there was a failure, so we can return status
 760                          * back to the initiator.
 761                          */
 762                         io = msg_info.hdr.serializing_sc;
 763                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
 764                         io->io_hdr.status = msg_info.hdr.status;
 765                         io->scsiio.scsi_status = msg_info.scsi.scsi_status;
 766                         io->scsiio.sense_len = msg_info.scsi.sense_len;
 767                         io->scsiio.sense_residual =msg_info.scsi.sense_residual;
 768                         io->io_hdr.port_status = msg_info.scsi.fetd_status;
 769                         io->scsiio.residual = msg_info.scsi.residual;
 770                         memcpy(&io->scsiio.sense_data,&msg_info.scsi.sense_data,
 771                                sizeof(io->scsiio.sense_data));
 772                         ctl_enqueue_isc(io);
 773                         break;
 774                 }
 775
 776                 /* Preformed on Originating SC, SER_ONLY mode */
 777                 case CTL_MSG_R2R:
 778                         io = msg_info.hdr.original_sc;
 779                         if (io == NULL) {
 780                                 printf("%s: Major Bummer\n", __func__);
 781                                 return;
 782                         } else {
 783 #if 0
 784                                 printf("pOrig %x\n",(int) ctsio);
 785 #endif
 786                         }
 787                         io->io_hdr.msg_type = CTL_MSG_R2R;
 788                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 789                         ctl_enqueue_isc(io);
 790                         break;
 791
 792                 /*
 793                  * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
 794                  * mode.
 795                  * Performed on the Originating (i.e. secondary) SC in XFER
 796                  * mode
 797                  */
 798                 case CTL_MSG_FINISH_IO:
 799                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER)
 800                                 ctl_isc_handler_finish_xfer(ctl_softc,
 801                                                             &msg_info);
 802                         else
 803                                 ctl_isc_handler_finish_ser_only(ctl_softc,
 804                                                                 &msg_info);
 805                         break;
 806
 807                 /* Preformed on Originating SC */
 808                 case CTL_MSG_BAD_JUJU:
 809                         io = msg_info.hdr.original_sc;
 810                         if (io == NULL) {
 811                                 printf("%s: Bad JUJU!, original_sc is NULL!\n",
 812                                        __func__);
 813                                 break;
 814                         }
 815                         ctl_copy_sense_data(&msg_info, io);
 816                         /*
 817                          * IO should have already been cleaned up on other
 818                          * SC so clear this flag so we won't send a message
 819                          * back to finish the IO there.
 820                          */
 821                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
 822                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 823
 824                         /* io = msg_info.hdr.serializing_sc; */
 825                         io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
 826                         ctl_enqueue_isc(io);
 827                         break;
 828
 829                 /* Handle resets sent from the other side */
 830                 case CTL_MSG_MANAGE_TASKS: {
 831                         struct ctl_taskio *taskio;
 832                         taskio = (struct ctl_taskio *)ctl_alloc_io(
 833                                 (void *)ctl_softc->othersc_pool);
 834                         if (taskio == NULL) {
 835                                 printf("ctl_isc_event_handler: can't allocate "
 836                                        "ctl_io!\n");
 837                                 /* Bad Juju */
 838                                 /* should I just call the proper reset func
 839                                    here??? */
 840                                 goto bailout;
 841                         }
 842                         ctl_zero_io((union ctl_io *)taskio);
 843                         taskio->io_hdr.io_type = CTL_IO_TASK;
 844                         taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
 845                         taskio->io_hdr.nexus = msg_info.hdr.nexus;
 846                         taskio->task_action = msg_info.task.task_action;
 847                         taskio->tag_num = msg_info.task.tag_num;
 848                         taskio->tag_type = msg_info.task.tag_type;
 849 #ifdef CTL_TIME_IO
 850                         taskio->io_hdr.start_time = time_uptime;
 851                         getbintime(&taskio->io_hdr.start_bt);
 852 #if 0
 853                         cs_prof_gettime(&taskio->io_hdr.start_ticks);
 854 #endif
 855 #endif /* CTL_TIME_IO */
 856                         ctl_run_task((union ctl_io *)taskio);
 857                         break;
 858                 }
 859                 /* Persistent Reserve action which needs attention */
 860                 case CTL_MSG_PERS_ACTION:
 861                         presio = (struct ctl_prio *)ctl_alloc_io(
 862                                 (void *)ctl_softc->othersc_pool);
 863                         if (presio == NULL) {
 864                                 printf("ctl_isc_event_handler: can't allocate "
 865                                        "ctl_io!\n");
 866                                 /* Bad Juju */
 867                                 /* Need to set busy and send msg back */
 868                                 goto bailout;
 869                         }
 870                         ctl_zero_io((union ctl_io *)presio);
 871                         presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
 872                         presio->pr_msg = msg_info.pr;
 873                         ctl_enqueue_isc((union ctl_io *)presio);
 874                         break;
 875                 case CTL_MSG_SYNC_FE:
 876                         rcv_sync_msg = 1;
 877                         break;
 878                 case CTL_MSG_APS_LOCK: {
 879                         // It's quicker to execute this then to
 880                         // queue it.
 881                         struct ctl_lun *lun;
 882                         struct ctl_page_index *page_index;
 883                         struct copan_aps_subpage *current_sp;
 884                         uint32_t targ_lun;
 885
 886                         targ_lun = msg_info.hdr.nexus.targ_mapped_lun;
 887                         lun = ctl_softc->ctl_luns[targ_lun];
 888                         mtx_lock(&lun->lun_lock);
 889                         page_index = &lun->mode_pages.index[index_to_aps_page];
 890                         current_sp = (struct copan_aps_subpage *)
 891                                      (page_index->page_data +
 892                                      (page_index->page_len * CTL_PAGE_CURRENT));
 893
 894                         current_sp->lock_active = msg_info.aps.lock_flag;
 895                         mtx_unlock(&lun->lun_lock);
 896                         break;
 897                 }
 898                 default:
 899                         printf("How did I get here?\n");
 900                 }
 901         } else if (event == CTL_HA_EVT_MSG_SENT) {
 902                 if (param != CTL_HA_STATUS_SUCCESS) {
 903                         printf("Bad status from ctl_ha_msg_send status %d\n",
 904                                param);
 905                 }
 906                 return;
 907         } else if (event == CTL_HA_EVT_DISCONNECT) {
 908                 printf("CTL: Got a disconnect from Isc\n");
 909                 return;
 910         } else {
 911                 printf("ctl_isc_event_handler: Unknown event %d\n", event);
 912                 return;
 913         }
 914
 915 bailout:
 916         return;
 917 }
 918
 919 static void
 920 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
 921 {
 922         struct scsi_sense_data *sense;
 923
 924         sense = &dest->scsiio.sense_data;
 925         bcopy(&src->scsi.sense_data, sense, sizeof(*sense));
 926         dest->scsiio.scsi_status = src->scsi.scsi_status;
 927         dest->scsiio.sense_len = src->scsi.sense_len;
 928         dest->io_hdr.status = src->hdr.status;
 929 }
 930
 931 static int
 932 ctl_init(void)
 933 {
 934         struct ctl_softc *softc;
 935         struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
 936         struct ctl_port *port;
 937         uint8_t sc_id =0;
 938         int i, error, retval;
 939         //int isc_retval;
 940
 941         retval = 0;
 942         ctl_pause_rtr = 0;
 943         rcv_sync_msg = 0;
 944
 945         control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
 946                                M_WAITOK | M_ZERO);
 947         softc = control_softc;
 948
 949         softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
 950                               "cam/ctl");
 951
 952         softc->dev->si_drv1 = softc;
 953
 954         /*
 955          * By default, return a "bad LUN" peripheral qualifier for unknown
 956          * LUNs.  The user can override this default using the tunable or
 957          * sysctl.  See the comment in ctl_inquiry_std() for more details.
 958          */
 959         softc->inquiry_pq_no_lun = 1;
 960         TUNABLE_INT_FETCH("kern.cam.ctl.inquiry_pq_no_lun",
 961                           &softc->inquiry_pq_no_lun);
 962         sysctl_ctx_init(&softc->sysctl_ctx);
 963         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
 964                 SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
 965                 CTLFLAG_RD, 0, "CAM Target Layer");
 966
 967         if (softc->sysctl_tree == NULL) {
 968                 printf("%s: unable to allocate sysctl tree\n", __func__);
 969                 destroy_dev(softc->dev);
 970                 free(control_softc, M_DEVBUF);
 971                 control_softc = NULL;
 972                 return (ENOMEM);
 973         }
 974
 975         SYSCTL_ADD_INT(&softc->sysctl_ctx,
 976                        SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
 977                        "inquiry_pq_no_lun", CTLFLAG_RW,
 978                        &softc->inquiry_pq_no_lun, 0,
 979                        "Report no lun possible for invalid LUNs");
 980
 981         mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
 982         mtx_init(&softc->pool_lock, "CTL pool mutex", NULL, MTX_DEF);
 983         softc->open_count = 0;
 984
 985         /*
 986          * Default to actually sending a SYNCHRONIZE CACHE command down to
 987          * the drive.
 988          */
 989         softc->flags = CTL_FLAG_REAL_SYNC;
 990
 991         /*
 992          * In Copan's HA scheme, the "master" and "slave" roles are
 993          * figured out through the slot the controller is in.  Although it
 994          * is an active/active system, someone has to be in charge.
 995          */
 996 #ifdef NEEDTOPORT
 997         scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
 998 #endif
 999
1000         if (sc_id == 0) {
1001                 softc->flags |= CTL_FLAG_MASTER_SHELF;
1002                 persis_offset = 0;
1003         } else
1004                 persis_offset = CTL_MAX_INITIATORS;
1005
1006         /*
1007          * XXX KDM need to figure out where we want to get our target ID
1008          * and WWID.  Is it different on each port?
1009          */
1010         softc->target.id = 0;
1011         softc->target.wwid[0] = 0x12345678;
1012         softc->target.wwid[1] = 0x87654321;
1013         STAILQ_INIT(&softc->lun_list);
1014         STAILQ_INIT(&softc->pending_lun_queue);
1015         STAILQ_INIT(&softc->fe_list);
1016         STAILQ_INIT(&softc->port_list);
1017         STAILQ_INIT(&softc->be_list);
1018         STAILQ_INIT(&softc->io_pools);
1019
1020         if (ctl_pool_create(softc, CTL_POOL_INTERNAL, CTL_POOL_ENTRIES_INTERNAL,
1021                             &internal_pool)!= 0){
1022                 printf("ctl: can't allocate %d entry internal pool, "
1023                        "exiting\n", CTL_POOL_ENTRIES_INTERNAL);
1024                 return (ENOMEM);
1025         }
1026
1027         if (ctl_pool_create(softc, CTL_POOL_EMERGENCY,
1028                             CTL_POOL_ENTRIES_EMERGENCY, &emergency_pool) != 0) {
1029                 printf("ctl: can't allocate %d entry emergency pool, "
1030                        "exiting\n", CTL_POOL_ENTRIES_EMERGENCY);
1031                 ctl_pool_free(internal_pool);
1032                 return (ENOMEM);
1033         }
1034
1035         if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC,
1036                             &other_pool) != 0)
1037         {
1038                 printf("ctl: can't allocate %d entry other SC pool, "
1039                        "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1040                 ctl_pool_free(internal_pool);
1041                 ctl_pool_free(emergency_pool);
1042                 return (ENOMEM);
1043         }
1044
1045         softc->internal_pool = internal_pool;
1046         softc->emergency_pool = emergency_pool;
1047         softc->othersc_pool = other_pool;
1048
1049         if (worker_threads <= 0)
1050                 worker_threads = max(1, mp_ncpus / 4);
1051         if (worker_threads > CTL_MAX_THREADS)
1052                 worker_threads = CTL_MAX_THREADS;
1053
1054         for (i = 0; i < worker_threads; i++) {
1055                 struct ctl_thread *thr = &softc->threads[i];
1056
1057                 mtx_init(&thr->queue_lock, "CTL queue mutex", NULL, MTX_DEF);
1058                 thr->ctl_softc = softc;
1059                 STAILQ_INIT(&thr->incoming_queue);
1060                 STAILQ_INIT(&thr->rtr_queue);
1061                 STAILQ_INIT(&thr->done_queue);
1062                 STAILQ_INIT(&thr->isc_queue);
1063
1064                 error = kproc_kthread_add(ctl_work_thread, thr,
1065                     &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i);
1066                 if (error != 0) {
1067                         printf("error creating CTL work thread!\n");
1068                         ctl_pool_free(internal_pool);
1069                         ctl_pool_free(emergency_pool);
1070                         ctl_pool_free(other_pool);
1071                         return (error);
1072                 }
1073         }
1074         error = kproc_kthread_add(ctl_lun_thread, softc,
1075             &softc->ctl_proc, NULL, 0, 0, "ctl", "lun");
1076         if (error != 0) {
1077                 printf("error creating CTL lun thread!\n");
1078                 ctl_pool_free(internal_pool);
1079                 ctl_pool_free(emergency_pool);
1080                 ctl_pool_free(other_pool);
1081                 return (error);
1082         }
1083         if (bootverbose)
1084                 printf("ctl: CAM Target Layer loaded\n");
1085
1086         /*
1087          * Initialize the initiator and portname mappings
1088          */
1089         memset(softc->wwpn_iid, 0, sizeof(softc->wwpn_iid));
1090
1091         /*
1092          * Initialize the ioctl front end.
1093          */
1094         ctl_frontend_register(&ioctl_frontend);
1095         port = &softc->ioctl_info.port;
1096         port->frontend = &ioctl_frontend;
1097         sprintf(softc->ioctl_info.port_name, "ioctl");
1098         port->port_type = CTL_PORT_IOCTL;
1099         port->num_requested_ctl_io = 100;
1100         port->port_name = softc->ioctl_info.port_name;
1101         port->port_online = ctl_ioctl_online;
1102         port->port_offline = ctl_ioctl_offline;
1103         port->onoff_arg = &softc->ioctl_info;
1104         port->lun_enable = ctl_ioctl_lun_enable;
1105         port->lun_disable = ctl_ioctl_lun_disable;
1106         port->targ_lun_arg = &softc->ioctl_info;
1107         port->fe_datamove = ctl_ioctl_datamove;
1108         port->fe_done = ctl_ioctl_done;
1109         port->max_targets = 15;
1110         port->max_target_id = 15;
1111
1112         if (ctl_port_register(&softc->ioctl_info.port,
1113                           (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
1114                 printf("ctl: ioctl front end registration failed, will "
1115                        "continue anyway\n");
1116         }
1117
1118 #ifdef CTL_IO_DELAY
1119         if (sizeof(struct callout) > CTL_TIMER_BYTES) {
1120                 printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
1121                        sizeof(struct callout), CTL_TIMER_BYTES);
1122                 return (EINVAL);
1123         }
1124 #endif /* CTL_IO_DELAY */
1125
1126         return (0);
1127 }
1128
1129 void
1130 ctl_shutdown(void)
1131 {
1132         struct ctl_softc *softc;
1133         struct ctl_lun *lun, *next_lun;
1134         struct ctl_io_pool *pool;
1135
1136         softc = (struct ctl_softc *)control_softc;
1137
1138         if (ctl_port_deregister(&softc->ioctl_info.port) != 0)
1139                 printf("ctl: ioctl front end deregistration failed\n");
1140
1141         mtx_lock(&softc->ctl_lock);
1142
1143         /*
1144          * Free up each LUN.
1145          */
1146         for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1147                 next_lun = STAILQ_NEXT(lun, links);
1148                 ctl_free_lun(lun);
1149         }
1150
1151         mtx_unlock(&softc->ctl_lock);
1152
1153         ctl_frontend_deregister(&ioctl_frontend);
1154
1155         /*
1156          * This will rip the rug out from under any FETDs or anyone else
1157          * that has a pool allocated.  Since we increment our module
1158          * refcount any time someone outside the main CTL module allocates
1159          * a pool, we shouldn't have any problems here.  The user won't be
1160          * able to unload the CTL module until client modules have
1161          * successfully unloaded.
1162          */
1163         while ((pool = STAILQ_FIRST(&softc->io_pools)) != NULL)
1164                 ctl_pool_free(pool);
1165
1166 #if 0
1167         ctl_shutdown_thread(softc->work_thread);
1168         mtx_destroy(&softc->queue_lock);
1169 #endif
1170
1171         mtx_destroy(&softc->pool_lock);
1172         mtx_destroy(&softc->ctl_lock);
1173
1174         destroy_dev(softc->dev);
1175
1176         sysctl_ctx_free(&softc->sysctl_ctx);
1177
1178         free(control_softc, M_DEVBUF);
1179         control_softc = NULL;
1180
1181         if (bootverbose)
1182                 printf("ctl: CAM Target Layer unloaded\n");
1183 }
1184
1185 static int
1186 ctl_module_event_handler(module_t mod, int what, void *arg)
1187 {
1188
1189         switch (what) {
1190         case MOD_LOAD:
1191                 return (ctl_init());
1192         case MOD_UNLOAD:
1193                 return (EBUSY);
1194         default:
1195                 return (EOPNOTSUPP);
1196         }
1197 }
1198
1199 /*
1200  * XXX KDM should we do some access checks here?  Bump a reference count to
1201  * prevent a CTL module from being unloaded while someone has it open?
1202  */
1203 static int
1204 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1205 {
1206         return (0);
1207 }
1208
1209 static int
1210 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1211 {
1212         return (0);
1213 }
1214
1215 int
1216 ctl_port_enable(ctl_port_type port_type)
1217 {
1218         struct ctl_softc *softc;
1219         struct ctl_port *port;
1220
1221         if (ctl_is_single == 0) {
1222                 union ctl_ha_msg msg_info;
1223                 int isc_retval;
1224
1225 #if 0
1226                 printf("%s: HA mode, synchronizing frontend enable\n",
1227                         __func__);
1228 #endif
1229                 msg_info.hdr.msg_type = CTL_MSG_SYNC_FE;
1230                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1231                         sizeof(msg_info), 1 )) > CTL_HA_STATUS_SUCCESS) {
1232                         printf("Sync msg send error retval %d\n", isc_retval);
1233                 }
1234                 if (!rcv_sync_msg) {
1235                         isc_retval=ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
1236                                 sizeof(msg_info), 1);
1237                 }
1238 #if 0
1239                 printf("CTL:Frontend Enable\n");
1240         } else {
1241                 printf("%s: single mode, skipping frontend synchronization\n",
1242                         __func__);
1243 #endif
1244         }
1245
1246         softc = control_softc;
1247
1248         STAILQ_FOREACH(port, &softc->port_list, links) {
1249                 if (port_type & port->port_type)
1250                 {
1251 #if 0
1252                         printf("port %d\n", port->targ_port);
1253 #endif
1254                         ctl_port_online(port);
1255                 }
1256         }
1257
1258         return (0);
1259 }
1260
1261 int
1262 ctl_port_disable(ctl_port_type port_type)
1263 {
1264         struct ctl_softc *softc;
1265         struct ctl_port *port;
1266
1267         softc = control_softc;
1268
1269         STAILQ_FOREACH(port, &softc->port_list, links) {
1270                 if (port_type & port->port_type)
1271                         ctl_port_offline(port);
1272         }
1273
1274         return (0);
1275 }
1276
1277 /*
1278  * Returns 0 for success, 1 for failure.
1279  * Currently the only failure mode is if there aren't enough entries
1280  * allocated.  So, in case of a failure, look at num_entries_dropped,
1281  * reallocate and try again.
1282  */
1283 int
1284 ctl_port_list(struct ctl_port_entry *entries, int num_entries_alloced,
1285               int *num_entries_filled, int *num_entries_dropped,
1286               ctl_port_type port_type, int no_virtual)
1287 {
1288         struct ctl_softc *softc;
1289         struct ctl_port *port;
1290         int entries_dropped, entries_filled;
1291         int retval;
1292         int i;
1293
1294         softc = control_softc;
1295
1296         retval = 0;
1297         entries_filled = 0;
1298         entries_dropped = 0;
1299
1300         i = 0;
1301         mtx_lock(&softc->ctl_lock);
1302         STAILQ_FOREACH(port, &softc->port_list, links) {
1303                 struct ctl_port_entry *entry;
1304
1305                 if ((port->port_type & port_type) == 0)
1306                         continue;
1307
1308                 if ((no_virtual != 0)
1309                  && (port->virtual_port != 0))
1310                         continue;
1311
1312                 if (entries_filled >= num_entries_alloced) {
1313                         entries_dropped++;
1314                         continue;
1315                 }
1316                 entry = &entries[i];
1317
1318                 entry->port_type = port->port_type;
1319                 strlcpy(entry->port_name, port->port_name,
1320                         sizeof(entry->port_name));
1321                 entry->physical_port = port->physical_port;
1322                 entry->virtual_port = port->virtual_port;
1323                 entry->wwnn = port->wwnn;
1324                 entry->wwpn = port->wwpn;
1325
1326                 i++;
1327                 entries_filled++;
1328         }
1329
1330         mtx_unlock(&softc->ctl_lock);
1331
1332         if (entries_dropped > 0)
1333                 retval = 1;
1334
1335         *num_entries_dropped = entries_dropped;
1336         *num_entries_filled = entries_filled;
1337
1338         return (retval);
1339 }
1340
1341 static void
1342 ctl_ioctl_online(void *arg)
1343 {
1344         struct ctl_ioctl_info *ioctl_info;
1345
1346         ioctl_info = (struct ctl_ioctl_info *)arg;
1347
1348         ioctl_info->flags |= CTL_IOCTL_FLAG_ENABLED;
1349 }
1350
1351 static void
1352 ctl_ioctl_offline(void *arg)
1353 {
1354         struct ctl_ioctl_info *ioctl_info;
1355
1356         ioctl_info = (struct ctl_ioctl_info *)arg;
1357
1358         ioctl_info->flags &= ~CTL_IOCTL_FLAG_ENABLED;
1359 }
1360
1361 /*
1362  * Remove an initiator by port number and initiator ID.
1363  * Returns 0 for success, 1 for failure.
1364  */
1365 int
1366 ctl_remove_initiator(int32_t targ_port, uint32_t iid)
1367 {
1368         struct ctl_softc *softc;
1369
1370         softc = control_softc;
1371
1372         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1373
1374         if ((targ_port < 0)
1375          || (targ_port > CTL_MAX_PORTS)) {
1376                 printf("%s: invalid port number %d\n", __func__, targ_port);
1377                 return (1);
1378         }
1379         if (iid > CTL_MAX_INIT_PER_PORT) {
1380                 printf("%s: initiator ID %u > maximun %u!\n",
1381                        __func__, iid, CTL_MAX_INIT_PER_PORT);
1382                 return (1);
1383         }
1384
1385         mtx_lock(&softc->ctl_lock);
1386
1387         softc->wwpn_iid[targ_port][iid].in_use = 0;
1388
1389         mtx_unlock(&softc->ctl_lock);
1390
1391         return (0);
1392 }
1393
1394 /*
1395  * Add an initiator to the initiator map.
1396  * Returns 0 for success, 1 for failure.
1397  */
1398 int
1399 ctl_add_initiator(uint64_t wwpn, int32_t targ_port, uint32_t iid)
1400 {
1401         struct ctl_softc *softc;
1402         int retval;
1403
1404         softc = control_softc;
1405
1406         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1407
1408         retval = 0;
1409
1410         if ((targ_port < 0)
1411          || (targ_port > CTL_MAX_PORTS)) {
1412                 printf("%s: invalid port number %d\n", __func__, targ_port);
1413                 return (1);
1414         }
1415         if (iid > CTL_MAX_INIT_PER_PORT) {
1416                 printf("%s: WWPN %#jx initiator ID %u > maximun %u!\n",
1417                        __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1418                 return (1);
1419         }
1420
1421         mtx_lock(&softc->ctl_lock);
1422
1423         if (softc->wwpn_iid[targ_port][iid].in_use != 0) {
1424                 /*
1425                  * We don't treat this as an error.
1426                  */
1427                 if (softc->wwpn_iid[targ_port][iid].wwpn == wwpn) {
1428                         printf("%s: port %d iid %u WWPN %#jx arrived again?\n",
1429                                __func__, targ_port, iid, (uintmax_t)wwpn);
1430                         goto bailout;
1431                 }
1432
1433                 /*
1434                  * This is an error, but what do we do about it?  The
1435                  * driver is telling us we have a new WWPN for this
1436                  * initiator ID, so we pretty much need to use it.
1437                  */
1438                 printf("%s: port %d iid %u WWPN %#jx arrived, WWPN %#jx is "
1439                        "still at that address\n", __func__, targ_port, iid,
1440                        (uintmax_t)wwpn,
1441                        (uintmax_t)softc->wwpn_iid[targ_port][iid].wwpn);
1442
1443                 /*
1444                  * XXX KDM clear have_ca and ua_pending on each LUN for
1445                  * this initiator.
1446                  */
1447         }
1448         softc->wwpn_iid[targ_port][iid].in_use = 1;
1449         softc->wwpn_iid[targ_port][iid].iid = iid;
1450         softc->wwpn_iid[targ_port][iid].wwpn = wwpn;
1451         softc->wwpn_iid[targ_port][iid].port = targ_port;
1452
1453 bailout:
1454
1455         mtx_unlock(&softc->ctl_lock);
1456
1457         return (retval);
1458 }
1459
1460 static int
1461 ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
1462 {
1463         return (0);
1464 }
1465
1466 static int
1467 ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
1468 {
1469         return (0);
1470 }
1471
1472 /*
1473  * Data movement routine for the CTL ioctl frontend port.
1474  */
1475 static int
1476 ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
1477 {
1478         struct ctl_sg_entry *ext_sglist, *kern_sglist;
1479         struct ctl_sg_entry ext_entry, kern_entry;
1480         int ext_sglen, ext_sg_entries, kern_sg_entries;
1481         int ext_sg_start, ext_offset;
1482         int len_to_copy, len_copied;
1483         int kern_watermark, ext_watermark;
1484         int ext_sglist_malloced;
1485         int i, j;
1486
1487         ext_sglist_malloced = 0;
1488         ext_sg_start = 0;
1489         ext_offset = 0;
1490
1491         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove\n"));
1492
1493         /*
1494          * If this flag is set, fake the data transfer.
1495          */
1496         if (ctsio->io_hdr.flags & CTL_FLAG_NO_DATAMOVE) {
1497                 ctsio->ext_data_filled = ctsio->ext_data_len;
1498                 goto bailout;
1499         }
1500
1501         /*
1502          * To simplify things here, if we have a single buffer, stick it in
1503          * a S/G entry and just make it a single entry S/G list.
1504          */
1505         if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
1506                 int len_seen;
1507
1508                 ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
1509
1510                 ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
1511                                                            M_WAITOK);
1512                 ext_sglist_malloced = 1;
1513                 if (copyin(ctsio->ext_data_ptr, ext_sglist,
1514                                    ext_sglen) != 0) {
1515                         ctl_set_internal_failure(ctsio,
1516                                                  /*sks_valid*/ 0,
1517                                                  /*retry_count*/ 0);
1518                         goto bailout;
1519                 }
1520                 ext_sg_entries = ctsio->ext_sg_entries;
1521                 len_seen = 0;
1522                 for (i = 0; i < ext_sg_entries; i++) {
1523                         if ((len_seen + ext_sglist[i].len) >=
1524                              ctsio->ext_data_filled) {
1525                                 ext_sg_start = i;
1526                                 ext_offset = ctsio->ext_data_filled - len_seen;
1527                                 break;
1528                         }
1529                         len_seen += ext_sglist[i].len;
1530                 }
1531         } else {
1532                 ext_sglist = &ext_entry;
1533                 ext_sglist->addr = ctsio->ext_data_ptr;
1534                 ext_sglist->len = ctsio->ext_data_len;
1535                 ext_sg_entries = 1;
1536                 ext_sg_start = 0;
1537                 ext_offset = ctsio->ext_data_filled;
1538         }
1539
1540         if (ctsio->kern_sg_entries > 0) {
1541                 kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
1542                 kern_sg_entries = ctsio->kern_sg_entries;
1543         } else {
1544                 kern_sglist = &kern_entry;
1545                 kern_sglist->addr = ctsio->kern_data_ptr;
1546                 kern_sglist->len = ctsio->kern_data_len;
1547                 kern_sg_entries = 1;
1548         }
1549
1550
1551         kern_watermark = 0;
1552         ext_watermark = ext_offset;
1553         len_copied = 0;
1554         for (i = ext_sg_start, j = 0;
1555              i < ext_sg_entries && j < kern_sg_entries;) {
1556                 uint8_t *ext_ptr, *kern_ptr;
1557
1558                 len_to_copy = ctl_min(ext_sglist[i].len - ext_watermark,
1559                                       kern_sglist[j].len - kern_watermark);
1560
1561                 ext_ptr = (uint8_t *)ext_sglist[i].addr;
1562                 ext_ptr = ext_ptr + ext_watermark;
1563                 if (ctsio->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
1564                         /*
1565                          * XXX KDM fix this!
1566                          */
1567                         panic("need to implement bus address support");
1568 #if 0
1569                         kern_ptr = bus_to_virt(kern_sglist[j].addr);
1570 #endif
1571                 } else
1572                         kern_ptr = (uint8_t *)kern_sglist[j].addr;
1573                 kern_ptr = kern_ptr + kern_watermark;
1574
1575                 kern_watermark += len_to_copy;
1576                 ext_watermark += len_to_copy;
1577
1578                 if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
1579                      CTL_FLAG_DATA_IN) {
1580                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1581                                          "bytes to user\n", len_to_copy));
1582                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1583                                          "to %p\n", kern_ptr, ext_ptr));
1584                         if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
1585                                 ctl_set_internal_failure(ctsio,
1586                                                          /*sks_valid*/ 0,
1587                                                          /*retry_count*/ 0);
1588                                 goto bailout;
1589                         }
1590                 } else {
1591                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1592                                          "bytes from user\n", len_to_copy));
1593                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1594                                          "to %p\n", ext_ptr, kern_ptr));
1595                         if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
1596                                 ctl_set_internal_failure(ctsio,
1597                                                          /*sks_valid*/ 0,
1598                                                          /*retry_count*/0);
1599                                 goto bailout;
1600                         }
1601                 }
1602
1603                 len_copied += len_to_copy;
1604
1605                 if (ext_sglist[i].len == ext_watermark) {
1606                         i++;
1607                         ext_watermark = 0;
1608                 }
1609
1610                 if (kern_sglist[j].len == kern_watermark) {
1611                         j++;
1612                         kern_watermark = 0;
1613                 }
1614         }
1615
1616         ctsio->ext_data_filled += len_copied;
1617
1618         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_sg_entries: %d, "
1619                          "kern_sg_entries: %d\n", ext_sg_entries,
1620                          kern_sg_entries));
1621         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_data_len = %d, "
1622                          "kern_data_len = %d\n", ctsio->ext_data_len,
1623                          ctsio->kern_data_len));
1624
1625
1626         /* XXX KDM set residual?? */
1627 bailout:
1628
1629         if (ext_sglist_malloced != 0)
1630                 free(ext_sglist, M_CTL);
1631
1632         return (CTL_RETVAL_COMPLETE);
1633 }
1634
1635 /*
1636  * Serialize a command that went down the "wrong" side, and so was sent to
1637  * this controller for execution.  The logic is a little different than the
1638  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1639  * sent back to the other side, but in the success case, we execute the
1640  * command on this side (XFER mode) or tell the other side to execute it
1641  * (SER_ONLY mode).
1642  */
1643 static int
1644 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
1645 {
1646         struct ctl_softc *ctl_softc;
1647         union ctl_ha_msg msg_info;
1648         struct ctl_lun *lun;
1649         int retval = 0;
1650         uint32_t targ_lun;
1651
1652         ctl_softc = control_softc;
1653
1654         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
1655         lun = ctl_softc->ctl_luns[targ_lun];
1656         if (lun==NULL)
1657         {
1658                 /*
1659                  * Why isn't LUN defined? The other side wouldn't
1660                  * send a cmd if the LUN is undefined.
1661                  */
1662                 printf("%s: Bad JUJU!, LUN is NULL!\n", __func__);
1663
1664                 /* "Logical unit not supported" */
1665                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1666                                    lun,
1667                                    /*sense_format*/SSD_TYPE_NONE,
1668                                    /*current_error*/ 1,
1669                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1670                                    /*asc*/ 0x25,
1671                                    /*ascq*/ 0x00,
1672                                    SSD_ELEM_NONE);
1673
1674                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1675                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1676                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1677                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1678                 msg_info.hdr.serializing_sc = NULL;
1679                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1680                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1681                                 sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1682                 }
1683                 return(1);
1684
1685         }
1686
1687         mtx_lock(&lun->lun_lock);
1688         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1689
1690         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1691                 (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1692                  ooa_links))) {
1693         case CTL_ACTION_BLOCK:
1694                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
1695                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
1696                                   blocked_links);
1697                 break;
1698         case CTL_ACTION_PASS:
1699         case CTL_ACTION_SKIP:
1700                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
1701                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
1702                         ctl_enqueue_rtr((union ctl_io *)ctsio);
1703                 } else {
1704
1705                         /* send msg back to other side */
1706                         msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1707                         msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
1708                         msg_info.hdr.msg_type = CTL_MSG_R2R;
1709 #if 0
1710                         printf("2. pOrig %x\n", (int)msg_info.hdr.original_sc);
1711 #endif
1712                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1713                             sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1714                         }
1715                 }
1716                 break;
1717         case CTL_ACTION_OVERLAP:
1718                 /* OVERLAPPED COMMANDS ATTEMPTED */
1719                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1720                                    lun,
1721                                    /*sense_format*/SSD_TYPE_NONE,
1722                                    /*current_error*/ 1,
1723                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1724                                    /*asc*/ 0x4E,
1725                                    /*ascq*/ 0x00,
1726                                    SSD_ELEM_NONE);
1727
1728                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1729                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1730                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1731                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1732                 msg_info.hdr.serializing_sc = NULL;
1733                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1734 #if 0
1735                 printf("BAD JUJU:Major Bummer Overlap\n");
1736 #endif
1737                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1738                 retval = 1;
1739                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1740                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1741                 }
1742                 break;
1743         case CTL_ACTION_OVERLAP_TAG:
1744                 /* TAGGED OVERLAPPED COMMANDS (NN = QUEUE TAG) */
1745                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1746                                    lun,
1747                                    /*sense_format*/SSD_TYPE_NONE,
1748                                    /*current_error*/ 1,
1749                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1750                                    /*asc*/ 0x4D,
1751                                    /*ascq*/ ctsio->tag_num & 0xff,
1752                                    SSD_ELEM_NONE);
1753
1754                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1755                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1756                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1757                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1758                 msg_info.hdr.serializing_sc = NULL;
1759                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1760 #if 0
1761                 printf("BAD JUJU:Major Bummer Overlap Tag\n");
1762 #endif
1763                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1764                 retval = 1;
1765                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1766                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1767                 }
1768                 break;
1769         case CTL_ACTION_ERROR:
1770         default:
1771                 /* "Internal target failure" */
1772                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1773                                    lun,
1774                                    /*sense_format*/SSD_TYPE_NONE,
1775                                    /*current_error*/ 1,
1776                                    /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
1777                                    /*asc*/ 0x44,
1778                                    /*ascq*/ 0x00,
1779                                    SSD_ELEM_NONE);
1780
1781                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1782                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1783                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1784                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1785                 msg_info.hdr.serializing_sc = NULL;
1786                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1787 #if 0
1788                 printf("BAD JUJU:Major Bummer HW Error\n");
1789 #endif
1790                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1791                 retval = 1;
1792                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1793                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1794                 }
1795                 break;
1796         }
1797         mtx_unlock(&lun->lun_lock);
1798         return (retval);
1799 }
1800
1801 static int
1802 ctl_ioctl_submit_wait(union ctl_io *io)
1803 {
1804         struct ctl_fe_ioctl_params params;
1805         ctl_fe_ioctl_state last_state;
1806         int done, retval;
1807
1808         retval = 0;
1809
1810         bzero(&params, sizeof(params));
1811
1812         mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
1813         cv_init(&params.sem, "ctlioccv");
1814         params.state = CTL_IOCTL_INPROG;
1815         last_state = params.state;
1816
1817         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = &params;
1818
1819         CTL_DEBUG_PRINT(("ctl_ioctl_submit_wait\n"));
1820
1821         /* This shouldn't happen */
1822         if ((retval = ctl_queue(io)) != CTL_RETVAL_COMPLETE)
1823                 return (retval);
1824
1825         done = 0;
1826
1827         do {
1828                 mtx_lock(&params.ioctl_mtx);
1829                 /*
1830                  * Check the state here, and don't sleep if the state has
1831                  * already changed (i.e. wakeup has already occured, but we
1832                  * weren't waiting yet).
1833                  */
1834                 if (params.state == last_state) {
1835                         /* XXX KDM cv_wait_sig instead? */
1836                         cv_wait(&params.sem, &params.ioctl_mtx);
1837                 }
1838                 last_state = params.state;
1839
1840                 switch (params.state) {
1841                 case CTL_IOCTL_INPROG:
1842                         /* Why did we wake up? */
1843                         /* XXX KDM error here? */
1844                         mtx_unlock(&params.ioctl_mtx);
1845                         break;
1846                 case CTL_IOCTL_DATAMOVE:
1847                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DATAMOVE\n"));
1848
1849                         /*
1850                          * change last_state back to INPROG to avoid
1851                          * deadlock on subsequent data moves.
1852                          */
1853                         params.state = last_state = CTL_IOCTL_INPROG;
1854
1855                         mtx_unlock(&params.ioctl_mtx);
1856                         ctl_ioctl_do_datamove(&io->scsiio);
1857                         /*
1858                          * Note that in some cases, most notably writes,
1859                          * this will queue the I/O and call us back later.
1860                          * In other cases, generally reads, this routine
1861                          * will immediately call back and wake us up,
1862                          * probably using our own context.
1863                          */
1864                         io->scsiio.be_move_done(io);
1865                         break;
1866                 case CTL_IOCTL_DONE:
1867                         mtx_unlock(&params.ioctl_mtx);
1868                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DONE\n"));
1869                         done = 1;
1870                         break;
1871                 default:
1872                         mtx_unlock(&params.ioctl_mtx);
1873                         /* XXX KDM error here? */
1874                         break;
1875                 }
1876         } while (done == 0);
1877
1878         mtx_destroy(&params.ioctl_mtx);
1879         cv_destroy(&params.sem);
1880
1881         return (CTL_RETVAL_COMPLETE);
1882 }
1883
1884 static void
1885 ctl_ioctl_datamove(union ctl_io *io)
1886 {
1887         struct ctl_fe_ioctl_params *params;
1888
1889         params = (struct ctl_fe_ioctl_params *)
1890                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1891
1892         mtx_lock(&params->ioctl_mtx);
1893         params->state = CTL_IOCTL_DATAMOVE;
1894         cv_broadcast(&params->sem);
1895         mtx_unlock(&params->ioctl_mtx);
1896 }
1897
1898 static void
1899 ctl_ioctl_done(union ctl_io *io)
1900 {
1901         struct ctl_fe_ioctl_params *params;
1902
1903         params = (struct ctl_fe_ioctl_params *)
1904                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1905
1906         mtx_lock(&params->ioctl_mtx);
1907         params->state = CTL_IOCTL_DONE;
1908         cv_broadcast(&params->sem);
1909         mtx_unlock(&params->ioctl_mtx);
1910 }
1911
1912 static void
1913 ctl_ioctl_hard_startstop_callback(void *arg, struct cfi_metatask *metatask)
1914 {
1915         struct ctl_fe_ioctl_startstop_info *sd_info;
1916
1917         sd_info = (struct ctl_fe_ioctl_startstop_info *)arg;
1918
1919         sd_info->hs_info.status = metatask->status;
1920         sd_info->hs_info.total_luns = metatask->taskinfo.startstop.total_luns;
1921         sd_info->hs_info.luns_complete =
1922                 metatask->taskinfo.startstop.luns_complete;
1923         sd_info->hs_info.luns_failed = metatask->taskinfo.startstop.luns_failed;
1924
1925         cv_broadcast(&sd_info->sem);
1926 }
1927
1928 static void
1929 ctl_ioctl_bbrread_callback(void *arg, struct cfi_metatask *metatask)
1930 {
1931         struct ctl_fe_ioctl_bbrread_info *fe_bbr_info;
1932
1933         fe_bbr_info = (struct ctl_fe_ioctl_bbrread_info *)arg;
1934
1935         mtx_lock(fe_bbr_info->lock);
1936         fe_bbr_info->bbr_info->status = metatask->status;
1937         fe_bbr_info->bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
1938         fe_bbr_info->wakeup_done = 1;
1939         mtx_unlock(fe_bbr_info->lock);
1940
1941         cv_broadcast(&fe_bbr_info->sem);
1942 }
1943
1944 /*
1945  * Returns 0 for success, errno for failure.
1946  */
1947 static int
1948 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
1949                    struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
1950 {
1951         union ctl_io *io;
1952         int retval;
1953
1954         retval = 0;
1955
1956         mtx_lock(&lun->lun_lock);
1957         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
1958              (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
1959              ooa_links)) {
1960                 struct ctl_ooa_entry *entry;
1961
1962                 /*
1963                  * If we've got more than we can fit, just count the
1964                  * remaining entries.
1965                  */
1966                 if (*cur_fill_num >= ooa_hdr->alloc_num)
1967                         continue;
1968
1969                 entry = &kern_entries[*cur_fill_num];
1970
1971                 entry->tag_num = io->scsiio.tag_num;
1972                 entry->lun_num = lun->lun;
1973 #ifdef CTL_TIME_IO
1974                 entry->start_bt = io->io_hdr.start_bt;
1975 #endif
1976                 bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
1977                 entry->cdb_len = io->scsiio.cdb_len;
1978                 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
1979                         entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
1980
1981                 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
1982                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
1983
1984                 if (io->io_hdr.flags & CTL_FLAG_ABORT)
1985                         entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
1986
1987                 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
1988                         entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
1989
1990                 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
1991                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
1992         }
1993         mtx_unlock(&lun->lun_lock);
1994
1995         return (retval);
1996 }
1997
1998 static void *
1999 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2000                  size_t error_str_len)
2001 {
2002         void *kptr;
2003
2004         kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2005
2006         if (copyin(user_addr, kptr, len) != 0) {
2007                 snprintf(error_str, error_str_len, "Error copying %d bytes "
2008                          "from user address %p to kernel address %p", len,
2009                          user_addr, kptr);
2010                 free(kptr, M_CTL);
2011                 return (NULL);
2012         }
2013
2014         return (kptr);
2015 }
2016
2017 static void
2018 ctl_free_args(int num_args, struct ctl_be_arg *args)
2019 {
2020         int i;
2021
2022         if (args == NULL)
2023                 return;
2024
2025         for (i = 0; i < num_args; i++) {
2026                 free(args[i].kname, M_CTL);
2027                 free(args[i].kvalue, M_CTL);
2028         }
2029
2030         free(args, M_CTL);
2031 }
2032
2033 static struct ctl_be_arg *
2034 ctl_copyin_args(int num_args, struct ctl_be_arg *uargs,
2035                 char *error_str, size_t error_str_len)
2036 {
2037         struct ctl_be_arg *args;
2038         int i;
2039
2040         args = ctl_copyin_alloc(uargs, num_args * sizeof(*args),
2041                                 error_str, error_str_len);
2042
2043         if (args == NULL)
2044                 goto bailout;
2045
2046         for (i = 0; i < num_args; i++) {
2047                 args[i].kname = NULL;
2048                 args[i].kvalue = NULL;
2049         }
2050
2051         for (i = 0; i < num_args; i++) {
2052                 uint8_t *tmpptr;
2053
2054                 args[i].kname = ctl_copyin_alloc(args[i].name,
2055                         args[i].namelen, error_str, error_str_len);
2056                 if (args[i].kname == NULL)
2057                         goto bailout;
2058
2059                 if (args[i].kname[args[i].namelen - 1] != '\0') {
2060                         snprintf(error_str, error_str_len, "Argument %d "
2061                                  "name is not NUL-terminated", i);
2062                         goto bailout;
2063                 }
2064
2065                 if (args[i].flags & CTL_BEARG_RD) {
2066                         tmpptr = ctl_copyin_alloc(args[i].value,
2067                                 args[i].vallen, error_str, error_str_len);
2068                         if (tmpptr == NULL)
2069                                 goto bailout;
2070                         if ((args[i].flags & CTL_BEARG_ASCII)
2071                          && (tmpptr[args[i].vallen - 1] != '\0')) {
2072                                 snprintf(error_str, error_str_len, "Argument "
2073                                     "%d value is not NUL-terminated", i);
2074                                 goto bailout;
2075                         }
2076                         args[i].kvalue = tmpptr;
2077                 } else {
2078                         args[i].kvalue = malloc(args[i].vallen,
2079                             M_CTL, M_WAITOK | M_ZERO);
2080                 }
2081         }
2082
2083         return (args);
2084 bailout:
2085
2086         ctl_free_args(num_args, args);
2087
2088         return (NULL);
2089 }
2090
2091 static void
2092 ctl_copyout_args(int num_args, struct ctl_be_arg *args)
2093 {
2094         int i;
2095
2096         for (i = 0; i < num_args; i++) {
2097                 if (args[i].flags & CTL_BEARG_WR)
2098                         copyout(args[i].kvalue, args[i].value, args[i].vallen);
2099         }
2100 }
2101
2102 /*
2103  * Escape characters that are illegal or not recommended in XML.
2104  */
2105 int
2106 ctl_sbuf_printf_esc(struct sbuf *sb, char *str)
2107 {
2108         int retval;
2109
2110         retval = 0;
2111
2112         for (; *str; str++) {
2113                 switch (*str) {
2114                 case '&':
2115                         retval = sbuf_printf(sb, "&amp;");
2116                         break;
2117                 case '>':
2118                         retval = sbuf_printf(sb, "&gt;");
2119                         break;
2120                 case '<':
2121                         retval = sbuf_printf(sb, "&lt;");
2122                         break;
2123                 default:
2124                         retval = sbuf_putc(sb, *str);
2125                         break;
2126                 }
2127
2128                 if (retval != 0)
2129                         break;
2130
2131         }
2132
2133         return (retval);
2134 }
2135
2136 static int
2137 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2138           struct thread *td)
2139 {
2140         struct ctl_softc *softc;
2141         int retval;
2142
2143         softc = control_softc;
2144
2145         retval = 0;
2146
2147         switch (cmd) {
2148         case CTL_IO: {
2149                 union ctl_io *io;
2150                 void *pool_tmp;
2151
2152                 /*
2153                  * If we haven't been "enabled", don't allow any SCSI I/O
2154                  * to this FETD.
2155                  */
2156                 if ((softc->ioctl_info.flags & CTL_IOCTL_FLAG_ENABLED) == 0) {
2157                         retval = EPERM;
2158                         break;
2159                 }
2160
2161                 io = ctl_alloc_io(softc->ioctl_info.port.ctl_pool_ref);
2162                 if (io == NULL) {
2163                         printf("ctl_ioctl: can't allocate ctl_io!\n");
2164                         retval = ENOSPC;
2165                         break;
2166                 }
2167
2168                 /*
2169                  * Need to save the pool reference so it doesn't get
2170                  * spammed by the user's ctl_io.
2171                  */
2172                 pool_tmp = io->io_hdr.pool;
2173
2174                 memcpy(io, (void *)addr, sizeof(*io));
2175
2176                 io->io_hdr.pool = pool_tmp;
2177                 /*
2178                  * No status yet, so make sure the status is set properly.
2179                  */
2180                 io->io_hdr.status = CTL_STATUS_NONE;
2181
2182                 /*
2183                  * The user sets the initiator ID, target and LUN IDs.
2184                  */
2185                 io->io_hdr.nexus.targ_port = softc->ioctl_info.port.targ_port;
2186                 io->io_hdr.flags |= CTL_FLAG_USER_REQ;
2187                 if ((io->io_hdr.io_type == CTL_IO_SCSI)
2188                  && (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
2189                         io->scsiio.tag_num = softc->ioctl_info.cur_tag_num++;
2190
2191                 retval = ctl_ioctl_submit_wait(io);
2192
2193                 if (retval != 0) {
2194                         ctl_free_io(io);
2195                         break;
2196                 }
2197
2198                 memcpy((void *)addr, io, sizeof(*io));
2199
2200                 /* return this to our pool */
2201                 ctl_free_io(io);
2202
2203                 break;
2204         }
2205         case CTL_ENABLE_PORT:
2206         case CTL_DISABLE_PORT:
2207         case CTL_SET_PORT_WWNS: {
2208                 struct ctl_port *port;
2209                 struct ctl_port_entry *entry;
2210
2211                 entry = (struct ctl_port_entry *)addr;
2212
2213                 mtx_lock(&softc->ctl_lock);
2214                 STAILQ_FOREACH(port, &softc->port_list, links) {
2215                         int action, done;
2216
2217                         action = 0;
2218                         done = 0;
2219
2220                         if ((entry->port_type == CTL_PORT_NONE)
2221                          && (entry->targ_port == port->targ_port)) {
2222                                 /*
2223                                  * If the user only wants to enable or
2224                                  * disable or set WWNs on a specific port,
2225                                  * do the operation and we're done.
2226                                  */
2227                                 action = 1;
2228                                 done = 1;
2229                         } else if (entry->port_type & port->port_type) {
2230                                 /*
2231                                  * Compare the user's type mask with the
2232                                  * particular frontend type to see if we
2233                                  * have a match.
2234                                  */
2235                                 action = 1;
2236                                 done = 0;
2237
2238                                 /*
2239                                  * Make sure the user isn't trying to set
2240                                  * WWNs on multiple ports at the same time.
2241                                  */
2242                                 if (cmd == CTL_SET_PORT_WWNS) {
2243                                         printf("%s: Can't set WWNs on "
2244                                                "multiple ports\n", __func__);
2245                                         retval = EINVAL;
2246                                         break;
2247                                 }
2248                         }
2249                         if (action != 0) {
2250                                 /*
2251                                  * XXX KDM we have to drop the lock here,
2252                                  * because the online/offline operations
2253                                  * can potentially block.  We need to
2254                                  * reference count the frontends so they
2255                                  * can't go away,
2256                                  */
2257                                 mtx_unlock(&softc->ctl_lock);
2258
2259                                 if (cmd == CTL_ENABLE_PORT) {
2260                                         struct ctl_lun *lun;
2261
2262                                         STAILQ_FOREACH(lun, &softc->lun_list,
2263                                                        links) {
2264                                                 port->lun_enable(port->targ_lun_arg,
2265                                                     lun->target,
2266                                                     lun->lun);
2267                                         }
2268
2269                                         ctl_port_online(port);
2270                                 } else if (cmd == CTL_DISABLE_PORT) {
2271                                         struct ctl_lun *lun;
2272
2273                                         ctl_port_offline(port);
2274
2275                                         STAILQ_FOREACH(lun, &softc->lun_list,
2276                                                        links) {
2277                                                 port->lun_disable(
2278                                                     port->targ_lun_arg,
2279                                                     lun->target,
2280                                                     lun->lun);
2281                                         }
2282                                 }
2283
2284                                 mtx_lock(&softc->ctl_lock);
2285
2286                                 if (cmd == CTL_SET_PORT_WWNS)
2287                                         ctl_port_set_wwns(port,
2288                                             (entry->flags & CTL_PORT_WWNN_VALID) ?
2289                                             1 : 0, entry->wwnn,
2290                                             (entry->flags & CTL_PORT_WWPN_VALID) ?
2291                                             1 : 0, entry->wwpn);
2292                         }
2293                         if (done != 0)
2294                                 break;
2295                 }
2296                 mtx_unlock(&softc->ctl_lock);
2297                 break;
2298         }
2299         case CTL_GET_PORT_LIST: {
2300                 struct ctl_port *port;
2301                 struct ctl_port_list *list;
2302                 int i;
2303
2304                 list = (struct ctl_port_list *)addr;
2305
2306                 if (list->alloc_len != (list->alloc_num *
2307                     sizeof(struct ctl_port_entry))) {
2308                         printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2309                                "alloc_num %u * sizeof(struct ctl_port_entry) "
2310                                "%zu\n", __func__, list->alloc_len,
2311                                list->alloc_num, sizeof(struct ctl_port_entry));
2312                         retval = EINVAL;
2313                         break;
2314                 }
2315                 list->fill_len = 0;
2316                 list->fill_num = 0;
2317                 list->dropped_num = 0;
2318                 i = 0;
2319                 mtx_lock(&softc->ctl_lock);
2320                 STAILQ_FOREACH(port, &softc->port_list, links) {
2321                         struct ctl_port_entry entry, *list_entry;
2322
2323                         if (list->fill_num >= list->alloc_num) {
2324                                 list->dropped_num++;
2325                                 continue;
2326                         }
2327
2328                         entry.port_type = port->port_type;
2329                         strlcpy(entry.port_name, port->port_name,
2330                                 sizeof(entry.port_name));
2331                         entry.targ_port = port->targ_port;
2332                         entry.physical_port = port->physical_port;
2333                         entry.virtual_port = port->virtual_port;
2334                         entry.wwnn = port->wwnn;
2335                         entry.wwpn = port->wwpn;
2336                         if (port->status & CTL_PORT_STATUS_ONLINE)
2337                                 entry.online = 1;
2338                         else
2339                                 entry.online = 0;
2340
2341                         list_entry = &list->entries[i];
2342
2343                         retval = copyout(&entry, list_entry, sizeof(entry));
2344                         if (retval != 0) {
2345                                 printf("%s: CTL_GET_PORT_LIST: copyout "
2346                                        "returned %d\n", __func__, retval);
2347                                 break;
2348                         }
2349                         i++;
2350                         list->fill_num++;
2351                         list->fill_len += sizeof(entry);
2352                 }
2353                 mtx_unlock(&softc->ctl_lock);
2354
2355                 /*
2356                  * If this is non-zero, we had a copyout fault, so there's
2357                  * probably no point in attempting to set the status inside
2358                  * the structure.
2359                  */
2360                 if (retval != 0)
2361                         break;
2362
2363                 if (list->dropped_num > 0)
2364                         list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2365                 else
2366                         list->status = CTL_PORT_LIST_OK;
2367                 break;
2368         }
2369         case CTL_DUMP_OOA: {
2370                 struct ctl_lun *lun;
2371                 union ctl_io *io;
2372                 char printbuf[128];
2373                 struct sbuf sb;
2374
2375                 mtx_lock(&softc->ctl_lock);
2376                 printf("Dumping OOA queues:\n");
2377                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2378                         mtx_lock(&lun->lun_lock);
2379                         for (io = (union ctl_io *)TAILQ_FIRST(
2380                              &lun->ooa_queue); io != NULL;
2381                              io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2382                              ooa_links)) {
2383                                 sbuf_new(&sb, printbuf, sizeof(printbuf),
2384                                          SBUF_FIXEDLEN);
2385                                 sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2386                                             (intmax_t)lun->lun,
2387                                             io->scsiio.tag_num,
2388                                             (io->io_hdr.flags &
2389                                             CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2390                                             (io->io_hdr.flags &
2391                                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2392                                             (io->io_hdr.flags &
2393                                             CTL_FLAG_ABORT) ? " ABORT" : "",
2394                                             (io->io_hdr.flags &
2395                                         CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2396                                 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2397                                 sbuf_finish(&sb);
2398                                 printf("%s\n", sbuf_data(&sb));
2399                         }
2400                         mtx_unlock(&lun->lun_lock);
2401                 }
2402                 printf("OOA queues dump done\n");
2403                 mtx_unlock(&softc->ctl_lock);
2404                 break;
2405         }
2406         case CTL_GET_OOA: {
2407                 struct ctl_lun *lun;
2408                 struct ctl_ooa *ooa_hdr;
2409                 struct ctl_ooa_entry *entries;
2410                 uint32_t cur_fill_num;
2411
2412                 ooa_hdr = (struct ctl_ooa *)addr;
2413
2414                 if ((ooa_hdr->alloc_len == 0)
2415                  || (ooa_hdr->alloc_num == 0)) {
2416                         printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2417                                "must be non-zero\n", __func__,
2418                                ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2419                         retval = EINVAL;
2420                         break;
2421                 }
2422
2423                 if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2424                     sizeof(struct ctl_ooa_entry))) {
2425                         printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2426                                "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2427                                __func__, ooa_hdr->alloc_len,
2428                                ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2429                         retval = EINVAL;
2430                         break;
2431                 }
2432
2433                 entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2434                 if (entries == NULL) {
2435                         printf("%s: could not allocate %d bytes for OOA "
2436                                "dump\n", __func__, ooa_hdr->alloc_len);
2437                         retval = ENOMEM;
2438                         break;
2439                 }
2440
2441                 mtx_lock(&softc->ctl_lock);
2442                 if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2443                  && ((ooa_hdr->lun_num > CTL_MAX_LUNS)
2444                   || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2445                         mtx_unlock(&softc->ctl_lock);
2446                         free(entries, M_CTL);
2447                         printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2448                                __func__, (uintmax_t)ooa_hdr->lun_num);
2449                         retval = EINVAL;
2450                         break;
2451                 }
2452
2453                 cur_fill_num = 0;
2454
2455                 if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2456                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
2457                                 retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2458                                         ooa_hdr, entries);
2459                                 if (retval != 0)
2460                                         break;
2461                         }
2462                         if (retval != 0) {
2463                                 mtx_unlock(&softc->ctl_lock);
2464                                 free(entries, M_CTL);
2465                                 break;
2466                         }
2467                 } else {
2468                         lun = softc->ctl_luns[ooa_hdr->lun_num];
2469
2470                         retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
2471                                                     entries);
2472                 }
2473                 mtx_unlock(&softc->ctl_lock);
2474
2475                 ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2476                 ooa_hdr->fill_len = ooa_hdr->fill_num *
2477                         sizeof(struct ctl_ooa_entry);
2478                 retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2479                 if (retval != 0) {
2480                         printf("%s: error copying out %d bytes for OOA dump\n",
2481                                __func__, ooa_hdr->fill_len);
2482                 }
2483
2484                 getbintime(&ooa_hdr->cur_bt);
2485
2486                 if (cur_fill_num > ooa_hdr->alloc_num) {
2487                         ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2488                         ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2489                 } else {
2490                         ooa_hdr->dropped_num = 0;
2491                         ooa_hdr->status = CTL_OOA_OK;
2492                 }
2493
2494                 free(entries, M_CTL);
2495                 break;
2496         }
2497         case CTL_CHECK_OOA: {
2498                 union ctl_io *io;
2499                 struct ctl_lun *lun;
2500                 struct ctl_ooa_info *ooa_info;
2501
2502
2503                 ooa_info = (struct ctl_ooa_info *)addr;
2504
2505                 if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2506                         ooa_info->status = CTL_OOA_INVALID_LUN;
2507                         break;
2508                 }
2509                 mtx_lock(&softc->ctl_lock);
2510                 lun = softc->ctl_luns[ooa_info->lun_id];
2511                 if (lun == NULL) {
2512                         mtx_unlock(&softc->ctl_lock);
2513                         ooa_info->status = CTL_OOA_INVALID_LUN;
2514                         break;
2515                 }
2516                 mtx_lock(&lun->lun_lock);
2517                 mtx_unlock(&softc->ctl_lock);
2518                 ooa_info->num_entries = 0;
2519                 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2520                      io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2521                      &io->io_hdr, ooa_links)) {
2522                         ooa_info->num_entries++;
2523                 }
2524                 mtx_unlock(&lun->lun_lock);
2525
2526                 ooa_info->status = CTL_OOA_SUCCESS;
2527
2528                 break;
2529         }
2530         case CTL_HARD_START:
2531         case CTL_HARD_STOP: {
2532                 struct ctl_fe_ioctl_startstop_info ss_info;
2533                 struct cfi_metatask *metatask;
2534                 struct mtx hs_mtx;
2535
2536                 mtx_init(&hs_mtx, "HS Mutex", NULL, MTX_DEF);
2537
2538                 cv_init(&ss_info.sem, "hard start/stop cv" );
2539
2540                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2541                 if (metatask == NULL) {
2542                         retval = ENOMEM;
2543                         mtx_destroy(&hs_mtx);
2544                         break;
2545                 }
2546
2547                 if (cmd == CTL_HARD_START)
2548                         metatask->tasktype = CFI_TASK_STARTUP;
2549                 else
2550                         metatask->tasktype = CFI_TASK_SHUTDOWN;
2551
2552                 metatask->callback = ctl_ioctl_hard_startstop_callback;
2553                 metatask->callback_arg = &ss_info;
2554
2555                 cfi_action(metatask);
2556
2557                 /* Wait for the callback */
2558                 mtx_lock(&hs_mtx);
2559                 cv_wait_sig(&ss_info.sem, &hs_mtx);
2560                 mtx_unlock(&hs_mtx);
2561
2562                 /*
2563                  * All information has been copied from the metatask by the
2564                  * time cv_broadcast() is called, so we free the metatask here.
2565                  */
2566                 cfi_free_metatask(metatask);
2567
2568                 memcpy((void *)addr, &ss_info.hs_info, sizeof(ss_info.hs_info));
2569
2570                 mtx_destroy(&hs_mtx);
2571                 break;
2572         }
2573         case CTL_BBRREAD: {
2574                 struct ctl_bbrread_info *bbr_info;
2575                 struct ctl_fe_ioctl_bbrread_info fe_bbr_info;
2576                 struct mtx bbr_mtx;
2577                 struct cfi_metatask *metatask;
2578
2579                 bbr_info = (struct ctl_bbrread_info *)addr;
2580
2581                 bzero(&fe_bbr_info, sizeof(fe_bbr_info));
2582
2583                 bzero(&bbr_mtx, sizeof(bbr_mtx));
2584                 mtx_init(&bbr_mtx, "BBR Mutex", NULL, MTX_DEF);
2585
2586                 fe_bbr_info.bbr_info = bbr_info;
2587                 fe_bbr_info.lock = &bbr_mtx;
2588
2589                 cv_init(&fe_bbr_info.sem, "BBR read cv");
2590                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2591
2592                 if (metatask == NULL) {
2593                         mtx_destroy(&bbr_mtx);
2594                         cv_destroy(&fe_bbr_info.sem);
2595                         retval = ENOMEM;
2596                         break;
2597                 }
2598                 metatask->tasktype = CFI_TASK_BBRREAD;
2599                 metatask->callback = ctl_ioctl_bbrread_callback;
2600                 metatask->callback_arg = &fe_bbr_info;
2601                 metatask->taskinfo.bbrread.lun_num = bbr_info->lun_num;
2602                 metatask->taskinfo.bbrread.lba = bbr_info->lba;
2603                 metatask->taskinfo.bbrread.len = bbr_info->len;
2604
2605                 cfi_action(metatask);
2606
2607                 mtx_lock(&bbr_mtx);
2608                 while (fe_bbr_info.wakeup_done == 0)
2609                         cv_wait_sig(&fe_bbr_info.sem, &bbr_mtx);
2610                 mtx_unlock(&bbr_mtx);
2611
2612                 bbr_info->status = metatask->status;
2613                 bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2614                 bbr_info->scsi_status = metatask->taskinfo.bbrread.scsi_status;
2615                 memcpy(&bbr_info->sense_data,
2616                        &metatask->taskinfo.bbrread.sense_data,
2617                        ctl_min(sizeof(bbr_info->sense_data),
2618                                sizeof(metatask->taskinfo.bbrread.sense_data)));
2619
2620                 cfi_free_metatask(metatask);
2621
2622                 mtx_destroy(&bbr_mtx);
2623                 cv_destroy(&fe_bbr_info.sem);
2624
2625                 break;
2626         }
2627         case CTL_DELAY_IO: {
2628                 struct ctl_io_delay_info *delay_info;
2629 #ifdef CTL_IO_DELAY
2630                 struct ctl_lun *lun;
2631 #endif /* CTL_IO_DELAY */
2632
2633                 delay_info = (struct ctl_io_delay_info *)addr;
2634
2635 #ifdef CTL_IO_DELAY
2636                 mtx_lock(&softc->ctl_lock);
2637
2638                 if ((delay_info->lun_id > CTL_MAX_LUNS)
2639                  || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2640                         delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2641                 } else {
2642                         lun = softc->ctl_luns[delay_info->lun_id];
2643                         mtx_lock(&lun->lun_lock);
2644
2645                         delay_info->status = CTL_DELAY_STATUS_OK;
2646
2647                         switch (delay_info->delay_type) {
2648                         case CTL_DELAY_TYPE_CONT:
2649                                 break;
2650                         case CTL_DELAY_TYPE_ONESHOT:
2651                                 break;
2652                         default:
2653                                 delay_info->status =
2654                                         CTL_DELAY_STATUS_INVALID_TYPE;
2655                                 break;
2656                         }
2657
2658                         switch (delay_info->delay_loc) {
2659                         case CTL_DELAY_LOC_DATAMOVE:
2660                                 lun->delay_info.datamove_type =
2661                                         delay_info->delay_type;
2662                                 lun->delay_info.datamove_delay =
2663                                         delay_info->delay_secs;
2664                                 break;
2665                         case CTL_DELAY_LOC_DONE:
2666                                 lun->delay_info.done_type =
2667                                         delay_info->delay_type;
2668                                 lun->delay_info.done_delay =
2669                                         delay_info->delay_secs;
2670                                 break;
2671                         default:
2672                                 delay_info->status =
2673                                         CTL_DELAY_STATUS_INVALID_LOC;
2674                                 break;
2675                         }
2676                         mtx_unlock(&lun->lun_lock);
2677                 }
2678
2679                 mtx_unlock(&softc->ctl_lock);
2680 #else
2681                 delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2682 #endif /* CTL_IO_DELAY */
2683                 break;
2684         }
2685         case CTL_REALSYNC_SET: {
2686                 int *syncstate;
2687
2688                 syncstate = (int *)addr;
2689
2690                 mtx_lock(&softc->ctl_lock);
2691                 switch (*syncstate) {
2692                 case 0:
2693                         softc->flags &= ~CTL_FLAG_REAL_SYNC;
2694                         break;
2695                 case 1:
2696                         softc->flags |= CTL_FLAG_REAL_SYNC;
2697                         break;
2698                 default:
2699                         retval = EINVAL;
2700                         break;
2701                 }
2702                 mtx_unlock(&softc->ctl_lock);
2703                 break;
2704         }
2705         case CTL_REALSYNC_GET: {
2706                 int *syncstate;
2707
2708                 syncstate = (int*)addr;
2709
2710                 mtx_lock(&softc->ctl_lock);
2711                 if (softc->flags & CTL_FLAG_REAL_SYNC)
2712                         *syncstate = 1;
2713                 else
2714                         *syncstate = 0;
2715                 mtx_unlock(&softc->ctl_lock);
2716
2717                 break;
2718         }
2719         case CTL_SETSYNC:
2720         case CTL_GETSYNC: {
2721                 struct ctl_sync_info *sync_info;
2722                 struct ctl_lun *lun;
2723
2724                 sync_info = (struct ctl_sync_info *)addr;
2725
2726                 mtx_lock(&softc->ctl_lock);
2727                 lun = softc->ctl_luns[sync_info->lun_id];
2728                 if (lun == NULL) {
2729                         mtx_unlock(&softc->ctl_lock);
2730                         sync_info->status = CTL_GS_SYNC_NO_LUN;
2731                 }
2732                 /*
2733                  * Get or set the sync interval.  We're not bounds checking
2734                  * in the set case, hopefully the user won't do something
2735                  * silly.
2736                  */
2737                 mtx_lock(&lun->lun_lock);
2738                 mtx_unlock(&softc->ctl_lock);
2739                 if (cmd == CTL_GETSYNC)
2740                         sync_info->sync_interval = lun->sync_interval;
2741                 else
2742                         lun->sync_interval = sync_info->sync_interval;
2743                 mtx_unlock(&lun->lun_lock);
2744
2745                 sync_info->status = CTL_GS_SYNC_OK;
2746
2747                 break;
2748         }
2749         case CTL_GETSTATS: {
2750                 struct ctl_stats *stats;
2751                 struct ctl_lun *lun;
2752                 int i;
2753
2754                 stats = (struct ctl_stats *)addr;
2755
2756                 if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2757                      stats->alloc_len) {
2758                         stats->status = CTL_SS_NEED_MORE_SPACE;
2759                         stats->num_luns = softc->num_luns;
2760                         break;
2761                 }
2762                 /*
2763                  * XXX KDM no locking here.  If the LUN list changes,
2764                  * things can blow up.
2765                  */
2766                 for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2767                      i++, lun = STAILQ_NEXT(lun, links)) {
2768                         retval = copyout(&lun->stats, &stats->lun_stats[i],
2769                                          sizeof(lun->stats));
2770                         if (retval != 0)
2771                                 break;
2772                 }
2773                 stats->num_luns = softc->num_luns;
2774                 stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2775                                  softc->num_luns;
2776                 stats->status = CTL_SS_OK;
2777 #ifdef CTL_TIME_IO
2778                 stats->flags = CTL_STATS_FLAG_TIME_VALID;
2779 #else
2780                 stats->flags = CTL_STATS_FLAG_NONE;
2781 #endif
2782                 getnanouptime(&stats->timestamp);
2783                 break;
2784         }
2785         case CTL_ERROR_INJECT: {
2786                 struct ctl_error_desc *err_desc, *new_err_desc;
2787                 struct ctl_lun *lun;
2788
2789                 err_desc = (struct ctl_error_desc *)addr;
2790
2791                 new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2792                                       M_WAITOK | M_ZERO);
2793                 bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2794
2795                 mtx_lock(&softc->ctl_lock);
2796                 lun = softc->ctl_luns[err_desc->lun_id];
2797                 if (lun == NULL) {
2798                         mtx_unlock(&softc->ctl_lock);
2799                         printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2800                                __func__, (uintmax_t)err_desc->lun_id);
2801                         retval = EINVAL;
2802                         break;
2803                 }
2804                 mtx_lock(&lun->lun_lock);
2805                 mtx_unlock(&softc->ctl_lock);
2806
2807                 /*
2808                  * We could do some checking here to verify the validity
2809                  * of the request, but given the complexity of error
2810                  * injection requests, the checking logic would be fairly
2811                  * complex.
2812                  *
2813                  * For now, if the request is invalid, it just won't get
2814                  * executed and might get deleted.
2815                  */
2816                 STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2817
2818                 /*
2819                  * XXX KDM check to make sure the serial number is unique,
2820                  * in case we somehow manage to wrap.  That shouldn't
2821                  * happen for a very long time, but it's the right thing to
2822                  * do.
2823                  */
2824                 new_err_desc->serial = lun->error_serial;
2825                 err_desc->serial = lun->error_serial;
2826                 lun->error_serial++;
2827
2828                 mtx_unlock(&lun->lun_lock);
2829                 break;
2830         }
2831         case CTL_ERROR_INJECT_DELETE: {
2832                 struct ctl_error_desc *delete_desc, *desc, *desc2;
2833                 struct ctl_lun *lun;
2834                 int delete_done;
2835
2836                 delete_desc = (struct ctl_error_desc *)addr;
2837                 delete_done = 0;
2838
2839                 mtx_lock(&softc->ctl_lock);
2840                 lun = softc->ctl_luns[delete_desc->lun_id];
2841                 if (lun == NULL) {
2842                         mtx_unlock(&softc->ctl_lock);
2843                         printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
2844                                __func__, (uintmax_t)delete_desc->lun_id);
2845                         retval = EINVAL;
2846                         break;
2847                 }
2848                 mtx_lock(&lun->lun_lock);
2849                 mtx_unlock(&softc->ctl_lock);
2850                 STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
2851                         if (desc->serial != delete_desc->serial)
2852                                 continue;
2853
2854                         STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
2855                                       links);
2856                         free(desc, M_CTL);
2857                         delete_done = 1;
2858                 }
2859                 mtx_unlock(&lun->lun_lock);
2860                 if (delete_done == 0) {
2861                         printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
2862                                "error serial %ju on LUN %u\n", __func__,
2863                                delete_desc->serial, delete_desc->lun_id);
2864                         retval = EINVAL;
2865                         break;
2866                 }
2867                 break;
2868         }
2869         case CTL_DUMP_STRUCTS: {
2870                 int i, j, k;
2871                 struct ctl_port *port;
2872                 struct ctl_frontend *fe;
2873
2874                 printf("CTL IID to WWPN map start:\n");
2875                 for (i = 0; i < CTL_MAX_PORTS; i++) {
2876                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
2877                                 if (softc->wwpn_iid[i][j].in_use == 0)
2878                                         continue;
2879
2880                                 printf("port %d iid %u WWPN %#jx\n",
2881                                        softc->wwpn_iid[i][j].port,
2882                                        softc->wwpn_iid[i][j].iid,
2883                                        (uintmax_t)softc->wwpn_iid[i][j].wwpn);
2884                         }
2885                 }
2886                 printf("CTL IID to WWPN map end\n");
2887                 printf("CTL Persistent Reservation information start:\n");
2888                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2889                         struct ctl_lun *lun;
2890
2891                         lun = softc->ctl_luns[i];
2892
2893                         if ((lun == NULL)
2894                          || ((lun->flags & CTL_LUN_DISABLED) != 0))
2895                                 continue;
2896
2897                         for (j = 0; j < (CTL_MAX_PORTS * 2); j++) {
2898                                 for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
2899                                         if (lun->per_res[j+k].registered == 0)
2900                                                 continue;
2901                                         printf("LUN %d port %d iid %d key "
2902                                                "%#jx\n", i, j, k,
2903                                                (uintmax_t)scsi_8btou64(
2904                                                lun->per_res[j+k].res_key.key));
2905                                 }
2906                         }
2907                 }
2908                 printf("CTL Persistent Reservation information end\n");
2909                 printf("CTL Ports:\n");
2910                 /*
2911                  * XXX KDM calling this without a lock.  We'd likely want
2912                  * to drop the lock before calling the frontend's dump
2913                  * routine anyway.
2914                  */
2915                 STAILQ_FOREACH(port, &softc->port_list, links) {
2916                         printf("Port %s Frontend %s Type %u pport %d vport %d WWNN "
2917                                "%#jx WWPN %#jx\n", port->port_name,
2918                                port->frontend->name, port->port_type,
2919                                port->physical_port, port->virtual_port,
2920                                (uintmax_t)port->wwnn, (uintmax_t)port->wwpn);
2921                 }
2922                 printf("CTL Port information end\n");
2923                 printf("CTL Frontends:\n");
2924                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2925                         printf("Frontend %s\n", fe->name);
2926                         if (fe->fe_dump != NULL)
2927                                 fe->fe_dump();
2928                 }
2929                 printf("CTL Frontend information end\n");
2930                 break;
2931         }
2932         case CTL_LUN_REQ: {
2933                 struct ctl_lun_req *lun_req;
2934                 struct ctl_backend_driver *backend;
2935
2936                 lun_req = (struct ctl_lun_req *)addr;
2937
2938                 backend = ctl_backend_find(lun_req->backend);
2939                 if (backend == NULL) {
2940                         lun_req->status = CTL_LUN_ERROR;
2941                         snprintf(lun_req->error_str,
2942                                  sizeof(lun_req->error_str),
2943                                  "Backend \"%s\" not found.",
2944                                  lun_req->backend);
2945                         break;
2946                 }
2947                 if (lun_req->num_be_args > 0) {
2948                         lun_req->kern_be_args = ctl_copyin_args(
2949                                 lun_req->num_be_args,
2950                                 lun_req->be_args,
2951                                 lun_req->error_str,
2952                                 sizeof(lun_req->error_str));
2953                         if (lun_req->kern_be_args == NULL) {
2954                                 lun_req->status = CTL_LUN_ERROR;
2955                                 break;
2956                         }
2957                 }
2958
2959                 retval = backend->ioctl(dev, cmd, addr, flag, td);
2960
2961                 if (lun_req->num_be_args > 0) {
2962                         ctl_copyout_args(lun_req->num_be_args,
2963                                       lun_req->kern_be_args);
2964                         ctl_free_args(lun_req->num_be_args,
2965                                       lun_req->kern_be_args);
2966                 }
2967                 break;
2968         }
2969         case CTL_LUN_LIST: {
2970                 struct sbuf *sb;
2971                 struct ctl_lun *lun;
2972                 struct ctl_lun_list *list;
2973                 struct ctl_option *opt;
2974
2975                 list = (struct ctl_lun_list *)addr;
2976
2977                 /*
2978                  * Allocate a fixed length sbuf here, based on the length
2979                  * of the user's buffer.  We could allocate an auto-extending
2980                  * buffer, and then tell the user how much larger our
2981                  * amount of data is than his buffer, but that presents
2982                  * some problems:
2983                  *
2984                  * 1.  The sbuf(9) routines use a blocking malloc, and so
2985                  *     we can't hold a lock while calling them with an
2986                  *     auto-extending buffer.
2987                  *
2988                  * 2.  There is not currently a LUN reference counting
2989                  *     mechanism, outside of outstanding transactions on
2990                  *     the LUN's OOA queue.  So a LUN could go away on us
2991                  *     while we're getting the LUN number, backend-specific
2992                  *     information, etc.  Thus, given the way things
2993                  *     currently work, we need to hold the CTL lock while
2994                  *     grabbing LUN information.
2995                  *
2996                  * So, from the user's standpoint, the best thing to do is
2997                  * allocate what he thinks is a reasonable buffer length,
2998                  * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
2999                  * double the buffer length and try again.  (And repeat
3000                  * that until he succeeds.)
3001                  */
3002                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3003                 if (sb == NULL) {
3004                         list->status = CTL_LUN_LIST_ERROR;
3005                         snprintf(list->error_str, sizeof(list->error_str),
3006                                  "Unable to allocate %d bytes for LUN list",
3007                                  list->alloc_len);
3008                         break;
3009                 }
3010
3011                 sbuf_printf(sb, "<ctllunlist>\n");
3012
3013                 mtx_lock(&softc->ctl_lock);
3014                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
3015                         mtx_lock(&lun->lun_lock);
3016                         retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3017                                              (uintmax_t)lun->lun);
3018
3019                         /*
3020                          * Bail out as soon as we see that we've overfilled
3021                          * the buffer.
3022                          */
3023                         if (retval != 0)
3024                                 break;
3025
3026                         retval = sbuf_printf(sb, "\t<backend_type>%s"
3027                                              "</backend_type>\n",
3028                                              (lun->backend == NULL) ?  "none" :
3029                                              lun->backend->name);
3030
3031                         if (retval != 0)
3032                                 break;
3033
3034                         retval = sbuf_printf(sb, "\t<lun_type>%d</lun_type>\n",
3035                                              lun->be_lun->lun_type);
3036
3037                         if (retval != 0)
3038                                 break;
3039
3040                         if (lun->backend == NULL) {
3041                                 retval = sbuf_printf(sb, "</lun>\n");
3042                                 if (retval != 0)
3043                                         break;
3044                                 continue;
3045                         }
3046
3047                         retval = sbuf_printf(sb, "\t<size>%ju</size>\n",
3048                                              (lun->be_lun->maxlba > 0) ?
3049                                              lun->be_lun->maxlba + 1 : 0);
3050
3051                         if (retval != 0)
3052                                 break;
3053
3054                         retval = sbuf_printf(sb, "\t<blocksize>%u</blocksize>\n",
3055                                              lun->be_lun->blocksize);
3056
3057                         if (retval != 0)
3058                                 break;
3059
3060                         retval = sbuf_printf(sb, "\t<serial_number>");
3061
3062                         if (retval != 0)
3063                                 break;
3064
3065                         retval = ctl_sbuf_printf_esc(sb,
3066                                                      lun->be_lun->serial_num);
3067
3068                         if (retval != 0)
3069                                 break;
3070
3071                         retval = sbuf_printf(sb, "</serial_number>\n");
3072
3073                         if (retval != 0)
3074                                 break;
3075
3076                         retval = sbuf_printf(sb, "\t<device_id>");
3077
3078                         if (retval != 0)
3079                                 break;
3080
3081                         retval = ctl_sbuf_printf_esc(sb,lun->be_lun->device_id);
3082
3083                         if (retval != 0)
3084                                 break;
3085
3086                         retval = sbuf_printf(sb, "</device_id>\n");
3087
3088                         if (retval != 0)
3089                                 break;
3090
3091                         if (lun->backend->lun_info != NULL) {
3092                                 retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3093                                 if (retval != 0)
3094                                         break;
3095                         }
3096                         STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3097                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3098                                     opt->name, opt->value, opt->name);
3099                                 if (retval != 0)
3100                                         break;
3101                         }
3102
3103                         retval = sbuf_printf(sb, "</lun>\n");
3104
3105                         if (retval != 0)
3106                                 break;
3107                         mtx_unlock(&lun->lun_lock);
3108                 }
3109                 if (lun != NULL)
3110                         mtx_unlock(&lun->lun_lock);
3111                 mtx_unlock(&softc->ctl_lock);
3112
3113                 if ((retval != 0)
3114                  || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3115                         retval = 0;
3116                         sbuf_delete(sb);
3117                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3118                         snprintf(list->error_str, sizeof(list->error_str),
3119                                  "Out of space, %d bytes is too small",
3120                                  list->alloc_len);
3121                         break;
3122                 }
3123
3124                 sbuf_finish(sb);
3125
3126                 retval = copyout(sbuf_data(sb), list->lun_xml,
3127                                  sbuf_len(sb) + 1);
3128
3129                 list->fill_len = sbuf_len(sb) + 1;
3130                 list->status = CTL_LUN_LIST_OK;
3131                 sbuf_delete(sb);
3132                 break;
3133         }
3134         case CTL_ISCSI: {
3135                 struct ctl_iscsi *ci;
3136                 struct ctl_frontend *fe;
3137
3138                 ci = (struct ctl_iscsi *)addr;
3139
3140                 fe = ctl_frontend_find("iscsi");
3141                 if (fe == NULL) {
3142                         ci->status = CTL_ISCSI_ERROR;
3143                         snprintf(ci->error_str, sizeof(ci->error_str),
3144                             "Frontend \"iscsi\" not found.");
3145                         break;
3146                 }
3147
3148                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3149                 break;
3150         }
3151         case CTL_PORT_REQ: {
3152                 struct ctl_req *req;
3153                 struct ctl_frontend *fe;
3154
3155                 req = (struct ctl_req *)addr;
3156
3157                 fe = ctl_frontend_find(req->driver);
3158                 if (fe == NULL) {
3159                         req->status = CTL_LUN_ERROR;
3160                         snprintf(req->error_str, sizeof(req->error_str),
3161                             "Frontend \"%s\" not found.", req->driver);
3162                         break;
3163                 }
3164                 if (req->num_args > 0) {
3165                         req->kern_args = ctl_copyin_args(req->num_args,
3166                             req->args, req->error_str, sizeof(req->error_str));
3167                         if (req->kern_args == NULL) {
3168                                 req->status = CTL_LUN_ERROR;
3169                                 break;
3170                         }
3171                 }
3172
3173                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3174
3175                 if (req->num_args > 0) {
3176                         ctl_copyout_args(req->num_args, req->kern_args);
3177                         ctl_free_args(req->num_args, req->kern_args);
3178                 }
3179                 break;
3180         }
3181         case CTL_PORT_LIST: {
3182                 struct sbuf *sb;
3183                 struct ctl_port *port;
3184                 struct ctl_lun_list *list;
3185                 struct ctl_option *opt;
3186
3187                 list = (struct ctl_lun_list *)addr;
3188
3189                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3190                 if (sb == NULL) {
3191                         list->status = CTL_LUN_LIST_ERROR;
3192                         snprintf(list->error_str, sizeof(list->error_str),
3193                                  "Unable to allocate %d bytes for LUN list",
3194                                  list->alloc_len);
3195                         break;
3196                 }
3197
3198                 sbuf_printf(sb, "<ctlportlist>\n");
3199
3200                 mtx_lock(&softc->ctl_lock);
3201                 STAILQ_FOREACH(port, &softc->port_list, links) {
3202                         retval = sbuf_printf(sb, "<targ_port id=\"%ju\">\n",
3203                                              (uintmax_t)port->targ_port);
3204
3205                         /*
3206                          * Bail out as soon as we see that we've overfilled
3207                          * the buffer.
3208                          */
3209                         if (retval != 0)
3210                                 break;
3211
3212                         retval = sbuf_printf(sb, "\t<frontend_type>%s"
3213                             "</frontend_type>\n", port->frontend->name);
3214                         if (retval != 0)
3215                                 break;
3216
3217                         retval = sbuf_printf(sb, "\t<port_type>%d</port_type>\n",
3218                                              port->port_type);
3219                         if (retval != 0)
3220                                 break;
3221
3222                         retval = sbuf_printf(sb, "\t<online>%s</online>\n",
3223                             (port->status & CTL_PORT_STATUS_ONLINE) ? "YES" : "NO");
3224                         if (retval != 0)
3225                                 break;
3226
3227                         retval = sbuf_printf(sb, "\t<port_name>%s</port_name>\n",
3228                             port->port_name);
3229                         if (retval != 0)
3230                                 break;
3231
3232                         retval = sbuf_printf(sb, "\t<physical_port>%d</physical_port>\n",
3233                             port->physical_port);
3234                         if (retval != 0)
3235                                 break;
3236
3237                         retval = sbuf_printf(sb, "\t<virtual_port>%d</virtual_port>\n",
3238                             port->virtual_port);
3239                         if (retval != 0)
3240                                 break;
3241
3242                         retval = sbuf_printf(sb, "\t<wwnn>%#jx</wwnn>\n",
3243                             (uintmax_t)port->wwnn);
3244                         if (retval != 0)
3245                                 break;
3246
3247                         retval = sbuf_printf(sb, "\t<wwpn>%#jx</wwpn>\n",
3248                             (uintmax_t)port->wwpn);
3249                         if (retval != 0)
3250                                 break;
3251
3252                         STAILQ_FOREACH(opt, &port->options, links) {
3253                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3254                                     opt->name, opt->value, opt->name);
3255                                 if (retval != 0)
3256                                         break;
3257                         }
3258
3259                         retval = sbuf_printf(sb, "</targ_port>\n");
3260                         if (retval != 0)
3261                                 break;
3262                 }
3263                 mtx_unlock(&softc->ctl_lock);
3264
3265                 if ((retval != 0)
3266                  || ((retval = sbuf_printf(sb, "</ctlportlist>\n")) != 0)) {
3267                         retval = 0;
3268                         sbuf_delete(sb);
3269                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3270                         snprintf(list->error_str, sizeof(list->error_str),
3271                                  "Out of space, %d bytes is too small",
3272                                  list->alloc_len);
3273                         break;
3274                 }
3275
3276                 sbuf_finish(sb);
3277
3278                 retval = copyout(sbuf_data(sb), list->lun_xml,
3279                                  sbuf_len(sb) + 1);
3280
3281                 list->fill_len = sbuf_len(sb) + 1;
3282                 list->status = CTL_LUN_LIST_OK;
3283                 sbuf_delete(sb);
3284                 break;
3285         }
3286         default: {
3287                 /* XXX KDM should we fix this? */
3288 #if 0
3289                 struct ctl_backend_driver *backend;
3290                 unsigned int type;
3291                 int found;
3292
3293                 found = 0;
3294
3295                 /*
3296                  * We encode the backend type as the ioctl type for backend
3297                  * ioctls.  So parse it out here, and then search for a
3298                  * backend of this type.
3299                  */
3300                 type = _IOC_TYPE(cmd);
3301
3302                 STAILQ_FOREACH(backend, &softc->be_list, links) {
3303                         if (backend->type == type) {
3304                                 found = 1;
3305                                 break;
3306                         }
3307                 }
3308                 if (found == 0) {
3309                         printf("ctl: unknown ioctl command %#lx or backend "
3310                                "%d\n", cmd, type);
3311                         retval = EINVAL;
3312                         break;
3313                 }
3314                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3315 #endif
3316                 retval = ENOTTY;
3317                 break;
3318         }
3319         }
3320         return (retval);
3321 }
3322
3323 uint32_t
3324 ctl_get_initindex(struct ctl_nexus *nexus)
3325 {
3326         if (nexus->targ_port < CTL_MAX_PORTS)
3327                 return (nexus->initid.id +
3328                         (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3329         else
3330                 return (nexus->initid.id +
3331                        ((nexus->targ_port - CTL_MAX_PORTS) *
3332                         CTL_MAX_INIT_PER_PORT));
3333 }
3334
3335 uint32_t
3336 ctl_get_resindex(struct ctl_nexus *nexus)
3337 {
3338         return (nexus->initid.id + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3339 }
3340
3341 uint32_t
3342 ctl_port_idx(int port_num)
3343 {
3344         if (port_num < CTL_MAX_PORTS)
3345                 return(port_num);
3346         else
3347                 return(port_num - CTL_MAX_PORTS);
3348 }
3349
3350 /*
3351  * Note:  This only works for bitmask sizes that are at least 32 bits, and
3352  * that are a power of 2.
3353  */
3354 int
3355 ctl_ffz(uint32_t *mask, uint32_t size)
3356 {
3357         uint32_t num_chunks, num_pieces;
3358         int i, j;
3359
3360         num_chunks = (size >> 5);
3361         if (num_chunks == 0)
3362                 num_chunks++;
3363         num_pieces = ctl_min((sizeof(uint32_t) * 8), size);
3364
3365         for (i = 0; i < num_chunks; i++) {
3366                 for (j = 0; j < num_pieces; j++) {
3367                         if ((mask[i] & (1 << j)) == 0)
3368                                 return ((i << 5) + j);
3369                 }
3370         }
3371
3372         return (-1);
3373 }
3374
3375 int
3376 ctl_set_mask(uint32_t *mask, uint32_t bit)
3377 {
3378         uint32_t chunk, piece;
3379
3380         chunk = bit >> 5;
3381         piece = bit % (sizeof(uint32_t) * 8);
3382
3383         if ((mask[chunk] & (1 << piece)) != 0)
3384                 return (-1);
3385         else
3386                 mask[chunk] |= (1 << piece);
3387
3388         return (0);
3389 }
3390
3391 int
3392 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3393 {
3394         uint32_t chunk, piece;
3395
3396         chunk = bit >> 5;
3397         piece = bit % (sizeof(uint32_t) * 8);
3398
3399         if ((mask[chunk] & (1 << piece)) == 0)
3400                 return (-1);
3401         else
3402                 mask[chunk] &= ~(1 << piece);
3403
3404         return (0);
3405 }
3406
3407 int
3408 ctl_is_set(uint32_t *mask, uint32_t bit)
3409 {
3410         uint32_t chunk, piece;
3411
3412         chunk = bit >> 5;
3413         piece = bit % (sizeof(uint32_t) * 8);
3414
3415         if ((mask[chunk] & (1 << piece)) == 0)
3416                 return (0);
3417         else
3418                 return (1);
3419 }
3420
3421 #ifdef unused
3422 /*
3423  * The bus, target and lun are optional, they can be filled in later.
3424  * can_wait is used to determine whether we can wait on the malloc or not.
3425  */
3426 union ctl_io*
3427 ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port, uint32_t targ_target,
3428               uint32_t targ_lun, int can_wait)
3429 {
3430         union ctl_io *io;
3431
3432         if (can_wait)
3433                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_WAITOK);
3434         else
3435                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3436
3437         if (io != NULL) {
3438                 io->io_hdr.io_type = io_type;
3439                 io->io_hdr.targ_port = targ_port;
3440                 /*
3441                  * XXX KDM this needs to change/go away.  We need to move
3442                  * to a preallocated pool of ctl_scsiio structures.
3443                  */
3444                 io->io_hdr.nexus.targ_target.id = targ_target;
3445                 io->io_hdr.nexus.targ_lun = targ_lun;
3446         }
3447
3448         return (io);
3449 }
3450
3451 void
3452 ctl_kfree_io(union ctl_io *io)
3453 {
3454         free(io, M_CTL);
3455 }
3456 #endif /* unused */
3457
3458 /*
3459  * ctl_softc, pool_type, total_ctl_io are passed in.
3460  * npool is passed out.
3461  */
3462 int
3463 ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type,
3464                 uint32_t total_ctl_io, struct ctl_io_pool **npool)
3465 {
3466         uint32_t i;
3467         union ctl_io *cur_io, *next_io;
3468         struct ctl_io_pool *pool;
3469         int retval;
3470
3471         retval = 0;
3472
3473         pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3474                                             M_NOWAIT | M_ZERO);
3475         if (pool == NULL) {
3476                 retval = ENOMEM;
3477                 goto bailout;
3478         }
3479
3480         pool->type = pool_type;
3481         pool->ctl_softc = ctl_softc;
3482
3483         mtx_lock(&ctl_softc->pool_lock);
3484         pool->id = ctl_softc->cur_pool_id++;
3485         mtx_unlock(&ctl_softc->pool_lock);
3486
3487         pool->flags = CTL_POOL_FLAG_NONE;
3488         pool->refcount = 1;             /* Reference for validity. */
3489         STAILQ_INIT(&pool->free_queue);
3490
3491         /*
3492          * XXX KDM other options here:
3493          * - allocate a page at a time
3494          * - allocate one big chunk of memory.
3495          * Page allocation might work well, but would take a little more
3496          * tracking.
3497          */
3498         for (i = 0; i < total_ctl_io; i++) {
3499                 cur_io = (union ctl_io *)malloc(sizeof(*cur_io), M_CTLIO,
3500                                                 M_NOWAIT);
3501                 if (cur_io == NULL) {
3502                         retval = ENOMEM;
3503                         break;
3504                 }
3505                 cur_io->io_hdr.pool = pool;
3506                 STAILQ_INSERT_TAIL(&pool->free_queue, &cur_io->io_hdr, links);
3507                 pool->total_ctl_io++;
3508                 pool->free_ctl_io++;
3509         }
3510
3511         if (retval != 0) {
3512                 for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3513                      cur_io != NULL; cur_io = next_io) {
3514                         next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr,
3515                                                               links);
3516                         STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr,
3517                                       ctl_io_hdr, links);
3518                         free(cur_io, M_CTLIO);
3519                 }
3520
3521                 free(pool, M_CTL);
3522                 goto bailout;
3523         }
3524         mtx_lock(&ctl_softc->pool_lock);
3525         ctl_softc->num_pools++;
3526         STAILQ_INSERT_TAIL(&ctl_softc->io_pools, pool, links);
3527         /*
3528          * Increment our usage count if this is an external consumer, so we
3529          * can't get unloaded until the external consumer (most likely a
3530          * FETD) unloads and frees his pool.
3531          *
3532          * XXX KDM will this increment the caller's module use count, or
3533          * mine?
3534          */
3535 #if 0
3536         if ((pool_type != CTL_POOL_EMERGENCY)
3537          && (pool_type != CTL_POOL_INTERNAL)
3538          && (pool_type != CTL_POOL_4OTHERSC))
3539                 MOD_INC_USE_COUNT;
3540 #endif
3541
3542         mtx_unlock(&ctl_softc->pool_lock);
3543
3544         *npool = pool;
3545
3546 bailout:
3547
3548         return (retval);
3549 }
3550
3551 static int
3552 ctl_pool_acquire(struct ctl_io_pool *pool)
3553 {
3554
3555         mtx_assert(&pool->ctl_softc->pool_lock, MA_OWNED);
3556
3557         if (pool->flags & CTL_POOL_FLAG_INVALID)
3558                 return (EINVAL);
3559
3560         pool->refcount++;
3561
3562         return (0);
3563 }
3564
3565 static void
3566 ctl_pool_release(struct ctl_io_pool *pool)
3567 {
3568         struct ctl_softc *ctl_softc = pool->ctl_softc;
3569         union ctl_io *io;
3570
3571         mtx_assert(&ctl_softc->pool_lock, MA_OWNED);
3572
3573         if (--pool->refcount != 0)
3574                 return;
3575
3576         while ((io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue)) != NULL) {
3577                 STAILQ_REMOVE(&pool->free_queue, &io->io_hdr, ctl_io_hdr,
3578                               links);
3579                 free(io, M_CTLIO);
3580         }
3581
3582         STAILQ_REMOVE(&ctl_softc->io_pools, pool, ctl_io_pool, links);
3583         ctl_softc->num_pools--;
3584
3585         /*
3586          * XXX KDM will this decrement the caller's usage count or mine?
3587          */
3588 #if 0
3589         if ((pool->type != CTL_POOL_EMERGENCY)
3590          && (pool->type != CTL_POOL_INTERNAL)
3591          && (pool->type != CTL_POOL_4OTHERSC))
3592                 MOD_DEC_USE_COUNT;
3593 #endif
3594
3595         free(pool, M_CTL);
3596 }
3597
3598 void
3599 ctl_pool_free(struct ctl_io_pool *pool)
3600 {
3601         struct ctl_softc *ctl_softc;
3602
3603         if (pool == NULL)
3604                 return;
3605
3606         ctl_softc = pool->ctl_softc;
3607         mtx_lock(&ctl_softc->pool_lock);
3608         pool->flags |= CTL_POOL_FLAG_INVALID;
3609         ctl_pool_release(pool);
3610         mtx_unlock(&ctl_softc->pool_lock);
3611 }
3612
3613 /*
3614  * This routine does not block (except for spinlocks of course).
3615  * It tries to allocate a ctl_io union from the caller's pool as quickly as
3616  * possible.
3617  */
3618 union ctl_io *
3619 ctl_alloc_io(void *pool_ref)
3620 {
3621         union ctl_io *io;
3622         struct ctl_softc *ctl_softc;
3623         struct ctl_io_pool *pool, *npool;
3624         struct ctl_io_pool *emergency_pool;
3625
3626         pool = (struct ctl_io_pool *)pool_ref;
3627
3628         if (pool == NULL) {
3629                 printf("%s: pool is NULL\n", __func__);
3630                 return (NULL);
3631         }
3632
3633         emergency_pool = NULL;
3634
3635         ctl_softc = pool->ctl_softc;
3636
3637         mtx_lock(&ctl_softc->pool_lock);
3638         /*
3639          * First, try to get the io structure from the user's pool.
3640          */
3641         if (ctl_pool_acquire(pool) == 0) {
3642                 io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3643                 if (io != NULL) {
3644                         STAILQ_REMOVE_HEAD(&pool->free_queue, links);
3645                         pool->total_allocated++;
3646                         pool->free_ctl_io--;
3647                         mtx_unlock(&ctl_softc->pool_lock);
3648                         return (io);
3649                 } else
3650                         ctl_pool_release(pool);
3651         }
3652         /*
3653          * If he doesn't have any io structures left, search for an
3654          * emergency pool and grab one from there.
3655          */
3656         STAILQ_FOREACH(npool, &ctl_softc->io_pools, links) {
3657                 if (npool->type != CTL_POOL_EMERGENCY)
3658                         continue;
3659
3660                 if (ctl_pool_acquire(npool) != 0)
3661                         continue;
3662
3663                 emergency_pool = npool;
3664
3665                 io = (union ctl_io *)STAILQ_FIRST(&npool->free_queue);
3666                 if (io != NULL) {
3667                         STAILQ_REMOVE_HEAD(&npool->free_queue, links);
3668                         npool->total_allocated++;
3669                         npool->free_ctl_io--;
3670                         mtx_unlock(&ctl_softc->pool_lock);
3671                         return (io);
3672                 } else
3673                         ctl_pool_release(npool);
3674         }
3675
3676         /* Drop the spinlock before we malloc */
3677         mtx_unlock(&ctl_softc->pool_lock);
3678
3679         /*
3680          * The emergency pool (if it exists) didn't have one, so try an
3681          * atomic (i.e. nonblocking) malloc and see if we get lucky.
3682          */
3683         io = (union ctl_io *)malloc(sizeof(*io), M_CTLIO, M_NOWAIT);
3684         if (io != NULL) {
3685                 /*
3686                  * If the emergency pool exists but is empty, add this
3687                  * ctl_io to its list when it gets freed.
3688                  */
3689                 if (emergency_pool != NULL) {
3690                         mtx_lock(&ctl_softc->pool_lock);
3691                         if (ctl_pool_acquire(emergency_pool) == 0) {
3692                                 io->io_hdr.pool = emergency_pool;
3693                                 emergency_pool->total_ctl_io++;
3694                                 /*
3695                                  * Need to bump this, otherwise
3696                                  * total_allocated and total_freed won't
3697                                  * match when we no longer have anything
3698                                  * outstanding.
3699                                  */
3700                                 emergency_pool->total_allocated++;
3701                         }
3702                         mtx_unlock(&ctl_softc->pool_lock);
3703                 } else
3704                         io->io_hdr.pool = NULL;
3705         }
3706
3707         return (io);
3708 }
3709
3710 void
3711 ctl_free_io(union ctl_io *io)
3712 {
3713         if (io == NULL)
3714                 return;
3715
3716         /*
3717          * If this ctl_io has a pool, return it to that pool.
3718          */
3719         if (io->io_hdr.pool != NULL) {
3720                 struct ctl_io_pool *pool;
3721
3722                 pool = (struct ctl_io_pool *)io->io_hdr.pool;
3723                 mtx_lock(&pool->ctl_softc->pool_lock);
3724                 io->io_hdr.io_type = 0xff;
3725                 STAILQ_INSERT_TAIL(&pool->free_queue, &io->io_hdr, links);
3726                 pool->total_freed++;
3727                 pool->free_ctl_io++;
3728                 ctl_pool_release(pool);
3729                 mtx_unlock(&pool->ctl_softc->pool_lock);
3730         } else {
3731                 /*
3732                  * Otherwise, just free it.  We probably malloced it and
3733                  * the emergency pool wasn't available.
3734                  */
3735                 free(io, M_CTLIO);
3736         }
3737
3738 }
3739
3740 void
3741 ctl_zero_io(union ctl_io *io)
3742 {
3743         void *pool_ref;
3744
3745         if (io == NULL)
3746                 return;
3747
3748         /*
3749          * May need to preserve linked list pointers at some point too.
3750          */
3751         pool_ref = io->io_hdr.pool;
3752
3753         memset(io, 0, sizeof(*io));
3754
3755         io->io_hdr.pool = pool_ref;
3756 }
3757
3758 /*
3759  * This routine is currently used for internal copies of ctl_ios that need
3760  * to persist for some reason after we've already returned status to the
3761  * FETD.  (Thus the flag set.)
3762  *
3763  * XXX XXX
3764  * Note that this makes a blind copy of all fields in the ctl_io, except
3765  * for the pool reference.  This includes any memory that has been
3766  * allocated!  That memory will no longer be valid after done has been
3767  * called, so this would be VERY DANGEROUS for command that actually does
3768  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3769  * start and stop commands, which don't transfer any data, so this is not a
3770  * problem.  If it is used for anything else, the caller would also need to
3771  * allocate data buffer space and this routine would need to be modified to
3772  * copy the data buffer(s) as well.
3773  */
3774 void
3775 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3776 {
3777         void *pool_ref;
3778
3779         if ((src == NULL)
3780          || (dest == NULL))
3781                 return;
3782
3783         /*
3784          * May need to preserve linked list pointers at some point too.
3785          */
3786         pool_ref = dest->io_hdr.pool;
3787
3788         memcpy(dest, src, ctl_min(sizeof(*src), sizeof(*dest)));
3789
3790         dest->io_hdr.pool = pool_ref;
3791         /*
3792          * We need to know that this is an internal copy, and doesn't need
3793          * to get passed back to the FETD that allocated it.
3794          */
3795         dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3796 }
3797
3798 #ifdef NEEDTOPORT
3799 static void
3800 ctl_update_power_subpage(struct copan_power_subpage *page)
3801 {
3802         int num_luns, num_partitions, config_type;
3803         struct ctl_softc *softc;
3804         cs_BOOL_t aor_present, shelf_50pct_power;
3805         cs_raidset_personality_t rs_type;
3806         int max_active_luns;
3807
3808         softc = control_softc;
3809
3810         /* subtract out the processor LUN */
3811         num_luns = softc->num_luns - 1;
3812         /*
3813          * Default to 7 LUNs active, which was the only number we allowed
3814          * in the past.
3815          */
3816         max_active_luns = 7;
3817
3818         num_partitions = config_GetRsPartitionInfo();
3819         config_type = config_GetConfigType();
3820         shelf_50pct_power = config_GetShelfPowerMode();
3821         aor_present = config_IsAorRsPresent();
3822
3823         rs_type = ddb_GetRsRaidType(1);
3824         if ((rs_type != CS_RAIDSET_PERSONALITY_RAID5)
3825          && (rs_type != CS_RAIDSET_PERSONALITY_RAID1)) {
3826                 EPRINT(0, "Unsupported RS type %d!", rs_type);
3827         }
3828
3829
3830         page->total_luns = num_luns;
3831
3832         switch (config_type) {
3833         case 40:
3834                 /*
3835                  * In a 40 drive configuration, it doesn't matter what DC
3836                  * cards we have, whether we have AOR enabled or not,
3837                  * partitioning or not, or what type of RAIDset we have.
3838                  * In that scenario, we can power up every LUN we present
3839                  * to the user.
3840                  */
3841                 max_active_luns = num_luns;
3842
3843                 break;
3844         case 64:
3845                 if (shelf_50pct_power == CS_FALSE) {
3846                         /* 25% power */
3847                         if (aor_present == CS_TRUE) {
3848                                 if (rs_type ==
3849                                      CS_RAIDSET_PERSONALITY_RAID5) {
3850                                         max_active_luns = 7;
3851                                 } else if (rs_type ==
3852                                          CS_RAIDSET_PERSONALITY_RAID1){
3853                                         max_active_luns = 14;
3854                                 } else {
3855                                         /* XXX KDM now what?? */
3856                                 }
3857                         } else {
3858                                 if (rs_type ==
3859                                      CS_RAIDSET_PERSONALITY_RAID5) {
3860                                         max_active_luns = 8;
3861                                 } else if (rs_type ==
3862                                          CS_RAIDSET_PERSONALITY_RAID1){
3863                                         max_active_luns = 16;
3864                                 } else {
3865                                         /* XXX KDM now what?? */
3866                                 }
3867                         }
3868                 } else {
3869                         /* 50% power */
3870                         /*
3871                          * With 50% power in a 64 drive configuration, we
3872                          * can power all LUNs we present.
3873                          */
3874                         max_active_luns = num_luns;
3875                 }
3876                 break;
3877         case 112:
3878                 if (shelf_50pct_power == CS_FALSE) {
3879                         /* 25% power */
3880                         if (aor_present == CS_TRUE) {
3881                                 if (rs_type ==
3882                                      CS_RAIDSET_PERSONALITY_RAID5) {
3883                                         max_active_luns = 7;
3884                                 } else if (rs_type ==
3885                                          CS_RAIDSET_PERSONALITY_RAID1){
3886                                         max_active_luns = 14;
3887                                 } else {
3888                                         /* XXX KDM now what?? */
3889                                 }
3890                         } else {
3891                                 if (rs_type ==
3892                                      CS_RAIDSET_PERSONALITY_RAID5) {
3893                                         max_active_luns = 8;
3894                                 } else if (rs_type ==
3895                                          CS_RAIDSET_PERSONALITY_RAID1){
3896                                         max_active_luns = 16;
3897                                 } else {
3898                                         /* XXX KDM now what?? */
3899                                 }
3900                         }
3901                 } else {
3902                         /* 50% power */
3903                         if (aor_present == CS_TRUE) {
3904                                 if (rs_type ==
3905                                      CS_RAIDSET_PERSONALITY_RAID5) {
3906                                         max_active_luns = 14;
3907                                 } else if (rs_type ==
3908                                          CS_RAIDSET_PERSONALITY_RAID1){
3909                                         /*
3910                                          * We're assuming here that disk
3911                                          * caching is enabled, and so we're
3912                                          * able to power up half of each
3913                                          * LUN, and cache all writes.
3914                                          */
3915                                         max_active_luns = num_luns;
3916                                 } else {
3917                                         /* XXX KDM now what?? */
3918                                 }
3919                         } else {
3920                                 if (rs_type ==
3921                                      CS_RAIDSET_PERSONALITY_RAID5) {
3922                                         max_active_luns = 15;
3923                                 } else if (rs_type ==
3924                                          CS_RAIDSET_PERSONALITY_RAID1){
3925                                         max_active_luns = 30;
3926                                 } else {
3927                                         /* XXX KDM now what?? */
3928                                 }
3929                         }
3930                 }
3931                 break;
3932         default:
3933                 /*
3934                  * In this case, we have an unknown configuration, so we
3935                  * just use the default from above.
3936                  */
3937                 break;
3938         }
3939
3940         page->max_active_luns = max_active_luns;
3941 #if 0
3942         printk("%s: total_luns = %d, max_active_luns = %d\n", __func__,
3943                page->total_luns, page->max_active_luns);
3944 #endif
3945 }
3946 #endif /* NEEDTOPORT */
3947
3948 /*
3949  * This routine could be used in the future to load default and/or saved
3950  * mode page parameters for a particuar lun.
3951  */
3952 static int
3953 ctl_init_page_index(struct ctl_lun *lun)
3954 {
3955         int i;
3956         struct ctl_page_index *page_index;
3957         struct ctl_softc *softc;
3958
3959         memcpy(&lun->mode_pages.index, page_index_template,
3960                sizeof(page_index_template));
3961
3962         softc = lun->ctl_softc;
3963
3964         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
3965
3966                 page_index = &lun->mode_pages.index[i];
3967                 /*
3968                  * If this is a disk-only mode page, there's no point in
3969                  * setting it up.  For some pages, we have to have some
3970                  * basic information about the disk in order to calculate the
3971                  * mode page data.
3972                  */
3973                 if ((lun->be_lun->lun_type != T_DIRECT)
3974                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
3975                         continue;
3976
3977                 switch (page_index->page_code & SMPH_PC_MASK) {
3978                 case SMS_FORMAT_DEVICE_PAGE: {
3979                         struct scsi_format_page *format_page;
3980
3981                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3982                                 panic("subpage is incorrect!");
3983
3984                         /*
3985                          * Sectors per track are set above.  Bytes per
3986                          * sector need to be set here on a per-LUN basis.
3987                          */
3988                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
3989                                &format_page_default,
3990                                sizeof(format_page_default));
3991                         memcpy(&lun->mode_pages.format_page[
3992                                CTL_PAGE_CHANGEABLE], &format_page_changeable,
3993                                sizeof(format_page_changeable));
3994                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
3995                                &format_page_default,
3996                                sizeof(format_page_default));
3997                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
3998                                &format_page_default,
3999                                sizeof(format_page_default));
4000
4001                         format_page = &lun->mode_pages.format_page[
4002                                 CTL_PAGE_CURRENT];
4003                         scsi_ulto2b(lun->be_lun->blocksize,
4004                                     format_page->bytes_per_sector);
4005
4006                         format_page = &lun->mode_pages.format_page[
4007                                 CTL_PAGE_DEFAULT];
4008                         scsi_ulto2b(lun->be_lun->blocksize,
4009                                     format_page->bytes_per_sector);
4010
4011                         format_page = &lun->mode_pages.format_page[
4012                                 CTL_PAGE_SAVED];
4013                         scsi_ulto2b(lun->be_lun->blocksize,
4014                                     format_page->bytes_per_sector);
4015
4016                         page_index->page_data =
4017                                 (uint8_t *)lun->mode_pages.format_page;
4018                         break;
4019                 }
4020                 case SMS_RIGID_DISK_PAGE: {
4021                         struct scsi_rigid_disk_page *rigid_disk_page;
4022                         uint32_t sectors_per_cylinder;
4023                         uint64_t cylinders;
4024 #ifndef __XSCALE__
4025                         int shift;
4026 #endif /* !__XSCALE__ */
4027
4028                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4029                                 panic("invalid subpage value %d",
4030                                       page_index->subpage);
4031
4032                         /*
4033                          * Rotation rate and sectors per track are set
4034                          * above.  We calculate the cylinders here based on
4035                          * capacity.  Due to the number of heads and
4036                          * sectors per track we're using, smaller arrays
4037                          * may turn out to have 0 cylinders.  Linux and
4038                          * FreeBSD don't pay attention to these mode pages
4039                          * to figure out capacity, but Solaris does.  It
4040                          * seems to deal with 0 cylinders just fine, and
4041                          * works out a fake geometry based on the capacity.
4042                          */
4043                         memcpy(&lun->mode_pages.rigid_disk_page[
4044                                CTL_PAGE_CURRENT], &rigid_disk_page_default,
4045                                sizeof(rigid_disk_page_default));
4046                         memcpy(&lun->mode_pages.rigid_disk_page[
4047                                CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
4048                                sizeof(rigid_disk_page_changeable));
4049                         memcpy(&lun->mode_pages.rigid_disk_page[
4050                                CTL_PAGE_DEFAULT], &rigid_disk_page_default,
4051                                sizeof(rigid_disk_page_default));
4052                         memcpy(&lun->mode_pages.rigid_disk_page[
4053                                CTL_PAGE_SAVED], &rigid_disk_page_default,
4054                                sizeof(rigid_disk_page_default));
4055
4056                         sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
4057                                 CTL_DEFAULT_HEADS;
4058
4059                         /*
4060                          * The divide method here will be more accurate,
4061                          * probably, but results in floating point being
4062                          * used in the kernel on i386 (__udivdi3()).  On the
4063                          * XScale, though, __udivdi3() is implemented in
4064                          * software.
4065                          *
4066                          * The shift method for cylinder calculation is
4067                          * accurate if sectors_per_cylinder is a power of
4068                          * 2.  Otherwise it might be slightly off -- you
4069                          * might have a bit of a truncation problem.
4070                          */
4071 #ifdef  __XSCALE__
4072                         cylinders = (lun->be_lun->maxlba + 1) /
4073                                 sectors_per_cylinder;
4074 #else
4075                         for (shift = 31; shift > 0; shift--) {
4076                                 if (sectors_per_cylinder & (1 << shift))
4077                                         break;
4078                         }
4079                         cylinders = (lun->be_lun->maxlba + 1) >> shift;
4080 #endif
4081
4082                         /*
4083                          * We've basically got 3 bytes, or 24 bits for the
4084                          * cylinder size in the mode page.  If we're over,
4085                          * just round down to 2^24.
4086                          */
4087                         if (cylinders > 0xffffff)
4088                                 cylinders = 0xffffff;
4089
4090                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4091                                 CTL_PAGE_CURRENT];
4092                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4093
4094                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4095                                 CTL_PAGE_DEFAULT];
4096                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4097
4098                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4099                                 CTL_PAGE_SAVED];
4100                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4101
4102                         page_index->page_data =
4103                                 (uint8_t *)lun->mode_pages.rigid_disk_page;
4104                         break;
4105                 }
4106                 case SMS_CACHING_PAGE: {
4107
4108                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4109                                 panic("invalid subpage value %d",
4110                                       page_index->subpage);
4111                         /*
4112                          * Defaults should be okay here, no calculations
4113                          * needed.
4114                          */
4115                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
4116                                &caching_page_default,
4117                                sizeof(caching_page_default));
4118                         memcpy(&lun->mode_pages.caching_page[
4119                                CTL_PAGE_CHANGEABLE], &caching_page_changeable,
4120                                sizeof(caching_page_changeable));
4121                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
4122                                &caching_page_default,
4123                                sizeof(caching_page_default));
4124                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4125                                &caching_page_default,
4126                                sizeof(caching_page_default));
4127                         page_index->page_data =
4128                                 (uint8_t *)lun->mode_pages.caching_page;
4129                         break;
4130                 }
4131                 case SMS_CONTROL_MODE_PAGE: {
4132
4133                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4134                                 panic("invalid subpage value %d",
4135                                       page_index->subpage);
4136
4137                         /*
4138                          * Defaults should be okay here, no calculations
4139                          * needed.
4140                          */
4141                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
4142                                &control_page_default,
4143                                sizeof(control_page_default));
4144                         memcpy(&lun->mode_pages.control_page[
4145                                CTL_PAGE_CHANGEABLE], &control_page_changeable,
4146                                sizeof(control_page_changeable));
4147                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
4148                                &control_page_default,
4149                                sizeof(control_page_default));
4150                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
4151                                &control_page_default,
4152                                sizeof(control_page_default));
4153                         page_index->page_data =
4154                                 (uint8_t *)lun->mode_pages.control_page;
4155                         break;
4156
4157                 }
4158                 case SMS_VENDOR_SPECIFIC_PAGE:{
4159                         switch (page_index->subpage) {
4160                         case PWR_SUBPAGE_CODE: {
4161                                 struct copan_power_subpage *current_page,
4162                                                            *saved_page;
4163
4164                                 memcpy(&lun->mode_pages.power_subpage[
4165                                        CTL_PAGE_CURRENT],
4166                                        &power_page_default,
4167                                        sizeof(power_page_default));
4168                                 memcpy(&lun->mode_pages.power_subpage[
4169                                        CTL_PAGE_CHANGEABLE],
4170                                        &power_page_changeable,
4171                                        sizeof(power_page_changeable));
4172                                 memcpy(&lun->mode_pages.power_subpage[
4173                                        CTL_PAGE_DEFAULT],
4174                                        &power_page_default,
4175                                        sizeof(power_page_default));
4176                                 memcpy(&lun->mode_pages.power_subpage[
4177                                        CTL_PAGE_SAVED],
4178                                        &power_page_default,
4179                                        sizeof(power_page_default));
4180                                 page_index->page_data =
4181                                     (uint8_t *)lun->mode_pages.power_subpage;
4182
4183                                 current_page = (struct copan_power_subpage *)
4184                                         (page_index->page_data +
4185                                          (page_index->page_len *
4186                                           CTL_PAGE_CURRENT));
4187                                 saved_page = (struct copan_power_subpage *)
4188                                         (page_index->page_data +
4189                                          (page_index->page_len *
4190                                           CTL_PAGE_SAVED));
4191                                 break;
4192                         }
4193                         case APS_SUBPAGE_CODE: {
4194                                 struct copan_aps_subpage *current_page,
4195                                                          *saved_page;
4196
4197                                 // This gets set multiple times but
4198                                 // it should always be the same. It's
4199                                 // only done during init so who cares.
4200                                 index_to_aps_page = i;
4201
4202                                 memcpy(&lun->mode_pages.aps_subpage[
4203                                        CTL_PAGE_CURRENT],
4204                                        &aps_page_default,
4205                                        sizeof(aps_page_default));
4206                                 memcpy(&lun->mode_pages.aps_subpage[
4207                                        CTL_PAGE_CHANGEABLE],
4208                                        &aps_page_changeable,
4209                                        sizeof(aps_page_changeable));
4210                                 memcpy(&lun->mode_pages.aps_subpage[
4211                                        CTL_PAGE_DEFAULT],
4212                                        &aps_page_default,
4213                                        sizeof(aps_page_default));
4214                                 memcpy(&lun->mode_pages.aps_subpage[
4215                                        CTL_PAGE_SAVED],
4216                                        &aps_page_default,
4217                                        sizeof(aps_page_default));
4218                                 page_index->page_data =
4219                                         (uint8_t *)lun->mode_pages.aps_subpage;
4220
4221                                 current_page = (struct copan_aps_subpage *)
4222                                         (page_index->page_data +
4223                                          (page_index->page_len *
4224                                           CTL_PAGE_CURRENT));
4225                                 saved_page = (struct copan_aps_subpage *)
4226                                         (page_index->page_data +
4227                                          (page_index->page_len *
4228                                           CTL_PAGE_SAVED));
4229                                 break;
4230                         }
4231                         case DBGCNF_SUBPAGE_CODE: {
4232                                 struct copan_debugconf_subpage *current_page,
4233                                                                *saved_page;
4234
4235                                 memcpy(&lun->mode_pages.debugconf_subpage[
4236                                        CTL_PAGE_CURRENT],
4237                                        &debugconf_page_default,
4238                                        sizeof(debugconf_page_default));
4239                                 memcpy(&lun->mode_pages.debugconf_subpage[
4240                                        CTL_PAGE_CHANGEABLE],
4241                                        &debugconf_page_changeable,
4242                                        sizeof(debugconf_page_changeable));
4243                                 memcpy(&lun->mode_pages.debugconf_subpage[
4244                                        CTL_PAGE_DEFAULT],
4245                                        &debugconf_page_default,
4246                                        sizeof(debugconf_page_default));
4247                                 memcpy(&lun->mode_pages.debugconf_subpage[
4248                                        CTL_PAGE_SAVED],
4249                                        &debugconf_page_default,
4250                                        sizeof(debugconf_page_default));
4251                                 page_index->page_data =
4252                                         (uint8_t *)lun->mode_pages.debugconf_subpage;
4253
4254                                 current_page = (struct copan_debugconf_subpage *)
4255                                         (page_index->page_data +
4256                                          (page_index->page_len *
4257                                           CTL_PAGE_CURRENT));
4258                                 saved_page = (struct copan_debugconf_subpage *)
4259                                         (page_index->page_data +
4260                                          (page_index->page_len *
4261                                           CTL_PAGE_SAVED));
4262                                 break;
4263                         }
4264                         default:
4265                                 panic("invalid subpage value %d",
4266                                       page_index->subpage);
4267                                 break;
4268                         }
4269                         break;
4270                 }
4271                 default:
4272                         panic("invalid page value %d",
4273                               page_index->page_code & SMPH_PC_MASK);
4274                         break;
4275         }
4276         }
4277
4278         return (CTL_RETVAL_COMPLETE);
4279 }
4280
4281 /*
4282  * LUN allocation.
4283  *
4284  * Requirements:
4285  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4286  *   wants us to allocate the LUN and he can block.
4287  * - ctl_softc is always set
4288  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4289  *
4290  * Returns 0 for success, non-zero (errno) for failure.
4291  */
4292 static int
4293 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4294               struct ctl_be_lun *const be_lun, struct ctl_id target_id)
4295 {
4296         struct ctl_lun *nlun, *lun;
4297         struct ctl_port *port;
4298         struct scsi_vpd_id_descriptor *desc;
4299         struct scsi_vpd_id_t10 *t10id;
4300         const char *scsiname, *vendor;
4301         int lun_number, i, lun_malloced;
4302         int devidlen, idlen1, idlen2, len;
4303
4304         if (be_lun == NULL)
4305                 return (EINVAL);
4306
4307         /*
4308          * We currently only support Direct Access or Processor LUN types.
4309          */
4310         switch (be_lun->lun_type) {
4311         case T_DIRECT:
4312                 break;
4313         case T_PROCESSOR:
4314                 break;
4315         case T_SEQUENTIAL:
4316         case T_CHANGER:
4317         default:
4318                 be_lun->lun_config_status(be_lun->be_lun,
4319                                           CTL_LUN_CONFIG_FAILURE);
4320                 break;
4321         }
4322         if (ctl_lun == NULL) {
4323                 lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4324                 lun_malloced = 1;
4325         } else {
4326                 lun_malloced = 0;
4327                 lun = ctl_lun;
4328         }
4329
4330         memset(lun, 0, sizeof(*lun));
4331         if (lun_malloced)
4332                 lun->flags = CTL_LUN_MALLOCED;
4333
4334         /* Generate LUN ID. */
4335         devidlen = max(CTL_DEVID_MIN_LEN,
4336             strnlen(be_lun->device_id, CTL_DEVID_LEN));
4337         idlen1 = sizeof(*t10id) + devidlen;
4338         len = sizeof(struct scsi_vpd_id_descriptor) + idlen1;
4339         scsiname = ctl_get_opt(&be_lun->options, "scsiname");
4340         if (scsiname != NULL) {
4341                 idlen2 = roundup2(strlen(scsiname) + 1, 4);
4342                 len += sizeof(struct scsi_vpd_id_descriptor) + idlen2;
4343         }
4344         lun->lun_devid = malloc(sizeof(struct ctl_devid) + len,
4345             M_CTL, M_WAITOK | M_ZERO);
4346         lun->lun_devid->len = len;
4347         desc = (struct scsi_vpd_id_descriptor *)lun->lun_devid->data;
4348         desc->proto_codeset = SVPD_ID_CODESET_ASCII;
4349         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
4350         desc->length = idlen1;
4351         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
4352         memset(t10id->vendor, ' ', sizeof(t10id->vendor));
4353         if ((vendor = ctl_get_opt(&be_lun->options, "vendor")) == NULL) {
4354                 strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
4355         } else {
4356                 strncpy(t10id->vendor, vendor,
4357                     min(sizeof(t10id->vendor), strlen(vendor)));
4358         }
4359         strncpy((char *)t10id->vendor_spec_id,
4360             (char *)be_lun->device_id, devidlen);
4361         if (scsiname != NULL) {
4362                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4363                     desc->length);
4364                 desc->proto_codeset = SVPD_ID_CODESET_UTF8;
4365                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4366                     SVPD_ID_TYPE_SCSI_NAME;
4367                 desc->length = idlen2;
4368                 strlcpy(desc->identifier, scsiname, idlen2);
4369         }
4370
4371         mtx_lock(&ctl_softc->ctl_lock);
4372         /*
4373          * See if the caller requested a particular LUN number.  If so, see
4374          * if it is available.  Otherwise, allocate the first available LUN.
4375          */
4376         if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4377                 if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4378                  || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4379                         mtx_unlock(&ctl_softc->ctl_lock);
4380                         if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4381                                 printf("ctl: requested LUN ID %d is higher "
4382                                        "than CTL_MAX_LUNS - 1 (%d)\n",
4383                                        be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4384                         } else {
4385                                 /*
4386                                  * XXX KDM return an error, or just assign
4387                                  * another LUN ID in this case??
4388                                  */
4389                                 printf("ctl: requested LUN ID %d is already "
4390                                        "in use\n", be_lun->req_lun_id);
4391                         }
4392                         if (lun->flags & CTL_LUN_MALLOCED)
4393                                 free(lun, M_CTL);
4394                         be_lun->lun_config_status(be_lun->be_lun,
4395                                                   CTL_LUN_CONFIG_FAILURE);
4396                         return (ENOSPC);
4397                 }
4398                 lun_number = be_lun->req_lun_id;
4399         } else {
4400                 lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, CTL_MAX_LUNS);
4401                 if (lun_number == -1) {
4402                         mtx_unlock(&ctl_softc->ctl_lock);
4403                         printf("ctl: can't allocate LUN on target %ju, out of "
4404                                "LUNs\n", (uintmax_t)target_id.id);
4405                         if (lun->flags & CTL_LUN_MALLOCED)
4406                                 free(lun, M_CTL);
4407                         be_lun->lun_config_status(be_lun->be_lun,
4408                                                   CTL_LUN_CONFIG_FAILURE);
4409                         return (ENOSPC);
4410                 }
4411         }
4412         ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4413
4414         mtx_init(&lun->lun_lock, "CTL LUN", NULL, MTX_DEF);
4415         lun->target = target_id;
4416         lun->lun = lun_number;
4417         lun->be_lun = be_lun;
4418         /*
4419          * The processor LUN is always enabled.  Disk LUNs come on line
4420          * disabled, and must be enabled by the backend.
4421          */
4422         lun->flags |= CTL_LUN_DISABLED;
4423         lun->backend = be_lun->be;
4424         be_lun->ctl_lun = lun;
4425         be_lun->lun_id = lun_number;
4426         atomic_add_int(&be_lun->be->num_luns, 1);
4427         if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4428                 lun->flags |= CTL_LUN_STOPPED;
4429
4430         if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4431                 lun->flags |= CTL_LUN_INOPERABLE;
4432
4433         if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4434                 lun->flags |= CTL_LUN_PRIMARY_SC;
4435
4436         lun->ctl_softc = ctl_softc;
4437         TAILQ_INIT(&lun->ooa_queue);
4438         TAILQ_INIT(&lun->blocked_queue);
4439         STAILQ_INIT(&lun->error_list);
4440
4441         /*
4442          * Initialize the mode page index.
4443          */
4444         ctl_init_page_index(lun);
4445
4446         /*
4447          * Set the poweron UA for all initiators on this LUN only.
4448          */
4449         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4450                 lun->pending_sense[i].ua_pending = CTL_UA_POWERON;
4451
4452         /*
4453          * Now, before we insert this lun on the lun list, set the lun
4454          * inventory changed UA for all other luns.
4455          */
4456         STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4457                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4458                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4459                 }
4460         }
4461
4462         STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4463
4464         ctl_softc->ctl_luns[lun_number] = lun;
4465
4466         ctl_softc->num_luns++;
4467
4468         /* Setup statistics gathering */
4469         lun->stats.device_type = be_lun->lun_type;
4470         lun->stats.lun_number = lun_number;
4471         if (lun->stats.device_type == T_DIRECT)
4472                 lun->stats.blocksize = be_lun->blocksize;
4473         else
4474                 lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4475         for (i = 0;i < CTL_MAX_PORTS;i++)
4476                 lun->stats.ports[i].targ_port = i;
4477
4478         mtx_unlock(&ctl_softc->ctl_lock);
4479
4480         lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4481
4482         /*
4483          * Run through each registered FETD and bring it online if it isn't
4484          * already.  Enable the target ID if it hasn't been enabled, and
4485          * enable this particular LUN.
4486          */
4487         STAILQ_FOREACH(port, &ctl_softc->port_list, links) {
4488                 int retval;
4489
4490                 retval = port->lun_enable(port->targ_lun_arg, target_id,lun_number);
4491                 if (retval != 0) {
4492                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4493                                "%d for lun_enable on target %ju lun %d\n",
4494                                port->port_name, port->targ_port, retval,
4495                                (uintmax_t)target_id.id, lun_number);
4496                 } else
4497                         port->status |= CTL_PORT_STATUS_LUN_ONLINE;
4498         }
4499         return (0);
4500 }
4501
4502 /*
4503  * Delete a LUN.
4504  * Assumptions:
4505  * - LUN has already been marked invalid and any pending I/O has been taken
4506  *   care of.
4507  */
4508 static int
4509 ctl_free_lun(struct ctl_lun *lun)
4510 {
4511         struct ctl_softc *softc;
4512 #if 0
4513         struct ctl_port *port;
4514 #endif
4515         struct ctl_lun *nlun;
4516         int i;
4517
4518         softc = lun->ctl_softc;
4519
4520         mtx_assert(&softc->ctl_lock, MA_OWNED);
4521
4522         STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4523
4524         ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4525
4526         softc->ctl_luns[lun->lun] = NULL;
4527
4528         if (!TAILQ_EMPTY(&lun->ooa_queue))
4529                 panic("Freeing a LUN %p with outstanding I/O!!\n", lun);
4530
4531         softc->num_luns--;
4532
4533         /*
4534          * XXX KDM this scheme only works for a single target/multiple LUN
4535          * setup.  It needs to be revamped for a multiple target scheme.
4536          *
4537          * XXX KDM this results in port->lun_disable() getting called twice,
4538          * once when ctl_disable_lun() is called, and a second time here.
4539          * We really need to re-think the LUN disable semantics.  There
4540          * should probably be several steps/levels to LUN removal:
4541          *  - disable
4542          *  - invalidate
4543          *  - free
4544          *
4545          * Right now we only have a disable method when communicating to
4546          * the front end ports, at least for individual LUNs.
4547          */
4548 #if 0
4549         STAILQ_FOREACH(port, &softc->port_list, links) {
4550                 int retval;
4551
4552                 retval = port->lun_disable(port->targ_lun_arg, lun->target,
4553                                          lun->lun);
4554                 if (retval != 0) {
4555                         printf("ctl_free_lun: FETD %s port %d returned error "
4556                                "%d for lun_disable on target %ju lun %jd\n",
4557                                port->port_name, port->targ_port, retval,
4558                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4559                 }
4560
4561                 if (STAILQ_FIRST(&softc->lun_list) == NULL) {
4562                         port->status &= ~CTL_PORT_STATUS_LUN_ONLINE;
4563
4564                         retval = port->targ_disable(port->targ_lun_arg,lun->target);
4565                         if (retval != 0) {
4566                                 printf("ctl_free_lun: FETD %s port %d "
4567                                        "returned error %d for targ_disable on "
4568                                        "target %ju\n", port->port_name,
4569                                        port->targ_port, retval,
4570                                        (uintmax_t)lun->target.id);
4571                         } else
4572                                 port->status &= ~CTL_PORT_STATUS_TARG_ONLINE;
4573
4574                         if ((port->status & CTL_PORT_STATUS_TARG_ONLINE) != 0)
4575                                 continue;
4576
4577 #if 0
4578                         port->port_offline(port->onoff_arg);
4579                         port->status &= ~CTL_PORT_STATUS_ONLINE;
4580 #endif
4581                 }
4582         }
4583 #endif
4584
4585         /*
4586          * Tell the backend to free resources, if this LUN has a backend.
4587          */
4588         atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4589         lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4590
4591         mtx_destroy(&lun->lun_lock);
4592         free(lun->lun_devid, M_CTL);
4593         if (lun->flags & CTL_LUN_MALLOCED)
4594                 free(lun, M_CTL);
4595
4596         STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4597                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4598                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4599                 }
4600         }
4601
4602         return (0);
4603 }
4604
4605 static void
4606 ctl_create_lun(struct ctl_be_lun *be_lun)
4607 {
4608         struct ctl_softc *ctl_softc;
4609
4610         ctl_softc = control_softc;
4611
4612         /*
4613          * ctl_alloc_lun() should handle all potential failure cases.
4614          */
4615         ctl_alloc_lun(ctl_softc, NULL, be_lun, ctl_softc->target);
4616 }
4617
4618 int
4619 ctl_add_lun(struct ctl_be_lun *be_lun)
4620 {
4621         struct ctl_softc *ctl_softc = control_softc;
4622
4623         mtx_lock(&ctl_softc->ctl_lock);
4624         STAILQ_INSERT_TAIL(&ctl_softc->pending_lun_queue, be_lun, links);
4625         mtx_unlock(&ctl_softc->ctl_lock);
4626         wakeup(&ctl_softc->pending_lun_queue);
4627
4628         return (0);
4629 }
4630
4631 int
4632 ctl_enable_lun(struct ctl_be_lun *be_lun)
4633 {
4634         struct ctl_softc *ctl_softc;
4635         struct ctl_port *port, *nport;
4636         struct ctl_lun *lun;
4637         int retval;
4638
4639         ctl_softc = control_softc;
4640
4641         lun = (struct ctl_lun *)be_lun->ctl_lun;
4642
4643         mtx_lock(&ctl_softc->ctl_lock);
4644         mtx_lock(&lun->lun_lock);
4645         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4646                 /*
4647                  * eh?  Why did we get called if the LUN is already
4648                  * enabled?
4649                  */
4650                 mtx_unlock(&lun->lun_lock);
4651                 mtx_unlock(&ctl_softc->ctl_lock);
4652                 return (0);
4653         }
4654         lun->flags &= ~CTL_LUN_DISABLED;
4655         mtx_unlock(&lun->lun_lock);
4656
4657         for (port = STAILQ_FIRST(&ctl_softc->port_list); port != NULL; port = nport) {
4658                 nport = STAILQ_NEXT(port, links);
4659
4660                 /*
4661                  * Drop the lock while we call the FETD's enable routine.
4662                  * This can lead to a callback into CTL (at least in the
4663                  * case of the internal initiator frontend.
4664                  */
4665                 mtx_unlock(&ctl_softc->ctl_lock);
4666                 retval = port->lun_enable(port->targ_lun_arg, lun->target,lun->lun);
4667                 mtx_lock(&ctl_softc->ctl_lock);
4668                 if (retval != 0) {
4669                         printf("%s: FETD %s port %d returned error "
4670                                "%d for lun_enable on target %ju lun %jd\n",
4671                                __func__, port->port_name, port->targ_port, retval,
4672                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4673                 }
4674 #if 0
4675                  else {
4676             /* NOTE:  TODO:  why does lun enable affect port status? */
4677                         port->status |= CTL_PORT_STATUS_LUN_ONLINE;
4678                 }
4679 #endif
4680         }
4681
4682         mtx_unlock(&ctl_softc->ctl_lock);
4683
4684         return (0);
4685 }
4686
4687 int
4688 ctl_disable_lun(struct ctl_be_lun *be_lun)
4689 {
4690         struct ctl_softc *ctl_softc;
4691         struct ctl_port *port;
4692         struct ctl_lun *lun;
4693         int retval;
4694
4695         ctl_softc = control_softc;
4696
4697         lun = (struct ctl_lun *)be_lun->ctl_lun;
4698
4699         mtx_lock(&ctl_softc->ctl_lock);
4700         mtx_lock(&lun->lun_lock);
4701         if (lun->flags & CTL_LUN_DISABLED) {
4702                 mtx_unlock(&lun->lun_lock);
4703                 mtx_unlock(&ctl_softc->ctl_lock);
4704                 return (0);
4705         }
4706         lun->flags |= CTL_LUN_DISABLED;
4707         mtx_unlock(&lun->lun_lock);
4708
4709         STAILQ_FOREACH(port, &ctl_softc->port_list, links) {
4710                 mtx_unlock(&ctl_softc->ctl_lock);
4711                 /*
4712                  * Drop the lock before we call the frontend's disable
4713                  * routine, to avoid lock order reversals.
4714                  *
4715                  * XXX KDM what happens if the frontend list changes while
4716                  * we're traversing it?  It's unlikely, but should be handled.
4717                  */
4718                 retval = port->lun_disable(port->targ_lun_arg, lun->target,
4719                                          lun->lun);
4720                 mtx_lock(&ctl_softc->ctl_lock);
4721                 if (retval != 0) {
4722                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4723                                "%d for lun_disable on target %ju lun %jd\n",
4724                                port->port_name, port->targ_port, retval,
4725                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4726                 }
4727         }
4728
4729         mtx_unlock(&ctl_softc->ctl_lock);
4730
4731         return (0);
4732 }
4733
4734 int
4735 ctl_start_lun(struct ctl_be_lun *be_lun)
4736 {
4737         struct ctl_softc *ctl_softc;
4738         struct ctl_lun *lun;
4739
4740         ctl_softc = control_softc;
4741
4742         lun = (struct ctl_lun *)be_lun->ctl_lun;
4743
4744         mtx_lock(&lun->lun_lock);
4745         lun->flags &= ~CTL_LUN_STOPPED;
4746         mtx_unlock(&lun->lun_lock);
4747
4748         return (0);
4749 }
4750
4751 int
4752 ctl_stop_lun(struct ctl_be_lun *be_lun)
4753 {
4754         struct ctl_softc *ctl_softc;
4755         struct ctl_lun *lun;
4756
4757         ctl_softc = control_softc;
4758
4759         lun = (struct ctl_lun *)be_lun->ctl_lun;
4760
4761         mtx_lock(&lun->lun_lock);
4762         lun->flags |= CTL_LUN_STOPPED;
4763         mtx_unlock(&lun->lun_lock);
4764
4765         return (0);
4766 }
4767
4768 int
4769 ctl_lun_offline(struct ctl_be_lun *be_lun)
4770 {
4771         struct ctl_softc *ctl_softc;
4772         struct ctl_lun *lun;
4773
4774         ctl_softc = control_softc;
4775
4776         lun = (struct ctl_lun *)be_lun->ctl_lun;
4777
4778         mtx_lock(&lun->lun_lock);
4779         lun->flags |= CTL_LUN_OFFLINE;
4780         mtx_unlock(&lun->lun_lock);
4781
4782         return (0);
4783 }
4784
4785 int
4786 ctl_lun_online(struct ctl_be_lun *be_lun)
4787 {
4788         struct ctl_softc *ctl_softc;
4789         struct ctl_lun *lun;
4790
4791         ctl_softc = control_softc;
4792
4793         lun = (struct ctl_lun *)be_lun->ctl_lun;
4794
4795         mtx_lock(&lun->lun_lock);
4796         lun->flags &= ~CTL_LUN_OFFLINE;
4797         mtx_unlock(&lun->lun_lock);
4798
4799         return (0);
4800 }
4801
4802 int
4803 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4804 {
4805         struct ctl_softc *ctl_softc;
4806         struct ctl_lun *lun;
4807
4808         ctl_softc = control_softc;
4809
4810         lun = (struct ctl_lun *)be_lun->ctl_lun;
4811
4812         mtx_lock(&lun->lun_lock);
4813
4814         /*
4815          * The LUN needs to be disabled before it can be marked invalid.
4816          */
4817         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4818                 mtx_unlock(&lun->lun_lock);
4819                 return (-1);
4820         }
4821         /*
4822          * Mark the LUN invalid.
4823          */
4824         lun->flags |= CTL_LUN_INVALID;
4825
4826         /*
4827          * If there is nothing in the OOA queue, go ahead and free the LUN.
4828          * If we have something in the OOA queue, we'll free it when the
4829          * last I/O completes.
4830          */
4831         if (TAILQ_EMPTY(&lun->ooa_queue)) {
4832                 mtx_unlock(&lun->lun_lock);
4833                 mtx_lock(&ctl_softc->ctl_lock);
4834                 ctl_free_lun(lun);
4835                 mtx_unlock(&ctl_softc->ctl_lock);
4836         } else
4837                 mtx_unlock(&lun->lun_lock);
4838
4839         return (0);
4840 }
4841
4842 int
4843 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
4844 {
4845         struct ctl_softc *ctl_softc;
4846         struct ctl_lun *lun;
4847
4848         ctl_softc = control_softc;
4849         lun = (struct ctl_lun *)be_lun->ctl_lun;
4850
4851         mtx_lock(&lun->lun_lock);
4852         lun->flags |= CTL_LUN_INOPERABLE;
4853         mtx_unlock(&lun->lun_lock);
4854
4855         return (0);
4856 }
4857
4858 int
4859 ctl_lun_operable(struct ctl_be_lun *be_lun)
4860 {
4861         struct ctl_softc *ctl_softc;
4862         struct ctl_lun *lun;
4863
4864         ctl_softc = control_softc;
4865         lun = (struct ctl_lun *)be_lun->ctl_lun;
4866
4867         mtx_lock(&lun->lun_lock);
4868         lun->flags &= ~CTL_LUN_INOPERABLE;
4869         mtx_unlock(&lun->lun_lock);
4870
4871         return (0);
4872 }
4873
4874 int
4875 ctl_lun_power_lock(struct ctl_be_lun *be_lun, struct ctl_nexus *nexus,
4876                    int lock)
4877 {
4878         struct ctl_softc *softc;
4879         struct ctl_lun *lun;
4880         struct copan_aps_subpage *current_sp;
4881         struct ctl_page_index *page_index;
4882         int i;
4883
4884         softc = control_softc;
4885
4886         mtx_lock(&softc->ctl_lock);
4887
4888         lun = (struct ctl_lun *)be_lun->ctl_lun;
4889         mtx_lock(&lun->lun_lock);
4890
4891         page_index = NULL;
4892         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4893                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
4894                      APS_PAGE_CODE)
4895                         continue;
4896
4897                 if (lun->mode_pages.index[i].subpage != APS_SUBPAGE_CODE)
4898                         continue;
4899                 page_index = &lun->mode_pages.index[i];
4900         }
4901
4902         if (page_index == NULL) {
4903                 mtx_unlock(&lun->lun_lock);
4904                 mtx_unlock(&softc->ctl_lock);
4905                 printf("%s: APS subpage not found for lun %ju!\n", __func__,
4906                        (uintmax_t)lun->lun);
4907                 return (1);
4908         }
4909 #if 0
4910         if ((softc->aps_locked_lun != 0)
4911          && (softc->aps_locked_lun != lun->lun)) {
4912                 printf("%s: attempt to lock LUN %llu when %llu is already "
4913                        "locked\n");
4914                 mtx_unlock(&lun->lun_lock);
4915                 mtx_unlock(&softc->ctl_lock);
4916                 return (1);
4917         }
4918 #endif
4919
4920         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
4921                 (page_index->page_len * CTL_PAGE_CURRENT));
4922
4923         if (lock != 0) {
4924                 current_sp->lock_active = APS_LOCK_ACTIVE;
4925                 softc->aps_locked_lun = lun->lun;
4926         } else {
4927                 current_sp->lock_active = 0;
4928                 softc->aps_locked_lun = 0;
4929         }
4930
4931
4932         /*
4933          * If we're in HA mode, try to send the lock message to the other
4934          * side.
4935          */
4936         if (ctl_is_single == 0) {
4937                 int isc_retval;
4938                 union ctl_ha_msg lock_msg;
4939
4940                 lock_msg.hdr.nexus = *nexus;
4941                 lock_msg.hdr.msg_type = CTL_MSG_APS_LOCK;
4942                 if (lock != 0)
4943                         lock_msg.aps.lock_flag = 1;
4944                 else
4945                         lock_msg.aps.lock_flag = 0;
4946                 isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &lock_msg,
4947                                          sizeof(lock_msg), 0);
4948                 if (isc_retval > CTL_HA_STATUS_SUCCESS) {
4949                         printf("%s: APS (lock=%d) error returned from "
4950                                "ctl_ha_msg_send: %d\n", __func__, lock, isc_retval);
4951                         mtx_unlock(&lun->lun_lock);
4952                         mtx_unlock(&softc->ctl_lock);
4953                         return (1);
4954                 }
4955         }
4956
4957         mtx_unlock(&lun->lun_lock);
4958         mtx_unlock(&softc->ctl_lock);
4959
4960         return (0);
4961 }
4962
4963 void
4964 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
4965 {
4966         struct ctl_lun *lun;
4967         struct ctl_softc *softc;
4968         int i;
4969
4970         softc = control_softc;
4971
4972         lun = (struct ctl_lun *)be_lun->ctl_lun;
4973
4974         mtx_lock(&lun->lun_lock);
4975
4976         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4977                 lun->pending_sense[i].ua_pending |= CTL_UA_CAPACITY_CHANGED;
4978
4979         mtx_unlock(&lun->lun_lock);
4980 }
4981
4982 /*
4983  * Backend "memory move is complete" callback for requests that never
4984  * make it down to say RAIDCore's configuration code.
4985  */
4986 int
4987 ctl_config_move_done(union ctl_io *io)
4988 {
4989         int retval;
4990
4991         retval = CTL_RETVAL_COMPLETE;
4992
4993
4994         CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
4995         /*
4996          * XXX KDM this shouldn't happen, but what if it does?
4997          */
4998         if (io->io_hdr.io_type != CTL_IO_SCSI)
4999                 panic("I/O type isn't CTL_IO_SCSI!");
5000
5001         if ((io->io_hdr.port_status == 0)
5002          && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
5003          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
5004                 io->io_hdr.status = CTL_SUCCESS;
5005         else if ((io->io_hdr.port_status != 0)
5006               && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
5007               && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)){
5008                 /*
5009                  * For hardware error sense keys, the sense key
5010                  * specific value is defined to be a retry count,
5011                  * but we use it to pass back an internal FETD
5012                  * error code.  XXX KDM  Hopefully the FETD is only
5013                  * using 16 bits for an error code, since that's
5014                  * all the space we have in the sks field.
5015                  */
5016                 ctl_set_internal_failure(&io->scsiio,
5017                                          /*sks_valid*/ 1,
5018                                          /*retry_count*/
5019                                          io->io_hdr.port_status);
5020                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5021                         free(io->scsiio.kern_data_ptr, M_CTL);
5022                 ctl_done(io);
5023                 goto bailout;
5024         }
5025
5026         if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
5027          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
5028          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
5029                 /*
5030                  * XXX KDM just assuming a single pointer here, and not a
5031                  * S/G list.  If we start using S/G lists for config data,
5032                  * we'll need to know how to clean them up here as well.
5033                  */
5034                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5035                         free(io->scsiio.kern_data_ptr, M_CTL);
5036                 /* Hopefully the user has already set the status... */
5037                 ctl_done(io);
5038         } else {
5039                 /*
5040                  * XXX KDM now we need to continue data movement.  Some
5041                  * options:
5042                  * - call ctl_scsiio() again?  We don't do this for data
5043                  *   writes, because for those at least we know ahead of
5044                  *   time where the write will go and how long it is.  For
5045                  *   config writes, though, that information is largely
5046                  *   contained within the write itself, thus we need to
5047                  *   parse out the data again.
5048                  *
5049                  * - Call some other function once the data is in?
5050                  */
5051
5052                 /*
5053                  * XXX KDM call ctl_scsiio() again for now, and check flag
5054                  * bits to see whether we're allocated or not.
5055                  */
5056                 retval = ctl_scsiio(&io->scsiio);
5057         }
5058 bailout:
5059         return (retval);
5060 }
5061
5062 /*
5063  * This gets called by a backend driver when it is done with a
5064  * data_submit method.
5065  */
5066 void
5067 ctl_data_submit_done(union ctl_io *io)
5068 {
5069         /*
5070          * If the IO_CONT flag is set, we need to call the supplied
5071          * function to continue processing the I/O, instead of completing
5072          * the I/O just yet.
5073          *
5074          * If there is an error, though, we don't want to keep processing.
5075          * Instead, just send status back to the initiator.
5076          */
5077         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5078             (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5079             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5080              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5081                 io->scsiio.io_cont(io);
5082                 return;
5083         }
5084         ctl_done(io);
5085 }
5086
5087 /*
5088  * This gets called by a backend driver when it is done with a
5089  * configuration write.
5090  */
5091 void
5092 ctl_config_write_done(union ctl_io *io)
5093 {
5094         /*
5095          * If the IO_CONT flag is set, we need to call the supplied
5096          * function to continue processing the I/O, instead of completing
5097          * the I/O just yet.
5098          *
5099          * If there is an error, though, we don't want to keep processing.
5100          * Instead, just send status back to the initiator.
5101          */
5102         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT)
5103          && (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)
5104           || ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))) {
5105                 io->scsiio.io_cont(io);
5106                 return;
5107         }
5108         /*
5109          * Since a configuration write can be done for commands that actually
5110          * have data allocated, like write buffer, and commands that have
5111          * no data, like start/stop unit, we need to check here.
5112          */
5113         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
5114                 free(io->scsiio.kern_data_ptr, M_CTL);
5115         ctl_done(io);
5116 }
5117
5118 /*
5119  * SCSI release command.
5120  */
5121 int
5122 ctl_scsi_release(struct ctl_scsiio *ctsio)
5123 {
5124         int length, longid, thirdparty_id, resv_id;
5125         struct ctl_softc *ctl_softc;
5126         struct ctl_lun *lun;
5127
5128         length = 0;
5129         resv_id = 0;
5130
5131         CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
5132
5133         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5134         ctl_softc = control_softc;
5135
5136         switch (ctsio->cdb[0]) {
5137         case RELEASE_10: {
5138                 struct scsi_release_10 *cdb;
5139
5140                 cdb = (struct scsi_release_10 *)ctsio->cdb;
5141
5142                 if (cdb->byte2 & SR10_LONGID)
5143                         longid = 1;
5144                 else
5145                         thirdparty_id = cdb->thirdparty_id;
5146
5147                 resv_id = cdb->resv_id;
5148                 length = scsi_2btoul(cdb->length);
5149                 break;
5150         }
5151         }
5152
5153
5154         /*
5155          * XXX KDM right now, we only support LUN reservation.  We don't
5156          * support 3rd party reservations, or extent reservations, which
5157          * might actually need the parameter list.  If we've gotten this
5158          * far, we've got a LUN reservation.  Anything else got kicked out
5159          * above.  So, according to SPC, ignore the length.
5160          */
5161         length = 0;
5162
5163         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5164          && (length > 0)) {
5165                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5166                 ctsio->kern_data_len = length;
5167                 ctsio->kern_total_len = length;
5168                 ctsio->kern_data_resid = 0;
5169                 ctsio->kern_rel_offset = 0;
5170                 ctsio->kern_sg_entries = 0;
5171                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5172                 ctsio->be_move_done = ctl_config_move_done;
5173                 ctl_datamove((union ctl_io *)ctsio);
5174
5175                 return (CTL_RETVAL_COMPLETE);
5176         }
5177
5178         if (length > 0)
5179                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5180
5181         mtx_lock(&lun->lun_lock);
5182
5183         /*
5184          * According to SPC, it is not an error for an intiator to attempt
5185          * to release a reservation on a LUN that isn't reserved, or that
5186          * is reserved by another initiator.  The reservation can only be
5187          * released, though, by the initiator who made it or by one of
5188          * several reset type events.
5189          */
5190         if (lun->flags & CTL_LUN_RESERVED) {
5191                 if ((ctsio->io_hdr.nexus.initid.id == lun->rsv_nexus.initid.id)
5192                  && (ctsio->io_hdr.nexus.targ_port == lun->rsv_nexus.targ_port)
5193                  && (ctsio->io_hdr.nexus.targ_target.id ==
5194                      lun->rsv_nexus.targ_target.id)) {
5195                         lun->flags &= ~CTL_LUN_RESERVED;
5196                 }
5197         }
5198
5199         mtx_unlock(&lun->lun_lock);
5200
5201         ctsio->scsi_status = SCSI_STATUS_OK;
5202         ctsio->io_hdr.status = CTL_SUCCESS;
5203
5204         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5205                 free(ctsio->kern_data_ptr, M_CTL);
5206                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5207         }
5208
5209         ctl_done((union ctl_io *)ctsio);
5210         return (CTL_RETVAL_COMPLETE);
5211 }
5212
5213 int
5214 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5215 {
5216         int extent, thirdparty, longid;
5217         int resv_id, length;
5218         uint64_t thirdparty_id;
5219         struct ctl_softc *ctl_softc;
5220         struct ctl_lun *lun;
5221
5222         extent = 0;
5223         thirdparty = 0;
5224         longid = 0;
5225         resv_id = 0;
5226         length = 0;
5227         thirdparty_id = 0;
5228
5229         CTL_DEBUG_PRINT(("ctl_reserve\n"));
5230
5231         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5232         ctl_softc = control_softc;
5233
5234         switch (ctsio->cdb[0]) {
5235         case RESERVE_10: {
5236                 struct scsi_reserve_10 *cdb;
5237
5238                 cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5239
5240                 if (cdb->byte2 & SR10_LONGID)
5241                         longid = 1;
5242                 else
5243                         thirdparty_id = cdb->thirdparty_id;
5244
5245                 resv_id = cdb->resv_id;
5246                 length = scsi_2btoul(cdb->length);
5247                 break;
5248         }
5249         }
5250
5251         /*
5252          * XXX KDM right now, we only support LUN reservation.  We don't
5253          * support 3rd party reservations, or extent reservations, which
5254          * might actually need the parameter list.  If we've gotten this
5255          * far, we've got a LUN reservation.  Anything else got kicked out
5256          * above.  So, according to SPC, ignore the length.
5257          */
5258         length = 0;
5259
5260         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5261          && (length > 0)) {
5262                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5263                 ctsio->kern_data_len = length;
5264                 ctsio->kern_total_len = length;
5265                 ctsio->kern_data_resid = 0;
5266                 ctsio->kern_rel_offset = 0;
5267                 ctsio->kern_sg_entries = 0;
5268                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5269                 ctsio->be_move_done = ctl_config_move_done;
5270                 ctl_datamove((union ctl_io *)ctsio);
5271
5272                 return (CTL_RETVAL_COMPLETE);
5273         }
5274
5275         if (length > 0)
5276                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5277
5278         mtx_lock(&lun->lun_lock);
5279         if (lun->flags & CTL_LUN_RESERVED) {
5280                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
5281                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
5282                  || (ctsio->io_hdr.nexus.targ_target.id !=
5283                      lun->rsv_nexus.targ_target.id)) {
5284                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
5285                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
5286                         goto bailout;
5287                 }
5288         }
5289
5290         lun->flags |= CTL_LUN_RESERVED;
5291         lun->rsv_nexus = ctsio->io_hdr.nexus;
5292
5293         ctsio->scsi_status = SCSI_STATUS_OK;
5294         ctsio->io_hdr.status = CTL_SUCCESS;
5295
5296 bailout:
5297         mtx_unlock(&lun->lun_lock);
5298
5299         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5300                 free(ctsio->kern_data_ptr, M_CTL);
5301                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5302         }
5303
5304         ctl_done((union ctl_io *)ctsio);
5305         return (CTL_RETVAL_COMPLETE);
5306 }
5307
5308 int
5309 ctl_start_stop(struct ctl_scsiio *ctsio)
5310 {
5311         struct scsi_start_stop_unit *cdb;
5312         struct ctl_lun *lun;
5313         struct ctl_softc *ctl_softc;
5314         int retval;
5315
5316         CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5317
5318         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5319         ctl_softc = control_softc;
5320         retval = 0;
5321
5322         cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5323
5324         /*
5325          * XXX KDM
5326          * We don't support the immediate bit on a stop unit.  In order to
5327          * do that, we would need to code up a way to know that a stop is
5328          * pending, and hold off any new commands until it completes, one
5329          * way or another.  Then we could accept or reject those commands
5330          * depending on its status.  We would almost need to do the reverse
5331          * of what we do below for an immediate start -- return the copy of
5332          * the ctl_io to the FETD with status to send to the host (and to
5333          * free the copy!) and then free the original I/O once the stop
5334          * actually completes.  That way, the OOA queue mechanism can work
5335          * to block commands that shouldn't proceed.  Another alternative
5336          * would be to put the copy in the queue in place of the original,
5337          * and return the original back to the caller.  That could be
5338          * slightly safer..
5339          */
5340         if ((cdb->byte2 & SSS_IMMED)
5341          && ((cdb->how & SSS_START) == 0)) {
5342                 ctl_set_invalid_field(ctsio,
5343                                       /*sks_valid*/ 1,
5344                                       /*command*/ 1,
5345                                       /*field*/ 1,
5346                                       /*bit_valid*/ 1,
5347                                       /*bit*/ 0);
5348                 ctl_done((union ctl_io *)ctsio);
5349                 return (CTL_RETVAL_COMPLETE);
5350         }
5351
5352         if ((lun->flags & CTL_LUN_PR_RESERVED)
5353          && ((cdb->how & SSS_START)==0)) {
5354                 uint32_t residx;
5355
5356                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5357                 if (!lun->per_res[residx].registered
5358                  || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5359
5360                         ctl_set_reservation_conflict(ctsio);
5361                         ctl_done((union ctl_io *)ctsio);
5362                         return (CTL_RETVAL_COMPLETE);
5363                 }
5364         }
5365
5366         /*
5367          * If there is no backend on this device, we can't start or stop
5368          * it.  In theory we shouldn't get any start/stop commands in the
5369          * first place at this level if the LUN doesn't have a backend.
5370          * That should get stopped by the command decode code.
5371          */
5372         if (lun->backend == NULL) {
5373                 ctl_set_invalid_opcode(ctsio);
5374                 ctl_done((union ctl_io *)ctsio);
5375                 return (CTL_RETVAL_COMPLETE);
5376         }
5377
5378         /*
5379          * XXX KDM Copan-specific offline behavior.
5380          * Figure out a reasonable way to port this?
5381          */
5382 #ifdef NEEDTOPORT
5383         mtx_lock(&lun->lun_lock);
5384
5385         if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5386          && (lun->flags & CTL_LUN_OFFLINE)) {
5387                 /*
5388                  * If the LUN is offline, and the on/offline bit isn't set,
5389                  * reject the start or stop.  Otherwise, let it through.
5390                  */
5391                 mtx_unlock(&lun->lun_lock);
5392                 ctl_set_lun_not_ready(ctsio);
5393                 ctl_done((union ctl_io *)ctsio);
5394         } else {
5395                 mtx_unlock(&lun->lun_lock);
5396 #endif /* NEEDTOPORT */
5397                 /*
5398                  * This could be a start or a stop when we're online,
5399                  * or a stop/offline or start/online.  A start or stop when
5400                  * we're offline is covered in the case above.
5401                  */
5402                 /*
5403                  * In the non-immediate case, we send the request to
5404                  * the backend and return status to the user when
5405                  * it is done.
5406                  *
5407                  * In the immediate case, we allocate a new ctl_io
5408                  * to hold a copy of the request, and send that to
5409                  * the backend.  We then set good status on the
5410                  * user's request and return it immediately.
5411                  */
5412                 if (cdb->byte2 & SSS_IMMED) {
5413                         union ctl_io *new_io;
5414
5415                         new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5416                         if (new_io == NULL) {
5417                                 ctl_set_busy(ctsio);
5418                                 ctl_done((union ctl_io *)ctsio);
5419                         } else {
5420                                 ctl_copy_io((union ctl_io *)ctsio,
5421                                             new_io);
5422                                 retval = lun->backend->config_write(new_io);
5423                                 ctl_set_success(ctsio);
5424                                 ctl_done((union ctl_io *)ctsio);
5425                         }
5426                 } else {
5427                         retval = lun->backend->config_write(
5428                                 (union ctl_io *)ctsio);
5429                 }
5430 #ifdef NEEDTOPORT
5431         }
5432 #endif
5433         return (retval);
5434 }
5435
5436 /*
5437  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5438  * we don't really do anything with the LBA and length fields if the user
5439  * passes them in.  Instead we'll just flush out the cache for the entire
5440  * LUN.
5441  */
5442 int
5443 ctl_sync_cache(struct ctl_scsiio *ctsio)
5444 {
5445         struct ctl_lun *lun;
5446         struct ctl_softc *ctl_softc;
5447         uint64_t starting_lba;
5448         uint32_t block_count;
5449         int retval;
5450
5451         CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5452
5453         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5454         ctl_softc = control_softc;
5455         retval = 0;
5456
5457         switch (ctsio->cdb[0]) {
5458         case SYNCHRONIZE_CACHE: {
5459                 struct scsi_sync_cache *cdb;
5460                 cdb = (struct scsi_sync_cache *)ctsio->cdb;
5461
5462                 starting_lba = scsi_4btoul(cdb->begin_lba);
5463                 block_count = scsi_2btoul(cdb->lb_count);
5464                 break;
5465         }
5466         case SYNCHRONIZE_CACHE_16: {
5467                 struct scsi_sync_cache_16 *cdb;
5468                 cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5469
5470                 starting_lba = scsi_8btou64(cdb->begin_lba);
5471                 block_count = scsi_4btoul(cdb->lb_count);
5472                 break;
5473         }
5474         default:
5475                 ctl_set_invalid_opcode(ctsio);
5476                 ctl_done((union ctl_io *)ctsio);
5477                 goto bailout;
5478                 break; /* NOTREACHED */
5479         }
5480
5481         /*
5482          * We check the LBA and length, but don't do anything with them.
5483          * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5484          * get flushed.  This check will just help satisfy anyone who wants
5485          * to see an error for an out of range LBA.
5486          */
5487         if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5488                 ctl_set_lba_out_of_range(ctsio);
5489                 ctl_done((union ctl_io *)ctsio);
5490                 goto bailout;
5491         }
5492
5493         /*
5494          * If this LUN has no backend, we can't flush the cache anyway.
5495          */
5496         if (lun->backend == NULL) {
5497                 ctl_set_invalid_opcode(ctsio);
5498                 ctl_done((union ctl_io *)ctsio);
5499                 goto bailout;
5500         }
5501
5502         /*
5503          * Check to see whether we're configured to send the SYNCHRONIZE
5504          * CACHE command directly to the back end.
5505          */
5506         mtx_lock(&lun->lun_lock);
5507         if ((ctl_softc->flags & CTL_FLAG_REAL_SYNC)
5508          && (++(lun->sync_count) >= lun->sync_interval)) {
5509                 lun->sync_count = 0;
5510                 mtx_unlock(&lun->lun_lock);
5511                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5512         } else {
5513                 mtx_unlock(&lun->lun_lock);
5514                 ctl_set_success(ctsio);
5515                 ctl_done((union ctl_io *)ctsio);
5516         }
5517
5518 bailout:
5519
5520         return (retval);
5521 }
5522
5523 int
5524 ctl_format(struct ctl_scsiio *ctsio)
5525 {
5526         struct scsi_format *cdb;
5527         struct ctl_lun *lun;
5528         struct ctl_softc *ctl_softc;
5529         int length, defect_list_len;
5530
5531         CTL_DEBUG_PRINT(("ctl_format\n"));
5532
5533         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5534         ctl_softc = control_softc;
5535
5536         cdb = (struct scsi_format *)ctsio->cdb;
5537
5538         length = 0;
5539         if (cdb->byte2 & SF_FMTDATA) {
5540                 if (cdb->byte2 & SF_LONGLIST)
5541                         length = sizeof(struct scsi_format_header_long);
5542                 else
5543                         length = sizeof(struct scsi_format_header_short);
5544         }
5545
5546         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5547          && (length > 0)) {
5548                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5549                 ctsio->kern_data_len = length;
5550                 ctsio->kern_total_len = length;
5551                 ctsio->kern_data_resid = 0;
5552                 ctsio->kern_rel_offset = 0;
5553                 ctsio->kern_sg_entries = 0;
5554                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5555                 ctsio->be_move_done = ctl_config_move_done;
5556                 ctl_datamove((union ctl_io *)ctsio);
5557
5558                 return (CTL_RETVAL_COMPLETE);
5559         }
5560
5561         defect_list_len = 0;
5562
5563         if (cdb->byte2 & SF_FMTDATA) {
5564                 if (cdb->byte2 & SF_LONGLIST) {
5565                         struct scsi_format_header_long *header;
5566
5567                         header = (struct scsi_format_header_long *)
5568                                 ctsio->kern_data_ptr;
5569
5570                         defect_list_len = scsi_4btoul(header->defect_list_len);
5571                         if (defect_list_len != 0) {
5572                                 ctl_set_invalid_field(ctsio,
5573                                                       /*sks_valid*/ 1,
5574                                                       /*command*/ 0,
5575                                                       /*field*/ 2,
5576                                                       /*bit_valid*/ 0,
5577                                                       /*bit*/ 0);
5578                                 goto bailout;
5579                         }
5580                 } else {
5581                         struct scsi_format_header_short *header;
5582
5583                         header = (struct scsi_format_header_short *)
5584                                 ctsio->kern_data_ptr;
5585
5586                         defect_list_len = scsi_2btoul(header->defect_list_len);
5587                         if (defect_list_len != 0) {
5588                                 ctl_set_invalid_field(ctsio,
5589                                                       /*sks_valid*/ 1,
5590                                                       /*command*/ 0,
5591                                                       /*field*/ 2,
5592                                                       /*bit_valid*/ 0,
5593                                                       /*bit*/ 0);
5594                                 goto bailout;
5595                         }
5596                 }
5597         }
5598
5599         /*
5600          * The format command will clear out the "Medium format corrupted"
5601          * status if set by the configuration code.  That status is really
5602          * just a way to notify the host that we have lost the media, and
5603          * get them to issue a command that will basically make them think
5604          * they're blowing away the media.
5605          */
5606         mtx_lock(&lun->lun_lock);
5607         lun->flags &= ~CTL_LUN_INOPERABLE;
5608         mtx_unlock(&lun->lun_lock);
5609
5610         ctsio->scsi_status = SCSI_STATUS_OK;
5611         ctsio->io_hdr.status = CTL_SUCCESS;
5612 bailout:
5613
5614         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5615                 free(ctsio->kern_data_ptr, M_CTL);
5616                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5617         }
5618
5619         ctl_done((union ctl_io *)ctsio);
5620         return (CTL_RETVAL_COMPLETE);
5621 }
5622
5623 int
5624 ctl_read_buffer(struct ctl_scsiio *ctsio)
5625 {
5626         struct scsi_read_buffer *cdb;
5627         struct ctl_lun *lun;
5628         int buffer_offset, len;
5629         static uint8_t descr[4];
5630         static uint8_t echo_descr[4] = { 0 };
5631
5632         CTL_DEBUG_PRINT(("ctl_read_buffer\n"));
5633
5634         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5635         cdb = (struct scsi_read_buffer *)ctsio->cdb;
5636
5637         if (lun->flags & CTL_LUN_PR_RESERVED) {
5638                 uint32_t residx;
5639
5640                 /*
5641                  * XXX KDM need a lock here.
5642                  */
5643                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5644                 if ((lun->res_type == SPR_TYPE_EX_AC
5645                   && residx != lun->pr_res_idx)
5646                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
5647                    || lun->res_type == SPR_TYPE_EX_AC_AR)
5648                   && !lun->per_res[residx].registered)) {
5649                         ctl_set_reservation_conflict(ctsio);
5650                         ctl_done((union ctl_io *)ctsio);
5651                         return (CTL_RETVAL_COMPLETE);
5652                 }
5653         }
5654
5655         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA &&
5656             (cdb->byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
5657             (cdb->byte2 & RWB_MODE) != RWB_MODE_DESCR) {
5658                 ctl_set_invalid_field(ctsio,
5659                                       /*sks_valid*/ 1,
5660                                       /*command*/ 1,
5661                                       /*field*/ 1,
5662                                       /*bit_valid*/ 1,
5663                                       /*bit*/ 4);
5664                 ctl_done((union ctl_io *)ctsio);
5665                 return (CTL_RETVAL_COMPLETE);
5666         }
5667
5668         len = scsi_3btoul(cdb->length);
5669         buffer_offset = scsi_3btoul(cdb->offset);
5670
5671         if (buffer_offset + len > sizeof(lun->write_buffer)) {
5672                 ctl_set_invalid_field(ctsio,
5673                                       /*sks_valid*/ 1,
5674                                       /*command*/ 1,
5675                                       /*field*/ 6,
5676                                       /*bit_valid*/ 0,
5677                                       /*bit*/ 0);
5678                 ctl_done((union ctl_io *)ctsio);
5679                 return (CTL_RETVAL_COMPLETE);
5680         }
5681
5682         if ((cdb->byte2 & RWB_MODE) == RWB_MODE_DESCR) {
5683                 descr[0] = 0;
5684                 scsi_ulto3b(sizeof(lun->write_buffer), &descr[1]);
5685                 ctsio->kern_data_ptr = descr;
5686                 len = min(len, sizeof(descr));
5687         } else if ((cdb->byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
5688                 ctsio->kern_data_ptr = echo_descr;
5689                 len = min(len, sizeof(echo_descr));
5690         } else
5691                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5692         ctsio->kern_data_len = len;
5693         ctsio->kern_total_len = len;
5694         ctsio->kern_data_resid = 0;
5695         ctsio->kern_rel_offset = 0;
5696         ctsio->kern_sg_entries = 0;
5697         ctsio->be_move_done = ctl_config_move_done;
5698         ctl_datamove((union ctl_io *)ctsio);
5699
5700         return (CTL_RETVAL_COMPLETE);
5701 }
5702
5703 int
5704 ctl_write_buffer(struct ctl_scsiio *ctsio)
5705 {
5706         struct scsi_write_buffer *cdb;
5707         struct ctl_lun *lun;
5708         int buffer_offset, len;
5709
5710         CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5711
5712         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5713         cdb = (struct scsi_write_buffer *)ctsio->cdb;
5714
5715         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5716                 ctl_set_invalid_field(ctsio,
5717                                       /*sks_valid*/ 1,
5718                                       /*command*/ 1,
5719                                       /*field*/ 1,
5720                                       /*bit_valid*/ 1,
5721                                       /*bit*/ 4);
5722                 ctl_done((union ctl_io *)ctsio);
5723                 return (CTL_RETVAL_COMPLETE);
5724         }
5725
5726         len = scsi_3btoul(cdb->length);
5727         buffer_offset = scsi_3btoul(cdb->offset);
5728
5729         if (buffer_offset + len > sizeof(lun->write_buffer)) {
5730                 ctl_set_invalid_field(ctsio,
5731                                       /*sks_valid*/ 1,
5732                                       /*command*/ 1,
5733                                       /*field*/ 6,
5734                                       /*bit_valid*/ 0,
5735                                       /*bit*/ 0);
5736                 ctl_done((union ctl_io *)ctsio);
5737                 return (CTL_RETVAL_COMPLETE);
5738         }
5739
5740         /*
5741          * If we've got a kernel request that hasn't been malloced yet,
5742          * malloc it and tell the caller the data buffer is here.
5743          */
5744         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5745                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5746                 ctsio->kern_data_len = len;
5747                 ctsio->kern_total_len = len;
5748                 ctsio->kern_data_resid = 0;
5749                 ctsio->kern_rel_offset = 0;
5750                 ctsio->kern_sg_entries = 0;
5751                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5752                 ctsio->be_move_done = ctl_config_move_done;
5753                 ctl_datamove((union ctl_io *)ctsio);
5754
5755                 return (CTL_RETVAL_COMPLETE);
5756         }
5757
5758         ctl_done((union ctl_io *)ctsio);
5759
5760         return (CTL_RETVAL_COMPLETE);
5761 }
5762
5763 int
5764 ctl_write_same(struct ctl_scsiio *ctsio)
5765 {
5766         struct ctl_lun *lun;
5767         struct ctl_lba_len_flags *lbalen;
5768         uint64_t lba;
5769         uint32_t num_blocks;
5770         int len, retval;
5771         uint8_t byte2;
5772
5773         retval = CTL_RETVAL_COMPLETE;
5774
5775         CTL_DEBUG_PRINT(("ctl_write_same\n"));
5776
5777         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5778
5779         switch (ctsio->cdb[0]) {
5780         case WRITE_SAME_10: {
5781                 struct scsi_write_same_10 *cdb;
5782
5783                 cdb = (struct scsi_write_same_10 *)ctsio->cdb;
5784
5785                 lba = scsi_4btoul(cdb->addr);
5786                 num_blocks = scsi_2btoul(cdb->length);
5787                 byte2 = cdb->byte2;
5788                 break;
5789         }
5790         case WRITE_SAME_16: {
5791                 struct scsi_write_same_16 *cdb;
5792
5793                 cdb = (struct scsi_write_same_16 *)ctsio->cdb;
5794
5795                 lba = scsi_8btou64(cdb->addr);
5796                 num_blocks = scsi_4btoul(cdb->length);
5797                 byte2 = cdb->byte2;
5798                 break;
5799         }
5800         default:
5801                 /*
5802                  * We got a command we don't support.  This shouldn't
5803                  * happen, commands should be filtered out above us.
5804                  */
5805                 ctl_set_invalid_opcode(ctsio);
5806                 ctl_done((union ctl_io *)ctsio);
5807
5808                 return (CTL_RETVAL_COMPLETE);
5809                 break; /* NOTREACHED */
5810         }
5811
5812         /*
5813          * The first check is to make sure we're in bounds, the second
5814          * check is to catch wrap-around problems.  If the lba + num blocks
5815          * is less than the lba, then we've wrapped around and the block
5816          * range is invalid anyway.
5817          */
5818         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5819          || ((lba + num_blocks) < lba)) {
5820                 ctl_set_lba_out_of_range(ctsio);
5821                 ctl_done((union ctl_io *)ctsio);
5822                 return (CTL_RETVAL_COMPLETE);
5823         }
5824
5825         /* Zero number of blocks means "to the last logical block" */
5826         if (num_blocks == 0) {
5827                 if ((lun->be_lun->maxlba + 1) - lba > UINT32_MAX) {
5828                         ctl_set_invalid_field(ctsio,
5829                                               /*sks_valid*/ 0,
5830                                               /*command*/ 1,
5831                                               /*field*/ 0,
5832                                               /*bit_valid*/ 0,
5833                                               /*bit*/ 0);
5834                         ctl_done((union ctl_io *)ctsio);
5835                         return (CTL_RETVAL_COMPLETE);
5836                 }
5837                 num_blocks = (lun->be_lun->maxlba + 1) - lba;
5838         }
5839
5840         len = lun->be_lun->blocksize;
5841
5842         /*
5843          * If we've got a kernel request that hasn't been malloced yet,
5844          * malloc it and tell the caller the data buffer is here.
5845          */
5846         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5847                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5848                 ctsio->kern_data_len = len;
5849                 ctsio->kern_total_len = len;
5850                 ctsio->kern_data_resid = 0;
5851                 ctsio->kern_rel_offset = 0;
5852                 ctsio->kern_sg_entries = 0;
5853                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5854                 ctsio->be_move_done = ctl_config_move_done;
5855                 ctl_datamove((union ctl_io *)ctsio);
5856
5857                 return (CTL_RETVAL_COMPLETE);
5858         }
5859
5860         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5861         lbalen->lba = lba;
5862         lbalen->len = num_blocks;
5863         lbalen->flags = byte2;
5864         retval = lun->backend->config_write((union ctl_io *)ctsio);
5865
5866         return (retval);
5867 }
5868
5869 int
5870 ctl_unmap(struct ctl_scsiio *ctsio)
5871 {
5872         struct ctl_lun *lun;
5873         struct scsi_unmap *cdb;
5874         struct ctl_ptr_len_flags *ptrlen;
5875         struct scsi_unmap_header *hdr;
5876         struct scsi_unmap_desc *buf, *end;
5877         uint64_t lba;
5878         uint32_t num_blocks;
5879         int len, retval;
5880         uint8_t byte2;
5881
5882         retval = CTL_RETVAL_COMPLETE;
5883
5884         CTL_DEBUG_PRINT(("ctl_unmap\n"));
5885
5886         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5887         cdb = (struct scsi_unmap *)ctsio->cdb;
5888
5889         len = scsi_2btoul(cdb->length);
5890         byte2 = cdb->byte2;
5891
5892         /*
5893          * If we've got a kernel request that hasn't been malloced yet,
5894          * malloc it and tell the caller the data buffer is here.
5895          */
5896         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5897                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5898                 ctsio->kern_data_len = len;
5899                 ctsio->kern_total_len = len;
5900                 ctsio->kern_data_resid = 0;
5901                 ctsio->kern_rel_offset = 0;
5902                 ctsio->kern_sg_entries = 0;
5903                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5904                 ctsio->be_move_done = ctl_config_move_done;
5905                 ctl_datamove((union ctl_io *)ctsio);
5906
5907                 return (CTL_RETVAL_COMPLETE);
5908         }
5909
5910         len = ctsio->kern_total_len - ctsio->kern_data_resid;
5911         hdr = (struct scsi_unmap_header *)ctsio->kern_data_ptr;
5912         if (len < sizeof (*hdr) ||
5913             len < (scsi_2btoul(hdr->length) + sizeof(hdr->length)) ||
5914             len < (scsi_2btoul(hdr->desc_length) + sizeof (*hdr)) ||
5915             scsi_2btoul(hdr->desc_length) % sizeof(*buf) != 0) {
5916                 ctl_set_invalid_field(ctsio,
5917                                       /*sks_valid*/ 0,
5918                                       /*command*/ 0,
5919                                       /*field*/ 0,
5920                                       /*bit_valid*/ 0,
5921                                       /*bit*/ 0);
5922                 ctl_done((union ctl_io *)ctsio);
5923                 return (CTL_RETVAL_COMPLETE);
5924         }
5925         len = scsi_2btoul(hdr->desc_length);
5926         buf = (struct scsi_unmap_desc *)(hdr + 1);
5927         end = buf + len / sizeof(*buf);
5928
5929         ptrlen = (struct ctl_ptr_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5930         ptrlen->ptr = (void *)buf;
5931         ptrlen->len = len;
5932         ptrlen->flags = byte2;
5933
5934         for (; buf < end; buf++) {
5935                 lba = scsi_8btou64(buf->lba);
5936                 num_blocks = scsi_4btoul(buf->length);
5937                 if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5938                  || ((lba + num_blocks) < lba)) {
5939                         ctl_set_lba_out_of_range(ctsio);
5940                         ctl_done((union ctl_io *)ctsio);
5941                         return (CTL_RETVAL_COMPLETE);
5942                 }
5943         }
5944
5945         retval = lun->backend->config_write((union ctl_io *)ctsio);
5946
5947         return (retval);
5948 }
5949
5950 /*
5951  * Note that this function currently doesn't actually do anything inside
5952  * CTL to enforce things if the DQue bit is turned on.
5953  *
5954  * Also note that this function can't be used in the default case, because
5955  * the DQue bit isn't set in the changeable mask for the control mode page
5956  * anyway.  This is just here as an example for how to implement a page
5957  * handler, and a placeholder in case we want to allow the user to turn
5958  * tagged queueing on and off.
5959  *
5960  * The D_SENSE bit handling is functional, however, and will turn
5961  * descriptor sense on and off for a given LUN.
5962  */
5963 int
5964 ctl_control_page_handler(struct ctl_scsiio *ctsio,
5965                          struct ctl_page_index *page_index, uint8_t *page_ptr)
5966 {
5967         struct scsi_control_page *current_cp, *saved_cp, *user_cp;
5968         struct ctl_lun *lun;
5969         struct ctl_softc *softc;
5970         int set_ua;
5971         uint32_t initidx;
5972
5973         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5974         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5975         set_ua = 0;
5976
5977         user_cp = (struct scsi_control_page *)page_ptr;
5978         current_cp = (struct scsi_control_page *)
5979                 (page_index->page_data + (page_index->page_len *
5980                 CTL_PAGE_CURRENT));
5981         saved_cp = (struct scsi_control_page *)
5982                 (page_index->page_data + (page_index->page_len *
5983                 CTL_PAGE_SAVED));
5984
5985         softc = control_softc;
5986
5987         mtx_lock(&lun->lun_lock);
5988         if (((current_cp->rlec & SCP_DSENSE) == 0)
5989          && ((user_cp->rlec & SCP_DSENSE) != 0)) {
5990                 /*
5991                  * Descriptor sense is currently turned off and the user
5992                  * wants to turn it on.
5993                  */
5994                 current_cp->rlec |= SCP_DSENSE;
5995                 saved_cp->rlec |= SCP_DSENSE;
5996                 lun->flags |= CTL_LUN_SENSE_DESC;
5997                 set_ua = 1;
5998         } else if (((current_cp->rlec & SCP_DSENSE) != 0)
5999                 && ((user_cp->rlec & SCP_DSENSE) == 0)) {
6000                 /*
6001                  * Descriptor sense is currently turned on, and the user
6002                  * wants to turn it off.
6003                  */
6004                 current_cp->rlec &= ~SCP_DSENSE;
6005                 saved_cp->rlec &= ~SCP_DSENSE;
6006                 lun->flags &= ~CTL_LUN_SENSE_DESC;
6007                 set_ua = 1;
6008         }
6009         if (current_cp->queue_flags & SCP_QUEUE_DQUE) {
6010                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
6011 #ifdef NEEDTOPORT
6012                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6013                                     CTL_UNTAG_TO_UNTAG,
6014                                     csevent_LogType_Trace,
6015                                     csevent_Severity_Information,
6016                                     csevent_AlertLevel_Green,
6017                                     csevent_FRU_Firmware,
6018                                     csevent_FRU_Unknown,
6019                                     "Received untagged to untagged transition");
6020 #endif /* NEEDTOPORT */
6021                 } else {
6022 #ifdef NEEDTOPORT
6023                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6024                                     CTL_UNTAG_TO_TAG,
6025                                     csevent_LogType_ConfigChange,
6026                                     csevent_Severity_Information,
6027                                     csevent_AlertLevel_Green,
6028                                     csevent_FRU_Firmware,
6029                                     csevent_FRU_Unknown,
6030                                     "Received untagged to tagged "
6031                                     "queueing transition");
6032 #endif /* NEEDTOPORT */
6033
6034                         current_cp->queue_flags &= ~SCP_QUEUE_DQUE;
6035                         saved_cp->queue_flags &= ~SCP_QUEUE_DQUE;
6036                         set_ua = 1;
6037                 }
6038         } else {
6039                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
6040 #ifdef NEEDTOPORT
6041                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6042                                     CTL_TAG_TO_UNTAG,
6043                                     csevent_LogType_ConfigChange,
6044                                     csevent_Severity_Warning,
6045                                     csevent_AlertLevel_Yellow,
6046                                     csevent_FRU_Firmware,
6047                                     csevent_FRU_Unknown,
6048                                     "Received tagged queueing to untagged "
6049                                     "transition");
6050 #endif /* NEEDTOPORT */
6051
6052                         current_cp->queue_flags |= SCP_QUEUE_DQUE;
6053                         saved_cp->queue_flags |= SCP_QUEUE_DQUE;
6054                         set_ua = 1;
6055                 } else {
6056 #ifdef NEEDTOPORT
6057                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6058                                     CTL_TAG_TO_TAG,
6059                                     csevent_LogType_Trace,
6060                                     csevent_Severity_Information,
6061                                     csevent_AlertLevel_Green,
6062                                     csevent_FRU_Firmware,
6063                                     csevent_FRU_Unknown,
6064                                     "Received tagged queueing to tagged "
6065                                     "queueing transition");
6066 #endif /* NEEDTOPORT */
6067                 }
6068         }
6069         if (set_ua != 0) {
6070                 int i;
6071                 /*
6072                  * Let other initiators know that the mode
6073                  * parameters for this LUN have changed.
6074                  */
6075                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
6076                         if (i == initidx)
6077                                 continue;
6078
6079                         lun->pending_sense[i].ua_pending |=
6080                                 CTL_UA_MODE_CHANGE;
6081                 }
6082         }
6083         mtx_unlock(&lun->lun_lock);
6084
6085         return (0);
6086 }
6087
6088 int
6089 ctl_power_sp_handler(struct ctl_scsiio *ctsio,
6090                      struct ctl_page_index *page_index, uint8_t *page_ptr)
6091 {
6092         return (0);
6093 }
6094
6095 int
6096 ctl_power_sp_sense_handler(struct ctl_scsiio *ctsio,
6097                            struct ctl_page_index *page_index, int pc)
6098 {
6099         struct copan_power_subpage *page;
6100
6101         page = (struct copan_power_subpage *)page_index->page_data +
6102                 (page_index->page_len * pc);
6103
6104         switch (pc) {
6105         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6106                 /*
6107                  * We don't update the changable bits for this page.
6108                  */
6109                 break;
6110         case SMS_PAGE_CTRL_CURRENT >> 6:
6111         case SMS_PAGE_CTRL_DEFAULT >> 6:
6112         case SMS_PAGE_CTRL_SAVED >> 6:
6113 #ifdef NEEDTOPORT
6114                 ctl_update_power_subpage(page);
6115 #endif
6116                 break;
6117         default:
6118 #ifdef NEEDTOPORT
6119                 EPRINT(0, "Invalid PC %d!!", pc);
6120 #endif
6121                 break;
6122         }
6123         return (0);
6124 }
6125
6126
6127 int
6128 ctl_aps_sp_handler(struct ctl_scsiio *ctsio,
6129                    struct ctl_page_index *page_index, uint8_t *page_ptr)
6130 {
6131         struct copan_aps_subpage *user_sp;
6132         struct copan_aps_subpage *current_sp;
6133         union ctl_modepage_info *modepage_info;
6134         struct ctl_softc *softc;
6135         struct ctl_lun *lun;
6136         int retval;
6137
6138         retval = CTL_RETVAL_COMPLETE;
6139         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
6140                      (page_index->page_len * CTL_PAGE_CURRENT));
6141         softc = control_softc;
6142         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6143
6144         user_sp = (struct copan_aps_subpage *)page_ptr;
6145
6146         modepage_info = (union ctl_modepage_info *)
6147                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6148
6149         modepage_info->header.page_code = page_index->page_code & SMPH_PC_MASK;
6150         modepage_info->header.subpage = page_index->subpage;
6151         modepage_info->aps.lock_active = user_sp->lock_active;
6152
6153         mtx_lock(&softc->ctl_lock);
6154
6155         /*
6156          * If there is a request to lock the LUN and another LUN is locked
6157          * this is an error. If the requested LUN is already locked ignore
6158          * the request. If no LUN is locked attempt to lock it.
6159          * if there is a request to unlock the LUN and the LUN is currently
6160          * locked attempt to unlock it. Otherwise ignore the request. i.e.
6161          * if another LUN is locked or no LUN is locked.
6162          */
6163         if (user_sp->lock_active & APS_LOCK_ACTIVE) {
6164                 if (softc->aps_locked_lun == lun->lun) {
6165                         /*
6166                          * This LUN is already locked, so we're done.
6167                          */
6168                         retval = CTL_RETVAL_COMPLETE;
6169                 } else if (softc->aps_locked_lun == 0) {
6170                         /*
6171                          * No one has the lock, pass the request to the
6172                          * backend.
6173                          */
6174                         retval = lun->backend->config_write(
6175                                 (union ctl_io *)ctsio);
6176                 } else {
6177                         /*
6178                          * Someone else has the lock, throw out the request.
6179                          */
6180                         ctl_set_already_locked(ctsio);
6181                         free(ctsio->kern_data_ptr, M_CTL);
6182                         ctl_done((union ctl_io *)ctsio);
6183
6184                         /*
6185                          * Set the return value so that ctl_do_mode_select()
6186                          * won't try to complete the command.  We already
6187                          * completed it here.
6188                          */
6189                         retval = CTL_RETVAL_ERROR;
6190                 }
6191         } else if (softc->aps_locked_lun == lun->lun) {
6192                 /*
6193                  * This LUN is locked, so pass the unlock request to the
6194                  * backend.
6195                  */
6196                 retval = lun->backend->config_write((union ctl_io *)ctsio);
6197         }
6198         mtx_unlock(&softc->ctl_lock);
6199
6200         return (retval);
6201 }
6202
6203 int
6204 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6205                                 struct ctl_page_index *page_index,
6206                                 uint8_t *page_ptr)
6207 {
6208         uint8_t *c;
6209         int i;
6210
6211         c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6212         ctl_time_io_secs =
6213                 (c[0] << 8) |
6214                 (c[1] << 0) |
6215                 0;
6216         CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6217         printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6218         printf("page data:");
6219         for (i=0; i<8; i++)
6220                 printf(" %.2x",page_ptr[i]);
6221         printf("\n");
6222         return (0);
6223 }
6224
6225 int
6226 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6227                                struct ctl_page_index *page_index,
6228                                int pc)
6229 {
6230         struct copan_debugconf_subpage *page;
6231
6232         page = (struct copan_debugconf_subpage *)page_index->page_data +
6233                 (page_index->page_len * pc);
6234
6235         switch (pc) {
6236         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6237         case SMS_PAGE_CTRL_DEFAULT >> 6:
6238         case SMS_PAGE_CTRL_SAVED >> 6:
6239                 /*
6240                  * We don't update the changable or default bits for this page.
6241                  */
6242                 break;
6243         case SMS_PAGE_CTRL_CURRENT >> 6:
6244                 page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6245                 page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6246                 break;
6247         default:
6248 #ifdef NEEDTOPORT
6249                 EPRINT(0, "Invalid PC %d!!", pc);
6250 #endif /* NEEDTOPORT */
6251                 break;
6252         }
6253         return (0);
6254 }
6255
6256
6257 static int
6258 ctl_do_mode_select(union ctl_io *io)
6259 {
6260         struct scsi_mode_page_header *page_header;
6261         struct ctl_page_index *page_index;
6262         struct ctl_scsiio *ctsio;
6263         int control_dev, page_len;
6264         int page_len_offset, page_len_size;
6265         union ctl_modepage_info *modepage_info;
6266         struct ctl_lun *lun;
6267         int *len_left, *len_used;
6268         int retval, i;
6269
6270         ctsio = &io->scsiio;
6271         page_index = NULL;
6272         page_len = 0;
6273         retval = CTL_RETVAL_COMPLETE;
6274
6275         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6276
6277         if (lun->be_lun->lun_type != T_DIRECT)
6278                 control_dev = 1;
6279         else
6280                 control_dev = 0;
6281
6282         modepage_info = (union ctl_modepage_info *)
6283                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6284         len_left = &modepage_info->header.len_left;
6285         len_used = &modepage_info->header.len_used;
6286
6287 do_next_page:
6288
6289         page_header = (struct scsi_mode_page_header *)
6290                 (ctsio->kern_data_ptr + *len_used);
6291
6292         if (*len_left == 0) {
6293                 free(ctsio->kern_data_ptr, M_CTL);
6294                 ctl_set_success(ctsio);
6295                 ctl_done((union ctl_io *)ctsio);
6296                 return (CTL_RETVAL_COMPLETE);
6297         } else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6298
6299                 free(ctsio->kern_data_ptr, M_CTL);
6300                 ctl_set_param_len_error(ctsio);
6301                 ctl_done((union ctl_io *)ctsio);
6302                 return (CTL_RETVAL_COMPLETE);
6303
6304         } else if ((page_header->page_code & SMPH_SPF)
6305                 && (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6306
6307                 free(ctsio->kern_data_ptr, M_CTL);
6308                 ctl_set_param_len_error(ctsio);
6309                 ctl_done((union ctl_io *)ctsio);
6310                 return (CTL_RETVAL_COMPLETE);
6311         }
6312
6313
6314         /*
6315          * XXX KDM should we do something with the block descriptor?
6316          */
6317         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6318
6319                 if ((control_dev != 0)
6320                  && (lun->mode_pages.index[i].page_flags &
6321                      CTL_PAGE_FLAG_DISK_ONLY))
6322                         continue;
6323
6324                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6325                     (page_header->page_code & SMPH_PC_MASK))
6326                         continue;
6327
6328                 /*
6329                  * If neither page has a subpage code, then we've got a
6330                  * match.
6331                  */
6332                 if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6333                  && ((page_header->page_code & SMPH_SPF) == 0)) {
6334                         page_index = &lun->mode_pages.index[i];
6335                         page_len = page_header->page_length;
6336                         break;
6337                 }
6338
6339                 /*
6340                  * If both pages have subpages, then the subpage numbers
6341                  * have to match.
6342                  */
6343                 if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6344                   && (page_header->page_code & SMPH_SPF)) {
6345                         struct scsi_mode_page_header_sp *sph;
6346
6347                         sph = (struct scsi_mode_page_header_sp *)page_header;
6348
6349                         if (lun->mode_pages.index[i].subpage ==
6350                             sph->subpage) {
6351                                 page_index = &lun->mode_pages.index[i];
6352                                 page_len = scsi_2btoul(sph->page_length);
6353                                 break;
6354                         }
6355                 }
6356         }
6357
6358         /*
6359          * If we couldn't find the page, or if we don't have a mode select
6360          * handler for it, send back an error to the user.
6361          */
6362         if ((page_index == NULL)
6363          || (page_index->select_handler == NULL)) {
6364                 ctl_set_invalid_field(ctsio,
6365                                       /*sks_valid*/ 1,
6366                                       /*command*/ 0,
6367                                       /*field*/ *len_used,
6368                                       /*bit_valid*/ 0,
6369                                       /*bit*/ 0);
6370                 free(ctsio->kern_data_ptr, M_CTL);
6371                 ctl_done((union ctl_io *)ctsio);
6372                 return (CTL_RETVAL_COMPLETE);
6373         }
6374
6375         if (page_index->page_code & SMPH_SPF) {
6376                 page_len_offset = 2;
6377                 page_len_size = 2;
6378         } else {
6379                 page_len_size = 1;
6380                 page_len_offset = 1;
6381         }
6382
6383         /*
6384          * If the length the initiator gives us isn't the one we specify in
6385          * the mode page header, or if they didn't specify enough data in
6386          * the CDB to avoid truncating this page, kick out the request.
6387          */
6388         if ((page_len != (page_index->page_len - page_len_offset -
6389                           page_len_size))
6390          || (*len_left < page_index->page_len)) {
6391
6392
6393                 ctl_set_invalid_field(ctsio,
6394                                       /*sks_valid*/ 1,
6395                                       /*command*/ 0,
6396                                       /*field*/ *len_used + page_len_offset,
6397                                       /*bit_valid*/ 0,
6398                                       /*bit*/ 0);
6399                 free(ctsio->kern_data_ptr, M_CTL);
6400                 ctl_done((union ctl_io *)ctsio);
6401                 return (CTL_RETVAL_COMPLETE);
6402         }
6403
6404         /*
6405          * Run through the mode page, checking to make sure that the bits
6406          * the user changed are actually legal for him to change.
6407          */
6408         for (i = 0; i < page_index->page_len; i++) {
6409                 uint8_t *user_byte, *change_mask, *current_byte;
6410                 int bad_bit;
6411                 int j;
6412
6413                 user_byte = (uint8_t *)page_header + i;
6414                 change_mask = page_index->page_data +
6415                               (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6416                 current_byte = page_index->page_data +
6417                                (page_index->page_len * CTL_PAGE_CURRENT) + i;
6418
6419                 /*
6420                  * Check to see whether the user set any bits in this byte
6421                  * that he is not allowed to set.
6422                  */
6423                 if ((*user_byte & ~(*change_mask)) ==
6424                     (*current_byte & ~(*change_mask)))
6425                         continue;
6426
6427                 /*
6428                  * Go through bit by bit to determine which one is illegal.
6429                  */
6430                 bad_bit = 0;
6431                 for (j = 7; j >= 0; j--) {
6432                         if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6433                             (((1 << i) & ~(*change_mask)) & *current_byte)) {
6434                                 bad_bit = i;
6435                                 break;
6436                         }
6437                 }
6438                 ctl_set_invalid_field(ctsio,
6439                                       /*sks_valid*/ 1,
6440                                       /*command*/ 0,
6441                                       /*field*/ *len_used + i,
6442                                       /*bit_valid*/ 1,
6443                                       /*bit*/ bad_bit);
6444                 free(ctsio->kern_data_ptr, M_CTL);
6445                 ctl_done((union ctl_io *)ctsio);
6446                 return (CTL_RETVAL_COMPLETE);
6447         }
6448
6449         /*
6450          * Decrement these before we call the page handler, since we may
6451          * end up getting called back one way or another before the handler
6452          * returns to this context.
6453          */
6454         *len_left -= page_index->page_len;
6455         *len_used += page_index->page_len;
6456
6457         retval = page_index->select_handler(ctsio, page_index,
6458                                             (uint8_t *)page_header);
6459
6460         /*
6461          * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6462          * wait until this queued command completes to finish processing
6463          * the mode page.  If it returns anything other than
6464          * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6465          * already set the sense information, freed the data pointer, and
6466          * completed the io for us.
6467          */
6468         if (retval != CTL_RETVAL_COMPLETE)
6469                 goto bailout_no_done;
6470
6471         /*
6472          * If the initiator sent us more than one page, parse the next one.
6473          */
6474         if (*len_left > 0)
6475                 goto do_next_page;
6476
6477         ctl_set_success(ctsio);
6478         free(ctsio->kern_data_ptr, M_CTL);
6479         ctl_done((union ctl_io *)ctsio);
6480
6481 bailout_no_done:
6482
6483         return (CTL_RETVAL_COMPLETE);
6484
6485 }
6486
6487 int
6488 ctl_mode_select(struct ctl_scsiio *ctsio)
6489 {
6490         int param_len, pf, sp;
6491         int header_size, bd_len;
6492         int len_left, len_used;
6493         struct ctl_page_index *page_index;
6494         struct ctl_lun *lun;
6495         int control_dev, page_len;
6496         union ctl_modepage_info *modepage_info;
6497         int retval;
6498
6499         pf = 0;
6500         sp = 0;
6501         page_len = 0;
6502         len_used = 0;
6503         len_left = 0;
6504         retval = 0;
6505         bd_len = 0;
6506         page_index = NULL;
6507
6508         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6509
6510         if (lun->be_lun->lun_type != T_DIRECT)
6511                 control_dev = 1;
6512         else
6513                 control_dev = 0;
6514
6515         switch (ctsio->cdb[0]) {
6516         case MODE_SELECT_6: {
6517                 struct scsi_mode_select_6 *cdb;
6518
6519                 cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6520
6521                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6522                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6523
6524                 param_len = cdb->length;
6525                 header_size = sizeof(struct scsi_mode_header_6);
6526                 break;
6527         }
6528         case MODE_SELECT_10: {
6529                 struct scsi_mode_select_10 *cdb;
6530
6531                 cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6532
6533                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6534                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6535
6536                 param_len = scsi_2btoul(cdb->length);
6537                 header_size = sizeof(struct scsi_mode_header_10);
6538                 break;
6539         }
6540         default:
6541                 ctl_set_invalid_opcode(ctsio);
6542                 ctl_done((union ctl_io *)ctsio);
6543                 return (CTL_RETVAL_COMPLETE);
6544                 break; /* NOTREACHED */
6545         }
6546
6547         /*
6548          * From SPC-3:
6549          * "A parameter list length of zero indicates that the Data-Out Buffer
6550          * shall be empty. This condition shall not be considered as an error."
6551          */
6552         if (param_len == 0) {
6553                 ctl_set_success(ctsio);
6554                 ctl_done((union ctl_io *)ctsio);
6555                 return (CTL_RETVAL_COMPLETE);
6556         }
6557
6558         /*
6559          * Since we'll hit this the first time through, prior to
6560          * allocation, we don't need to free a data buffer here.
6561          */
6562         if (param_len < header_size) {
6563                 ctl_set_param_len_error(ctsio);
6564                 ctl_done((union ctl_io *)ctsio);
6565                 return (CTL_RETVAL_COMPLETE);
6566         }
6567
6568         /*
6569          * Allocate the data buffer and grab the user's data.  In theory,
6570          * we shouldn't have to sanity check the parameter list length here
6571          * because the maximum size is 64K.  We should be able to malloc
6572          * that much without too many problems.
6573          */
6574         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6575                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6576                 ctsio->kern_data_len = param_len;
6577                 ctsio->kern_total_len = param_len;
6578                 ctsio->kern_data_resid = 0;
6579                 ctsio->kern_rel_offset = 0;
6580                 ctsio->kern_sg_entries = 0;
6581                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6582                 ctsio->be_move_done = ctl_config_move_done;
6583                 ctl_datamove((union ctl_io *)ctsio);
6584
6585                 return (CTL_RETVAL_COMPLETE);
6586         }
6587
6588         switch (ctsio->cdb[0]) {
6589         case MODE_SELECT_6: {
6590                 struct scsi_mode_header_6 *mh6;
6591
6592                 mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6593                 bd_len = mh6->blk_desc_len;
6594                 break;
6595         }
6596         case MODE_SELECT_10: {
6597                 struct scsi_mode_header_10 *mh10;
6598
6599                 mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6600                 bd_len = scsi_2btoul(mh10->blk_desc_len);
6601                 break;
6602         }
6603         default:
6604                 panic("Invalid CDB type %#x", ctsio->cdb[0]);
6605                 break;
6606         }
6607
6608         if (param_len < (header_size + bd_len)) {
6609                 free(ctsio->kern_data_ptr, M_CTL);
6610                 ctl_set_param_len_error(ctsio);
6611                 ctl_done((union ctl_io *)ctsio);
6612                 return (CTL_RETVAL_COMPLETE);
6613         }
6614
6615         /*
6616          * Set the IO_CONT flag, so that if this I/O gets passed to
6617          * ctl_config_write_done(), it'll get passed back to
6618          * ctl_do_mode_select() for further processing, or completion if
6619          * we're all done.
6620          */
6621         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6622         ctsio->io_cont = ctl_do_mode_select;
6623
6624         modepage_info = (union ctl_modepage_info *)
6625                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6626
6627         memset(modepage_info, 0, sizeof(*modepage_info));
6628
6629         len_left = param_len - header_size - bd_len;
6630         len_used = header_size + bd_len;
6631
6632         modepage_info->header.len_left = len_left;
6633         modepage_info->header.len_used = len_used;
6634
6635         return (ctl_do_mode_select((union ctl_io *)ctsio));
6636 }
6637
6638 int
6639 ctl_mode_sense(struct ctl_scsiio *ctsio)
6640 {
6641         struct ctl_lun *lun;
6642         int pc, page_code, dbd, llba, subpage;
6643         int alloc_len, page_len, header_len, total_len;
6644         struct scsi_mode_block_descr *block_desc;
6645         struct ctl_page_index *page_index;
6646         int control_dev;
6647
6648         dbd = 0;
6649         llba = 0;
6650         block_desc = NULL;
6651         page_index = NULL;
6652
6653         CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6654
6655         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6656
6657         if (lun->be_lun->lun_type != T_DIRECT)
6658                 control_dev = 1;
6659         else
6660                 control_dev = 0;
6661
6662         if (lun->flags & CTL_LUN_PR_RESERVED) {
6663                 uint32_t residx;
6664
6665                 /*
6666                  * XXX KDM need a lock here.
6667                  */
6668                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
6669                 if ((lun->res_type == SPR_TYPE_EX_AC
6670                   && residx != lun->pr_res_idx)
6671                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
6672                    || lun->res_type == SPR_TYPE_EX_AC_AR)
6673                   && !lun->per_res[residx].registered)) {
6674                         ctl_set_reservation_conflict(ctsio);
6675                         ctl_done((union ctl_io *)ctsio);
6676                         return (CTL_RETVAL_COMPLETE);
6677                 }
6678         }
6679
6680         switch (ctsio->cdb[0]) {
6681         case MODE_SENSE_6: {
6682                 struct scsi_mode_sense_6 *cdb;
6683
6684                 cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6685
6686                 header_len = sizeof(struct scsi_mode_hdr_6);
6687                 if (cdb->byte2 & SMS_DBD)
6688                         dbd = 1;
6689                 else
6690                         header_len += sizeof(struct scsi_mode_block_descr);
6691
6692                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6693                 page_code = cdb->page & SMS_PAGE_CODE;
6694                 subpage = cdb->subpage;
6695                 alloc_len = cdb->length;
6696                 break;
6697         }
6698         case MODE_SENSE_10: {
6699                 struct scsi_mode_sense_10 *cdb;
6700
6701                 cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6702
6703                 header_len = sizeof(struct scsi_mode_hdr_10);
6704
6705                 if (cdb->byte2 & SMS_DBD)
6706                         dbd = 1;
6707                 else
6708                         header_len += sizeof(struct scsi_mode_block_descr);
6709                 if (cdb->byte2 & SMS10_LLBAA)
6710                         llba = 1;
6711                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6712                 page_code = cdb->page & SMS_PAGE_CODE;
6713                 subpage = cdb->subpage;
6714                 alloc_len = scsi_2btoul(cdb->length);
6715                 break;
6716         }
6717         default:
6718                 ctl_set_invalid_opcode(ctsio);
6719                 ctl_done((union ctl_io *)ctsio);
6720                 return (CTL_RETVAL_COMPLETE);
6721                 break; /* NOTREACHED */
6722         }
6723
6724         /*
6725          * We have to make a first pass through to calculate the size of
6726          * the pages that match the user's query.  Then we allocate enough
6727          * memory to hold it, and actually copy the data into the buffer.
6728          */
6729         switch (page_code) {
6730         case SMS_ALL_PAGES_PAGE: {
6731                 int i;
6732
6733                 page_len = 0;
6734
6735                 /*
6736                  * At the moment, values other than 0 and 0xff here are
6737                  * reserved according to SPC-3.
6738                  */
6739                 if ((subpage != SMS_SUBPAGE_PAGE_0)
6740                  && (subpage != SMS_SUBPAGE_ALL)) {
6741                         ctl_set_invalid_field(ctsio,
6742                                               /*sks_valid*/ 1,
6743                                               /*command*/ 1,
6744                                               /*field*/ 3,
6745                                               /*bit_valid*/ 0,
6746                                               /*bit*/ 0);
6747                         ctl_done((union ctl_io *)ctsio);
6748                         return (CTL_RETVAL_COMPLETE);
6749                 }
6750
6751                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6752                         if ((control_dev != 0)
6753                          && (lun->mode_pages.index[i].page_flags &
6754                              CTL_PAGE_FLAG_DISK_ONLY))
6755                                 continue;
6756
6757                         /*
6758                          * We don't use this subpage if the user didn't
6759                          * request all subpages.
6760                          */
6761                         if ((lun->mode_pages.index[i].subpage != 0)
6762                          && (subpage == SMS_SUBPAGE_PAGE_0))
6763                                 continue;
6764
6765 #if 0
6766                         printf("found page %#x len %d\n",
6767                                lun->mode_pages.index[i].page_code &
6768                                SMPH_PC_MASK,
6769                                lun->mode_pages.index[i].page_len);
6770 #endif
6771                         page_len += lun->mode_pages.index[i].page_len;
6772                 }
6773                 break;
6774         }
6775         default: {
6776                 int i;
6777
6778                 page_len = 0;
6779
6780                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6781                         /* Look for the right page code */
6782                         if ((lun->mode_pages.index[i].page_code &
6783                              SMPH_PC_MASK) != page_code)
6784                                 continue;
6785
6786                         /* Look for the right subpage or the subpage wildcard*/
6787                         if ((lun->mode_pages.index[i].subpage != subpage)
6788                          && (subpage != SMS_SUBPAGE_ALL))
6789                                 continue;
6790
6791                         /* Make sure the page is supported for this dev type */
6792                         if ((control_dev != 0)
6793                          && (lun->mode_pages.index[i].page_flags &
6794                              CTL_PAGE_FLAG_DISK_ONLY))
6795                                 continue;
6796
6797 #if 0
6798                         printf("found page %#x len %d\n",
6799                                lun->mode_pages.index[i].page_code &
6800                                SMPH_PC_MASK,
6801                                lun->mode_pages.index[i].page_len);
6802 #endif
6803
6804                         page_len += lun->mode_pages.index[i].page_len;
6805                 }
6806
6807                 if (page_len == 0) {
6808                         ctl_set_invalid_field(ctsio,
6809                                               /*sks_valid*/ 1,
6810                                               /*command*/ 1,
6811                                               /*field*/ 2,
6812                                               /*bit_valid*/ 1,
6813                                               /*bit*/ 5);
6814                         ctl_done((union ctl_io *)ctsio);
6815                         return (CTL_RETVAL_COMPLETE);
6816                 }
6817                 break;
6818         }
6819         }
6820
6821         total_len = header_len + page_len;
6822 #if 0
6823         printf("header_len = %d, page_len = %d, total_len = %d\n",
6824                header_len, page_len, total_len);
6825 #endif
6826
6827         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6828         ctsio->kern_sg_entries = 0;
6829         ctsio->kern_data_resid = 0;
6830         ctsio->kern_rel_offset = 0;
6831         if (total_len < alloc_len) {
6832                 ctsio->residual = alloc_len - total_len;
6833                 ctsio->kern_data_len = total_len;
6834                 ctsio->kern_total_len = total_len;
6835         } else {
6836                 ctsio->residual = 0;
6837                 ctsio->kern_data_len = alloc_len;
6838                 ctsio->kern_total_len = alloc_len;
6839         }
6840
6841         switch (ctsio->cdb[0]) {
6842         case MODE_SENSE_6: {
6843                 struct scsi_mode_hdr_6 *header;
6844
6845                 header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6846
6847                 header->datalen = ctl_min(total_len - 1, 254);
6848
6849                 if (dbd)
6850                         header->block_descr_len = 0;
6851                 else
6852                         header->block_descr_len =
6853                                 sizeof(struct scsi_mode_block_descr);
6854                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6855                 break;
6856         }
6857         case MODE_SENSE_10: {
6858                 struct scsi_mode_hdr_10 *header;
6859                 int datalen;
6860
6861                 header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6862
6863                 datalen = ctl_min(total_len - 2, 65533);
6864                 scsi_ulto2b(datalen, header->datalen);
6865                 if (dbd)
6866                         scsi_ulto2b(0, header->block_descr_len);
6867                 else
6868                         scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
6869                                     header->block_descr_len);
6870                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6871                 break;
6872         }
6873         default:
6874                 panic("invalid CDB type %#x", ctsio->cdb[0]);
6875                 break; /* NOTREACHED */
6876         }
6877
6878         /*
6879          * If we've got a disk, use its blocksize in the block
6880          * descriptor.  Otherwise, just set it to 0.
6881          */
6882         if (dbd == 0) {
6883                 if (control_dev != 0)
6884                         scsi_ulto3b(lun->be_lun->blocksize,
6885                                     block_desc->block_len);
6886                 else
6887                         scsi_ulto3b(0, block_desc->block_len);
6888         }
6889
6890         switch (page_code) {
6891         case SMS_ALL_PAGES_PAGE: {
6892                 int i, data_used;
6893
6894                 data_used = header_len;
6895                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6896                         struct ctl_page_index *page_index;
6897
6898                         page_index = &lun->mode_pages.index[i];
6899
6900                         if ((control_dev != 0)
6901                          && (page_index->page_flags &
6902                             CTL_PAGE_FLAG_DISK_ONLY))
6903                                 continue;
6904
6905                         /*
6906                          * We don't use this subpage if the user didn't
6907                          * request all subpages.  We already checked (above)
6908                          * to make sure the user only specified a subpage
6909                          * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
6910                          */
6911                         if ((page_index->subpage != 0)
6912                          && (subpage == SMS_SUBPAGE_PAGE_0))
6913                                 continue;
6914
6915                         /*
6916                          * Call the handler, if it exists, to update the
6917                          * page to the latest values.
6918                          */
6919                         if (page_index->sense_handler != NULL)
6920                                 page_index->sense_handler(ctsio, page_index,pc);
6921
6922                         memcpy(ctsio->kern_data_ptr + data_used,
6923                                page_index->page_data +
6924                                (page_index->page_len * pc),
6925                                page_index->page_len);
6926                         data_used += page_index->page_len;
6927                 }
6928                 break;
6929         }
6930         default: {
6931                 int i, data_used;
6932
6933                 data_used = header_len;
6934
6935                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6936                         struct ctl_page_index *page_index;
6937
6938                         page_index = &lun->mode_pages.index[i];
6939
6940                         /* Look for the right page code */
6941                         if ((page_index->page_code & SMPH_PC_MASK) != page_code)
6942                                 continue;
6943
6944                         /* Look for the right subpage or the subpage wildcard*/
6945                         if ((page_index->subpage != subpage)
6946                          && (subpage != SMS_SUBPAGE_ALL))
6947                                 continue;
6948
6949                         /* Make sure the page is supported for this dev type */
6950                         if ((control_dev != 0)
6951                          && (page_index->page_flags &
6952                              CTL_PAGE_FLAG_DISK_ONLY))
6953                                 continue;
6954
6955                         /*
6956                          * Call the handler, if it exists, to update the
6957                          * page to the latest values.
6958                          */
6959                         if (page_index->sense_handler != NULL)
6960                                 page_index->sense_handler(ctsio, page_index,pc);
6961
6962                         memcpy(ctsio->kern_data_ptr + data_used,
6963                                page_index->page_data +
6964                                (page_index->page_len * pc),
6965                                page_index->page_len);
6966                         data_used += page_index->page_len;
6967                 }
6968                 break;
6969         }
6970         }
6971
6972         ctsio->scsi_status = SCSI_STATUS_OK;
6973
6974         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6975         ctsio->be_move_done = ctl_config_move_done;
6976         ctl_datamove((union ctl_io *)ctsio);
6977
6978         return (CTL_RETVAL_COMPLETE);
6979 }
6980
6981 int
6982 ctl_read_capacity(struct ctl_scsiio *ctsio)
6983 {
6984         struct scsi_read_capacity *cdb;
6985         struct scsi_read_capacity_data *data;
6986         struct ctl_lun *lun;
6987         uint32_t lba;
6988
6989         CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
6990
6991         cdb = (struct scsi_read_capacity *)ctsio->cdb;
6992
6993         lba = scsi_4btoul(cdb->addr);
6994         if (((cdb->pmi & SRC_PMI) == 0)
6995          && (lba != 0)) {
6996                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6997                                       /*sks_valid*/ 1,
6998                                       /*command*/ 1,
6999                                       /*field*/ 2,
7000                                       /*bit_valid*/ 0,
7001                                       /*bit*/ 0);
7002                 ctl_done((union ctl_io *)ctsio);
7003                 return (CTL_RETVAL_COMPLETE);
7004         }
7005
7006         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7007
7008         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7009         data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
7010         ctsio->residual = 0;
7011         ctsio->kern_data_len = sizeof(*data);
7012         ctsio->kern_total_len = sizeof(*data);
7013         ctsio->kern_data_resid = 0;
7014         ctsio->kern_rel_offset = 0;
7015         ctsio->kern_sg_entries = 0;
7016
7017         /*
7018          * If the maximum LBA is greater than 0xfffffffe, the user must
7019          * issue a SERVICE ACTION IN (16) command, with the read capacity
7020          * serivce action set.
7021          */
7022         if (lun->be_lun->maxlba > 0xfffffffe)
7023                 scsi_ulto4b(0xffffffff, data->addr);
7024         else
7025                 scsi_ulto4b(lun->be_lun->maxlba, data->addr);
7026
7027         /*
7028          * XXX KDM this may not be 512 bytes...
7029          */
7030         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7031
7032         ctsio->scsi_status = SCSI_STATUS_OK;
7033
7034         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7035         ctsio->be_move_done = ctl_config_move_done;
7036         ctl_datamove((union ctl_io *)ctsio);
7037
7038         return (CTL_RETVAL_COMPLETE);
7039 }
7040
7041 int
7042 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
7043 {
7044         struct scsi_read_capacity_16 *cdb;
7045         struct scsi_read_capacity_data_long *data;
7046         struct ctl_lun *lun;
7047         uint64_t lba;
7048         uint32_t alloc_len;
7049
7050         CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
7051
7052         cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
7053
7054         alloc_len = scsi_4btoul(cdb->alloc_len);
7055         lba = scsi_8btou64(cdb->addr);
7056
7057         if ((cdb->reladr & SRC16_PMI)
7058          && (lba != 0)) {
7059                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7060                                       /*sks_valid*/ 1,
7061                                       /*command*/ 1,
7062                                       /*field*/ 2,
7063                                       /*bit_valid*/ 0,
7064                                       /*bit*/ 0);
7065                 ctl_done((union ctl_io *)ctsio);
7066                 return (CTL_RETVAL_COMPLETE);
7067         }
7068
7069         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7070
7071         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7072         data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
7073
7074         if (sizeof(*data) < alloc_len) {
7075                 ctsio->residual = alloc_len - sizeof(*data);
7076                 ctsio->kern_data_len = sizeof(*data);
7077                 ctsio->kern_total_len = sizeof(*data);
7078         } else {
7079                 ctsio->residual = 0;
7080                 ctsio->kern_data_len = alloc_len;
7081                 ctsio->kern_total_len = alloc_len;
7082         }
7083         ctsio->kern_data_resid = 0;
7084         ctsio->kern_rel_offset = 0;
7085         ctsio->kern_sg_entries = 0;
7086
7087         scsi_u64to8b(lun->be_lun->maxlba, data->addr);
7088         /* XXX KDM this may not be 512 bytes... */
7089         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7090         data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
7091         scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
7092         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
7093                 data->lalba_lbp[0] |= SRC16_LBPME;
7094
7095         ctsio->scsi_status = SCSI_STATUS_OK;
7096
7097         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7098         ctsio->be_move_done = ctl_config_move_done;
7099         ctl_datamove((union ctl_io *)ctsio);
7100
7101         return (CTL_RETVAL_COMPLETE);
7102 }
7103
7104 int
7105 ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
7106 {
7107         struct scsi_maintenance_in *cdb;
7108         int retval;
7109         int alloc_len, total_len = 0;
7110         int num_target_port_groups, single;
7111         struct ctl_lun *lun;
7112         struct ctl_softc *softc;
7113         struct scsi_target_group_data *rtg_ptr;
7114         struct scsi_target_port_group_descriptor *tpg_desc_ptr1, *tpg_desc_ptr2;
7115         struct scsi_target_port_descriptor  *tp_desc_ptr1_1, *tp_desc_ptr1_2,
7116                                             *tp_desc_ptr2_1, *tp_desc_ptr2_2;
7117
7118         CTL_DEBUG_PRINT(("ctl_report_tagret_port_groups\n"));
7119
7120         cdb = (struct scsi_maintenance_in *)ctsio->cdb;
7121         softc = control_softc;
7122         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7123
7124         retval = CTL_RETVAL_COMPLETE;
7125
7126         single = ctl_is_single;
7127         if (single)
7128                 num_target_port_groups = NUM_TARGET_PORT_GROUPS - 1;
7129         else
7130                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
7131
7132         total_len = sizeof(struct scsi_target_group_data) +
7133                 sizeof(struct scsi_target_port_group_descriptor) *
7134                 num_target_port_groups +
7135                 sizeof(struct scsi_target_port_descriptor) *
7136                 NUM_PORTS_PER_GRP * num_target_port_groups;
7137
7138         alloc_len = scsi_4btoul(cdb->length);
7139
7140         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7141
7142         ctsio->kern_sg_entries = 0;
7143
7144         if (total_len < alloc_len) {
7145                 ctsio->residual = alloc_len - total_len;
7146                 ctsio->kern_data_len = total_len;
7147                 ctsio->kern_total_len = total_len;
7148         } else {
7149                 ctsio->residual = 0;
7150                 ctsio->kern_data_len = alloc_len;
7151                 ctsio->kern_total_len = alloc_len;
7152         }
7153         ctsio->kern_data_resid = 0;
7154         ctsio->kern_rel_offset = 0;
7155
7156         rtg_ptr = (struct scsi_target_group_data *)ctsio->kern_data_ptr;
7157
7158         tpg_desc_ptr1 = &rtg_ptr->groups[0];
7159         tp_desc_ptr1_1 = &tpg_desc_ptr1->descriptors[0];
7160         tp_desc_ptr1_2 = (struct scsi_target_port_descriptor *)
7161                 &tp_desc_ptr1_1->desc_list[0];
7162
7163         if (single == 0) {
7164                 tpg_desc_ptr2 = (struct scsi_target_port_group_descriptor *)
7165                         &tp_desc_ptr1_2->desc_list[0];
7166                 tp_desc_ptr2_1 = &tpg_desc_ptr2->descriptors[0];
7167                 tp_desc_ptr2_2 = (struct scsi_target_port_descriptor *)
7168                         &tp_desc_ptr2_1->desc_list[0];
7169         } else {
7170                 tpg_desc_ptr2 = NULL;
7171                 tp_desc_ptr2_1 = NULL;
7172                 tp_desc_ptr2_2 = NULL;
7173         }
7174
7175         scsi_ulto4b(total_len - 4, rtg_ptr->length);
7176         if (single == 0) {
7177                 if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7178                         if (lun->flags & CTL_LUN_PRIMARY_SC) {
7179                                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7180                                 tpg_desc_ptr2->pref_state =
7181                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7182                         } else {
7183                                 tpg_desc_ptr1->pref_state =
7184                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7185                                 tpg_desc_ptr2->pref_state = TPG_PRIMARY;
7186                         }
7187                 } else {
7188                         if (lun->flags & CTL_LUN_PRIMARY_SC) {
7189                                 tpg_desc_ptr1->pref_state =
7190                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7191                                 tpg_desc_ptr2->pref_state = TPG_PRIMARY;
7192                         } else {
7193                                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7194                                 tpg_desc_ptr2->pref_state =
7195                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7196                         }
7197                 }
7198         } else {
7199                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7200         }
7201         tpg_desc_ptr1->support = 0;
7202         tpg_desc_ptr1->target_port_group[1] = 1;
7203         tpg_desc_ptr1->status = TPG_IMPLICIT;
7204         tpg_desc_ptr1->target_port_count= NUM_PORTS_PER_GRP;
7205
7206         if (single == 0) {
7207                 tpg_desc_ptr2->support = 0;
7208                 tpg_desc_ptr2->target_port_group[1] = 2;
7209                 tpg_desc_ptr2->status = TPG_IMPLICIT;
7210                 tpg_desc_ptr2->target_port_count = NUM_PORTS_PER_GRP;
7211
7212                 tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7213                 tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7214
7215                 tp_desc_ptr2_1->relative_target_port_identifier[1] = 9;
7216                 tp_desc_ptr2_2->relative_target_port_identifier[1] = 10;
7217         } else {
7218                 if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7219                         tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7220                         tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7221                 } else {
7222                         tp_desc_ptr1_1->relative_target_port_identifier[1] = 9;
7223                         tp_desc_ptr1_2->relative_target_port_identifier[1] = 10;
7224                 }
7225         }
7226
7227         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7228         ctsio->be_move_done = ctl_config_move_done;
7229
7230         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7231                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7232                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7233                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7234                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7235
7236         ctl_datamove((union ctl_io *)ctsio);
7237         return(retval);
7238 }
7239
7240 int
7241 ctl_report_supported_opcodes(struct ctl_scsiio *ctsio)
7242 {
7243         struct ctl_lun *lun;
7244         struct scsi_report_supported_opcodes *cdb;
7245         const struct ctl_cmd_entry *entry, *sentry;
7246         struct scsi_report_supported_opcodes_all *all;
7247         struct scsi_report_supported_opcodes_descr *descr;
7248         struct scsi_report_supported_opcodes_one *one;
7249         int retval;
7250         int alloc_len, total_len;
7251         int opcode, service_action, i, j, num;
7252
7253         CTL_DEBUG_PRINT(("ctl_report_supported_opcodes\n"));
7254
7255         cdb = (struct scsi_report_supported_opcodes *)ctsio->cdb;
7256         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7257
7258         retval = CTL_RETVAL_COMPLETE;
7259
7260         opcode = cdb->requested_opcode;
7261         service_action = scsi_2btoul(cdb->requested_service_action);
7262         switch (cdb->options & RSO_OPTIONS_MASK) {
7263         case RSO_OPTIONS_ALL:
7264                 num = 0;
7265                 for (i = 0; i < 256; i++) {
7266                         entry = &ctl_cmd_table[i];
7267                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7268                                 for (j = 0; j < 32; j++) {
7269                                         sentry = &((const struct ctl_cmd_entry *)
7270                                             entry->execute)[j];
7271                                         if (ctl_cmd_applicable(
7272                                             lun->be_lun->lun_type, sentry))
7273                                                 num++;
7274                                 }
7275                         } else {
7276                                 if (ctl_cmd_applicable(lun->be_lun->lun_type,
7277                                     entry))
7278                                         num++;
7279                         }
7280                 }
7281                 total_len = sizeof(struct scsi_report_supported_opcodes_all) +
7282                     num * sizeof(struct scsi_report_supported_opcodes_descr);
7283                 break;
7284         case RSO_OPTIONS_OC:
7285                 if (ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) {
7286                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7287                                               /*sks_valid*/ 1,
7288                                               /*command*/ 1,
7289                                               /*field*/ 2,
7290                                               /*bit_valid*/ 1,
7291                                               /*bit*/ 2);
7292                         ctl_done((union ctl_io *)ctsio);
7293                         return (CTL_RETVAL_COMPLETE);
7294                 }
7295                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7296                 break;
7297         case RSO_OPTIONS_OC_SA:
7298                 if ((ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) == 0 ||
7299                     service_action >= 32) {
7300                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7301                                               /*sks_valid*/ 1,
7302                                               /*command*/ 1,
7303                                               /*field*/ 2,
7304                                               /*bit_valid*/ 1,
7305                                               /*bit*/ 2);
7306                         ctl_done((union ctl_io *)ctsio);
7307                         return (CTL_RETVAL_COMPLETE);
7308                 }
7309                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7310                 break;
7311         default:
7312                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7313                                       /*sks_valid*/ 1,
7314                                       /*command*/ 1,
7315                                       /*field*/ 2,
7316                                       /*bit_valid*/ 1,
7317                                       /*bit*/ 2);
7318                 ctl_done((union ctl_io *)ctsio);
7319                 return (CTL_RETVAL_COMPLETE);
7320         }
7321
7322         alloc_len = scsi_4btoul(cdb->length);
7323
7324         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7325
7326         ctsio->kern_sg_entries = 0;
7327
7328         if (total_len < alloc_len) {
7329                 ctsio->residual = alloc_len - total_len;
7330                 ctsio->kern_data_len = total_len;
7331                 ctsio->kern_total_len = total_len;
7332         } else {
7333                 ctsio->residual = 0;
7334                 ctsio->kern_data_len = alloc_len;
7335                 ctsio->kern_total_len = alloc_len;
7336         }
7337         ctsio->kern_data_resid = 0;
7338         ctsio->kern_rel_offset = 0;
7339
7340         switch (cdb->options & RSO_OPTIONS_MASK) {
7341         case RSO_OPTIONS_ALL:
7342                 all = (struct scsi_report_supported_opcodes_all *)
7343                     ctsio->kern_data_ptr;
7344                 num = 0;
7345                 for (i = 0; i < 256; i++) {
7346                         entry = &ctl_cmd_table[i];
7347                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7348                                 for (j = 0; j < 32; j++) {
7349                                         sentry = &((const struct ctl_cmd_entry *)
7350                                             entry->execute)[j];
7351                                         if (!ctl_cmd_applicable(
7352                                             lun->be_lun->lun_type, sentry))
7353                                                 continue;
7354                                         descr = &all->descr[num++];
7355                                         descr->opcode = i;
7356                                         scsi_ulto2b(j, descr->service_action);
7357                                         descr->flags = RSO_SERVACTV;
7358                                         scsi_ulto2b(sentry->length,
7359                                             descr->cdb_length);
7360                                 }
7361                         } else {
7362                                 if (!ctl_cmd_applicable(lun->be_lun->lun_type,
7363                                     entry))
7364                                         continue;
7365                                 descr = &all->descr[num++];
7366                                 descr->opcode = i;
7367                                 scsi_ulto2b(0, descr->service_action);
7368                                 descr->flags = 0;
7369                                 scsi_ulto2b(entry->length, descr->cdb_length);
7370                         }
7371                 }
7372                 scsi_ulto4b(
7373                     num * sizeof(struct scsi_report_supported_opcodes_descr),
7374                     all->length);
7375                 break;
7376         case RSO_OPTIONS_OC:
7377                 one = (struct scsi_report_supported_opcodes_one *)
7378                     ctsio->kern_data_ptr;
7379                 entry = &ctl_cmd_table[opcode];
7380                 goto fill_one;
7381         case RSO_OPTIONS_OC_SA:
7382                 one = (struct scsi_report_supported_opcodes_one *)
7383                     ctsio->kern_data_ptr;
7384                 entry = &ctl_cmd_table[opcode];
7385                 entry = &((const struct ctl_cmd_entry *)
7386                     entry->execute)[service_action];
7387 fill_one:
7388                 if (ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
7389                         one->support = 3;
7390                         scsi_ulto2b(entry->length, one->cdb_length);
7391                         one->cdb_usage[0] = opcode;
7392                         memcpy(&one->cdb_usage[1], entry->usage,
7393                             entry->length - 1);
7394                 } else
7395                         one->support = 1;
7396                 break;
7397         }
7398
7399         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7400         ctsio->be_move_done = ctl_config_move_done;
7401
7402         ctl_datamove((union ctl_io *)ctsio);
7403         return(retval);
7404 }
7405
7406 int
7407 ctl_report_supported_tmf(struct ctl_scsiio *ctsio)
7408 {
7409         struct ctl_lun *lun;
7410         struct scsi_report_supported_tmf *cdb;
7411         struct scsi_report_supported_tmf_data *data;
7412         int retval;
7413         int alloc_len, total_len;
7414
7415         CTL_DEBUG_PRINT(("ctl_report_supported_tmf\n"));
7416
7417         cdb = (struct scsi_report_supported_tmf *)ctsio->cdb;
7418         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7419
7420         retval = CTL_RETVAL_COMPLETE;
7421
7422         total_len = sizeof(struct scsi_report_supported_tmf_data);
7423         alloc_len = scsi_4btoul(cdb->length);
7424
7425         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7426
7427         ctsio->kern_sg_entries = 0;
7428
7429         if (total_len < alloc_len) {
7430                 ctsio->residual = alloc_len - total_len;
7431                 ctsio->kern_data_len = total_len;
7432                 ctsio->kern_total_len = total_len;
7433         } else {
7434                 ctsio->residual = 0;
7435                 ctsio->kern_data_len = alloc_len;
7436                 ctsio->kern_total_len = alloc_len;
7437         }
7438         ctsio->kern_data_resid = 0;
7439         ctsio->kern_rel_offset = 0;
7440
7441         data = (struct scsi_report_supported_tmf_data *)ctsio->kern_data_ptr;
7442         data->byte1 |= RST_ATS | RST_LURS | RST_TRS;
7443
7444         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7445         ctsio->be_move_done = ctl_config_move_done;
7446
7447         ctl_datamove((union ctl_io *)ctsio);
7448         return (retval);
7449 }
7450
7451 int
7452 ctl_report_timestamp(struct ctl_scsiio *ctsio)
7453 {
7454         struct ctl_lun *lun;
7455         struct scsi_report_timestamp *cdb;
7456         struct scsi_report_timestamp_data *data;
7457         struct timeval tv;
7458         int64_t timestamp;
7459         int retval;
7460         int alloc_len, total_len;
7461
7462         CTL_DEBUG_PRINT(("ctl_report_timestamp\n"));
7463
7464         cdb = (struct scsi_report_timestamp *)ctsio->cdb;
7465         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7466
7467         retval = CTL_RETVAL_COMPLETE;
7468
7469         total_len = sizeof(struct scsi_report_timestamp_data);
7470         alloc_len = scsi_4btoul(cdb->length);
7471
7472         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7473
7474         ctsio->kern_sg_entries = 0;
7475
7476         if (total_len < alloc_len) {
7477                 ctsio->residual = alloc_len - total_len;
7478                 ctsio->kern_data_len = total_len;
7479                 ctsio->kern_total_len = total_len;
7480         } else {
7481                 ctsio->residual = 0;
7482                 ctsio->kern_data_len = alloc_len;
7483                 ctsio->kern_total_len = alloc_len;
7484         }
7485         ctsio->kern_data_resid = 0;
7486         ctsio->kern_rel_offset = 0;
7487
7488         data = (struct scsi_report_timestamp_data *)ctsio->kern_data_ptr;
7489         scsi_ulto2b(sizeof(*data) - 2, data->length);
7490         data->origin = RTS_ORIG_OUTSIDE;
7491         getmicrotime(&tv);
7492         timestamp = (int64_t)tv.tv_sec * 1000 + tv.tv_usec / 1000;
7493         scsi_ulto4b(timestamp >> 16, data->timestamp);
7494         scsi_ulto2b(timestamp & 0xffff, &data->timestamp[4]);
7495
7496         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7497         ctsio->be_move_done = ctl_config_move_done;
7498
7499         ctl_datamove((union ctl_io *)ctsio);
7500         return (retval);
7501 }
7502
7503 int
7504 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7505 {
7506         struct scsi_per_res_in *cdb;
7507         int alloc_len, total_len = 0;
7508         /* struct scsi_per_res_in_rsrv in_data; */
7509         struct ctl_lun *lun;
7510         struct ctl_softc *softc;
7511
7512         CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7513
7514         softc = control_softc;
7515
7516         cdb = (struct scsi_per_res_in *)ctsio->cdb;
7517
7518         alloc_len = scsi_2btoul(cdb->length);
7519
7520         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7521
7522 retry:
7523         mtx_lock(&lun->lun_lock);
7524         switch (cdb->action) {
7525         case SPRI_RK: /* read keys */
7526                 total_len = sizeof(struct scsi_per_res_in_keys) +
7527                         lun->pr_key_count *
7528                         sizeof(struct scsi_per_res_key);
7529                 break;
7530         case SPRI_RR: /* read reservation */
7531                 if (lun->flags & CTL_LUN_PR_RESERVED)
7532                         total_len = sizeof(struct scsi_per_res_in_rsrv);
7533                 else
7534                         total_len = sizeof(struct scsi_per_res_in_header);
7535                 break;
7536         case SPRI_RC: /* report capabilities */
7537                 total_len = sizeof(struct scsi_per_res_cap);
7538                 break;
7539         default:
7540                 panic("Invalid PR type %x", cdb->action);
7541         }
7542         mtx_unlock(&lun->lun_lock);
7543
7544         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7545
7546         if (total_len < alloc_len) {
7547                 ctsio->residual = alloc_len - total_len;
7548                 ctsio->kern_data_len = total_len;
7549                 ctsio->kern_total_len = total_len;
7550         } else {
7551                 ctsio->residual = 0;
7552                 ctsio->kern_data_len = alloc_len;
7553                 ctsio->kern_total_len = alloc_len;
7554         }
7555
7556         ctsio->kern_data_resid = 0;
7557         ctsio->kern_rel_offset = 0;
7558         ctsio->kern_sg_entries = 0;
7559
7560         mtx_lock(&lun->lun_lock);
7561         switch (cdb->action) {
7562         case SPRI_RK: { // read keys
7563         struct scsi_per_res_in_keys *res_keys;
7564                 int i, key_count;
7565
7566                 res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7567
7568                 /*
7569                  * We had to drop the lock to allocate our buffer, which
7570                  * leaves time for someone to come in with another
7571                  * persistent reservation.  (That is unlikely, though,
7572                  * since this should be the only persistent reservation
7573                  * command active right now.)
7574                  */
7575                 if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7576                     (lun->pr_key_count *
7577                      sizeof(struct scsi_per_res_key)))){
7578                         mtx_unlock(&lun->lun_lock);
7579                         free(ctsio->kern_data_ptr, M_CTL);
7580                         printf("%s: reservation length changed, retrying\n",
7581                                __func__);
7582                         goto retry;
7583                 }
7584
7585                 scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7586
7587                 scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7588                              lun->pr_key_count, res_keys->header.length);
7589
7590                 for (i = 0, key_count = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7591                         if (!lun->per_res[i].registered)
7592                                 continue;
7593
7594                         /*
7595                          * We used lun->pr_key_count to calculate the
7596                          * size to allocate.  If it turns out the number of
7597                          * initiators with the registered flag set is
7598                          * larger than that (i.e. they haven't been kept in
7599                          * sync), we've got a problem.
7600                          */
7601                         if (key_count >= lun->pr_key_count) {
7602 #ifdef NEEDTOPORT
7603                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
7604                                             CTL_PR_ERROR,
7605                                             csevent_LogType_Fault,
7606                                             csevent_AlertLevel_Yellow,
7607                                             csevent_FRU_ShelfController,
7608                                             csevent_FRU_Firmware,
7609                                         csevent_FRU_Unknown,
7610                                             "registered keys %d >= key "
7611                                             "count %d", key_count,
7612                                             lun->pr_key_count);
7613 #endif
7614                                 key_count++;
7615                                 continue;
7616                         }
7617                         memcpy(res_keys->keys[key_count].key,
7618                                lun->per_res[i].res_key.key,
7619                                ctl_min(sizeof(res_keys->keys[key_count].key),
7620                                sizeof(lun->per_res[i].res_key)));
7621                         key_count++;
7622                 }
7623                 break;
7624         }
7625         case SPRI_RR: { // read reservation
7626                 struct scsi_per_res_in_rsrv *res;
7627                 int tmp_len, header_only;
7628
7629                 res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7630
7631                 scsi_ulto4b(lun->PRGeneration, res->header.generation);
7632
7633                 if (lun->flags & CTL_LUN_PR_RESERVED)
7634                 {
7635                         tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7636                         scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7637                                     res->header.length);
7638                         header_only = 0;
7639                 } else {
7640                         tmp_len = sizeof(struct scsi_per_res_in_header);
7641                         scsi_ulto4b(0, res->header.length);
7642                         header_only = 1;
7643                 }
7644
7645                 /*
7646                  * We had to drop the lock to allocate our buffer, which
7647                  * leaves time for someone to come in with another
7648                  * persistent reservation.  (That is unlikely, though,
7649                  * since this should be the only persistent reservation
7650                  * command active right now.)
7651                  */
7652                 if (tmp_len != total_len) {
7653                         mtx_unlock(&lun->lun_lock);
7654                         free(ctsio->kern_data_ptr, M_CTL);
7655                         printf("%s: reservation status changed, retrying\n",
7656                                __func__);
7657                         goto retry;
7658                 }
7659
7660                 /*
7661                  * No reservation held, so we're done.
7662                  */
7663                 if (header_only != 0)
7664                         break;
7665
7666                 /*
7667                  * If the registration is an All Registrants type, the key
7668                  * is 0, since it doesn't really matter.
7669                  */
7670                 if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7671                         memcpy(res->data.reservation,
7672                                &lun->per_res[lun->pr_res_idx].res_key,
7673                                sizeof(struct scsi_per_res_key));
7674                 }
7675                 res->data.scopetype = lun->res_type;
7676                 break;
7677         }
7678         case SPRI_RC:     //report capabilities
7679         {
7680                 struct scsi_per_res_cap *res_cap;
7681                 uint16_t type_mask;
7682
7683                 res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7684                 scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7685                 res_cap->flags2 |= SPRI_TMV | SPRI_ALLOW_3;
7686                 type_mask = SPRI_TM_WR_EX_AR |
7687                             SPRI_TM_EX_AC_RO |
7688                             SPRI_TM_WR_EX_RO |
7689                             SPRI_TM_EX_AC |
7690                             SPRI_TM_WR_EX |
7691                             SPRI_TM_EX_AC_AR;
7692                 scsi_ulto2b(type_mask, res_cap->type_mask);
7693                 break;
7694         }
7695         case SPRI_RS: //read full status
7696         default:
7697                 /*
7698                  * This is a bug, because we just checked for this above,
7699                  * and should have returned an error.
7700                  */
7701                 panic("Invalid PR type %x", cdb->action);
7702                 break; /* NOTREACHED */
7703         }
7704         mtx_unlock(&lun->lun_lock);
7705
7706         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7707         ctsio->be_move_done = ctl_config_move_done;
7708
7709         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7710                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7711                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7712                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7713                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7714
7715         ctl_datamove((union ctl_io *)ctsio);
7716
7717         return (CTL_RETVAL_COMPLETE);
7718 }
7719
7720 /*
7721  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
7722  * it should return.
7723  */
7724 static int
7725 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
7726                 uint64_t sa_res_key, uint8_t type, uint32_t residx,
7727                 struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
7728                 struct scsi_per_res_out_parms* param)
7729 {
7730         union ctl_ha_msg persis_io;
7731         int retval, i;
7732         int isc_retval;
7733
7734         retval = 0;
7735
7736         mtx_lock(&lun->lun_lock);
7737         if (sa_res_key == 0) {
7738                 if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7739                         /* validate scope and type */
7740                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7741                              SPR_LU_SCOPE) {
7742                                 mtx_unlock(&lun->lun_lock);
7743                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7744                                                       /*sks_valid*/ 1,
7745                                                       /*command*/ 1,
7746                                                       /*field*/ 2,
7747                                                       /*bit_valid*/ 1,
7748                                                       /*bit*/ 4);
7749                                 ctl_done((union ctl_io *)ctsio);
7750                                 return (1);
7751                         }
7752
7753                         if (type>8 || type==2 || type==4 || type==0) {
7754                                 mtx_unlock(&lun->lun_lock);
7755                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7756                                                       /*sks_valid*/ 1,
7757                                                       /*command*/ 1,
7758                                                       /*field*/ 2,
7759                                                       /*bit_valid*/ 1,
7760                                                       /*bit*/ 0);
7761                                 ctl_done((union ctl_io *)ctsio);
7762                                 return (1);
7763                         }
7764
7765                         /* temporarily unregister this nexus */
7766                         lun->per_res[residx].registered = 0;
7767
7768                         /*
7769                          * Unregister everybody else and build UA for
7770                          * them
7771                          */
7772                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7773                                 if (lun->per_res[i].registered == 0)
7774                                         continue;
7775
7776                                 if (!persis_offset
7777                                  && i <CTL_MAX_INITIATORS)
7778                                         lun->pending_sense[i].ua_pending |=
7779                                                 CTL_UA_REG_PREEMPT;
7780                                 else if (persis_offset
7781                                       && i >= persis_offset)
7782                                         lun->pending_sense[i-persis_offset
7783                                                 ].ua_pending |=
7784                                                 CTL_UA_REG_PREEMPT;
7785                                 lun->per_res[i].registered = 0;
7786                                 memset(&lun->per_res[i].res_key, 0,
7787                                        sizeof(struct scsi_per_res_key));
7788                         }
7789                         lun->per_res[residx].registered = 1;
7790                         lun->pr_key_count = 1;
7791                         lun->res_type = type;
7792                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7793                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7794                                 lun->pr_res_idx = residx;
7795
7796                         /* send msg to other side */
7797                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7798                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7799                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7800                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7801                         persis_io.pr.pr_info.res_type = type;
7802                         memcpy(persis_io.pr.pr_info.sa_res_key,
7803                                param->serv_act_res_key,
7804                                sizeof(param->serv_act_res_key));
7805                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7806                              &persis_io, sizeof(persis_io), 0)) >
7807                              CTL_HA_STATUS_SUCCESS) {
7808                                 printf("CTL:Persis Out error returned "
7809                                        "from ctl_ha_msg_send %d\n",
7810                                        isc_retval);
7811                         }
7812                 } else {
7813                         /* not all registrants */
7814                         mtx_unlock(&lun->lun_lock);
7815                         free(ctsio->kern_data_ptr, M_CTL);
7816                         ctl_set_invalid_field(ctsio,
7817                                               /*sks_valid*/ 1,
7818                                               /*command*/ 0,
7819                                               /*field*/ 8,
7820                                               /*bit_valid*/ 0,
7821                                               /*bit*/ 0);
7822                         ctl_done((union ctl_io *)ctsio);
7823                         return (1);
7824                 }
7825         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7826                 || !(lun->flags & CTL_LUN_PR_RESERVED)) {
7827                 int found = 0;
7828
7829                 if (res_key == sa_res_key) {
7830                         /* special case */
7831                         /*
7832                          * The spec implies this is not good but doesn't
7833                          * say what to do. There are two choices either
7834                          * generate a res conflict or check condition
7835                          * with illegal field in parameter data. Since
7836                          * that is what is done when the sa_res_key is
7837                          * zero I'll take that approach since this has
7838                          * to do with the sa_res_key.
7839                          */
7840                         mtx_unlock(&lun->lun_lock);
7841                         free(ctsio->kern_data_ptr, M_CTL);
7842                         ctl_set_invalid_field(ctsio,
7843                                               /*sks_valid*/ 1,
7844                                               /*command*/ 0,
7845                                               /*field*/ 8,
7846                                               /*bit_valid*/ 0,
7847                                               /*bit*/ 0);
7848                         ctl_done((union ctl_io *)ctsio);
7849                         return (1);
7850                 }
7851
7852                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7853                         if (lun->per_res[i].registered
7854                          && memcmp(param->serv_act_res_key,
7855                             lun->per_res[i].res_key.key,
7856                             sizeof(struct scsi_per_res_key)) != 0)
7857                                 continue;
7858
7859                         found = 1;
7860                         lun->per_res[i].registered = 0;
7861                         memset(&lun->per_res[i].res_key, 0,
7862                                sizeof(struct scsi_per_res_key));
7863                         lun->pr_key_count--;
7864
7865                         if (!persis_offset
7866                          && i < CTL_MAX_INITIATORS)
7867                                 lun->pending_sense[i].ua_pending |=
7868                                         CTL_UA_REG_PREEMPT;
7869                         else if (persis_offset
7870                               && i >= persis_offset)
7871                                 lun->pending_sense[i-persis_offset].ua_pending|=
7872                                         CTL_UA_REG_PREEMPT;
7873                 }
7874                 if (!found) {
7875                         mtx_unlock(&lun->lun_lock);
7876                         free(ctsio->kern_data_ptr, M_CTL);
7877                         ctl_set_reservation_conflict(ctsio);
7878                         ctl_done((union ctl_io *)ctsio);
7879                         return (CTL_RETVAL_COMPLETE);
7880                 }
7881                 /* send msg to other side */
7882                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7883                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7884                 persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7885                 persis_io.pr.pr_info.residx = lun->pr_res_idx;
7886                 persis_io.pr.pr_info.res_type = type;
7887                 memcpy(persis_io.pr.pr_info.sa_res_key,
7888                        param->serv_act_res_key,
7889                        sizeof(param->serv_act_res_key));
7890                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7891                      &persis_io, sizeof(persis_io), 0)) >
7892                      CTL_HA_STATUS_SUCCESS) {
7893                         printf("CTL:Persis Out error returned from "
7894                                "ctl_ha_msg_send %d\n", isc_retval);
7895                 }
7896         } else {
7897                 /* Reserved but not all registrants */
7898                 /* sa_res_key is res holder */
7899                 if (memcmp(param->serv_act_res_key,
7900                    lun->per_res[lun->pr_res_idx].res_key.key,
7901                    sizeof(struct scsi_per_res_key)) == 0) {
7902                         /* validate scope and type */
7903                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7904                              SPR_LU_SCOPE) {
7905                                 mtx_unlock(&lun->lun_lock);
7906                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7907                                                       /*sks_valid*/ 1,
7908                                                       /*command*/ 1,
7909                                                       /*field*/ 2,
7910                                                       /*bit_valid*/ 1,
7911                                                       /*bit*/ 4);
7912                                 ctl_done((union ctl_io *)ctsio);
7913                                 return (1);
7914                         }
7915
7916                         if (type>8 || type==2 || type==4 || type==0) {
7917                                 mtx_unlock(&lun->lun_lock);
7918                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7919                                                       /*sks_valid*/ 1,
7920                                                       /*command*/ 1,
7921                                                       /*field*/ 2,
7922                                                       /*bit_valid*/ 1,
7923                                                       /*bit*/ 0);
7924                                 ctl_done((union ctl_io *)ctsio);
7925                                 return (1);
7926                         }
7927
7928                         /*
7929                          * Do the following:
7930                          * if sa_res_key != res_key remove all
7931                          * registrants w/sa_res_key and generate UA
7932                          * for these registrants(Registrations
7933                          * Preempted) if it wasn't an exclusive
7934                          * reservation generate UA(Reservations
7935                          * Preempted) for all other registered nexuses
7936                          * if the type has changed. Establish the new
7937                          * reservation and holder. If res_key and
7938                          * sa_res_key are the same do the above
7939                          * except don't unregister the res holder.
7940                          */
7941
7942                         /*
7943                          * Temporarily unregister so it won't get
7944                          * removed or UA generated
7945                          */
7946                         lun->per_res[residx].registered = 0;
7947                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7948                                 if (lun->per_res[i].registered == 0)
7949                                         continue;
7950
7951                                 if (memcmp(param->serv_act_res_key,
7952                                     lun->per_res[i].res_key.key,
7953                                     sizeof(struct scsi_per_res_key)) == 0) {
7954                                         lun->per_res[i].registered = 0;
7955                                         memset(&lun->per_res[i].res_key,
7956                                                0,
7957                                                sizeof(struct scsi_per_res_key));
7958                                         lun->pr_key_count--;
7959
7960                                         if (!persis_offset
7961                                          && i < CTL_MAX_INITIATORS)
7962                                                 lun->pending_sense[i
7963                                                         ].ua_pending |=
7964                                                         CTL_UA_REG_PREEMPT;
7965                                         else if (persis_offset
7966                                               && i >= persis_offset)
7967                                                 lun->pending_sense[
7968                                                   i-persis_offset].ua_pending |=
7969                                                   CTL_UA_REG_PREEMPT;
7970                                 } else if (type != lun->res_type
7971                                         && (lun->res_type == SPR_TYPE_WR_EX_RO
7972                                          || lun->res_type ==SPR_TYPE_EX_AC_RO)){
7973                                                 if (!persis_offset
7974                                                  && i < CTL_MAX_INITIATORS)
7975                                                         lun->pending_sense[i
7976                                                         ].ua_pending |=
7977                                                         CTL_UA_RES_RELEASE;
7978                                                 else if (persis_offset
7979                                                       && i >= persis_offset)
7980                                                         lun->pending_sense[
7981                                                         i-persis_offset
7982                                                         ].ua_pending |=
7983                                                         CTL_UA_RES_RELEASE;
7984                                 }
7985                         }
7986                         lun->per_res[residx].registered = 1;
7987                         lun->res_type = type;
7988                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7989                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7990                                 lun->pr_res_idx = residx;
7991                         else
7992                                 lun->pr_res_idx =
7993                                         CTL_PR_ALL_REGISTRANTS;
7994
7995                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7996                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7997                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7998                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7999                         persis_io.pr.pr_info.res_type = type;
8000                         memcpy(persis_io.pr.pr_info.sa_res_key,
8001                                param->serv_act_res_key,
8002                                sizeof(param->serv_act_res_key));
8003                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8004                              &persis_io, sizeof(persis_io), 0)) >
8005                              CTL_HA_STATUS_SUCCESS) {
8006                                 printf("CTL:Persis Out error returned "
8007                                        "from ctl_ha_msg_send %d\n",
8008                                        isc_retval);
8009                         }
8010                 } else {
8011                         /*
8012                          * sa_res_key is not the res holder just
8013                          * remove registrants
8014                          */
8015                         int found=0;
8016
8017                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8018                                 if (memcmp(param->serv_act_res_key,
8019                                     lun->per_res[i].res_key.key,
8020                                     sizeof(struct scsi_per_res_key)) != 0)
8021                                         continue;
8022
8023                                 found = 1;
8024                                 lun->per_res[i].registered = 0;
8025                                 memset(&lun->per_res[i].res_key, 0,
8026                                        sizeof(struct scsi_per_res_key));
8027                                 lun->pr_key_count--;
8028
8029                                 if (!persis_offset
8030                                  && i < CTL_MAX_INITIATORS)
8031                                         lun->pending_sense[i].ua_pending |=
8032                                                 CTL_UA_REG_PREEMPT;
8033                                 else if (persis_offset
8034                                       && i >= persis_offset)
8035                                         lun->pending_sense[
8036                                                 i-persis_offset].ua_pending |=
8037                                                 CTL_UA_REG_PREEMPT;
8038                         }
8039
8040                         if (!found) {
8041                                 mtx_unlock(&lun->lun_lock);
8042                                 free(ctsio->kern_data_ptr, M_CTL);
8043                                 ctl_set_reservation_conflict(ctsio);
8044                                 ctl_done((union ctl_io *)ctsio);
8045                                 return (1);
8046                         }
8047                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8048                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8049                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8050                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8051                         persis_io.pr.pr_info.res_type = type;
8052                         memcpy(persis_io.pr.pr_info.sa_res_key,
8053                                param->serv_act_res_key,
8054                                sizeof(param->serv_act_res_key));
8055                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8056                              &persis_io, sizeof(persis_io), 0)) >
8057                              CTL_HA_STATUS_SUCCESS) {
8058                                 printf("CTL:Persis Out error returned "
8059                                        "from ctl_ha_msg_send %d\n",
8060                                 isc_retval);
8061                         }
8062                 }
8063         }
8064
8065         lun->PRGeneration++;
8066         mtx_unlock(&lun->lun_lock);
8067
8068         return (retval);
8069 }
8070
8071 static void
8072 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
8073 {
8074         int i;
8075
8076         if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8077          || lun->pr_res_idx == CTL_PR_NO_RESERVATION
8078          || memcmp(&lun->per_res[lun->pr_res_idx].res_key,
8079                    msg->pr.pr_info.sa_res_key,
8080                    sizeof(struct scsi_per_res_key)) != 0) {
8081                 uint64_t sa_res_key;
8082                 sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
8083
8084                 if (sa_res_key == 0) {
8085                         /* temporarily unregister this nexus */
8086                         lun->per_res[msg->pr.pr_info.residx].registered = 0;
8087
8088                         /*
8089                          * Unregister everybody else and build UA for
8090                          * them
8091                          */
8092                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8093                                 if (lun->per_res[i].registered == 0)
8094                                         continue;
8095
8096                                 if (!persis_offset
8097                                  && i < CTL_MAX_INITIATORS)
8098                                         lun->pending_sense[i].ua_pending |=
8099                                                 CTL_UA_REG_PREEMPT;
8100                                 else if (persis_offset && i >= persis_offset)
8101                                         lun->pending_sense[i -
8102                                                 persis_offset].ua_pending |=
8103                                                 CTL_UA_REG_PREEMPT;
8104                                 lun->per_res[i].registered = 0;
8105                                 memset(&lun->per_res[i].res_key, 0,
8106                                        sizeof(struct scsi_per_res_key));
8107                         }
8108
8109                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
8110                         lun->pr_key_count = 1;
8111                         lun->res_type = msg->pr.pr_info.res_type;
8112                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8113                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8114                                 lun->pr_res_idx = msg->pr.pr_info.residx;
8115                 } else {
8116                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8117                                 if (memcmp(msg->pr.pr_info.sa_res_key,
8118                                    lun->per_res[i].res_key.key,
8119                                    sizeof(struct scsi_per_res_key)) != 0)
8120                                         continue;
8121
8122                                 lun->per_res[i].registered = 0;
8123                                 memset(&lun->per_res[i].res_key, 0,
8124                                        sizeof(struct scsi_per_res_key));
8125                                 lun->pr_key_count--;
8126
8127                                 if (!persis_offset
8128                                  && i < persis_offset)
8129                                         lun->pending_sense[i].ua_pending |=
8130                                                 CTL_UA_REG_PREEMPT;
8131                                 else if (persis_offset
8132                                       && i >= persis_offset)
8133                                         lun->pending_sense[i -
8134                                                 persis_offset].ua_pending |=
8135                                                 CTL_UA_REG_PREEMPT;
8136                         }
8137                 }
8138         } else {
8139                 /*
8140                  * Temporarily unregister so it won't get removed
8141                  * or UA generated
8142                  */
8143                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
8144                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8145                         if (lun->per_res[i].registered == 0)
8146                                 continue;
8147
8148                         if (memcmp(msg->pr.pr_info.sa_res_key,
8149                            lun->per_res[i].res_key.key,
8150                            sizeof(struct scsi_per_res_key)) == 0) {
8151                                 lun->per_res[i].registered = 0;
8152                                 memset(&lun->per_res[i].res_key, 0,
8153                                        sizeof(struct scsi_per_res_key));
8154                                 lun->pr_key_count--;
8155                                 if (!persis_offset
8156                                  && i < CTL_MAX_INITIATORS)
8157                                         lun->pending_sense[i].ua_pending |=
8158                                                 CTL_UA_REG_PREEMPT;
8159                                 else if (persis_offset
8160                                       && i >= persis_offset)
8161                                         lun->pending_sense[i -
8162                                                 persis_offset].ua_pending |=
8163                                                 CTL_UA_REG_PREEMPT;
8164                         } else if (msg->pr.pr_info.res_type != lun->res_type
8165                                 && (lun->res_type == SPR_TYPE_WR_EX_RO
8166                                  || lun->res_type == SPR_TYPE_EX_AC_RO)) {
8167                                         if (!persis_offset
8168                                          && i < persis_offset)
8169                                                 lun->pending_sense[i
8170                                                         ].ua_pending |=
8171                                                         CTL_UA_RES_RELEASE;
8172                                         else if (persis_offset
8173                                               && i >= persis_offset)
8174                                         lun->pending_sense[i -
8175                                                 persis_offset].ua_pending |=
8176                                                 CTL_UA_RES_RELEASE;
8177                         }
8178                 }
8179                 lun->per_res[msg->pr.pr_info.residx].registered = 1;
8180                 lun->res_type = msg->pr.pr_info.res_type;
8181                 if (lun->res_type != SPR_TYPE_WR_EX_AR
8182                  && lun->res_type != SPR_TYPE_EX_AC_AR)
8183                         lun->pr_res_idx = msg->pr.pr_info.residx;
8184                 else
8185                         lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8186         }
8187         lun->PRGeneration++;
8188
8189 }
8190
8191
8192 int
8193 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
8194 {
8195         int retval;
8196         int isc_retval;
8197         u_int32_t param_len;
8198         struct scsi_per_res_out *cdb;
8199         struct ctl_lun *lun;
8200         struct scsi_per_res_out_parms* param;
8201         struct ctl_softc *softc;
8202         uint32_t residx;
8203         uint64_t res_key, sa_res_key;
8204         uint8_t type;
8205         union ctl_ha_msg persis_io;
8206         int    i;
8207
8208         CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
8209
8210         retval = CTL_RETVAL_COMPLETE;
8211
8212         softc = control_softc;
8213
8214         cdb = (struct scsi_per_res_out *)ctsio->cdb;
8215         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8216
8217         /*
8218          * We only support whole-LUN scope.  The scope & type are ignored for
8219          * register, register and ignore existing key and clear.
8220          * We sometimes ignore scope and type on preempts too!!
8221          * Verify reservation type here as well.
8222          */
8223         type = cdb->scope_type & SPR_TYPE_MASK;
8224         if ((cdb->action == SPRO_RESERVE)
8225          || (cdb->action == SPRO_RELEASE)) {
8226                 if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
8227                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8228                                               /*sks_valid*/ 1,
8229                                               /*command*/ 1,
8230                                               /*field*/ 2,
8231                                               /*bit_valid*/ 1,
8232                                               /*bit*/ 4);
8233                         ctl_done((union ctl_io *)ctsio);
8234                         return (CTL_RETVAL_COMPLETE);
8235                 }
8236
8237                 if (type>8 || type==2 || type==4 || type==0) {
8238                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8239                                               /*sks_valid*/ 1,
8240                                               /*command*/ 1,
8241                                               /*field*/ 2,
8242                                               /*bit_valid*/ 1,
8243                                               /*bit*/ 0);
8244                         ctl_done((union ctl_io *)ctsio);
8245                         return (CTL_RETVAL_COMPLETE);
8246                 }
8247         }
8248
8249         param_len = scsi_4btoul(cdb->length);
8250
8251         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
8252                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
8253                 ctsio->kern_data_len = param_len;
8254                 ctsio->kern_total_len = param_len;
8255                 ctsio->kern_data_resid = 0;
8256                 ctsio->kern_rel_offset = 0;
8257                 ctsio->kern_sg_entries = 0;
8258                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8259                 ctsio->be_move_done = ctl_config_move_done;
8260                 ctl_datamove((union ctl_io *)ctsio);
8261
8262                 return (CTL_RETVAL_COMPLETE);
8263         }
8264
8265         param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
8266
8267         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8268         res_key = scsi_8btou64(param->res_key.key);
8269         sa_res_key = scsi_8btou64(param->serv_act_res_key);
8270
8271         /*
8272          * Validate the reservation key here except for SPRO_REG_IGNO
8273          * This must be done for all other service actions
8274          */
8275         if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
8276                 mtx_lock(&lun->lun_lock);
8277                 if (lun->per_res[residx].registered) {
8278                     if (memcmp(param->res_key.key,
8279                                lun->per_res[residx].res_key.key,
8280                                ctl_min(sizeof(param->res_key),
8281                                sizeof(lun->per_res[residx].res_key))) != 0) {
8282                                 /*
8283                                  * The current key passed in doesn't match
8284                                  * the one the initiator previously
8285                                  * registered.
8286                                  */
8287                                 mtx_unlock(&lun->lun_lock);
8288                                 free(ctsio->kern_data_ptr, M_CTL);
8289                                 ctl_set_reservation_conflict(ctsio);
8290                                 ctl_done((union ctl_io *)ctsio);
8291                                 return (CTL_RETVAL_COMPLETE);
8292                         }
8293                 } else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
8294                         /*
8295                          * We are not registered
8296                          */
8297                         mtx_unlock(&lun->lun_lock);
8298                         free(ctsio->kern_data_ptr, M_CTL);
8299                         ctl_set_reservation_conflict(ctsio);
8300                         ctl_done((union ctl_io *)ctsio);
8301                         return (CTL_RETVAL_COMPLETE);
8302                 } else if (res_key != 0) {
8303                         /*
8304                          * We are not registered and trying to register but
8305                          * the register key isn't zero.
8306                          */
8307                         mtx_unlock(&lun->lun_lock);
8308                         free(ctsio->kern_data_ptr, M_CTL);
8309                         ctl_set_reservation_conflict(ctsio);
8310                         ctl_done((union ctl_io *)ctsio);
8311                         return (CTL_RETVAL_COMPLETE);
8312                 }
8313                 mtx_unlock(&lun->lun_lock);
8314         }
8315
8316         switch (cdb->action & SPRO_ACTION_MASK) {
8317         case SPRO_REGISTER:
8318         case SPRO_REG_IGNO: {
8319
8320 #if 0
8321                 printf("Registration received\n");
8322 #endif
8323
8324                 /*
8325                  * We don't support any of these options, as we report in
8326                  * the read capabilities request (see
8327                  * ctl_persistent_reserve_in(), above).
8328                  */
8329                 if ((param->flags & SPR_SPEC_I_PT)
8330                  || (param->flags & SPR_ALL_TG_PT)
8331                  || (param->flags & SPR_APTPL)) {
8332                         int bit_ptr;
8333
8334                         if (param->flags & SPR_APTPL)
8335                                 bit_ptr = 0;
8336                         else if (param->flags & SPR_ALL_TG_PT)
8337                                 bit_ptr = 2;
8338                         else /* SPR_SPEC_I_PT */
8339                                 bit_ptr = 3;
8340
8341                         free(ctsio->kern_data_ptr, M_CTL);
8342                         ctl_set_invalid_field(ctsio,
8343                                               /*sks_valid*/ 1,
8344                                               /*command*/ 0,
8345                                               /*field*/ 20,
8346                                               /*bit_valid*/ 1,
8347                                               /*bit*/ bit_ptr);
8348                         ctl_done((union ctl_io *)ctsio);
8349                         return (CTL_RETVAL_COMPLETE);
8350                 }
8351
8352                 mtx_lock(&lun->lun_lock);
8353
8354                 /*
8355                  * The initiator wants to clear the
8356                  * key/unregister.
8357                  */
8358                 if (sa_res_key == 0) {
8359                         if ((res_key == 0
8360                           && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
8361                          || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
8362                           && !lun->per_res[residx].registered)) {
8363                                 mtx_unlock(&lun->lun_lock);
8364                                 goto done;
8365                         }
8366
8367                         lun->per_res[residx].registered = 0;
8368                         memset(&lun->per_res[residx].res_key,
8369                                0, sizeof(lun->per_res[residx].res_key));
8370                         lun->pr_key_count--;
8371
8372                         if (residx == lun->pr_res_idx) {
8373                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8374                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8375
8376                                 if ((lun->res_type == SPR_TYPE_WR_EX_RO
8377                                   || lun->res_type == SPR_TYPE_EX_AC_RO)
8378                                  && lun->pr_key_count) {
8379                                         /*
8380                                          * If the reservation is a registrants
8381                                          * only type we need to generate a UA
8382                                          * for other registered inits.  The
8383                                          * sense code should be RESERVATIONS
8384                                          * RELEASED
8385                                          */
8386
8387                                         for (i = 0; i < CTL_MAX_INITIATORS;i++){
8388                                                 if (lun->per_res[
8389                                                     i+persis_offset].registered
8390                                                     == 0)
8391                                                         continue;
8392                                                 lun->pending_sense[i
8393                                                         ].ua_pending |=
8394                                                         CTL_UA_RES_RELEASE;
8395                                         }
8396                                 }
8397                                 lun->res_type = 0;
8398                         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8399                                 if (lun->pr_key_count==0) {
8400                                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8401                                         lun->res_type = 0;
8402                                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8403                                 }
8404                         }
8405                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8406                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8407                         persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8408                         persis_io.pr.pr_info.residx = residx;
8409                         if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8410                              &persis_io, sizeof(persis_io), 0 )) >
8411                              CTL_HA_STATUS_SUCCESS) {
8412                                 printf("CTL:Persis Out error returned from "
8413                                        "ctl_ha_msg_send %d\n", isc_retval);
8414                         }
8415                 } else /* sa_res_key != 0 */ {
8416
8417                         /*
8418                          * If we aren't registered currently then increment
8419                          * the key count and set the registered flag.
8420                          */
8421                         if (!lun->per_res[residx].registered) {
8422                                 lun->pr_key_count++;
8423                                 lun->per_res[residx].registered = 1;
8424                         }
8425
8426                         memcpy(&lun->per_res[residx].res_key,
8427                                param->serv_act_res_key,
8428                                ctl_min(sizeof(param->serv_act_res_key),
8429                                sizeof(lun->per_res[residx].res_key)));
8430
8431                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8432                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8433                         persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8434                         persis_io.pr.pr_info.residx = residx;
8435                         memcpy(persis_io.pr.pr_info.sa_res_key,
8436                                param->serv_act_res_key,
8437                                sizeof(param->serv_act_res_key));
8438                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8439                              &persis_io, sizeof(persis_io), 0)) >
8440                              CTL_HA_STATUS_SUCCESS) {
8441                                 printf("CTL:Persis Out error returned from "
8442                                        "ctl_ha_msg_send %d\n", isc_retval);
8443                         }
8444                 }
8445                 lun->PRGeneration++;
8446                 mtx_unlock(&lun->lun_lock);
8447
8448                 break;
8449         }
8450         case SPRO_RESERVE:
8451 #if 0
8452                 printf("Reserve executed type %d\n", type);
8453 #endif
8454                 mtx_lock(&lun->lun_lock);
8455                 if (lun->flags & CTL_LUN_PR_RESERVED) {
8456                         /*
8457                          * if this isn't the reservation holder and it's
8458                          * not a "all registrants" type or if the type is
8459                          * different then we have a conflict
8460                          */
8461                         if ((lun->pr_res_idx != residx
8462                           && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8463                          || lun->res_type != type) {
8464                                 mtx_unlock(&lun->lun_lock);
8465                                 free(ctsio->kern_data_ptr, M_CTL);
8466                                 ctl_set_reservation_conflict(ctsio);
8467                                 ctl_done((union ctl_io *)ctsio);
8468                                 return (CTL_RETVAL_COMPLETE);
8469                         }
8470                         mtx_unlock(&lun->lun_lock);
8471                 } else /* create a reservation */ {
8472                         /*
8473                          * If it's not an "all registrants" type record
8474                          * reservation holder
8475                          */
8476                         if (type != SPR_TYPE_WR_EX_AR
8477                          && type != SPR_TYPE_EX_AC_AR)
8478                                 lun->pr_res_idx = residx; /* Res holder */
8479                         else
8480                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8481
8482                         lun->flags |= CTL_LUN_PR_RESERVED;
8483                         lun->res_type = type;
8484
8485                         mtx_unlock(&lun->lun_lock);
8486
8487                         /* send msg to other side */
8488                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8489                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8490                         persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8491                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8492                         persis_io.pr.pr_info.res_type = type;
8493                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8494                              &persis_io, sizeof(persis_io), 0)) >
8495                              CTL_HA_STATUS_SUCCESS) {
8496                                 printf("CTL:Persis Out error returned from "
8497                                        "ctl_ha_msg_send %d\n", isc_retval);
8498                         }
8499                 }
8500                 break;
8501
8502         case SPRO_RELEASE:
8503                 mtx_lock(&lun->lun_lock);
8504                 if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8505                         /* No reservation exists return good status */
8506                         mtx_unlock(&lun->lun_lock);
8507                         goto done;
8508                 }
8509                 /*
8510                  * Is this nexus a reservation holder?
8511                  */
8512                 if (lun->pr_res_idx != residx
8513                  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8514                         /*
8515                          * not a res holder return good status but
8516                          * do nothing
8517                          */
8518                         mtx_unlock(&lun->lun_lock);
8519                         goto done;
8520                 }
8521
8522                 if (lun->res_type != type) {
8523                         mtx_unlock(&lun->lun_lock);
8524                         free(ctsio->kern_data_ptr, M_CTL);
8525                         ctl_set_illegal_pr_release(ctsio);
8526                         ctl_done((union ctl_io *)ctsio);
8527                         return (CTL_RETVAL_COMPLETE);
8528                 }
8529
8530                 /* okay to release */
8531                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8532                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8533                 lun->res_type = 0;
8534
8535                 /*
8536                  * if this isn't an exclusive access
8537                  * res generate UA for all other
8538                  * registrants.
8539                  */
8540                 if (type != SPR_TYPE_EX_AC
8541                  && type != SPR_TYPE_WR_EX) {
8542                         /*
8543                          * temporarily unregister so we don't generate UA
8544                          */
8545                         lun->per_res[residx].registered = 0;
8546
8547                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8548                                 if (lun->per_res[i+persis_offset].registered
8549                                     == 0)
8550                                         continue;
8551                                 lun->pending_sense[i].ua_pending |=
8552                                         CTL_UA_RES_RELEASE;
8553                         }
8554
8555                         lun->per_res[residx].registered = 1;
8556                 }
8557                 mtx_unlock(&lun->lun_lock);
8558                 /* Send msg to other side */
8559                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8560                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8561                 persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8562                 if ((isc_retval=ctl_ha_msg_send( CTL_HA_CHAN_CTL, &persis_io,
8563                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8564                         printf("CTL:Persis Out error returned from "
8565                                "ctl_ha_msg_send %d\n", isc_retval);
8566                 }
8567                 break;
8568
8569         case SPRO_CLEAR:
8570                 /* send msg to other side */
8571
8572                 mtx_lock(&lun->lun_lock);
8573                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8574                 lun->res_type = 0;
8575                 lun->pr_key_count = 0;
8576                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8577
8578
8579                 memset(&lun->per_res[residx].res_key,
8580                        0, sizeof(lun->per_res[residx].res_key));
8581                 lun->per_res[residx].registered = 0;
8582
8583                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++)
8584                         if (lun->per_res[i].registered) {
8585                                 if (!persis_offset && i < CTL_MAX_INITIATORS)
8586                                         lun->pending_sense[i].ua_pending |=
8587                                                 CTL_UA_RES_PREEMPT;
8588                                 else if (persis_offset && i >= persis_offset)
8589                                         lun->pending_sense[i-persis_offset
8590                                             ].ua_pending |= CTL_UA_RES_PREEMPT;
8591
8592                                 memset(&lun->per_res[i].res_key,
8593                                        0, sizeof(struct scsi_per_res_key));
8594                                 lun->per_res[i].registered = 0;
8595                         }
8596                 lun->PRGeneration++;
8597                 mtx_unlock(&lun->lun_lock);
8598                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8599                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8600                 persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8601                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8602                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8603                         printf("CTL:Persis Out error returned from "
8604                                "ctl_ha_msg_send %d\n", isc_retval);
8605                 }
8606                 break;
8607
8608         case SPRO_PREEMPT: {
8609                 int nretval;
8610
8611                 nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8612                                           residx, ctsio, cdb, param);
8613                 if (nretval != 0)
8614                         return (CTL_RETVAL_COMPLETE);
8615                 break;
8616         }
8617         default:
8618                 panic("Invalid PR type %x", cdb->action);
8619         }
8620
8621 done:
8622         free(ctsio->kern_data_ptr, M_CTL);
8623         ctl_set_success(ctsio);
8624         ctl_done((union ctl_io *)ctsio);
8625
8626         return (retval);
8627 }
8628
8629 /*
8630  * This routine is for handling a message from the other SC pertaining to
8631  * persistent reserve out. All the error checking will have been done
8632  * so only perorming the action need be done here to keep the two
8633  * in sync.
8634  */
8635 static void
8636 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8637 {
8638         struct ctl_lun *lun;
8639         struct ctl_softc *softc;
8640         int i;
8641         uint32_t targ_lun;
8642
8643         softc = control_softc;
8644
8645         targ_lun = msg->hdr.nexus.targ_mapped_lun;
8646         lun = softc->ctl_luns[targ_lun];
8647         mtx_lock(&lun->lun_lock);
8648         switch(msg->pr.pr_info.action) {
8649         case CTL_PR_REG_KEY:
8650                 if (!lun->per_res[msg->pr.pr_info.residx].registered) {
8651                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
8652                         lun->pr_key_count++;
8653                 }
8654                 lun->PRGeneration++;
8655                 memcpy(&lun->per_res[msg->pr.pr_info.residx].res_key,
8656                        msg->pr.pr_info.sa_res_key,
8657                        sizeof(struct scsi_per_res_key));
8658                 break;
8659
8660         case CTL_PR_UNREG_KEY:
8661                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
8662                 memset(&lun->per_res[msg->pr.pr_info.residx].res_key,
8663                        0, sizeof(struct scsi_per_res_key));
8664                 lun->pr_key_count--;
8665
8666                 /* XXX Need to see if the reservation has been released */
8667                 /* if so do we need to generate UA? */
8668                 if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8669                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8670                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8671
8672                         if ((lun->res_type == SPR_TYPE_WR_EX_RO
8673                           || lun->res_type == SPR_TYPE_EX_AC_RO)
8674                          && lun->pr_key_count) {
8675                                 /*
8676                                  * If the reservation is a registrants
8677                                  * only type we need to generate a UA
8678                                  * for other registered inits.  The
8679                                  * sense code should be RESERVATIONS
8680                                  * RELEASED
8681                                  */
8682
8683                                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8684                                         if (lun->per_res[i+
8685                                             persis_offset].registered == 0)
8686                                                 continue;
8687
8688                                         lun->pending_sense[i
8689                                                 ].ua_pending |=
8690                                                 CTL_UA_RES_RELEASE;
8691                                 }
8692                         }
8693                         lun->res_type = 0;
8694                 } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8695                         if (lun->pr_key_count==0) {
8696                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8697                                 lun->res_type = 0;
8698                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8699                         }
8700                 }
8701                 lun->PRGeneration++;
8702                 break;
8703
8704         case CTL_PR_RESERVE:
8705                 lun->flags |= CTL_LUN_PR_RESERVED;
8706                 lun->res_type = msg->pr.pr_info.res_type;
8707                 lun->pr_res_idx = msg->pr.pr_info.residx;
8708
8709                 break;
8710
8711         case CTL_PR_RELEASE:
8712                 /*
8713                  * if this isn't an exclusive access res generate UA for all
8714                  * other registrants.
8715                  */
8716                 if (lun->res_type != SPR_TYPE_EX_AC
8717                  && lun->res_type != SPR_TYPE_WR_EX) {
8718                         for (i = 0; i < CTL_MAX_INITIATORS; i++)
8719                                 if (lun->per_res[i+persis_offset].registered)
8720                                         lun->pending_sense[i].ua_pending |=
8721                                                 CTL_UA_RES_RELEASE;
8722                 }
8723
8724                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8725                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8726                 lun->res_type = 0;
8727                 break;
8728
8729         case CTL_PR_PREEMPT:
8730                 ctl_pro_preempt_other(lun, msg);
8731                 break;
8732         case CTL_PR_CLEAR:
8733                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8734                 lun->res_type = 0;
8735                 lun->pr_key_count = 0;
8736                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8737
8738                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8739                         if (lun->per_res[i].registered == 0)
8740                                 continue;
8741                         if (!persis_offset
8742                          && i < CTL_MAX_INITIATORS)
8743                                 lun->pending_sense[i].ua_pending |=
8744                                         CTL_UA_RES_PREEMPT;
8745                         else if (persis_offset
8746                               && i >= persis_offset)
8747                                 lun->pending_sense[i-persis_offset].ua_pending|=
8748                                         CTL_UA_RES_PREEMPT;
8749                         memset(&lun->per_res[i].res_key, 0,
8750                                sizeof(struct scsi_per_res_key));
8751                         lun->per_res[i].registered = 0;
8752                 }
8753                 lun->PRGeneration++;
8754                 break;
8755         }
8756
8757         mtx_unlock(&lun->lun_lock);
8758 }
8759
8760 int
8761 ctl_read_write(struct ctl_scsiio *ctsio)
8762 {
8763         struct ctl_lun *lun;
8764         struct ctl_lba_len_flags *lbalen;
8765         uint64_t lba;
8766         uint32_t num_blocks;
8767         int fua, dpo;
8768         int retval;
8769         int isread;
8770
8771         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8772
8773         CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8774
8775         fua = 0;
8776         dpo = 0;
8777
8778         retval = CTL_RETVAL_COMPLETE;
8779
8780         isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8781               || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8782         if (lun->flags & CTL_LUN_PR_RESERVED && isread) {
8783                 uint32_t residx;
8784
8785                 /*
8786                  * XXX KDM need a lock here.
8787                  */
8788                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8789                 if ((lun->res_type == SPR_TYPE_EX_AC
8790                   && residx != lun->pr_res_idx)
8791                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
8792                    || lun->res_type == SPR_TYPE_EX_AC_AR)
8793                   && !lun->per_res[residx].registered)) {
8794                         ctl_set_reservation_conflict(ctsio);
8795                         ctl_done((union ctl_io *)ctsio);
8796                         return (CTL_RETVAL_COMPLETE);
8797                 }
8798         }
8799
8800         switch (ctsio->cdb[0]) {
8801         case READ_6:
8802         case WRITE_6: {
8803                 struct scsi_rw_6 *cdb;
8804
8805                 cdb = (struct scsi_rw_6 *)ctsio->cdb;
8806
8807                 lba = scsi_3btoul(cdb->addr);
8808                 /* only 5 bits are valid in the most significant address byte */
8809                 lba &= 0x1fffff;
8810                 num_blocks = cdb->length;
8811                 /*
8812                  * This is correct according to SBC-2.
8813                  */
8814                 if (num_blocks == 0)
8815                         num_blocks = 256;
8816                 break;
8817         }
8818         case READ_10:
8819         case WRITE_10: {
8820                 struct scsi_rw_10 *cdb;
8821
8822                 cdb = (struct scsi_rw_10 *)ctsio->cdb;
8823
8824                 if (cdb->byte2 & SRW10_FUA)
8825                         fua = 1;
8826                 if (cdb->byte2 & SRW10_DPO)
8827                         dpo = 1;
8828
8829                 lba = scsi_4btoul(cdb->addr);
8830                 num_blocks = scsi_2btoul(cdb->length);
8831                 break;
8832         }
8833         case WRITE_VERIFY_10: {
8834                 struct scsi_write_verify_10 *cdb;
8835
8836                 cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
8837
8838                 /*
8839                  * XXX KDM we should do actual write verify support at some
8840                  * point.  This is obviously fake, we're just translating
8841                  * things to a write.  So we don't even bother checking the
8842                  * BYTCHK field, since we don't do any verification.  If
8843                  * the user asks for it, we'll just pretend we did it.
8844                  */
8845                 if (cdb->byte2 & SWV_DPO)
8846                         dpo = 1;
8847
8848                 lba = scsi_4btoul(cdb->addr);
8849                 num_blocks = scsi_2btoul(cdb->length);
8850                 break;
8851         }
8852         case READ_12:
8853         case WRITE_12: {
8854                 struct scsi_rw_12 *cdb;
8855
8856                 cdb = (struct scsi_rw_12 *)ctsio->cdb;
8857
8858                 if (cdb->byte2 & SRW12_FUA)
8859                         fua = 1;
8860                 if (cdb->byte2 & SRW12_DPO)
8861                         dpo = 1;
8862                 lba = scsi_4btoul(cdb->addr);
8863                 num_blocks = scsi_4btoul(cdb->length);
8864                 break;
8865         }
8866         case WRITE_VERIFY_12: {
8867                 struct scsi_write_verify_12 *cdb;
8868
8869                 cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
8870
8871                 if (cdb->byte2 & SWV_DPO)
8872                         dpo = 1;
8873
8874                 lba = scsi_4btoul(cdb->addr);
8875                 num_blocks = scsi_4btoul(cdb->length);
8876
8877                 break;
8878         }
8879         case READ_16:
8880         case WRITE_16: {
8881                 struct scsi_rw_16 *cdb;
8882
8883                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
8884
8885                 if (cdb->byte2 & SRW12_FUA)
8886                         fua = 1;
8887                 if (cdb->byte2 & SRW12_DPO)
8888                         dpo = 1;
8889
8890                 lba = scsi_8btou64(cdb->addr);
8891                 num_blocks = scsi_4btoul(cdb->length);
8892                 break;
8893         }
8894         case WRITE_VERIFY_16: {
8895                 struct scsi_write_verify_16 *cdb;
8896
8897                 cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
8898
8899                 if (cdb->byte2 & SWV_DPO)
8900                         dpo = 1;
8901
8902                 lba = scsi_8btou64(cdb->addr);
8903                 num_blocks = scsi_4btoul(cdb->length);
8904                 break;
8905         }
8906         default:
8907                 /*
8908                  * We got a command we don't support.  This shouldn't
8909                  * happen, commands should be filtered out above us.
8910                  */
8911                 ctl_set_invalid_opcode(ctsio);
8912                 ctl_done((union ctl_io *)ctsio);
8913
8914                 return (CTL_RETVAL_COMPLETE);
8915                 break; /* NOTREACHED */
8916         }
8917
8918         /*
8919          * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
8920          * interesting for us, but if RAIDCore is in write-back mode,
8921          * getting it to do write-through for a particular transaction may
8922          * not be possible.
8923          */
8924
8925         /*
8926          * The first check is to make sure we're in bounds, the second
8927          * check is to catch wrap-around problems.  If the lba + num blocks
8928          * is less than the lba, then we've wrapped around and the block
8929          * range is invalid anyway.
8930          */
8931         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
8932          || ((lba + num_blocks) < lba)) {
8933                 ctl_set_lba_out_of_range(ctsio);
8934                 ctl_done((union ctl_io *)ctsio);
8935                 return (CTL_RETVAL_COMPLETE);
8936         }
8937
8938         /*
8939          * According to SBC-3, a transfer length of 0 is not an error.
8940          * Note that this cannot happen with WRITE(6) or READ(6), since 0
8941          * translates to 256 blocks for those commands.
8942          */
8943         if (num_blocks == 0) {
8944                 ctl_set_success(ctsio);
8945                 ctl_done((union ctl_io *)ctsio);
8946                 return (CTL_RETVAL_COMPLETE);
8947         }
8948
8949         lbalen = (struct ctl_lba_len_flags *)
8950             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
8951         lbalen->lba = lba;
8952         lbalen->len = num_blocks;
8953         lbalen->flags = isread ? CTL_LLF_READ : CTL_LLF_WRITE;
8954
8955         ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
8956         ctsio->kern_rel_offset = 0;
8957
8958         CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
8959
8960         retval = lun->backend->data_submit((union ctl_io *)ctsio);
8961
8962         return (retval);
8963 }
8964
8965 static int
8966 ctl_cnw_cont(union ctl_io *io)
8967 {
8968         struct ctl_scsiio *ctsio;
8969         struct ctl_lun *lun;
8970         struct ctl_lba_len_flags *lbalen;
8971         int retval;
8972
8973         ctsio = &io->scsiio;
8974         ctsio->io_hdr.status = CTL_STATUS_NONE;
8975         ctsio->io_hdr.flags &= ~CTL_FLAG_IO_CONT;
8976         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8977         lbalen = (struct ctl_lba_len_flags *)
8978             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
8979         lbalen->flags = CTL_LLF_WRITE;
8980
8981         CTL_DEBUG_PRINT(("ctl_cnw_cont: calling data_submit()\n"));
8982         retval = lun->backend->data_submit((union ctl_io *)ctsio);
8983         return (retval);
8984 }
8985
8986 int
8987 ctl_cnw(struct ctl_scsiio *ctsio)
8988 {
8989         struct ctl_lun *lun;
8990         struct ctl_lba_len_flags *lbalen;
8991         uint64_t lba;
8992         uint32_t num_blocks;
8993         int fua, dpo;
8994         int retval;
8995
8996         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8997
8998         CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
8999
9000         fua = 0;
9001         dpo = 0;
9002
9003         retval = CTL_RETVAL_COMPLETE;
9004
9005         switch (ctsio->cdb[0]) {
9006         case COMPARE_AND_WRITE: {
9007                 struct scsi_compare_and_write *cdb;
9008
9009                 cdb = (struct scsi_compare_and_write *)ctsio->cdb;
9010
9011                 if (cdb->byte2 & SRW10_FUA)
9012                         fua = 1;
9013                 if (cdb->byte2 & SRW10_DPO)
9014                         dpo = 1;
9015                 lba = scsi_8btou64(cdb->addr);
9016                 num_blocks = cdb->length;
9017                 break;
9018         }
9019         default:
9020                 /*
9021                  * We got a command we don't support.  This shouldn't
9022                  * happen, commands should be filtered out above us.
9023                  */
9024                 ctl_set_invalid_opcode(ctsio);
9025                 ctl_done((union ctl_io *)ctsio);
9026
9027                 return (CTL_RETVAL_COMPLETE);
9028                 break; /* NOTREACHED */
9029         }
9030
9031         /*
9032          * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
9033          * interesting for us, but if RAIDCore is in write-back mode,
9034          * getting it to do write-through for a particular transaction may
9035          * not be possible.
9036          */
9037
9038         /*
9039          * The first check is to make sure we're in bounds, the second
9040          * check is to catch wrap-around problems.  If the lba + num blocks
9041          * is less than the lba, then we've wrapped around and the block
9042          * range is invalid anyway.
9043          */
9044         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9045          || ((lba + num_blocks) < lba)) {
9046                 ctl_set_lba_out_of_range(ctsio);
9047                 ctl_done((union ctl_io *)ctsio);
9048                 return (CTL_RETVAL_COMPLETE);
9049         }
9050
9051         /*
9052          * According to SBC-3, a transfer length of 0 is not an error.
9053          */
9054         if (num_blocks == 0) {
9055                 ctl_set_success(ctsio);
9056                 ctl_done((union ctl_io *)ctsio);
9057                 return (CTL_RETVAL_COMPLETE);
9058         }
9059
9060         ctsio->kern_total_len = 2 * num_blocks * lun->be_lun->blocksize;
9061         ctsio->kern_rel_offset = 0;
9062
9063         /*
9064          * Set the IO_CONT flag, so that if this I/O gets passed to
9065          * ctl_data_submit_done(), it'll get passed back to
9066          * ctl_ctl_cnw_cont() for further processing.
9067          */
9068         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
9069         ctsio->io_cont = ctl_cnw_cont;
9070
9071         lbalen = (struct ctl_lba_len_flags *)
9072             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9073         lbalen->lba = lba;
9074         lbalen->len = num_blocks;
9075         lbalen->flags = CTL_LLF_COMPARE;
9076
9077         CTL_DEBUG_PRINT(("ctl_cnw: calling data_submit()\n"));
9078         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9079         return (retval);
9080 }
9081
9082 int
9083 ctl_verify(struct ctl_scsiio *ctsio)
9084 {
9085         struct ctl_lun *lun;
9086         struct ctl_lba_len_flags *lbalen;
9087         uint64_t lba;
9088         uint32_t num_blocks;
9089         int bytchk, dpo;
9090         int retval;
9091
9092         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9093
9094         CTL_DEBUG_PRINT(("ctl_verify: command: %#x\n", ctsio->cdb[0]));
9095
9096         bytchk = 0;
9097         dpo = 0;
9098         retval = CTL_RETVAL_COMPLETE;
9099
9100         switch (ctsio->cdb[0]) {
9101         case VERIFY_10: {
9102                 struct scsi_verify_10 *cdb;
9103
9104                 cdb = (struct scsi_verify_10 *)ctsio->cdb;
9105                 if (cdb->byte2 & SVFY_BYTCHK)
9106                         bytchk = 1;
9107                 if (cdb->byte2 & SVFY_DPO)
9108                         dpo = 1;
9109                 lba = scsi_4btoul(cdb->addr);
9110                 num_blocks = scsi_2btoul(cdb->length);
9111                 break;
9112         }
9113         case VERIFY_12: {
9114                 struct scsi_verify_12 *cdb;
9115
9116                 cdb = (struct scsi_verify_12 *)ctsio->cdb;
9117                 if (cdb->byte2 & SVFY_BYTCHK)
9118                         bytchk = 1;
9119                 if (cdb->byte2 & SVFY_DPO)
9120                         dpo = 1;
9121                 lba = scsi_4btoul(cdb->addr);
9122                 num_blocks = scsi_4btoul(cdb->length);
9123                 break;
9124         }
9125         case VERIFY_16: {
9126                 struct scsi_rw_16 *cdb;
9127
9128                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
9129                 if (cdb->byte2 & SVFY_BYTCHK)
9130                         bytchk = 1;
9131                 if (cdb->byte2 & SVFY_DPO)
9132                         dpo = 1;
9133                 lba = scsi_8btou64(cdb->addr);
9134                 num_blocks = scsi_4btoul(cdb->length);
9135                 break;
9136         }
9137         default:
9138                 /*
9139                  * We got a command we don't support.  This shouldn't
9140                  * happen, commands should be filtered out above us.
9141                  */
9142                 ctl_set_invalid_opcode(ctsio);
9143                 ctl_done((union ctl_io *)ctsio);
9144                 return (CTL_RETVAL_COMPLETE);
9145         }
9146
9147         /*
9148          * The first check is to make sure we're in bounds, the second
9149          * check is to catch wrap-around problems.  If the lba + num blocks
9150          * is less than the lba, then we've wrapped around and the block
9151          * range is invalid anyway.
9152          */
9153         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9154          || ((lba + num_blocks) < lba)) {
9155                 ctl_set_lba_out_of_range(ctsio);
9156                 ctl_done((union ctl_io *)ctsio);
9157                 return (CTL_RETVAL_COMPLETE);
9158         }
9159
9160         /*
9161          * According to SBC-3, a transfer length of 0 is not an error.
9162          */
9163         if (num_blocks == 0) {
9164                 ctl_set_success(ctsio);
9165                 ctl_done((union ctl_io *)ctsio);
9166                 return (CTL_RETVAL_COMPLETE);
9167         }
9168
9169         lbalen = (struct ctl_lba_len_flags *)
9170             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9171         lbalen->lba = lba;
9172         lbalen->len = num_blocks;
9173         if (bytchk) {
9174                 lbalen->flags = CTL_LLF_COMPARE;
9175                 ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9176         } else {
9177                 lbalen->flags = CTL_LLF_VERIFY;
9178                 ctsio->kern_total_len = 0;
9179         }
9180         ctsio->kern_rel_offset = 0;
9181
9182         CTL_DEBUG_PRINT(("ctl_verify: calling data_submit()\n"));
9183         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9184         return (retval);
9185 }
9186
9187 int
9188 ctl_report_luns(struct ctl_scsiio *ctsio)
9189 {
9190         struct scsi_report_luns *cdb;
9191         struct scsi_report_luns_data *lun_data;
9192         struct ctl_lun *lun, *request_lun;
9193         int num_luns, retval;
9194         uint32_t alloc_len, lun_datalen;
9195         int num_filled, well_known;
9196         uint32_t initidx, targ_lun_id, lun_id;
9197
9198         retval = CTL_RETVAL_COMPLETE;
9199         well_known = 0;
9200
9201         cdb = (struct scsi_report_luns *)ctsio->cdb;
9202
9203         CTL_DEBUG_PRINT(("ctl_report_luns\n"));
9204
9205         mtx_lock(&control_softc->ctl_lock);
9206         num_luns = control_softc->num_luns;
9207         mtx_unlock(&control_softc->ctl_lock);
9208
9209         switch (cdb->select_report) {
9210         case RPL_REPORT_DEFAULT:
9211         case RPL_REPORT_ALL:
9212                 break;
9213         case RPL_REPORT_WELLKNOWN:
9214                 well_known = 1;
9215                 num_luns = 0;
9216                 break;
9217         default:
9218                 ctl_set_invalid_field(ctsio,
9219                                       /*sks_valid*/ 1,
9220                                       /*command*/ 1,
9221                                       /*field*/ 2,
9222                                       /*bit_valid*/ 0,
9223                                       /*bit*/ 0);
9224                 ctl_done((union ctl_io *)ctsio);
9225                 return (retval);
9226                 break; /* NOTREACHED */
9227         }
9228
9229         alloc_len = scsi_4btoul(cdb->length);
9230         /*
9231          * The initiator has to allocate at least 16 bytes for this request,
9232          * so he can at least get the header and the first LUN.  Otherwise
9233          * we reject the request (per SPC-3 rev 14, section 6.21).
9234          */
9235         if (alloc_len < (sizeof(struct scsi_report_luns_data) +
9236             sizeof(struct scsi_report_luns_lundata))) {
9237                 ctl_set_invalid_field(ctsio,
9238                                       /*sks_valid*/ 1,
9239                                       /*command*/ 1,
9240                                       /*field*/ 6,
9241                                       /*bit_valid*/ 0,
9242                                       /*bit*/ 0);
9243                 ctl_done((union ctl_io *)ctsio);
9244                 return (retval);
9245         }
9246
9247         request_lun = (struct ctl_lun *)
9248                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9249
9250         lun_datalen = sizeof(*lun_data) +
9251                 (num_luns * sizeof(struct scsi_report_luns_lundata));
9252
9253         ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
9254         lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
9255         ctsio->kern_sg_entries = 0;
9256
9257         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9258
9259         mtx_lock(&control_softc->ctl_lock);
9260         for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
9261                 lun_id = targ_lun_id;
9262                 if (ctsio->io_hdr.nexus.lun_map_fn != NULL)
9263                         lun_id = ctsio->io_hdr.nexus.lun_map_fn(ctsio->io_hdr.nexus.lun_map_arg, lun_id);
9264                 if (lun_id >= CTL_MAX_LUNS)
9265                         continue;
9266                 lun = control_softc->ctl_luns[lun_id];
9267                 if (lun == NULL)
9268                         continue;
9269
9270                 if (targ_lun_id <= 0xff) {
9271                         /*
9272                          * Peripheral addressing method, bus number 0.
9273                          */
9274                         lun_data->luns[num_filled].lundata[0] =
9275                                 RPL_LUNDATA_ATYP_PERIPH;
9276                         lun_data->luns[num_filled].lundata[1] = targ_lun_id;
9277                         num_filled++;
9278                 } else if (targ_lun_id <= 0x3fff) {
9279                         /*
9280                          * Flat addressing method.
9281                          */
9282                         lun_data->luns[num_filled].lundata[0] =
9283                                 RPL_LUNDATA_ATYP_FLAT |
9284                                 (targ_lun_id & RPL_LUNDATA_FLAT_LUN_MASK);
9285 #ifdef OLDCTLHEADERS
9286                                 (SRLD_ADDR_FLAT << SRLD_ADDR_SHIFT) |
9287                                 (targ_lun_id & SRLD_BUS_LUN_MASK);
9288 #endif
9289                         lun_data->luns[num_filled].lundata[1] =
9290 #ifdef OLDCTLHEADERS
9291                                 targ_lun_id >> SRLD_BUS_LUN_BITS;
9292 #endif
9293                                 targ_lun_id >> RPL_LUNDATA_FLAT_LUN_BITS;
9294                         num_filled++;
9295                 } else {
9296                         printf("ctl_report_luns: bogus LUN number %jd, "
9297                                "skipping\n", (intmax_t)targ_lun_id);
9298                 }
9299                 /*
9300                  * According to SPC-3, rev 14 section 6.21:
9301                  *
9302                  * "The execution of a REPORT LUNS command to any valid and
9303                  * installed logical unit shall clear the REPORTED LUNS DATA
9304                  * HAS CHANGED unit attention condition for all logical
9305                  * units of that target with respect to the requesting
9306                  * initiator. A valid and installed logical unit is one
9307                  * having a PERIPHERAL QUALIFIER of 000b in the standard
9308                  * INQUIRY data (see 6.4.2)."
9309                  *
9310                  * If request_lun is NULL, the LUN this report luns command
9311                  * was issued to is either disabled or doesn't exist. In that
9312                  * case, we shouldn't clear any pending lun change unit
9313                  * attention.
9314                  */
9315                 if (request_lun != NULL) {
9316                         mtx_lock(&lun->lun_lock);
9317                         lun->pending_sense[initidx].ua_pending &=
9318                                 ~CTL_UA_LUN_CHANGE;
9319                         mtx_unlock(&lun->lun_lock);
9320                 }
9321         }
9322         mtx_unlock(&control_softc->ctl_lock);
9323
9324         /*
9325          * It's quite possible that we've returned fewer LUNs than we allocated
9326          * space for.  Trim it.
9327          */
9328         lun_datalen = sizeof(*lun_data) +
9329                 (num_filled * sizeof(struct scsi_report_luns_lundata));
9330
9331         if (lun_datalen < alloc_len) {
9332                 ctsio->residual = alloc_len - lun_datalen;
9333                 ctsio->kern_data_len = lun_datalen;
9334                 ctsio->kern_total_len = lun_datalen;
9335         } else {
9336                 ctsio->residual = 0;
9337                 ctsio->kern_data_len = alloc_len;
9338                 ctsio->kern_total_len = alloc_len;
9339         }
9340         ctsio->kern_data_resid = 0;
9341         ctsio->kern_rel_offset = 0;
9342         ctsio->kern_sg_entries = 0;
9343
9344         /*
9345          * We set this to the actual data length, regardless of how much
9346          * space we actually have to return results.  If the user looks at
9347          * this value, he'll know whether or not he allocated enough space
9348          * and reissue the command if necessary.  We don't support well
9349          * known logical units, so if the user asks for that, return none.
9350          */
9351         scsi_ulto4b(lun_datalen - 8, lun_data->length);
9352
9353         /*
9354          * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
9355          * this request.
9356          */
9357         ctsio->scsi_status = SCSI_STATUS_OK;
9358
9359         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9360         ctsio->be_move_done = ctl_config_move_done;
9361         ctl_datamove((union ctl_io *)ctsio);
9362
9363         return (retval);
9364 }
9365
9366 int
9367 ctl_request_sense(struct ctl_scsiio *ctsio)
9368 {
9369         struct scsi_request_sense *cdb;
9370         struct scsi_sense_data *sense_ptr;
9371         struct ctl_lun *lun;
9372         uint32_t initidx;
9373         int have_error;
9374         scsi_sense_data_type sense_format;
9375
9376         cdb = (struct scsi_request_sense *)ctsio->cdb;
9377
9378         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9379
9380         CTL_DEBUG_PRINT(("ctl_request_sense\n"));
9381
9382         /*
9383          * Determine which sense format the user wants.
9384          */
9385         if (cdb->byte2 & SRS_DESC)
9386                 sense_format = SSD_TYPE_DESC;
9387         else
9388                 sense_format = SSD_TYPE_FIXED;
9389
9390         ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
9391         sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
9392         ctsio->kern_sg_entries = 0;
9393
9394         /*
9395          * struct scsi_sense_data, which is currently set to 256 bytes, is
9396          * larger than the largest allowed value for the length field in the
9397          * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
9398          */
9399         ctsio->residual = 0;
9400         ctsio->kern_data_len = cdb->length;
9401         ctsio->kern_total_len = cdb->length;
9402
9403         ctsio->kern_data_resid = 0;
9404         ctsio->kern_rel_offset = 0;
9405         ctsio->kern_sg_entries = 0;
9406
9407         /*
9408          * If we don't have a LUN, we don't have any pending sense.
9409          */
9410         if (lun == NULL)
9411                 goto no_sense;
9412
9413         have_error = 0;
9414         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9415         /*
9416          * Check for pending sense, and then for pending unit attentions.
9417          * Pending sense gets returned first, then pending unit attentions.
9418          */
9419         mtx_lock(&lun->lun_lock);
9420         if (ctl_is_set(lun->have_ca, initidx)) {
9421                 scsi_sense_data_type stored_format;
9422
9423                 /*
9424                  * Check to see which sense format was used for the stored
9425                  * sense data.
9426                  */
9427                 stored_format = scsi_sense_type(
9428                     &lun->pending_sense[initidx].sense);
9429
9430                 /*
9431                  * If the user requested a different sense format than the
9432                  * one we stored, then we need to convert it to the other
9433                  * format.  If we're going from descriptor to fixed format
9434                  * sense data, we may lose things in translation, depending
9435                  * on what options were used.
9436                  *
9437                  * If the stored format is SSD_TYPE_NONE (i.e. invalid),
9438                  * for some reason we'll just copy it out as-is.
9439                  */
9440                 if ((stored_format == SSD_TYPE_FIXED)
9441                  && (sense_format == SSD_TYPE_DESC))
9442                         ctl_sense_to_desc((struct scsi_sense_data_fixed *)
9443                             &lun->pending_sense[initidx].sense,
9444                             (struct scsi_sense_data_desc *)sense_ptr);
9445                 else if ((stored_format == SSD_TYPE_DESC)
9446                       && (sense_format == SSD_TYPE_FIXED))
9447                         ctl_sense_to_fixed((struct scsi_sense_data_desc *)
9448                             &lun->pending_sense[initidx].sense,
9449                             (struct scsi_sense_data_fixed *)sense_ptr);
9450                 else
9451                         memcpy(sense_ptr, &lun->pending_sense[initidx].sense,
9452                                ctl_min(sizeof(*sense_ptr),
9453                                sizeof(lun->pending_sense[initidx].sense)));
9454
9455                 ctl_clear_mask(lun->have_ca, initidx);
9456                 have_error = 1;
9457         } else if (lun->pending_sense[initidx].ua_pending != CTL_UA_NONE) {
9458                 ctl_ua_type ua_type;
9459
9460                 ua_type = ctl_build_ua(lun->pending_sense[initidx].ua_pending,
9461                                        sense_ptr, sense_format);
9462                 if (ua_type != CTL_UA_NONE) {
9463                         have_error = 1;
9464                         /* We're reporting this UA, so clear it */
9465                         lun->pending_sense[initidx].ua_pending &= ~ua_type;
9466                 }
9467         }
9468         mtx_unlock(&lun->lun_lock);
9469
9470         /*
9471          * We already have a pending error, return it.
9472          */
9473         if (have_error != 0) {
9474                 /*
9475                  * We report the SCSI status as OK, since the status of the
9476                  * request sense command itself is OK.
9477                  */
9478                 ctsio->scsi_status = SCSI_STATUS_OK;
9479
9480                 /*
9481                  * We report 0 for the sense length, because we aren't doing
9482                  * autosense in this case.  We're reporting sense as
9483                  * parameter data.
9484                  */
9485                 ctsio->sense_len = 0;
9486                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9487                 ctsio->be_move_done = ctl_config_move_done;
9488                 ctl_datamove((union ctl_io *)ctsio);
9489
9490                 return (CTL_RETVAL_COMPLETE);
9491         }
9492
9493 no_sense:
9494
9495         /*
9496          * No sense information to report, so we report that everything is
9497          * okay.
9498          */
9499         ctl_set_sense_data(sense_ptr,
9500                            lun,
9501                            sense_format,
9502                            /*current_error*/ 1,
9503                            /*sense_key*/ SSD_KEY_NO_SENSE,
9504                            /*asc*/ 0x00,
9505                            /*ascq*/ 0x00,
9506                            SSD_ELEM_NONE);
9507
9508         ctsio->scsi_status = SCSI_STATUS_OK;
9509
9510         /*
9511          * We report 0 for the sense length, because we aren't doing
9512          * autosense in this case.  We're reporting sense as parameter data.
9513          */
9514         ctsio->sense_len = 0;
9515         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9516         ctsio->be_move_done = ctl_config_move_done;
9517         ctl_datamove((union ctl_io *)ctsio);
9518
9519         return (CTL_RETVAL_COMPLETE);
9520 }
9521
9522 int
9523 ctl_tur(struct ctl_scsiio *ctsio)
9524 {
9525         struct ctl_lun *lun;
9526
9527         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9528
9529         CTL_DEBUG_PRINT(("ctl_tur\n"));
9530
9531         if (lun == NULL)
9532                 return (EINVAL);
9533
9534         ctsio->scsi_status = SCSI_STATUS_OK;
9535         ctsio->io_hdr.status = CTL_SUCCESS;
9536
9537         ctl_done((union ctl_io *)ctsio);
9538
9539         return (CTL_RETVAL_COMPLETE);
9540 }
9541
9542 #ifdef notyet
9543 static int
9544 ctl_cmddt_inquiry(struct ctl_scsiio *ctsio)
9545 {
9546
9547 }
9548 #endif
9549
9550 static int
9551 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
9552 {
9553         struct scsi_vpd_supported_pages *pages;
9554         int sup_page_size;
9555         struct ctl_lun *lun;
9556
9557         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9558
9559         sup_page_size = sizeof(struct scsi_vpd_supported_pages) *
9560             SCSI_EVPD_NUM_SUPPORTED_PAGES;
9561         ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9562         pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9563         ctsio->kern_sg_entries = 0;
9564
9565         if (sup_page_size < alloc_len) {
9566                 ctsio->residual = alloc_len - sup_page_size;
9567                 ctsio->kern_data_len = sup_page_size;
9568                 ctsio->kern_total_len = sup_page_size;
9569         } else {
9570                 ctsio->residual = 0;
9571                 ctsio->kern_data_len = alloc_len;
9572                 ctsio->kern_total_len = alloc_len;
9573         }
9574         ctsio->kern_data_resid = 0;
9575         ctsio->kern_rel_offset = 0;
9576         ctsio->kern_sg_entries = 0;
9577
9578         /*
9579          * The control device is always connected.  The disk device, on the
9580          * other hand, may not be online all the time.  Need to change this
9581          * to figure out whether the disk device is actually online or not.
9582          */
9583         if (lun != NULL)
9584                 pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9585                                 lun->be_lun->lun_type;
9586         else
9587                 pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9588
9589         pages->length = SCSI_EVPD_NUM_SUPPORTED_PAGES;
9590         /* Supported VPD pages */
9591         pages->page_list[0] = SVPD_SUPPORTED_PAGES;
9592         /* Serial Number */
9593         pages->page_list[1] = SVPD_UNIT_SERIAL_NUMBER;
9594         /* Device Identification */
9595         pages->page_list[2] = SVPD_DEVICE_ID;
9596         /* Block limits */
9597         pages->page_list[3] = SVPD_BLOCK_LIMITS;
9598         /* Logical Block Provisioning */
9599         pages->page_list[4] = SVPD_LBP;
9600
9601         ctsio->scsi_status = SCSI_STATUS_OK;
9602
9603         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9604         ctsio->be_move_done = ctl_config_move_done;
9605         ctl_datamove((union ctl_io *)ctsio);
9606
9607         return (CTL_RETVAL_COMPLETE);
9608 }
9609
9610 static int
9611 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9612 {
9613         struct scsi_vpd_unit_serial_number *sn_ptr;
9614         struct ctl_lun *lun;
9615
9616         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9617
9618         ctsio->kern_data_ptr = malloc(sizeof(*sn_ptr), M_CTL, M_WAITOK | M_ZERO);
9619         sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9620         ctsio->kern_sg_entries = 0;
9621
9622         if (sizeof(*sn_ptr) < alloc_len) {
9623                 ctsio->residual = alloc_len - sizeof(*sn_ptr);
9624                 ctsio->kern_data_len = sizeof(*sn_ptr);
9625                 ctsio->kern_total_len = sizeof(*sn_ptr);
9626         } else {
9627                 ctsio->residual = 0;
9628                 ctsio->kern_data_len = alloc_len;
9629                 ctsio->kern_total_len = alloc_len;
9630         }
9631         ctsio->kern_data_resid = 0;
9632         ctsio->kern_rel_offset = 0;
9633         ctsio->kern_sg_entries = 0;
9634
9635         /*
9636          * The control device is always connected.  The disk device, on the
9637          * other hand, may not be online all the time.  Need to change this
9638          * to figure out whether the disk device is actually online or not.
9639          */
9640         if (lun != NULL)
9641                 sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9642                                   lun->be_lun->lun_type;
9643         else
9644                 sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9645
9646         sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9647         sn_ptr->length = ctl_min(sizeof(*sn_ptr) - 4, CTL_SN_LEN);
9648         /*
9649          * If we don't have a LUN, we just leave the serial number as
9650          * all spaces.
9651          */
9652         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9653         if (lun != NULL) {
9654                 strncpy((char *)sn_ptr->serial_num,
9655                         (char *)lun->be_lun->serial_num, CTL_SN_LEN);
9656         }
9657         ctsio->scsi_status = SCSI_STATUS_OK;
9658
9659         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9660         ctsio->be_move_done = ctl_config_move_done;
9661         ctl_datamove((union ctl_io *)ctsio);
9662
9663         return (CTL_RETVAL_COMPLETE);
9664 }
9665
9666
9667 static int
9668 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9669 {
9670         struct scsi_vpd_device_id *devid_ptr;
9671         struct scsi_vpd_id_descriptor *desc;
9672         struct ctl_softc *ctl_softc;
9673         struct ctl_lun *lun;
9674         struct ctl_port *port;
9675         int data_len;
9676         uint8_t proto;
9677
9678         ctl_softc = control_softc;
9679
9680         port = ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)];
9681         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9682
9683         data_len = sizeof(struct scsi_vpd_device_id) +
9684             sizeof(struct scsi_vpd_id_descriptor) +
9685                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9686             sizeof(struct scsi_vpd_id_descriptor) +
9687                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9688         if (lun && lun->lun_devid)
9689                 data_len += lun->lun_devid->len;
9690         if (port->port_devid)
9691                 data_len += port->port_devid->len;
9692         if (port->target_devid)
9693                 data_len += port->target_devid->len;
9694
9695         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9696         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9697         ctsio->kern_sg_entries = 0;
9698
9699         if (data_len < alloc_len) {
9700                 ctsio->residual = alloc_len - data_len;
9701                 ctsio->kern_data_len = data_len;
9702                 ctsio->kern_total_len = data_len;
9703         } else {
9704                 ctsio->residual = 0;
9705                 ctsio->kern_data_len = alloc_len;
9706                 ctsio->kern_total_len = alloc_len;
9707         }
9708         ctsio->kern_data_resid = 0;
9709         ctsio->kern_rel_offset = 0;
9710         ctsio->kern_sg_entries = 0;
9711
9712         /*
9713          * The control device is always connected.  The disk device, on the
9714          * other hand, may not be online all the time.
9715          */
9716         if (lun != NULL)
9717                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9718                                      lun->be_lun->lun_type;
9719         else
9720                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9721         devid_ptr->page_code = SVPD_DEVICE_ID;
9722         scsi_ulto2b(data_len - 4, devid_ptr->length);
9723
9724         if (port->port_type == CTL_PORT_FC)
9725                 proto = SCSI_PROTO_FC << 4;
9726         else if (port->port_type == CTL_PORT_ISCSI)
9727                 proto = SCSI_PROTO_ISCSI << 4;
9728         else
9729                 proto = SCSI_PROTO_SPI << 4;
9730         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
9731
9732         /*
9733          * We're using a LUN association here.  i.e., this device ID is a
9734          * per-LUN identifier.
9735          */
9736         if (lun && lun->lun_devid) {
9737                 memcpy(desc, lun->lun_devid->data, lun->lun_devid->len);
9738                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
9739                     lun->lun_devid->len);
9740         }
9741
9742         /*
9743          * This is for the WWPN which is a port association.
9744          */
9745         if (port->port_devid) {
9746                 memcpy(desc, port->port_devid->data, port->port_devid->len);
9747                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
9748                     port->port_devid->len);
9749         }
9750
9751         /*
9752          * This is for the Relative Target Port(type 4h) identifier
9753          */
9754         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
9755         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
9756             SVPD_ID_TYPE_RELTARG;
9757         desc->length = 4;
9758         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port, &desc->identifier[2]);
9759         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9760             sizeof(struct scsi_vpd_id_rel_trgt_port_id));
9761
9762         /*
9763          * This is for the Target Port Group(type 5h) identifier
9764          */
9765         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
9766         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
9767             SVPD_ID_TYPE_TPORTGRP;
9768         desc->length = 4;
9769         if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS || ctl_is_single)
9770                 scsi_ulto2b(1, &desc->identifier[2]);
9771         else
9772                 scsi_ulto2b(2, &desc->identifier[2]);
9773         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9774             sizeof(struct scsi_vpd_id_trgt_port_grp_id));
9775
9776         /*
9777          * This is for the Target identifier
9778          */
9779         if (port->target_devid) {
9780                 memcpy(desc, port->target_devid->data, port->target_devid->len);
9781         }
9782
9783         ctsio->scsi_status = SCSI_STATUS_OK;
9784         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9785         ctsio->be_move_done = ctl_config_move_done;
9786         ctl_datamove((union ctl_io *)ctsio);
9787
9788         return (CTL_RETVAL_COMPLETE);
9789 }
9790
9791 static int
9792 ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
9793 {
9794         struct scsi_vpd_block_limits *bl_ptr;
9795         struct ctl_lun *lun;
9796         int bs;
9797
9798         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9799         bs = lun->be_lun->blocksize;
9800
9801         ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
9802         bl_ptr = (struct scsi_vpd_block_limits *)ctsio->kern_data_ptr;
9803         ctsio->kern_sg_entries = 0;
9804
9805         if (sizeof(*bl_ptr) < alloc_len) {
9806                 ctsio->residual = alloc_len - sizeof(*bl_ptr);
9807                 ctsio->kern_data_len = sizeof(*bl_ptr);
9808                 ctsio->kern_total_len = sizeof(*bl_ptr);
9809         } else {
9810                 ctsio->residual = 0;
9811                 ctsio->kern_data_len = alloc_len;
9812                 ctsio->kern_total_len = alloc_len;
9813         }
9814         ctsio->kern_data_resid = 0;
9815         ctsio->kern_rel_offset = 0;
9816         ctsio->kern_sg_entries = 0;
9817
9818         /*
9819          * The control device is always connected.  The disk device, on the
9820          * other hand, may not be online all the time.  Need to change this
9821          * to figure out whether the disk device is actually online or not.
9822          */
9823         if (lun != NULL)
9824                 bl_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9825                                   lun->be_lun->lun_type;
9826         else
9827                 bl_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9828
9829         bl_ptr->page_code = SVPD_BLOCK_LIMITS;
9830         scsi_ulto2b(sizeof(*bl_ptr), bl_ptr->page_length);
9831         bl_ptr->max_cmp_write_len = 0xff;
9832         scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
9833         scsi_ulto4b(MAXPHYS / bs, bl_ptr->opt_txfer_len);
9834         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
9835                 scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
9836                 scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_blk_cnt);
9837         }
9838         scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
9839
9840         ctsio->scsi_status = SCSI_STATUS_OK;
9841         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9842         ctsio->be_move_done = ctl_config_move_done;
9843         ctl_datamove((union ctl_io *)ctsio);
9844
9845         return (CTL_RETVAL_COMPLETE);
9846 }
9847
9848 static int
9849 ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
9850 {
9851         struct scsi_vpd_logical_block_prov *lbp_ptr;
9852         struct ctl_lun *lun;
9853         int bs;
9854
9855         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9856         bs = lun->be_lun->blocksize;
9857
9858         ctsio->kern_data_ptr = malloc(sizeof(*lbp_ptr), M_CTL, M_WAITOK | M_ZERO);
9859         lbp_ptr = (struct scsi_vpd_logical_block_prov *)ctsio->kern_data_ptr;
9860         ctsio->kern_sg_entries = 0;
9861
9862         if (sizeof(*lbp_ptr) < alloc_len) {
9863                 ctsio->residual = alloc_len - sizeof(*lbp_ptr);
9864                 ctsio->kern_data_len = sizeof(*lbp_ptr);
9865                 ctsio->kern_total_len = sizeof(*lbp_ptr);
9866         } else {
9867                 ctsio->residual = 0;
9868                 ctsio->kern_data_len = alloc_len;
9869                 ctsio->kern_total_len = alloc_len;
9870         }
9871         ctsio->kern_data_resid = 0;
9872         ctsio->kern_rel_offset = 0;
9873         ctsio->kern_sg_entries = 0;
9874
9875         /*
9876          * The control device is always connected.  The disk device, on the
9877          * other hand, may not be online all the time.  Need to change this
9878          * to figure out whether the disk device is actually online or not.
9879          */
9880         if (lun != NULL)
9881                 lbp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9882                                   lun->be_lun->lun_type;
9883         else
9884                 lbp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9885
9886         lbp_ptr->page_code = SVPD_LBP;
9887         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
9888                 lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 | SVPD_LBP_WS10;
9889
9890         ctsio->scsi_status = SCSI_STATUS_OK;
9891         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9892         ctsio->be_move_done = ctl_config_move_done;
9893         ctl_datamove((union ctl_io *)ctsio);
9894
9895         return (CTL_RETVAL_COMPLETE);
9896 }
9897
9898 static int
9899 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
9900 {
9901         struct scsi_inquiry *cdb;
9902         struct ctl_lun *lun;
9903         int alloc_len, retval;
9904
9905         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9906         cdb = (struct scsi_inquiry *)ctsio->cdb;
9907
9908         retval = CTL_RETVAL_COMPLETE;
9909
9910         alloc_len = scsi_2btoul(cdb->length);
9911
9912         switch (cdb->page_code) {
9913         case SVPD_SUPPORTED_PAGES:
9914                 retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
9915                 break;
9916         case SVPD_UNIT_SERIAL_NUMBER:
9917                 retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
9918                 break;
9919         case SVPD_DEVICE_ID:
9920                 retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
9921                 break;
9922         case SVPD_BLOCK_LIMITS:
9923                 retval = ctl_inquiry_evpd_block_limits(ctsio, alloc_len);
9924                 break;
9925         case SVPD_LBP:
9926                 retval = ctl_inquiry_evpd_lbp(ctsio, alloc_len);
9927                 break;
9928         default:
9929                 ctl_set_invalid_field(ctsio,
9930                                       /*sks_valid*/ 1,
9931                                       /*command*/ 1,
9932                                       /*field*/ 2,
9933                                       /*bit_valid*/ 0,
9934                                       /*bit*/ 0);
9935                 ctl_done((union ctl_io *)ctsio);
9936                 retval = CTL_RETVAL_COMPLETE;
9937                 break;
9938         }
9939
9940         return (retval);
9941 }
9942
9943 static int
9944 ctl_inquiry_std(struct ctl_scsiio *ctsio)
9945 {
9946         struct scsi_inquiry_data *inq_ptr;
9947         struct scsi_inquiry *cdb;
9948         struct ctl_softc *ctl_softc;
9949         struct ctl_lun *lun;
9950         char *val;
9951         uint32_t alloc_len;
9952         int is_fc;
9953
9954         ctl_softc = control_softc;
9955
9956         /*
9957          * Figure out whether we're talking to a Fibre Channel port or not.
9958          * We treat the ioctl front end, and any SCSI adapters, as packetized
9959          * SCSI front ends.
9960          */
9961         if (ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)]->port_type !=
9962             CTL_PORT_FC)
9963                 is_fc = 0;
9964         else
9965                 is_fc = 1;
9966
9967         lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9968         cdb = (struct scsi_inquiry *)ctsio->cdb;
9969         alloc_len = scsi_2btoul(cdb->length);
9970
9971         /*
9972          * We malloc the full inquiry data size here and fill it
9973          * in.  If the user only asks for less, we'll give him
9974          * that much.
9975          */
9976         ctsio->kern_data_ptr = malloc(sizeof(*inq_ptr), M_CTL, M_WAITOK | M_ZERO);
9977         inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
9978         ctsio->kern_sg_entries = 0;
9979         ctsio->kern_data_resid = 0;
9980         ctsio->kern_rel_offset = 0;
9981
9982         if (sizeof(*inq_ptr) < alloc_len) {
9983                 ctsio->residual = alloc_len - sizeof(*inq_ptr);
9984                 ctsio->kern_data_len = sizeof(*inq_ptr);
9985                 ctsio->kern_total_len = sizeof(*inq_ptr);
9986         } else {
9987                 ctsio->residual = 0;
9988                 ctsio->kern_data_len = alloc_len;
9989                 ctsio->kern_total_len = alloc_len;
9990         }
9991
9992         /*
9993          * If we have a LUN configured, report it as connected.  Otherwise,
9994          * report that it is offline or no device is supported, depending
9995          * on the value of inquiry_pq_no_lun.
9996          *
9997          * According to the spec (SPC-4 r34), the peripheral qualifier
9998          * SID_QUAL_LU_OFFLINE (001b) is used in the following scenario:
9999          *
10000          * "A peripheral device having the specified peripheral device type
10001          * is not connected to this logical unit. However, the device
10002          * server is capable of supporting the specified peripheral device
10003          * type on this logical unit."
10004          *
10005          * According to the same spec, the peripheral qualifier
10006          * SID_QUAL_BAD_LU (011b) is used in this scenario:
10007          *
10008          * "The device server is not capable of supporting a peripheral
10009          * device on this logical unit. For this peripheral qualifier the
10010          * peripheral device type shall be set to 1Fh. All other peripheral
10011          * device type values are reserved for this peripheral qualifier."
10012          *
10013          * Given the text, it would seem that we probably want to report that
10014          * the LUN is offline here.  There is no LUN connected, but we can
10015          * support a LUN at the given LUN number.
10016          *
10017          * In the real world, though, it sounds like things are a little
10018          * different:
10019          *
10020          * - Linux, when presented with a LUN with the offline peripheral
10021          *   qualifier, will create an sg driver instance for it.  So when
10022          *   you attach it to CTL, you wind up with a ton of sg driver
10023          *   instances.  (One for every LUN that Linux bothered to probe.)
10024          *   Linux does this despite the fact that it issues a REPORT LUNs
10025          *   to LUN 0 to get the inventory of supported LUNs.
10026          *
10027          * - There is other anecdotal evidence (from Emulex folks) about
10028          *   arrays that use the offline peripheral qualifier for LUNs that
10029          *   are on the "passive" path in an active/passive array.
10030          *
10031          * So the solution is provide a hopefully reasonable default
10032          * (return bad/no LUN) and allow the user to change the behavior
10033          * with a tunable/sysctl variable.
10034          */
10035         if (lun != NULL)
10036                 inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10037                                   lun->be_lun->lun_type;
10038         else if (ctl_softc->inquiry_pq_no_lun == 0)
10039                 inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10040         else
10041                 inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
10042
10043         /* RMB in byte 2 is 0 */
10044         inq_ptr->version = SCSI_REV_SPC3;
10045
10046         /*
10047          * According to SAM-3, even if a device only supports a single
10048          * level of LUN addressing, it should still set the HISUP bit:
10049          *
10050          * 4.9.1 Logical unit numbers overview
10051          *
10052          * All logical unit number formats described in this standard are
10053          * hierarchical in structure even when only a single level in that
10054          * hierarchy is used. The HISUP bit shall be set to one in the
10055          * standard INQUIRY data (see SPC-2) when any logical unit number
10056          * format described in this standard is used.  Non-hierarchical
10057          * formats are outside the scope of this standard.
10058          *
10059          * Therefore we set the HiSup bit here.
10060          *
10061          * The reponse format is 2, per SPC-3.
10062          */
10063         inq_ptr->response_format = SID_HiSup | 2;
10064
10065         inq_ptr->additional_length = sizeof(*inq_ptr) - 4;
10066         CTL_DEBUG_PRINT(("additional_length = %d\n",
10067                          inq_ptr->additional_length));
10068
10069         inq_ptr->spc3_flags = SPC3_SID_TPGS_IMPLICIT;
10070         /* 16 bit addressing */
10071         if (is_fc == 0)
10072                 inq_ptr->spc2_flags = SPC2_SID_ADDR16;
10073         /* XXX set the SID_MultiP bit here if we're actually going to
10074            respond on multiple ports */
10075         inq_ptr->spc2_flags |= SPC2_SID_MultiP;
10076
10077         /* 16 bit data bus, synchronous transfers */
10078         /* XXX these flags don't apply for FC */
10079         if (is_fc == 0)
10080                 inq_ptr->flags = SID_WBus16 | SID_Sync;
10081         /*
10082          * XXX KDM do we want to support tagged queueing on the control
10083          * device at all?
10084          */
10085         if ((lun == NULL)
10086          || (lun->be_lun->lun_type != T_PROCESSOR))
10087                 inq_ptr->flags |= SID_CmdQue;
10088         /*
10089          * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
10090          * We have 8 bytes for the vendor name, and 16 bytes for the device
10091          * name and 4 bytes for the revision.
10092          */
10093         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10094             "vendor")) == NULL) {
10095                 strcpy(inq_ptr->vendor, CTL_VENDOR);
10096         } else {
10097                 memset(inq_ptr->vendor, ' ', sizeof(inq_ptr->vendor));
10098                 strncpy(inq_ptr->vendor, val,
10099                     min(sizeof(inq_ptr->vendor), strlen(val)));
10100         }
10101         if (lun == NULL) {
10102                 strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
10103         } else if ((val = ctl_get_opt(&lun->be_lun->options, "product")) == NULL) {
10104                 switch (lun->be_lun->lun_type) {
10105                 case T_DIRECT:
10106                         strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
10107                         break;
10108                 case T_PROCESSOR:
10109                         strcpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT);
10110                         break;
10111                 default:
10112                         strcpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT);
10113                         break;
10114                 }
10115         } else {
10116                 memset(inq_ptr->product, ' ', sizeof(inq_ptr->product));
10117                 strncpy(inq_ptr->product, val,
10118                     min(sizeof(inq_ptr->product), strlen(val)));
10119         }
10120
10121         /*
10122          * XXX make this a macro somewhere so it automatically gets
10123          * incremented when we make changes.
10124          */
10125         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10126             "revision")) == NULL) {
10127                 strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
10128         } else {
10129                 memset(inq_ptr->revision, ' ', sizeof(inq_ptr->revision));
10130                 strncpy(inq_ptr->revision, val,
10131                     min(sizeof(inq_ptr->revision), strlen(val)));
10132         }
10133
10134         /*
10135          * For parallel SCSI, we support double transition and single
10136          * transition clocking.  We also support QAS (Quick Arbitration
10137          * and Selection) and Information Unit transfers on both the
10138          * control and array devices.
10139          */
10140         if (is_fc == 0)
10141                 inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
10142                                     SID_SPI_IUS;
10143
10144         /* SAM-3 */
10145         scsi_ulto2b(0x0060, inq_ptr->version1);
10146         /* SPC-3 (no version claimed) XXX should we claim a version? */
10147         scsi_ulto2b(0x0300, inq_ptr->version2);
10148         if (is_fc) {
10149                 /* FCP-2 ANSI INCITS.350:2003 */
10150                 scsi_ulto2b(0x0917, inq_ptr->version3);
10151         } else {
10152                 /* SPI-4 ANSI INCITS.362:200x */
10153                 scsi_ulto2b(0x0B56, inq_ptr->version3);
10154         }
10155
10156         if (lun == NULL) {
10157                 /* SBC-2 (no version claimed) XXX should we claim a version? */
10158                 scsi_ulto2b(0x0320, inq_ptr->version4);
10159         } else {
10160                 switch (lun->be_lun->lun_type) {
10161                 case T_DIRECT:
10162                         /*
10163                          * SBC-2 (no version claimed) XXX should we claim a
10164                          * version?
10165                          */
10166                         scsi_ulto2b(0x0320, inq_ptr->version4);
10167                         break;
10168                 case T_PROCESSOR:
10169                 default:
10170                         break;
10171                 }
10172         }
10173
10174         ctsio->scsi_status = SCSI_STATUS_OK;
10175         if (ctsio->kern_data_len > 0) {
10176                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10177                 ctsio->be_move_done = ctl_config_move_done;
10178                 ctl_datamove((union ctl_io *)ctsio);
10179         } else {
10180                 ctsio->io_hdr.status = CTL_SUCCESS;
10181                 ctl_done((union ctl_io *)ctsio);
10182         }
10183
10184         return (CTL_RETVAL_COMPLETE);
10185 }
10186
10187 int
10188 ctl_inquiry(struct ctl_scsiio *ctsio)
10189 {
10190         struct scsi_inquiry *cdb;
10191         int retval;
10192
10193         cdb = (struct scsi_inquiry *)ctsio->cdb;
10194
10195         retval = 0;
10196
10197         CTL_DEBUG_PRINT(("ctl_inquiry\n"));
10198
10199         /*
10200          * Right now, we don't support the CmdDt inquiry information.
10201          * This would be nice to support in the future.  When we do
10202          * support it, we should change this test so that it checks to make
10203          * sure SI_EVPD and SI_CMDDT aren't both set at the same time.
10204          */
10205 #ifdef notyet
10206         if (((cdb->byte2 & SI_EVPD)
10207          && (cdb->byte2 & SI_CMDDT)))
10208 #endif
10209         if (cdb->byte2 & SI_CMDDT) {
10210                 /*
10211                  * Point to the SI_CMDDT bit.  We might change this
10212                  * when we support SI_CMDDT, but since both bits would be
10213                  * "wrong", this should probably just stay as-is then.
10214                  */
10215                 ctl_set_invalid_field(ctsio,
10216                                       /*sks_valid*/ 1,
10217                                       /*command*/ 1,
10218                                       /*field*/ 1,
10219                                       /*bit_valid*/ 1,
10220                                       /*bit*/ 1);
10221                 ctl_done((union ctl_io *)ctsio);
10222                 return (CTL_RETVAL_COMPLETE);
10223         }
10224         if (cdb->byte2 & SI_EVPD)
10225                 retval = ctl_inquiry_evpd(ctsio);
10226 #ifdef notyet
10227         else if (cdb->byte2 & SI_CMDDT)
10228                 retval = ctl_inquiry_cmddt(ctsio);
10229 #endif
10230         else
10231                 retval = ctl_inquiry_std(ctsio);
10232
10233         return (retval);
10234 }
10235
10236 /*
10237  * For known CDB types, parse the LBA and length.
10238  */
10239 static int
10240 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len)
10241 {
10242         if (io->io_hdr.io_type != CTL_IO_SCSI)
10243                 return (1);
10244
10245         switch (io->scsiio.cdb[0]) {
10246         case COMPARE_AND_WRITE: {
10247                 struct scsi_compare_and_write *cdb;
10248
10249                 cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
10250
10251                 *lba = scsi_8btou64(cdb->addr);
10252                 *len = cdb->length;
10253                 break;
10254         }
10255         case READ_6:
10256         case WRITE_6: {
10257                 struct scsi_rw_6 *cdb;
10258
10259                 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
10260
10261                 *lba = scsi_3btoul(cdb->addr);
10262                 /* only 5 bits are valid in the most significant address byte */
10263                 *lba &= 0x1fffff;
10264                 *len = cdb->length;
10265                 break;
10266         }
10267         case READ_10:
10268         case WRITE_10: {
10269                 struct scsi_rw_10 *cdb;
10270
10271                 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
10272
10273                 *lba = scsi_4btoul(cdb->addr);
10274                 *len = scsi_2btoul(cdb->length);
10275                 break;
10276         }
10277         case WRITE_VERIFY_10: {
10278                 struct scsi_write_verify_10 *cdb;
10279
10280                 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
10281
10282                 *lba = scsi_4btoul(cdb->addr);
10283                 *len = scsi_2btoul(cdb->length);
10284                 break;
10285         }
10286         case READ_12:
10287         case WRITE_12: {
10288                 struct scsi_rw_12 *cdb;
10289
10290                 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
10291
10292                 *lba = scsi_4btoul(cdb->addr);
10293                 *len = scsi_4btoul(cdb->length);
10294                 break;
10295         }
10296         case WRITE_VERIFY_12: {
10297                 struct scsi_write_verify_12 *cdb;
10298
10299                 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
10300
10301                 *lba = scsi_4btoul(cdb->addr);
10302                 *len = scsi_4btoul(cdb->length);
10303                 break;
10304         }
10305         case READ_16:
10306         case WRITE_16: {
10307                 struct scsi_rw_16 *cdb;
10308
10309                 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
10310
10311                 *lba = scsi_8btou64(cdb->addr);
10312                 *len = scsi_4btoul(cdb->length);
10313                 break;
10314         }
10315         case WRITE_VERIFY_16: {
10316                 struct scsi_write_verify_16 *cdb;
10317
10318                 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
10319
10320
10321                 *lba = scsi_8btou64(cdb->addr);
10322                 *len = scsi_4btoul(cdb->length);
10323                 break;
10324         }
10325         case WRITE_SAME_10: {
10326                 struct scsi_write_same_10 *cdb;
10327
10328                 cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10329
10330                 *lba = scsi_4btoul(cdb->addr);
10331                 *len = scsi_2btoul(cdb->length);
10332                 break;
10333         }
10334         case WRITE_SAME_16: {
10335                 struct scsi_write_same_16 *cdb;
10336
10337                 cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10338
10339                 *lba = scsi_8btou64(cdb->addr);
10340                 *len = scsi_4btoul(cdb->length);
10341                 break;
10342         }
10343         case VERIFY_10: {
10344                 struct scsi_verify_10 *cdb;
10345
10346                 cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
10347
10348                 *lba = scsi_4btoul(cdb->addr);
10349                 *len = scsi_2btoul(cdb->length);
10350                 break;
10351         }
10352         case VERIFY_12: {
10353                 struct scsi_verify_12 *cdb;
10354
10355                 cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
10356
10357                 *lba = scsi_4btoul(cdb->addr);
10358                 *len = scsi_4btoul(cdb->length);
10359                 break;
10360         }
10361         case VERIFY_16: {
10362                 struct scsi_verify_16 *cdb;
10363
10364                 cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
10365
10366                 *lba = scsi_8btou64(cdb->addr);
10367                 *len = scsi_4btoul(cdb->length);
10368                 break;
10369         }
10370         default:
10371                 return (1);
10372                 break; /* NOTREACHED */
10373         }
10374
10375         return (0);
10376 }
10377
10378 static ctl_action
10379 ctl_extent_check_lba(uint64_t lba1, uint32_t len1, uint64_t lba2, uint32_t len2)
10380 {
10381         uint64_t endlba1, endlba2;
10382
10383         endlba1 = lba1 + len1 - 1;
10384         endlba2 = lba2 + len2 - 1;
10385
10386         if ((endlba1 < lba2)
10387          || (endlba2 < lba1))
10388                 return (CTL_ACTION_PASS);
10389         else
10390                 return (CTL_ACTION_BLOCK);
10391 }
10392
10393 static ctl_action
10394 ctl_extent_check(union ctl_io *io1, union ctl_io *io2)
10395 {
10396         uint64_t lba1, lba2;
10397         uint32_t len1, len2;
10398         int retval;
10399
10400         retval = ctl_get_lba_len(io1, &lba1, &len1);
10401         if (retval != 0)
10402                 return (CTL_ACTION_ERROR);
10403
10404         retval = ctl_get_lba_len(io2, &lba2, &len2);
10405         if (retval != 0)
10406                 return (CTL_ACTION_ERROR);
10407
10408         return (ctl_extent_check_lba(lba1, len1, lba2, len2));
10409 }
10410
10411 static ctl_action
10412 ctl_check_for_blockage(union ctl_io *pending_io, union ctl_io *ooa_io)
10413 {
10414         const struct ctl_cmd_entry *pending_entry, *ooa_entry;
10415         ctl_serialize_action *serialize_row;
10416
10417         /*
10418          * The initiator attempted multiple untagged commands at the same
10419          * time.  Can't do that.
10420          */
10421         if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10422          && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10423          && ((pending_io->io_hdr.nexus.targ_port ==
10424               ooa_io->io_hdr.nexus.targ_port)
10425           && (pending_io->io_hdr.nexus.initid.id ==
10426               ooa_io->io_hdr.nexus.initid.id))
10427          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
10428                 return (CTL_ACTION_OVERLAP);
10429
10430         /*
10431          * The initiator attempted to send multiple tagged commands with
10432          * the same ID.  (It's fine if different initiators have the same
10433          * tag ID.)
10434          *
10435          * Even if all of those conditions are true, we don't kill the I/O
10436          * if the command ahead of us has been aborted.  We won't end up
10437          * sending it to the FETD, and it's perfectly legal to resend a
10438          * command with the same tag number as long as the previous
10439          * instance of this tag number has been aborted somehow.
10440          */
10441         if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10442          && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10443          && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
10444          && ((pending_io->io_hdr.nexus.targ_port ==
10445               ooa_io->io_hdr.nexus.targ_port)
10446           && (pending_io->io_hdr.nexus.initid.id ==
10447               ooa_io->io_hdr.nexus.initid.id))
10448          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
10449                 return (CTL_ACTION_OVERLAP_TAG);
10450
10451         /*
10452          * If we get a head of queue tag, SAM-3 says that we should
10453          * immediately execute it.
10454          *
10455          * What happens if this command would normally block for some other
10456          * reason?  e.g. a request sense with a head of queue tag
10457          * immediately after a write.  Normally that would block, but this
10458          * will result in its getting executed immediately...
10459          *
10460          * We currently return "pass" instead of "skip", so we'll end up
10461          * going through the rest of the queue to check for overlapped tags.
10462          *
10463          * XXX KDM check for other types of blockage first??
10464          */
10465         if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10466                 return (CTL_ACTION_PASS);
10467
10468         /*
10469          * Ordered tags have to block until all items ahead of them
10470          * have completed.  If we get called with an ordered tag, we always
10471          * block, if something else is ahead of us in the queue.
10472          */
10473         if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
10474                 return (CTL_ACTION_BLOCK);
10475
10476         /*
10477          * Simple tags get blocked until all head of queue and ordered tags
10478          * ahead of them have completed.  I'm lumping untagged commands in
10479          * with simple tags here.  XXX KDM is that the right thing to do?
10480          */
10481         if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10482           || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
10483          && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10484           || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
10485                 return (CTL_ACTION_BLOCK);
10486
10487         pending_entry = ctl_get_cmd_entry(&pending_io->scsiio);
10488         ooa_entry = ctl_get_cmd_entry(&ooa_io->scsiio);
10489
10490         serialize_row = ctl_serialize_table[ooa_entry->seridx];
10491
10492         switch (serialize_row[pending_entry->seridx]) {
10493         case CTL_SER_BLOCK:
10494                 return (CTL_ACTION_BLOCK);
10495                 break; /* NOTREACHED */
10496         case CTL_SER_EXTENT:
10497                 return (ctl_extent_check(pending_io, ooa_io));
10498                 break; /* NOTREACHED */
10499         case CTL_SER_PASS:
10500                 return (CTL_ACTION_PASS);
10501                 break; /* NOTREACHED */
10502         case CTL_SER_SKIP:
10503                 return (CTL_ACTION_SKIP);
10504                 break;
10505         default:
10506                 panic("invalid serialization value %d",
10507                       serialize_row[pending_entry->seridx]);
10508                 break; /* NOTREACHED */
10509         }
10510
10511         return (CTL_ACTION_ERROR);
10512 }
10513
10514 /*
10515  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
10516  * Assumptions:
10517  * - pending_io is generally either incoming, or on the blocked queue
10518  * - starting I/O is the I/O we want to start the check with.
10519  */
10520 static ctl_action
10521 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
10522               union ctl_io *starting_io)
10523 {
10524         union ctl_io *ooa_io;
10525         ctl_action action;
10526
10527         mtx_assert(&lun->lun_lock, MA_OWNED);
10528
10529         /*
10530          * Run back along the OOA queue, starting with the current
10531          * blocked I/O and going through every I/O before it on the
10532          * queue.  If starting_io is NULL, we'll just end up returning
10533          * CTL_ACTION_PASS.
10534          */
10535         for (ooa_io = starting_io; ooa_io != NULL;
10536              ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
10537              ooa_links)){
10538
10539                 /*
10540                  * This routine just checks to see whether
10541                  * cur_blocked is blocked by ooa_io, which is ahead
10542                  * of it in the queue.  It doesn't queue/dequeue
10543                  * cur_blocked.
10544                  */
10545                 action = ctl_check_for_blockage(pending_io, ooa_io);
10546                 switch (action) {
10547                 case CTL_ACTION_BLOCK:
10548                 case CTL_ACTION_OVERLAP:
10549                 case CTL_ACTION_OVERLAP_TAG:
10550                 case CTL_ACTION_SKIP:
10551                 case CTL_ACTION_ERROR:
10552                         return (action);
10553                         break; /* NOTREACHED */
10554                 case CTL_ACTION_PASS:
10555                         break;
10556                 default:
10557                         panic("invalid action %d", action);
10558                         break;  /* NOTREACHED */
10559                 }
10560         }
10561
10562         return (CTL_ACTION_PASS);
10563 }
10564
10565 /*
10566  * Assumptions:
10567  * - An I/O has just completed, and has been removed from the per-LUN OOA
10568  *   queue, so some items on the blocked queue may now be unblocked.
10569  */
10570 static int
10571 ctl_check_blocked(struct ctl_lun *lun)
10572 {
10573         union ctl_io *cur_blocked, *next_blocked;
10574
10575         mtx_assert(&lun->lun_lock, MA_OWNED);
10576
10577         /*
10578          * Run forward from the head of the blocked queue, checking each
10579          * entry against the I/Os prior to it on the OOA queue to see if
10580          * there is still any blockage.
10581          *
10582          * We cannot use the TAILQ_FOREACH() macro, because it can't deal
10583          * with our removing a variable on it while it is traversing the
10584          * list.
10585          */
10586         for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
10587              cur_blocked != NULL; cur_blocked = next_blocked) {
10588                 union ctl_io *prev_ooa;
10589                 ctl_action action;
10590
10591                 next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
10592                                                           blocked_links);
10593
10594                 prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
10595                                                       ctl_ooaq, ooa_links);
10596
10597                 /*
10598                  * If cur_blocked happens to be the first item in the OOA
10599                  * queue now, prev_ooa will be NULL, and the action
10600                  * returned will just be CTL_ACTION_PASS.
10601                  */
10602                 action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
10603
10604                 switch (action) {
10605                 case CTL_ACTION_BLOCK:
10606                         /* Nothing to do here, still blocked */
10607                         break;
10608                 case CTL_ACTION_OVERLAP:
10609                 case CTL_ACTION_OVERLAP_TAG:
10610                         /*
10611                          * This shouldn't happen!  In theory we've already
10612                          * checked this command for overlap...
10613                          */
10614                         break;
10615                 case CTL_ACTION_PASS:
10616                 case CTL_ACTION_SKIP: {
10617                         struct ctl_softc *softc;
10618                         const struct ctl_cmd_entry *entry;
10619                         uint32_t initidx;
10620                         int isc_retval;
10621
10622                         /*
10623                          * The skip case shouldn't happen, this transaction
10624                          * should have never made it onto the blocked queue.
10625                          */
10626                         /*
10627                          * This I/O is no longer blocked, we can remove it
10628                          * from the blocked queue.  Since this is a TAILQ
10629                          * (doubly linked list), we can do O(1) removals
10630                          * from any place on the list.
10631                          */
10632                         TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
10633                                      blocked_links);
10634                         cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10635
10636                         if (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC){
10637                                 /*
10638                                  * Need to send IO back to original side to
10639                                  * run
10640                                  */
10641                                 union ctl_ha_msg msg_info;
10642
10643                                 msg_info.hdr.original_sc =
10644                                         cur_blocked->io_hdr.original_sc;
10645                                 msg_info.hdr.serializing_sc = cur_blocked;
10646                                 msg_info.hdr.msg_type = CTL_MSG_R2R;
10647                                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10648                                      &msg_info, sizeof(msg_info), 0)) >
10649                                      CTL_HA_STATUS_SUCCESS) {
10650                                         printf("CTL:Check Blocked error from "
10651                                                "ctl_ha_msg_send %d\n",
10652                                                isc_retval);
10653                                 }
10654                                 break;
10655                         }
10656                         entry = ctl_get_cmd_entry(&cur_blocked->scsiio);
10657                         softc = control_softc;
10658
10659                         initidx = ctl_get_initindex(&cur_blocked->io_hdr.nexus);
10660
10661                         /*
10662                          * Check this I/O for LUN state changes that may
10663                          * have happened while this command was blocked.
10664                          * The LUN state may have been changed by a command
10665                          * ahead of us in the queue, so we need to re-check
10666                          * for any states that can be caused by SCSI
10667                          * commands.
10668                          */
10669                         if (ctl_scsiio_lun_check(softc, lun, entry,
10670                                                  &cur_blocked->scsiio) == 0) {
10671                                 cur_blocked->io_hdr.flags |=
10672                                                       CTL_FLAG_IS_WAS_ON_RTR;
10673                                 ctl_enqueue_rtr(cur_blocked);
10674                         } else
10675                                 ctl_done(cur_blocked);
10676                         break;
10677                 }
10678                 default:
10679                         /*
10680                          * This probably shouldn't happen -- we shouldn't
10681                          * get CTL_ACTION_ERROR, or anything else.
10682                          */
10683                         break;
10684                 }
10685         }
10686
10687         return (CTL_RETVAL_COMPLETE);
10688 }
10689
10690 /*
10691  * This routine (with one exception) checks LUN flags that can be set by
10692  * commands ahead of us in the OOA queue.  These flags have to be checked
10693  * when a command initially comes in, and when we pull a command off the
10694  * blocked queue and are preparing to execute it.  The reason we have to
10695  * check these flags for commands on the blocked queue is that the LUN
10696  * state may have been changed by a command ahead of us while we're on the
10697  * blocked queue.
10698  *
10699  * Ordering is somewhat important with these checks, so please pay
10700  * careful attention to the placement of any new checks.
10701  */
10702 static int
10703 ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
10704     const struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
10705 {
10706         int retval;
10707
10708         retval = 0;
10709
10710         mtx_assert(&lun->lun_lock, MA_OWNED);
10711
10712         /*
10713          * If this shelf is a secondary shelf controller, we have to reject
10714          * any media access commands.
10715          */
10716 #if 0
10717         /* No longer needed for HA */
10718         if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
10719          && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
10720                 ctl_set_lun_standby(ctsio);
10721                 retval = 1;
10722                 goto bailout;
10723         }
10724 #endif
10725
10726         /*
10727          * Check for a reservation conflict.  If this command isn't allowed
10728          * even on reserved LUNs, and if this initiator isn't the one who
10729          * reserved us, reject the command with a reservation conflict.
10730          */
10731         if ((lun->flags & CTL_LUN_RESERVED)
10732          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
10733                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
10734                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
10735                  || (ctsio->io_hdr.nexus.targ_target.id !=
10736                      lun->rsv_nexus.targ_target.id)) {
10737                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10738                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
10739                         retval = 1;
10740                         goto bailout;
10741                 }
10742         }
10743
10744         if ( (lun->flags & CTL_LUN_PR_RESERVED)
10745          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV) == 0)) {
10746                 uint32_t residx;
10747
10748                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
10749                 /*
10750                  * if we aren't registered or it's a res holder type
10751                  * reservation and this isn't the res holder then set a
10752                  * conflict.
10753                  * NOTE: Commands which might be allowed on write exclusive
10754                  * type reservations are checked in the particular command
10755                  * for a conflict. Read and SSU are the only ones.
10756                  */
10757                 if (!lun->per_res[residx].registered
10758                  || (residx != lun->pr_res_idx && lun->res_type < 4)) {
10759                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10760                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
10761                         retval = 1;
10762                         goto bailout;
10763                 }
10764
10765         }
10766
10767         if ((lun->flags & CTL_LUN_OFFLINE)
10768          && ((entry->flags & CTL_CMD_FLAG_OK_ON_OFFLINE) == 0)) {
10769                 ctl_set_lun_not_ready(ctsio);
10770                 retval = 1;
10771                 goto bailout;
10772         }
10773
10774         /*
10775          * If the LUN is stopped, see if this particular command is allowed
10776          * for a stopped lun.  Otherwise, reject it with 0x04,0x02.
10777          */
10778         if ((lun->flags & CTL_LUN_STOPPED)
10779          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
10780                 /* "Logical unit not ready, initializing cmd. required" */
10781                 ctl_set_lun_stopped(ctsio);
10782                 retval = 1;
10783                 goto bailout;
10784         }
10785
10786         if ((lun->flags & CTL_LUN_INOPERABLE)
10787          && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
10788                 /* "Medium format corrupted" */
10789                 ctl_set_medium_format_corrupted(ctsio);
10790                 retval = 1;
10791                 goto bailout;
10792         }
10793
10794 bailout:
10795         return (retval);
10796
10797 }
10798
10799 static void
10800 ctl_failover_io(union ctl_io *io, int have_lock)
10801 {
10802         ctl_set_busy(&io->scsiio);
10803         ctl_done(io);
10804 }
10805
10806 static void
10807 ctl_failover(void)
10808 {
10809         struct ctl_lun *lun;
10810         struct ctl_softc *ctl_softc;
10811         union ctl_io *next_io, *pending_io;
10812         union ctl_io *io;
10813         int lun_idx;
10814         int i;
10815
10816         ctl_softc = control_softc;
10817
10818         mtx_lock(&ctl_softc->ctl_lock);
10819         /*
10820          * Remove any cmds from the other SC from the rtr queue.  These
10821          * will obviously only be for LUNs for which we're the primary.
10822          * We can't send status or get/send data for these commands.
10823          * Since they haven't been executed yet, we can just remove them.
10824          * We'll either abort them or delete them below, depending on
10825          * which HA mode we're in.
10826          */
10827 #ifdef notyet
10828         mtx_lock(&ctl_softc->queue_lock);
10829         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->rtr_queue);
10830              io != NULL; io = next_io) {
10831                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
10832                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10833                         STAILQ_REMOVE(&ctl_softc->rtr_queue, &io->io_hdr,
10834                                       ctl_io_hdr, links);
10835         }
10836         mtx_unlock(&ctl_softc->queue_lock);
10837 #endif
10838
10839         for (lun_idx=0; lun_idx < ctl_softc->num_luns; lun_idx++) {
10840                 lun = ctl_softc->ctl_luns[lun_idx];
10841                 if (lun==NULL)
10842                         continue;
10843
10844                 /*
10845                  * Processor LUNs are primary on both sides.
10846                  * XXX will this always be true?
10847                  */
10848                 if (lun->be_lun->lun_type == T_PROCESSOR)
10849                         continue;
10850
10851                 if ((lun->flags & CTL_LUN_PRIMARY_SC)
10852                  && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10853                         printf("FAILOVER: primary lun %d\n", lun_idx);
10854                         /*
10855                          * Remove all commands from the other SC. First from the
10856                          * blocked queue then from the ooa queue. Once we have
10857                          * removed them. Call ctl_check_blocked to see if there
10858                          * is anything that can run.
10859                          */
10860                         for (io = (union ctl_io *)TAILQ_FIRST(
10861                              &lun->blocked_queue); io != NULL; io = next_io) {
10862
10863                                 next_io = (union ctl_io *)TAILQ_NEXT(
10864                                     &io->io_hdr, blocked_links);
10865
10866                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10867                                         TAILQ_REMOVE(&lun->blocked_queue,
10868                                                      &io->io_hdr,blocked_links);
10869                                         io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10870                                         TAILQ_REMOVE(&lun->ooa_queue,
10871                                                      &io->io_hdr, ooa_links);
10872
10873                                         ctl_free_io(io);
10874                                 }
10875                         }
10876
10877                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10878                              io != NULL; io = next_io) {
10879
10880                                 next_io = (union ctl_io *)TAILQ_NEXT(
10881                                     &io->io_hdr, ooa_links);
10882
10883                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10884
10885                                         TAILQ_REMOVE(&lun->ooa_queue,
10886                                                 &io->io_hdr,
10887                                                 ooa_links);
10888
10889                                         ctl_free_io(io);
10890                                 }
10891                         }
10892                         ctl_check_blocked(lun);
10893                 } else if ((lun->flags & CTL_LUN_PRIMARY_SC)
10894                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10895
10896                         printf("FAILOVER: primary lun %d\n", lun_idx);
10897                         /*
10898                          * Abort all commands from the other SC.  We can't
10899                          * send status back for them now.  These should get
10900                          * cleaned up when they are completed or come out
10901                          * for a datamove operation.
10902                          */
10903                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10904                              io != NULL; io = next_io) {
10905                                 next_io = (union ctl_io *)TAILQ_NEXT(
10906                                         &io->io_hdr, ooa_links);
10907
10908                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10909                                         io->io_hdr.flags |= CTL_FLAG_ABORT;
10910                         }
10911                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10912                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10913
10914                         printf("FAILOVER: secondary lun %d\n", lun_idx);
10915
10916                         lun->flags |= CTL_LUN_PRIMARY_SC;
10917
10918                         /*
10919                          * We send all I/O that was sent to this controller
10920                          * and redirected to the other side back with
10921                          * busy status, and have the initiator retry it.
10922                          * Figuring out how much data has been transferred,
10923                          * etc. and picking up where we left off would be
10924                          * very tricky.
10925                          *
10926                          * XXX KDM need to remove I/O from the blocked
10927                          * queue as well!
10928                          */
10929                         for (pending_io = (union ctl_io *)TAILQ_FIRST(
10930                              &lun->ooa_queue); pending_io != NULL;
10931                              pending_io = next_io) {
10932
10933                                 next_io =  (union ctl_io *)TAILQ_NEXT(
10934                                         &pending_io->io_hdr, ooa_links);
10935
10936                                 pending_io->io_hdr.flags &=
10937                                         ~CTL_FLAG_SENT_2OTHER_SC;
10938
10939                                 if (pending_io->io_hdr.flags &
10940                                     CTL_FLAG_IO_ACTIVE) {
10941                                         pending_io->io_hdr.flags |=
10942                                                 CTL_FLAG_FAILOVER;
10943                                 } else {
10944                                         ctl_set_busy(&pending_io->scsiio);
10945                                         ctl_done(pending_io);
10946                                 }
10947                         }
10948
10949                         /*
10950                          * Build Unit Attention
10951                          */
10952                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10953                                 lun->pending_sense[i].ua_pending |=
10954                                                      CTL_UA_ASYM_ACC_CHANGE;
10955                         }
10956                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10957                         && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10958                         printf("FAILOVER: secondary lun %d\n", lun_idx);
10959                         /*
10960                          * if the first io on the OOA is not on the RtR queue
10961                          * add it.
10962                          */
10963                         lun->flags |= CTL_LUN_PRIMARY_SC;
10964
10965                         pending_io = (union ctl_io *)TAILQ_FIRST(
10966                             &lun->ooa_queue);
10967                         if (pending_io==NULL) {
10968                                 printf("Nothing on OOA queue\n");
10969                                 continue;
10970                         }
10971
10972                         pending_io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
10973                         if ((pending_io->io_hdr.flags &
10974                              CTL_FLAG_IS_WAS_ON_RTR) == 0) {
10975                                 pending_io->io_hdr.flags |=
10976                                     CTL_FLAG_IS_WAS_ON_RTR;
10977                                 ctl_enqueue_rtr(pending_io);
10978                         }
10979 #if 0
10980                         else
10981                         {
10982                                 printf("Tag 0x%04x is running\n",
10983                                       pending_io->scsiio.tag_num);
10984                         }
10985 #endif
10986
10987                         next_io = (union ctl_io *)TAILQ_NEXT(
10988                             &pending_io->io_hdr, ooa_links);
10989                         for (pending_io=next_io; pending_io != NULL;
10990                              pending_io = next_io) {
10991                                 pending_io->io_hdr.flags &=
10992                                     ~CTL_FLAG_SENT_2OTHER_SC;
10993                                 next_io = (union ctl_io *)TAILQ_NEXT(
10994                                         &pending_io->io_hdr, ooa_links);
10995                                 if (pending_io->io_hdr.flags &
10996                                     CTL_FLAG_IS_WAS_ON_RTR) {
10997 #if 0
10998                                         printf("Tag 0x%04x is running\n",
10999                                                 pending_io->scsiio.tag_num);
11000 #endif
11001                                         continue;
11002                                 }
11003
11004                                 switch (ctl_check_ooa(lun, pending_io,
11005                                     (union ctl_io *)TAILQ_PREV(
11006                                     &pending_io->io_hdr, ctl_ooaq,
11007                                     ooa_links))) {
11008
11009                                 case CTL_ACTION_BLOCK:
11010                                         TAILQ_INSERT_TAIL(&lun->blocked_queue,
11011                                                           &pending_io->io_hdr,
11012                                                           blocked_links);
11013                                         pending_io->io_hdr.flags |=
11014                                             CTL_FLAG_BLOCKED;
11015                                         break;
11016                                 case CTL_ACTION_PASS:
11017                                 case CTL_ACTION_SKIP:
11018                                         pending_io->io_hdr.flags |=
11019                                             CTL_FLAG_IS_WAS_ON_RTR;
11020                                         ctl_enqueue_rtr(pending_io);
11021                                         break;
11022                                 case CTL_ACTION_OVERLAP:
11023                                         ctl_set_overlapped_cmd(
11024                                             (struct ctl_scsiio *)pending_io);
11025                                         ctl_done(pending_io);
11026                                         break;
11027                                 case CTL_ACTION_OVERLAP_TAG:
11028                                         ctl_set_overlapped_tag(
11029                                             (struct ctl_scsiio *)pending_io,
11030                                             pending_io->scsiio.tag_num & 0xff);
11031                                         ctl_done(pending_io);
11032                                         break;
11033                                 case CTL_ACTION_ERROR:
11034                                 default:
11035                                         ctl_set_internal_failure(
11036                                                 (struct ctl_scsiio *)pending_io,
11037                                                 0,  // sks_valid
11038                                                 0); //retry count
11039                                         ctl_done(pending_io);
11040                                         break;
11041                                 }
11042                         }
11043
11044                         /*
11045                          * Build Unit Attention
11046                          */
11047                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11048                                 lun->pending_sense[i].ua_pending |=
11049                                                      CTL_UA_ASYM_ACC_CHANGE;
11050                         }
11051                 } else {
11052                         panic("Unhandled HA mode failover, LUN flags = %#x, "
11053                               "ha_mode = #%x", lun->flags, ctl_softc->ha_mode);
11054                 }
11055         }
11056         ctl_pause_rtr = 0;
11057         mtx_unlock(&ctl_softc->ctl_lock);
11058 }
11059
11060 static int
11061 ctl_scsiio_precheck(struct ctl_softc *ctl_softc, struct ctl_scsiio *ctsio)
11062 {
11063         struct ctl_lun *lun;
11064         const struct ctl_cmd_entry *entry;
11065         uint32_t initidx, targ_lun;
11066         int retval;
11067
11068         retval = 0;
11069
11070         lun = NULL;
11071
11072         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
11073         if ((targ_lun < CTL_MAX_LUNS)
11074          && (ctl_softc->ctl_luns[targ_lun] != NULL)) {
11075                 lun = ctl_softc->ctl_luns[targ_lun];
11076                 /*
11077                  * If the LUN is invalid, pretend that it doesn't exist.
11078                  * It will go away as soon as all pending I/O has been
11079                  * completed.
11080                  */
11081                 if (lun->flags & CTL_LUN_DISABLED) {
11082                         lun = NULL;
11083                 } else {
11084                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
11085                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
11086                                 lun->be_lun;
11087                         if (lun->be_lun->lun_type == T_PROCESSOR) {
11088                                 ctsio->io_hdr.flags |= CTL_FLAG_CONTROL_DEV;
11089                         }
11090
11091                         /*
11092                          * Every I/O goes into the OOA queue for a
11093                          * particular LUN, and stays there until completion.
11094                          */
11095                         mtx_lock(&lun->lun_lock);
11096                         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr,
11097                             ooa_links);
11098                 }
11099         } else {
11100                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11101                 ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11102         }
11103
11104         /* Get command entry and return error if it is unsuppotyed. */
11105         entry = ctl_validate_command(ctsio);
11106         if (entry == NULL) {
11107                 if (lun)
11108                         mtx_unlock(&lun->lun_lock);
11109                 return (retval);
11110         }
11111
11112         ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
11113         ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
11114
11115         /*
11116          * Check to see whether we can send this command to LUNs that don't
11117          * exist.  This should pretty much only be the case for inquiry
11118          * and request sense.  Further checks, below, really require having
11119          * a LUN, so we can't really check the command anymore.  Just put
11120          * it on the rtr queue.
11121          */
11122         if (lun == NULL) {
11123                 if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) {
11124                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11125                         ctl_enqueue_rtr((union ctl_io *)ctsio);
11126                         return (retval);
11127                 }
11128
11129                 ctl_set_unsupported_lun(ctsio);
11130                 ctl_done((union ctl_io *)ctsio);
11131                 CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
11132                 return (retval);
11133         } else {
11134                 /*
11135                  * Make sure we support this particular command on this LUN.
11136                  * e.g., we don't support writes to the control LUN.
11137                  */
11138                 if (!ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
11139                         mtx_unlock(&lun->lun_lock);
11140                         ctl_set_invalid_opcode(ctsio);
11141                         ctl_done((union ctl_io *)ctsio);
11142                         return (retval);
11143                 }
11144         }
11145
11146         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11147
11148         /*
11149          * If we've got a request sense, it'll clear the contingent
11150          * allegiance condition.  Otherwise, if we have a CA condition for
11151          * this initiator, clear it, because it sent down a command other
11152          * than request sense.
11153          */
11154         if ((ctsio->cdb[0] != REQUEST_SENSE)
11155          && (ctl_is_set(lun->have_ca, initidx)))
11156                 ctl_clear_mask(lun->have_ca, initidx);
11157
11158         /*
11159          * If the command has this flag set, it handles its own unit
11160          * attention reporting, we shouldn't do anything.  Otherwise we
11161          * check for any pending unit attentions, and send them back to the
11162          * initiator.  We only do this when a command initially comes in,
11163          * not when we pull it off the blocked queue.
11164          *
11165          * According to SAM-3, section 5.3.2, the order that things get
11166          * presented back to the host is basically unit attentions caused
11167          * by some sort of reset event, busy status, reservation conflicts
11168          * or task set full, and finally any other status.
11169          *
11170          * One issue here is that some of the unit attentions we report
11171          * don't fall into the "reset" category (e.g. "reported luns data
11172          * has changed").  So reporting it here, before the reservation
11173          * check, may be technically wrong.  I guess the only thing to do
11174          * would be to check for and report the reset events here, and then
11175          * check for the other unit attention types after we check for a
11176          * reservation conflict.
11177          *
11178          * XXX KDM need to fix this
11179          */
11180         if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
11181                 ctl_ua_type ua_type;
11182
11183                 ua_type = lun->pending_sense[initidx].ua_pending;
11184                 if (ua_type != CTL_UA_NONE) {
11185                         scsi_sense_data_type sense_format;
11186
11187                         if (lun != NULL)
11188                                 sense_format = (lun->flags &
11189                                     CTL_LUN_SENSE_DESC) ? SSD_TYPE_DESC :
11190                                     SSD_TYPE_FIXED;
11191                         else
11192                                 sense_format = SSD_TYPE_FIXED;
11193
11194                         ua_type = ctl_build_ua(ua_type, &ctsio->sense_data,
11195                                                sense_format);
11196                         if (ua_type != CTL_UA_NONE) {
11197                                 ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
11198                                 ctsio->io_hdr.status = CTL_SCSI_ERROR |
11199                                                        CTL_AUTOSENSE;
11200                                 ctsio->sense_len = SSD_FULL_SIZE;
11201                                 lun->pending_sense[initidx].ua_pending &=
11202                                         ~ua_type;
11203                                 mtx_unlock(&lun->lun_lock);
11204                                 ctl_done((union ctl_io *)ctsio);
11205                                 return (retval);
11206                         }
11207                 }
11208         }
11209
11210
11211         if (ctl_scsiio_lun_check(ctl_softc, lun, entry, ctsio) != 0) {
11212                 mtx_unlock(&lun->lun_lock);
11213                 ctl_done((union ctl_io *)ctsio);
11214                 return (retval);
11215         }
11216
11217         /*
11218          * XXX CHD this is where we want to send IO to other side if
11219          * this LUN is secondary on this SC. We will need to make a copy
11220          * of the IO and flag the IO on this side as SENT_2OTHER and the flag
11221          * the copy we send as FROM_OTHER.
11222          * We also need to stuff the address of the original IO so we can
11223          * find it easily. Something similar will need be done on the other
11224          * side so when we are done we can find the copy.
11225          */
11226         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
11227                 union ctl_ha_msg msg_info;
11228                 int isc_retval;
11229
11230                 ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11231
11232                 msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
11233                 msg_info.hdr.original_sc = (union ctl_io *)ctsio;
11234 #if 0
11235                 printf("1. ctsio %p\n", ctsio);
11236 #endif
11237                 msg_info.hdr.serializing_sc = NULL;
11238                 msg_info.hdr.nexus = ctsio->io_hdr.nexus;
11239                 msg_info.scsi.tag_num = ctsio->tag_num;
11240                 msg_info.scsi.tag_type = ctsio->tag_type;
11241                 memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
11242
11243                 ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11244
11245                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11246                     (void *)&msg_info, sizeof(msg_info), 0)) >
11247                     CTL_HA_STATUS_SUCCESS) {
11248                         printf("CTL:precheck, ctl_ha_msg_send returned %d\n",
11249                                isc_retval);
11250                         printf("CTL:opcode is %x\n", ctsio->cdb[0]);
11251                 } else {
11252 #if 0
11253                         printf("CTL:Precheck sent msg, opcode is %x\n",opcode);
11254 #endif
11255                 }
11256
11257                 /*
11258                  * XXX KDM this I/O is off the incoming queue, but hasn't
11259                  * been inserted on any other queue.  We may need to come
11260                  * up with a holding queue while we wait for serialization
11261                  * so that we have an idea of what we're waiting for from
11262                  * the other side.
11263                  */
11264                 mtx_unlock(&lun->lun_lock);
11265                 return (retval);
11266         }
11267
11268         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
11269                               (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
11270                               ctl_ooaq, ooa_links))) {
11271         case CTL_ACTION_BLOCK:
11272                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
11273                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
11274                                   blocked_links);
11275                 mtx_unlock(&lun->lun_lock);
11276                 return (retval);
11277         case CTL_ACTION_PASS:
11278         case CTL_ACTION_SKIP:
11279                 ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11280                 mtx_unlock(&lun->lun_lock);
11281                 ctl_enqueue_rtr((union ctl_io *)ctsio);
11282                 break;
11283         case CTL_ACTION_OVERLAP:
11284                 mtx_unlock(&lun->lun_lock);
11285                 ctl_set_overlapped_cmd(ctsio);
11286                 ctl_done((union ctl_io *)ctsio);
11287                 break;
11288         case CTL_ACTION_OVERLAP_TAG:
11289                 mtx_unlock(&lun->lun_lock);
11290                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
11291                 ctl_done((union ctl_io *)ctsio);
11292                 break;
11293         case CTL_ACTION_ERROR:
11294         default:
11295                 mtx_unlock(&lun->lun_lock);
11296                 ctl_set_internal_failure(ctsio,
11297                                          /*sks_valid*/ 0,
11298                                          /*retry_count*/ 0);
11299                 ctl_done((union ctl_io *)ctsio);
11300                 break;
11301         }
11302         return (retval);
11303 }
11304
11305 const struct ctl_cmd_entry *
11306 ctl_get_cmd_entry(struct ctl_scsiio *ctsio)
11307 {
11308         const struct ctl_cmd_entry *entry;
11309         int service_action;
11310
11311         entry = &ctl_cmd_table[ctsio->cdb[0]];
11312         if (entry->flags & CTL_CMD_FLAG_SA5) {
11313                 service_action = ctsio->cdb[1] & SERVICE_ACTION_MASK;
11314                 entry = &((const struct ctl_cmd_entry *)
11315                     entry->execute)[service_action];
11316         }
11317         return (entry);
11318 }
11319
11320 const struct ctl_cmd_entry *
11321 ctl_validate_command(struct ctl_scsiio *ctsio)
11322 {
11323         const struct ctl_cmd_entry *entry;
11324         int i;
11325         uint8_t diff;
11326
11327         entry = ctl_get_cmd_entry(ctsio);
11328         if (entry->execute == NULL) {
11329                 ctl_set_invalid_opcode(ctsio);
11330                 ctl_done((union ctl_io *)ctsio);
11331                 return (NULL);
11332         }
11333         KASSERT(entry->length > 0,
11334             ("Not defined length for command 0x%02x/0x%02x",
11335              ctsio->cdb[0], ctsio->cdb[1]));
11336         for (i = 1; i < entry->length; i++) {
11337                 diff = ctsio->cdb[i] & ~entry->usage[i - 1];
11338                 if (diff == 0)
11339                         continue;
11340                 ctl_set_invalid_field(ctsio,
11341                                       /*sks_valid*/ 1,
11342                                       /*command*/ 1,
11343                                       /*field*/ i,
11344                                       /*bit_valid*/ 1,
11345                                       /*bit*/ fls(diff) - 1);
11346                 ctl_done((union ctl_io *)ctsio);
11347                 return (NULL);
11348         }
11349         return (entry);
11350 }
11351
11352 static int
11353 ctl_cmd_applicable(uint8_t lun_type, const struct ctl_cmd_entry *entry)
11354 {
11355
11356         switch (lun_type) {
11357         case T_PROCESSOR:
11358                 if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0) &&
11359                     ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
11360                         return (0);
11361                 break;
11362         case T_DIRECT:
11363                 if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0) &&
11364                     ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
11365                         return (0);
11366                 break;
11367         default:
11368                 return (0);
11369         }
11370         return (1);
11371 }
11372
11373 static int
11374 ctl_scsiio(struct ctl_scsiio *ctsio)
11375 {
11376         int retval;
11377         const struct ctl_cmd_entry *entry;
11378
11379         retval = CTL_RETVAL_COMPLETE;
11380
11381         CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
11382
11383         entry = ctl_get_cmd_entry(ctsio);
11384
11385         /*
11386          * If this I/O has been aborted, just send it straight to
11387          * ctl_done() without executing it.
11388          */
11389         if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
11390                 ctl_done((union ctl_io *)ctsio);
11391                 goto bailout;
11392         }
11393
11394         /*
11395          * All the checks should have been handled by ctl_scsiio_precheck().
11396          * We should be clear now to just execute the I/O.
11397          */
11398         retval = entry->execute(ctsio);
11399
11400 bailout:
11401         return (retval);
11402 }
11403
11404 /*
11405  * Since we only implement one target right now, a bus reset simply resets
11406  * our single target.
11407  */
11408 static int
11409 ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io)
11410 {
11411         return(ctl_target_reset(ctl_softc, io, CTL_UA_BUS_RESET));
11412 }
11413
11414 static int
11415 ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
11416                  ctl_ua_type ua_type)
11417 {
11418         struct ctl_lun *lun;
11419         int retval;
11420
11421         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11422                 union ctl_ha_msg msg_info;
11423
11424                 io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11425                 msg_info.hdr.nexus = io->io_hdr.nexus;
11426                 if (ua_type==CTL_UA_TARG_RESET)
11427                         msg_info.task.task_action = CTL_TASK_TARGET_RESET;
11428                 else
11429                         msg_info.task.task_action = CTL_TASK_BUS_RESET;
11430                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11431                 msg_info.hdr.original_sc = NULL;
11432                 msg_info.hdr.serializing_sc = NULL;
11433                 if (CTL_HA_STATUS_SUCCESS != ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11434                     (void *)&msg_info, sizeof(msg_info), 0)) {
11435                 }
11436         }
11437         retval = 0;
11438
11439         mtx_lock(&ctl_softc->ctl_lock);
11440         STAILQ_FOREACH(lun, &ctl_softc->lun_list, links)
11441                 retval += ctl_lun_reset(lun, io, ua_type);
11442         mtx_unlock(&ctl_softc->ctl_lock);
11443
11444         return (retval);
11445 }
11446
11447 /*
11448  * The LUN should always be set.  The I/O is optional, and is used to
11449  * distinguish between I/Os sent by this initiator, and by other
11450  * initiators.  We set unit attention for initiators other than this one.
11451  * SAM-3 is vague on this point.  It does say that a unit attention should
11452  * be established for other initiators when a LUN is reset (see section
11453  * 5.7.3), but it doesn't specifically say that the unit attention should
11454  * be established for this particular initiator when a LUN is reset.  Here
11455  * is the relevant text, from SAM-3 rev 8:
11456  *
11457  * 5.7.2 When a SCSI initiator port aborts its own tasks
11458  *
11459  * When a SCSI initiator port causes its own task(s) to be aborted, no
11460  * notification that the task(s) have been aborted shall be returned to
11461  * the SCSI initiator port other than the completion response for the
11462  * command or task management function action that caused the task(s) to
11463  * be aborted and notification(s) associated with related effects of the
11464  * action (e.g., a reset unit attention condition).
11465  *
11466  * XXX KDM for now, we're setting unit attention for all initiators.
11467  */
11468 static int
11469 ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
11470 {
11471         union ctl_io *xio;
11472 #if 0
11473         uint32_t initindex;
11474 #endif
11475         int i;
11476
11477         mtx_lock(&lun->lun_lock);
11478         /*
11479          * Run through the OOA queue and abort each I/O.
11480          */
11481 #if 0
11482         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
11483 #endif
11484         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11485              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11486                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
11487         }
11488
11489         /*
11490          * This version sets unit attention for every
11491          */
11492 #if 0
11493         initindex = ctl_get_initindex(&io->io_hdr.nexus);
11494         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11495                 if (initindex == i)
11496                         continue;
11497                 lun->pending_sense[i].ua_pending |= ua_type;
11498         }
11499 #endif
11500
11501         /*
11502          * A reset (any kind, really) clears reservations established with
11503          * RESERVE/RELEASE.  It does not clear reservations established
11504          * with PERSISTENT RESERVE OUT, but we don't support that at the
11505          * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
11506          * reservations made with the RESERVE/RELEASE commands, because
11507          * those commands are obsolete in SPC-3.
11508          */
11509         lun->flags &= ~CTL_LUN_RESERVED;
11510
11511         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11512                 ctl_clear_mask(lun->have_ca, i);
11513                 lun->pending_sense[i].ua_pending |= ua_type;
11514         }
11515         mtx_unlock(&lun->lun_lock);
11516
11517         return (0);
11518 }
11519
11520 static int
11521 ctl_abort_task(union ctl_io *io)
11522 {
11523         union ctl_io *xio;
11524         struct ctl_lun *lun;
11525         struct ctl_softc *ctl_softc;
11526 #if 0
11527         struct sbuf sb;
11528         char printbuf[128];
11529 #endif
11530         int found;
11531         uint32_t targ_lun;
11532
11533         ctl_softc = control_softc;
11534         found = 0;
11535
11536         /*
11537          * Look up the LUN.
11538          */
11539         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11540         mtx_lock(&ctl_softc->ctl_lock);
11541         if ((targ_lun < CTL_MAX_LUNS)
11542          && (ctl_softc->ctl_luns[targ_lun] != NULL))
11543                 lun = ctl_softc->ctl_luns[targ_lun];
11544         else {
11545                 mtx_unlock(&ctl_softc->ctl_lock);
11546                 goto bailout;
11547         }
11548
11549 #if 0
11550         printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
11551                lun->lun, io->taskio.tag_num, io->taskio.tag_type);
11552 #endif
11553
11554         mtx_lock(&lun->lun_lock);
11555         mtx_unlock(&ctl_softc->ctl_lock);
11556         /*
11557          * Run through the OOA queue and attempt to find the given I/O.
11558          * The target port, initiator ID, tag type and tag number have to
11559          * match the values that we got from the initiator.  If we have an
11560          * untagged command to abort, simply abort the first untagged command
11561          * we come to.  We only allow one untagged command at a time of course.
11562          */
11563 #if 0
11564         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
11565 #endif
11566         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11567              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11568 #if 0
11569                 sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
11570
11571                 sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
11572                             lun->lun, xio->scsiio.tag_num,
11573                             xio->scsiio.tag_type,
11574                             (xio->io_hdr.blocked_links.tqe_prev
11575                             == NULL) ? "" : " BLOCKED",
11576                             (xio->io_hdr.flags &
11577                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
11578                             (xio->io_hdr.flags &
11579                             CTL_FLAG_ABORT) ? " ABORT" : "",
11580                             (xio->io_hdr.flags &
11581                             CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
11582                 ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
11583                 sbuf_finish(&sb);
11584                 printf("%s\n", sbuf_data(&sb));
11585 #endif
11586
11587                 if ((xio->io_hdr.nexus.targ_port == io->io_hdr.nexus.targ_port)
11588                  && (xio->io_hdr.nexus.initid.id ==
11589                      io->io_hdr.nexus.initid.id)) {
11590                         /*
11591                          * If the abort says that the task is untagged, the
11592                          * task in the queue must be untagged.  Otherwise,
11593                          * we just check to see whether the tag numbers
11594                          * match.  This is because the QLogic firmware
11595                          * doesn't pass back the tag type in an abort
11596                          * request.
11597                          */
11598 #if 0
11599                         if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
11600                           && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
11601                          || (xio->scsiio.tag_num == io->taskio.tag_num)) {
11602 #endif
11603                         /*
11604                          * XXX KDM we've got problems with FC, because it
11605                          * doesn't send down a tag type with aborts.  So we
11606                          * can only really go by the tag number...
11607                          * This may cause problems with parallel SCSI.
11608                          * Need to figure that out!!
11609                          */
11610                         if (xio->scsiio.tag_num == io->taskio.tag_num) {
11611                                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
11612                                 found = 1;
11613                                 if ((io->io_hdr.flags &
11614                                      CTL_FLAG_FROM_OTHER_SC) == 0 &&
11615                                     !(lun->flags & CTL_LUN_PRIMARY_SC)) {
11616                                         union ctl_ha_msg msg_info;
11617
11618                                         io->io_hdr.flags |=
11619                                                         CTL_FLAG_SENT_2OTHER_SC;
11620                                         msg_info.hdr.nexus = io->io_hdr.nexus;
11621                                         msg_info.task.task_action =
11622                                                 CTL_TASK_ABORT_TASK;
11623                                         msg_info.task.tag_num =
11624                                                 io->taskio.tag_num;
11625                                         msg_info.task.tag_type =
11626                                                 io->taskio.tag_type;
11627                                         msg_info.hdr.msg_type =
11628                                                 CTL_MSG_MANAGE_TASKS;
11629                                         msg_info.hdr.original_sc = NULL;
11630                                         msg_info.hdr.serializing_sc = NULL;
11631 #if 0
11632                                         printf("Sent Abort to other side\n");
11633 #endif
11634                                         if (CTL_HA_STATUS_SUCCESS !=
11635                                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11636                                                 (void *)&msg_info,
11637                                                 sizeof(msg_info), 0)) {
11638                                         }
11639                                 }
11640 #if 0
11641                                 printf("ctl_abort_task: found I/O to abort\n");
11642 #endif
11643                                 break;
11644                         }
11645                 }
11646         }
11647         mtx_unlock(&lun->lun_lock);
11648
11649 bailout:
11650
11651         if (found == 0) {
11652                 /*
11653                  * This isn't really an error.  It's entirely possible for
11654                  * the abort and command completion to cross on the wire.
11655                  * This is more of an informative/diagnostic error.
11656                  */
11657 #if 0
11658                 printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
11659                        "%d:%d:%d:%d tag %d type %d\n",
11660                        io->io_hdr.nexus.initid.id,
11661                        io->io_hdr.nexus.targ_port,
11662                        io->io_hdr.nexus.targ_target.id,
11663                        io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
11664                        io->taskio.tag_type);
11665 #endif
11666                 return (1);
11667         } else
11668                 return (0);
11669 }
11670
11671 static void
11672 ctl_run_task(union ctl_io *io)
11673 {
11674         struct ctl_softc *ctl_softc;
11675         int retval;
11676         const char *task_desc;
11677
11678         CTL_DEBUG_PRINT(("ctl_run_task\n"));
11679
11680         ctl_softc = control_softc;
11681         retval = 0;
11682
11683         KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
11684             ("ctl_run_task: Unextected io_type %d\n",
11685              io->io_hdr.io_type));
11686
11687         task_desc = ctl_scsi_task_string(&io->taskio);
11688         if (task_desc != NULL) {
11689 #ifdef NEEDTOPORT
11690                 csevent_log(CSC_CTL | CSC_SHELF_SW |
11691                             CTL_TASK_REPORT,
11692                             csevent_LogType_Trace,
11693                             csevent_Severity_Information,
11694                             csevent_AlertLevel_Green,
11695                             csevent_FRU_Firmware,
11696                             csevent_FRU_Unknown,
11697                             "CTL: received task: %s",task_desc);
11698 #endif
11699         } else {
11700 #ifdef NEEDTOPORT
11701                 csevent_log(CSC_CTL | CSC_SHELF_SW |
11702                             CTL_TASK_REPORT,
11703                             csevent_LogType_Trace,
11704                             csevent_Severity_Information,
11705                             csevent_AlertLevel_Green,
11706                             csevent_FRU_Firmware,
11707                             csevent_FRU_Unknown,
11708                             "CTL: received unknown task "
11709                             "type: %d (%#x)",
11710                             io->taskio.task_action,
11711                             io->taskio.task_action);
11712 #endif
11713         }
11714         switch (io->taskio.task_action) {
11715         case CTL_TASK_ABORT_TASK:
11716                 retval = ctl_abort_task(io);
11717                 break;
11718         case CTL_TASK_ABORT_TASK_SET:
11719                 break;
11720         case CTL_TASK_CLEAR_ACA:
11721                 break;
11722         case CTL_TASK_CLEAR_TASK_SET:
11723                 break;
11724         case CTL_TASK_LUN_RESET: {
11725                 struct ctl_lun *lun;
11726                 uint32_t targ_lun;
11727                 int retval;
11728
11729                 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11730                 mtx_lock(&ctl_softc->ctl_lock);
11731                 if ((targ_lun < CTL_MAX_LUNS)
11732                  && (ctl_softc->ctl_luns[targ_lun] != NULL))
11733                         lun = ctl_softc->ctl_luns[targ_lun];
11734                 else {
11735                         mtx_unlock(&ctl_softc->ctl_lock);
11736                         retval = 1;
11737                         break;
11738                 }
11739
11740                 if (!(io->io_hdr.flags &
11741                     CTL_FLAG_FROM_OTHER_SC)) {
11742                         union ctl_ha_msg msg_info;
11743
11744                         io->io_hdr.flags |=
11745                                 CTL_FLAG_SENT_2OTHER_SC;
11746                         msg_info.hdr.msg_type =
11747                                 CTL_MSG_MANAGE_TASKS;
11748                         msg_info.hdr.nexus = io->io_hdr.nexus;
11749                         msg_info.task.task_action =
11750                                 CTL_TASK_LUN_RESET;
11751                         msg_info.hdr.original_sc = NULL;
11752                         msg_info.hdr.serializing_sc = NULL;
11753                         if (CTL_HA_STATUS_SUCCESS !=
11754                             ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11755                             (void *)&msg_info,
11756                             sizeof(msg_info), 0)) {
11757                         }
11758                 }
11759
11760                 retval = ctl_lun_reset(lun, io,
11761                                        CTL_UA_LUN_RESET);
11762                 mtx_unlock(&ctl_softc->ctl_lock);
11763                 break;
11764         }
11765         case CTL_TASK_TARGET_RESET:
11766                 retval = ctl_target_reset(ctl_softc, io, CTL_UA_TARG_RESET);
11767                 break;
11768         case CTL_TASK_BUS_RESET:
11769                 retval = ctl_bus_reset(ctl_softc, io);
11770                 break;
11771         case CTL_TASK_PORT_LOGIN:
11772                 break;
11773         case CTL_TASK_PORT_LOGOUT:
11774                 break;
11775         default:
11776                 printf("ctl_run_task: got unknown task management event %d\n",
11777                        io->taskio.task_action);
11778                 break;
11779         }
11780         if (retval == 0)
11781                 io->io_hdr.status = CTL_SUCCESS;
11782         else
11783                 io->io_hdr.status = CTL_ERROR;
11784
11785         /*
11786          * This will queue this I/O to the done queue, but the
11787          * work thread won't be able to process it until we
11788          * return and the lock is released.
11789          */
11790         ctl_done(io);
11791 }
11792
11793 /*
11794  * For HA operation.  Handle commands that come in from the other
11795  * controller.
11796  */
11797 static void
11798 ctl_handle_isc(union ctl_io *io)
11799 {
11800         int free_io;
11801         struct ctl_lun *lun;
11802         struct ctl_softc *ctl_softc;
11803         uint32_t targ_lun;
11804
11805         ctl_softc = control_softc;
11806
11807         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11808         lun = ctl_softc->ctl_luns[targ_lun];
11809
11810         switch (io->io_hdr.msg_type) {
11811         case CTL_MSG_SERIALIZE:
11812                 free_io = ctl_serialize_other_sc_cmd(&io->scsiio);
11813                 break;
11814         case CTL_MSG_R2R: {
11815                 const struct ctl_cmd_entry *entry;
11816
11817                 /*
11818                  * This is only used in SER_ONLY mode.
11819                  */
11820                 free_io = 0;
11821                 entry = ctl_get_cmd_entry(&io->scsiio);
11822                 mtx_lock(&lun->lun_lock);
11823                 if (ctl_scsiio_lun_check(ctl_softc, lun,
11824                     entry, (struct ctl_scsiio *)io) != 0) {
11825                         mtx_unlock(&lun->lun_lock);
11826                         ctl_done(io);
11827                         break;
11828                 }
11829                 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11830                 mtx_unlock(&lun->lun_lock);
11831                 ctl_enqueue_rtr(io);
11832                 break;
11833         }
11834         case CTL_MSG_FINISH_IO:
11835                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
11836                         free_io = 0;
11837                         ctl_done(io);
11838                 } else {
11839                         free_io = 1;
11840                         mtx_lock(&lun->lun_lock);
11841                         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
11842                                      ooa_links);
11843                         ctl_check_blocked(lun);
11844                         mtx_unlock(&lun->lun_lock);
11845                 }
11846                 break;
11847         case CTL_MSG_PERS_ACTION:
11848                 ctl_hndl_per_res_out_on_other_sc(
11849                         (union ctl_ha_msg *)&io->presio.pr_msg);
11850                 free_io = 1;
11851                 break;
11852         case CTL_MSG_BAD_JUJU:
11853                 free_io = 0;
11854                 ctl_done(io);
11855                 break;
11856         case CTL_MSG_DATAMOVE:
11857                 /* Only used in XFER mode */
11858                 free_io = 0;
11859                 ctl_datamove_remote(io);
11860                 break;
11861         case CTL_MSG_DATAMOVE_DONE:
11862                 /* Only used in XFER mode */
11863                 free_io = 0;
11864                 io->scsiio.be_move_done(io);
11865                 break;
11866         default:
11867                 free_io = 1;
11868                 printf("%s: Invalid message type %d\n",
11869                        __func__, io->io_hdr.msg_type);
11870                 break;
11871         }
11872         if (free_io)
11873                 ctl_free_io(io);
11874
11875 }
11876
11877
11878 /*
11879  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
11880  * there is no match.
11881  */
11882 static ctl_lun_error_pattern
11883 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
11884 {
11885         const struct ctl_cmd_entry *entry;
11886         ctl_lun_error_pattern filtered_pattern, pattern;
11887
11888         pattern = desc->error_pattern;
11889
11890         /*
11891          * XXX KDM we need more data passed into this function to match a
11892          * custom pattern, and we actually need to implement custom pattern
11893          * matching.
11894          */
11895         if (pattern & CTL_LUN_PAT_CMD)
11896                 return (CTL_LUN_PAT_CMD);
11897
11898         if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
11899                 return (CTL_LUN_PAT_ANY);
11900
11901         entry = ctl_get_cmd_entry(ctsio);
11902
11903         filtered_pattern = entry->pattern & pattern;
11904
11905         /*
11906          * If the user requested specific flags in the pattern (e.g.
11907          * CTL_LUN_PAT_RANGE), make sure the command supports all of those
11908          * flags.
11909          *
11910          * If the user did not specify any flags, it doesn't matter whether
11911          * or not the command supports the flags.
11912          */
11913         if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
11914              (pattern & ~CTL_LUN_PAT_MASK))
11915                 return (CTL_LUN_PAT_NONE);
11916
11917         /*
11918          * If the user asked for a range check, see if the requested LBA
11919          * range overlaps with this command's LBA range.
11920          */
11921         if (filtered_pattern & CTL_LUN_PAT_RANGE) {
11922                 uint64_t lba1;
11923                 uint32_t len1;
11924                 ctl_action action;
11925                 int retval;
11926
11927                 retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
11928                 if (retval != 0)
11929                         return (CTL_LUN_PAT_NONE);
11930
11931                 action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
11932                                               desc->lba_range.len);
11933                 /*
11934                  * A "pass" means that the LBA ranges don't overlap, so
11935                  * this doesn't match the user's range criteria.
11936                  */
11937                 if (action == CTL_ACTION_PASS)
11938                         return (CTL_LUN_PAT_NONE);
11939         }
11940
11941         return (filtered_pattern);
11942 }
11943
11944 static void
11945 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
11946 {
11947         struct ctl_error_desc *desc, *desc2;
11948
11949         mtx_assert(&lun->lun_lock, MA_OWNED);
11950
11951         STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
11952                 ctl_lun_error_pattern pattern;
11953                 /*
11954                  * Check to see whether this particular command matches
11955                  * the pattern in the descriptor.
11956                  */
11957                 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
11958                 if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
11959                         continue;
11960
11961                 switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
11962                 case CTL_LUN_INJ_ABORTED:
11963                         ctl_set_aborted(&io->scsiio);
11964                         break;
11965                 case CTL_LUN_INJ_MEDIUM_ERR:
11966                         ctl_set_medium_error(&io->scsiio);
11967                         break;
11968                 case CTL_LUN_INJ_UA:
11969                         /* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
11970                          * OCCURRED */
11971                         ctl_set_ua(&io->scsiio, 0x29, 0x00);
11972                         break;
11973                 case CTL_LUN_INJ_CUSTOM:
11974                         /*
11975                          * We're assuming the user knows what he is doing.
11976                          * Just copy the sense information without doing
11977                          * checks.
11978                          */
11979                         bcopy(&desc->custom_sense, &io->scsiio.sense_data,
11980                               ctl_min(sizeof(desc->custom_sense),
11981                                       sizeof(io->scsiio.sense_data)));
11982                         io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
11983                         io->scsiio.sense_len = SSD_FULL_SIZE;
11984                         io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
11985                         break;
11986                 case CTL_LUN_INJ_NONE:
11987                 default:
11988                         /*
11989                          * If this is an error injection type we don't know
11990                          * about, clear the continuous flag (if it is set)
11991                          * so it will get deleted below.
11992                          */
11993                         desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
11994                         break;
11995                 }
11996                 /*
11997                  * By default, each error injection action is a one-shot
11998                  */
11999                 if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
12000                         continue;
12001
12002                 STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
12003
12004                 free(desc, M_CTL);
12005         }
12006 }
12007
12008 #ifdef CTL_IO_DELAY
12009 static void
12010 ctl_datamove_timer_wakeup(void *arg)
12011 {
12012         union ctl_io *io;
12013
12014         io = (union ctl_io *)arg;
12015
12016         ctl_datamove(io);
12017 }
12018 #endif /* CTL_IO_DELAY */
12019
12020 void
12021 ctl_datamove(union ctl_io *io)
12022 {
12023         void (*fe_datamove)(union ctl_io *io);
12024
12025         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
12026
12027         CTL_DEBUG_PRINT(("ctl_datamove\n"));
12028
12029 #ifdef CTL_TIME_IO
12030         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12031                 char str[256];
12032                 char path_str[64];
12033                 struct sbuf sb;
12034
12035                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12036                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12037
12038                 sbuf_cat(&sb, path_str);
12039                 switch (io->io_hdr.io_type) {
12040                 case CTL_IO_SCSI:
12041                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12042                         sbuf_printf(&sb, "\n");
12043                         sbuf_cat(&sb, path_str);
12044                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12045                                     io->scsiio.tag_num, io->scsiio.tag_type);
12046                         break;
12047                 case CTL_IO_TASK:
12048                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12049                                     "Tag Type: %d\n", io->taskio.task_action,
12050                                     io->taskio.tag_num, io->taskio.tag_type);
12051                         break;
12052                 default:
12053                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12054                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12055                         break;
12056                 }
12057                 sbuf_cat(&sb, path_str);
12058                 sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
12059                             (intmax_t)time_uptime - io->io_hdr.start_time);
12060                 sbuf_finish(&sb);
12061                 printf("%s", sbuf_data(&sb));
12062         }
12063 #endif /* CTL_TIME_IO */
12064
12065 #ifdef CTL_IO_DELAY
12066         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12067                 struct ctl_lun *lun;
12068
12069                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12070
12071                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12072         } else {
12073                 struct ctl_lun *lun;
12074
12075                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12076                 if ((lun != NULL)
12077                  && (lun->delay_info.datamove_delay > 0)) {
12078                         struct callout *callout;
12079
12080                         callout = (struct callout *)&io->io_hdr.timer_bytes;
12081                         callout_init(callout, /*mpsafe*/ 1);
12082                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12083                         callout_reset(callout,
12084                                       lun->delay_info.datamove_delay * hz,
12085                                       ctl_datamove_timer_wakeup, io);
12086                         if (lun->delay_info.datamove_type ==
12087                             CTL_DELAY_TYPE_ONESHOT)
12088                                 lun->delay_info.datamove_delay = 0;
12089                         return;
12090                 }
12091         }
12092 #endif
12093
12094         /*
12095          * This command has been aborted.  Set the port status, so we fail
12096          * the data move.
12097          */
12098         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12099                 printf("ctl_datamove: tag 0x%04x on (%ju:%d:%ju:%d) aborted\n",
12100                        io->scsiio.tag_num,(uintmax_t)io->io_hdr.nexus.initid.id,
12101                        io->io_hdr.nexus.targ_port,
12102                        (uintmax_t)io->io_hdr.nexus.targ_target.id,
12103                        io->io_hdr.nexus.targ_lun);
12104                 io->io_hdr.status = CTL_CMD_ABORTED;
12105                 io->io_hdr.port_status = 31337;
12106                 /*
12107                  * Note that the backend, in this case, will get the
12108                  * callback in its context.  In other cases it may get
12109                  * called in the frontend's interrupt thread context.
12110                  */
12111                 io->scsiio.be_move_done(io);
12112                 return;
12113         }
12114
12115         /*
12116          * If we're in XFER mode and this I/O is from the other shelf
12117          * controller, we need to send the DMA to the other side to
12118          * actually transfer the data to/from the host.  In serialize only
12119          * mode the transfer happens below CTL and ctl_datamove() is only
12120          * called on the machine that originally received the I/O.
12121          */
12122         if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
12123          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12124                 union ctl_ha_msg msg;
12125                 uint32_t sg_entries_sent;
12126                 int do_sg_copy;
12127                 int i;
12128
12129                 memset(&msg, 0, sizeof(msg));
12130                 msg.hdr.msg_type = CTL_MSG_DATAMOVE;
12131                 msg.hdr.original_sc = io->io_hdr.original_sc;
12132                 msg.hdr.serializing_sc = io;
12133                 msg.hdr.nexus = io->io_hdr.nexus;
12134                 msg.dt.flags = io->io_hdr.flags;
12135                 /*
12136                  * We convert everything into a S/G list here.  We can't
12137                  * pass by reference, only by value between controllers.
12138                  * So we can't pass a pointer to the S/G list, only as many
12139                  * S/G entries as we can fit in here.  If it's possible for
12140                  * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
12141                  * then we need to break this up into multiple transfers.
12142                  */
12143                 if (io->scsiio.kern_sg_entries == 0) {
12144                         msg.dt.kern_sg_entries = 1;
12145                         /*
12146                          * If this is in cached memory, flush the cache
12147                          * before we send the DMA request to the other
12148                          * controller.  We want to do this in either the
12149                          * read or the write case.  The read case is
12150                          * straightforward.  In the write case, we want to
12151                          * make sure nothing is in the local cache that
12152                          * could overwrite the DMAed data.
12153                          */
12154                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12155                                 /*
12156                                  * XXX KDM use bus_dmamap_sync() here.
12157                                  */
12158                         }
12159
12160                         /*
12161                          * Convert to a physical address if this is a
12162                          * virtual address.
12163                          */
12164                         if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
12165                                 msg.dt.sg_list[0].addr =
12166                                         io->scsiio.kern_data_ptr;
12167                         } else {
12168                                 /*
12169                                  * XXX KDM use busdma here!
12170                                  */
12171 #if 0
12172                                 msg.dt.sg_list[0].addr = (void *)
12173                                         vtophys(io->scsiio.kern_data_ptr);
12174 #endif
12175                         }
12176
12177                         msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
12178                         do_sg_copy = 0;
12179                 } else {
12180                         struct ctl_sg_entry *sgl;
12181
12182                         do_sg_copy = 1;
12183                         msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
12184                         sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
12185                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12186                                 /*
12187                                  * XXX KDM use bus_dmamap_sync() here.
12188                                  */
12189                         }
12190                 }
12191
12192                 msg.dt.kern_data_len = io->scsiio.kern_data_len;
12193                 msg.dt.kern_total_len = io->scsiio.kern_total_len;
12194                 msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
12195                 msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
12196                 msg.dt.sg_sequence = 0;
12197
12198                 /*
12199                  * Loop until we've sent all of the S/G entries.  On the
12200                  * other end, we'll recompose these S/G entries into one
12201                  * contiguous list before passing it to the
12202                  */
12203                 for (sg_entries_sent = 0; sg_entries_sent <
12204                      msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
12205                         msg.dt.cur_sg_entries = ctl_min((sizeof(msg.dt.sg_list)/
12206                                 sizeof(msg.dt.sg_list[0])),
12207                                 msg.dt.kern_sg_entries - sg_entries_sent);
12208
12209                         if (do_sg_copy != 0) {
12210                                 struct ctl_sg_entry *sgl;
12211                                 int j;
12212
12213                                 sgl = (struct ctl_sg_entry *)
12214                                         io->scsiio.kern_data_ptr;
12215                                 /*
12216                                  * If this is in cached memory, flush the cache
12217                                  * before we send the DMA request to the other
12218                                  * controller.  We want to do this in either
12219                                  * the * read or the write case.  The read
12220                                  * case is straightforward.  In the write
12221                                  * case, we want to make sure nothing is
12222                                  * in the local cache that could overwrite
12223                                  * the DMAed data.
12224                                  */
12225
12226                                 for (i = sg_entries_sent, j = 0;
12227                                      i < msg.dt.cur_sg_entries; i++, j++) {
12228                                         if ((io->io_hdr.flags &
12229                                              CTL_FLAG_NO_DATASYNC) == 0) {
12230                                                 /*
12231                                                  * XXX KDM use bus_dmamap_sync()
12232                                                  */
12233                                         }
12234                                         if ((io->io_hdr.flags &
12235                                              CTL_FLAG_BUS_ADDR) == 0) {
12236                                                 /*
12237                                                  * XXX KDM use busdma.
12238                                                  */
12239 #if 0
12240                                                 msg.dt.sg_list[j].addr =(void *)
12241                                                        vtophys(sgl[i].addr);
12242 #endif
12243                                         } else {
12244                                                 msg.dt.sg_list[j].addr =
12245                                                         sgl[i].addr;
12246                                         }
12247                                         msg.dt.sg_list[j].len = sgl[i].len;
12248                                 }
12249                         }
12250
12251                         sg_entries_sent += msg.dt.cur_sg_entries;
12252                         if (sg_entries_sent >= msg.dt.kern_sg_entries)
12253                                 msg.dt.sg_last = 1;
12254                         else
12255                                 msg.dt.sg_last = 0;
12256
12257                         /*
12258                          * XXX KDM drop and reacquire the lock here?
12259                          */
12260                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12261                             sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
12262                                 /*
12263                                  * XXX do something here.
12264                                  */
12265                         }
12266
12267                         msg.dt.sent_sg_entries = sg_entries_sent;
12268                 }
12269                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12270                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER)
12271                         ctl_failover_io(io, /*have_lock*/ 0);
12272
12273         } else {
12274
12275                 /*
12276                  * Lookup the fe_datamove() function for this particular
12277                  * front end.
12278                  */
12279                 fe_datamove =
12280                     control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12281
12282                 fe_datamove(io);
12283         }
12284 }
12285
12286 static void
12287 ctl_send_datamove_done(union ctl_io *io, int have_lock)
12288 {
12289         union ctl_ha_msg msg;
12290         int isc_status;
12291
12292         memset(&msg, 0, sizeof(msg));
12293
12294         msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
12295         msg.hdr.original_sc = io;
12296         msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
12297         msg.hdr.nexus = io->io_hdr.nexus;
12298         msg.hdr.status = io->io_hdr.status;
12299         msg.scsi.tag_num = io->scsiio.tag_num;
12300         msg.scsi.tag_type = io->scsiio.tag_type;
12301         msg.scsi.scsi_status = io->scsiio.scsi_status;
12302         memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12303                sizeof(io->scsiio.sense_data));
12304         msg.scsi.sense_len = io->scsiio.sense_len;
12305         msg.scsi.sense_residual = io->scsiio.sense_residual;
12306         msg.scsi.fetd_status = io->io_hdr.port_status;
12307         msg.scsi.residual = io->scsiio.residual;
12308         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12309
12310         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12311                 ctl_failover_io(io, /*have_lock*/ have_lock);
12312                 return;
12313         }
12314
12315         isc_status = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0);
12316         if (isc_status > CTL_HA_STATUS_SUCCESS) {
12317                 /* XXX do something if this fails */
12318         }
12319
12320 }
12321
12322 /*
12323  * The DMA to the remote side is done, now we need to tell the other side
12324  * we're done so it can continue with its data movement.
12325  */
12326 static void
12327 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
12328 {
12329         union ctl_io *io;
12330
12331         io = rq->context;
12332
12333         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12334                 printf("%s: ISC DMA write failed with error %d", __func__,
12335                        rq->ret);
12336                 ctl_set_internal_failure(&io->scsiio,
12337                                          /*sks_valid*/ 1,
12338                                          /*retry_count*/ rq->ret);
12339         }
12340
12341         ctl_dt_req_free(rq);
12342
12343         /*
12344          * In this case, we had to malloc the memory locally.  Free it.
12345          */
12346         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
12347                 int i;
12348                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12349                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12350         }
12351         /*
12352          * The data is in local and remote memory, so now we need to send
12353          * status (good or back) back to the other side.
12354          */
12355         ctl_send_datamove_done(io, /*have_lock*/ 0);
12356 }
12357
12358 /*
12359  * We've moved the data from the host/controller into local memory.  Now we
12360  * need to push it over to the remote controller's memory.
12361  */
12362 static int
12363 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
12364 {
12365         int retval;
12366
12367         retval = 0;
12368
12369         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
12370                                           ctl_datamove_remote_write_cb);
12371
12372         return (retval);
12373 }
12374
12375 static void
12376 ctl_datamove_remote_write(union ctl_io *io)
12377 {
12378         int retval;
12379         void (*fe_datamove)(union ctl_io *io);
12380
12381         /*
12382          * - Get the data from the host/HBA into local memory.
12383          * - DMA memory from the local controller to the remote controller.
12384          * - Send status back to the remote controller.
12385          */
12386
12387         retval = ctl_datamove_remote_sgl_setup(io);
12388         if (retval != 0)
12389                 return;
12390
12391         /* Switch the pointer over so the FETD knows what to do */
12392         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12393
12394         /*
12395          * Use a custom move done callback, since we need to send completion
12396          * back to the other controller, not to the backend on this side.
12397          */
12398         io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
12399
12400         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12401
12402         fe_datamove(io);
12403
12404         return;
12405
12406 }
12407
12408 static int
12409 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
12410 {
12411 #if 0
12412         char str[256];
12413         char path_str[64];
12414         struct sbuf sb;
12415 #endif
12416
12417         /*
12418          * In this case, we had to malloc the memory locally.  Free it.
12419          */
12420         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
12421                 int i;
12422                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12423                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12424         }
12425
12426 #if 0
12427         scsi_path_string(io, path_str, sizeof(path_str));
12428         sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12429         sbuf_cat(&sb, path_str);
12430         scsi_command_string(&io->scsiio, NULL, &sb);
12431         sbuf_printf(&sb, "\n");
12432         sbuf_cat(&sb, path_str);
12433         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12434                     io->scsiio.tag_num, io->scsiio.tag_type);
12435         sbuf_cat(&sb, path_str);
12436         sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
12437                     io->io_hdr.flags, io->io_hdr.status);
12438         sbuf_finish(&sb);
12439         printk("%s", sbuf_data(&sb));
12440 #endif
12441
12442
12443         /*
12444          * The read is done, now we need to send status (good or bad) back
12445          * to the other side.
12446          */
12447         ctl_send_datamove_done(io, /*have_lock*/ 0);
12448
12449         return (0);
12450 }
12451
12452 static void
12453 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
12454 {
12455         union ctl_io *io;
12456         void (*fe_datamove)(union ctl_io *io);
12457
12458         io = rq->context;
12459
12460         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12461                 printf("%s: ISC DMA read failed with error %d", __func__,
12462                        rq->ret);
12463                 ctl_set_internal_failure(&io->scsiio,
12464                                          /*sks_valid*/ 1,
12465                                          /*retry_count*/ rq->ret);
12466         }
12467
12468         ctl_dt_req_free(rq);
12469
12470         /* Switch the pointer over so the FETD knows what to do */
12471         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12472
12473         /*
12474          * Use a custom move done callback, since we need to send completion
12475          * back to the other controller, not to the backend on this side.
12476          */
12477         io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
12478
12479         /* XXX KDM add checks like the ones in ctl_datamove? */
12480
12481         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12482
12483         fe_datamove(io);
12484 }
12485
12486 static int
12487 ctl_datamove_remote_sgl_setup(union ctl_io *io)
12488 {
12489         struct ctl_sg_entry *local_sglist, *remote_sglist;
12490         struct ctl_sg_entry *local_dma_sglist, *remote_dma_sglist;
12491         struct ctl_softc *softc;
12492         int retval;
12493         int i;
12494
12495         retval = 0;
12496         softc = control_softc;
12497
12498         local_sglist = io->io_hdr.local_sglist;
12499         local_dma_sglist = io->io_hdr.local_dma_sglist;
12500         remote_sglist = io->io_hdr.remote_sglist;
12501         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
12502
12503         if (io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) {
12504                 for (i = 0; i < io->scsiio.kern_sg_entries; i++) {
12505                         local_sglist[i].len = remote_sglist[i].len;
12506
12507                         /*
12508                          * XXX Detect the situation where the RS-level I/O
12509                          * redirector on the other side has already read the
12510                          * data off of the AOR RS on this side, and
12511                          * transferred it to remote (mirror) memory on the
12512                          * other side.  Since we already have the data in
12513                          * memory here, we just need to use it.
12514                          *
12515                          * XXX KDM this can probably be removed once we
12516                          * get the cache device code in and take the
12517                          * current AOR implementation out.
12518                          */
12519 #ifdef NEEDTOPORT
12520                         if ((remote_sglist[i].addr >=
12521                              (void *)vtophys(softc->mirr->addr))
12522                          && (remote_sglist[i].addr <
12523                              ((void *)vtophys(softc->mirr->addr) +
12524                              CacheMirrorOffset))) {
12525                                 local_sglist[i].addr = remote_sglist[i].addr -
12526                                         CacheMirrorOffset;
12527                                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
12528                                      CTL_FLAG_DATA_IN)
12529                                         io->io_hdr.flags |= CTL_FLAG_REDIR_DONE;
12530                         } else {
12531                                 local_sglist[i].addr = remote_sglist[i].addr +
12532                                         CacheMirrorOffset;
12533                         }
12534 #endif
12535 #if 0
12536                         printf("%s: local %p, remote %p, len %d\n",
12537                                __func__, local_sglist[i].addr,
12538                                remote_sglist[i].addr, local_sglist[i].len);
12539 #endif
12540                 }
12541         } else {
12542                 uint32_t len_to_go;
12543
12544                 /*
12545                  * In this case, we don't have automatically allocated
12546                  * memory for this I/O on this controller.  This typically
12547                  * happens with internal CTL I/O -- e.g. inquiry, mode
12548                  * sense, etc.  Anything coming from RAIDCore will have
12549                  * a mirror area available.
12550                  */
12551                 len_to_go = io->scsiio.kern_data_len;
12552
12553                 /*
12554                  * Clear the no datasync flag, we have to use malloced
12555                  * buffers.
12556                  */
12557                 io->io_hdr.flags &= ~CTL_FLAG_NO_DATASYNC;
12558
12559                 /*
12560                  * The difficult thing here is that the size of the various
12561                  * S/G segments may be different than the size from the
12562                  * remote controller.  That'll make it harder when DMAing
12563                  * the data back to the other side.
12564                  */
12565                 for (i = 0; (i < sizeof(io->io_hdr.remote_sglist) /
12566                      sizeof(io->io_hdr.remote_sglist[0])) &&
12567                      (len_to_go > 0); i++) {
12568                         local_sglist[i].len = ctl_min(len_to_go, 131072);
12569                         CTL_SIZE_8B(local_dma_sglist[i].len,
12570                                     local_sglist[i].len);
12571                         local_sglist[i].addr =
12572                                 malloc(local_dma_sglist[i].len, M_CTL,M_WAITOK);
12573
12574                         local_dma_sglist[i].addr = local_sglist[i].addr;
12575
12576                         if (local_sglist[i].addr == NULL) {
12577                                 int j;
12578
12579                                 printf("malloc failed for %zd bytes!",
12580                                        local_dma_sglist[i].len);
12581                                 for (j = 0; j < i; j++) {
12582                                         free(local_sglist[j].addr, M_CTL);
12583                                 }
12584                                 ctl_set_internal_failure(&io->scsiio,
12585                                                          /*sks_valid*/ 1,
12586                                                          /*retry_count*/ 4857);
12587                                 retval = 1;
12588                                 goto bailout_error;
12589
12590                         }
12591                         /* XXX KDM do we need a sync here? */
12592
12593                         len_to_go -= local_sglist[i].len;
12594                 }
12595                 /*
12596                  * Reset the number of S/G entries accordingly.  The
12597                  * original number of S/G entries is available in
12598                  * rem_sg_entries.
12599                  */
12600                 io->scsiio.kern_sg_entries = i;
12601
12602 #if 0
12603                 printf("%s: kern_sg_entries = %d\n", __func__,
12604                        io->scsiio.kern_sg_entries);
12605                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12606                         printf("%s: sg[%d] = %p, %d (DMA: %d)\n", __func__, i,
12607                                local_sglist[i].addr, local_sglist[i].len,
12608                                local_dma_sglist[i].len);
12609 #endif
12610         }
12611
12612
12613         return (retval);
12614
12615 bailout_error:
12616
12617         ctl_send_datamove_done(io, /*have_lock*/ 0);
12618
12619         return (retval);
12620 }
12621
12622 static int
12623 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
12624                          ctl_ha_dt_cb callback)
12625 {
12626         struct ctl_ha_dt_req *rq;
12627         struct ctl_sg_entry *remote_sglist, *local_sglist;
12628         struct ctl_sg_entry *remote_dma_sglist, *local_dma_sglist;
12629         uint32_t local_used, remote_used, total_used;
12630         int retval;
12631         int i, j;
12632
12633         retval = 0;
12634
12635         rq = ctl_dt_req_alloc();
12636
12637         /*
12638          * If we failed to allocate the request, and if the DMA didn't fail
12639          * anyway, set busy status.  This is just a resource allocation
12640          * failure.
12641          */
12642         if ((rq == NULL)
12643          && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
12644                 ctl_set_busy(&io->scsiio);
12645
12646         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
12647
12648                 if (rq != NULL)
12649                         ctl_dt_req_free(rq);
12650
12651                 /*
12652                  * The data move failed.  We need to return status back
12653                  * to the other controller.  No point in trying to DMA
12654                  * data to the remote controller.
12655                  */
12656
12657                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12658
12659                 retval = 1;
12660
12661                 goto bailout;
12662         }
12663
12664         local_sglist = io->io_hdr.local_sglist;
12665         local_dma_sglist = io->io_hdr.local_dma_sglist;
12666         remote_sglist = io->io_hdr.remote_sglist;
12667         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
12668         local_used = 0;
12669         remote_used = 0;
12670         total_used = 0;
12671
12672         if (io->io_hdr.flags & CTL_FLAG_REDIR_DONE) {
12673                 rq->ret = CTL_HA_STATUS_SUCCESS;
12674                 rq->context = io;
12675                 callback(rq);
12676                 goto bailout;
12677         }
12678
12679         /*
12680          * Pull/push the data over the wire from/to the other controller.
12681          * This takes into account the possibility that the local and
12682          * remote sglists may not be identical in terms of the size of
12683          * the elements and the number of elements.
12684          *
12685          * One fundamental assumption here is that the length allocated for
12686          * both the local and remote sglists is identical.  Otherwise, we've
12687          * essentially got a coding error of some sort.
12688          */
12689         for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
12690                 int isc_ret;
12691                 uint32_t cur_len, dma_length;
12692                 uint8_t *tmp_ptr;
12693
12694                 rq->id = CTL_HA_DATA_CTL;
12695                 rq->command = command;
12696                 rq->context = io;
12697
12698                 /*
12699                  * Both pointers should be aligned.  But it is possible
12700                  * that the allocation length is not.  They should both
12701                  * also have enough slack left over at the end, though,
12702                  * to round up to the next 8 byte boundary.
12703                  */
12704                 cur_len = ctl_min(local_sglist[i].len - local_used,
12705                                   remote_sglist[j].len - remote_used);
12706
12707                 /*
12708                  * In this case, we have a size issue and need to decrease
12709                  * the size, except in the case where we actually have less
12710                  * than 8 bytes left.  In that case, we need to increase
12711                  * the DMA length to get the last bit.
12712                  */
12713                 if ((cur_len & 0x7) != 0) {
12714                         if (cur_len > 0x7) {
12715                                 cur_len = cur_len - (cur_len & 0x7);
12716                                 dma_length = cur_len;
12717                         } else {
12718                                 CTL_SIZE_8B(dma_length, cur_len);
12719                         }
12720
12721                 } else
12722                         dma_length = cur_len;
12723
12724                 /*
12725                  * If we had to allocate memory for this I/O, instead of using
12726                  * the non-cached mirror memory, we'll need to flush the cache
12727                  * before trying to DMA to the other controller.
12728                  *
12729                  * We could end up doing this multiple times for the same
12730                  * segment if we have a larger local segment than remote
12731                  * segment.  That shouldn't be an issue.
12732                  */
12733                 if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12734                         /*
12735                          * XXX KDM use bus_dmamap_sync() here.
12736                          */
12737                 }
12738
12739                 rq->size = dma_length;
12740
12741                 tmp_ptr = (uint8_t *)local_sglist[i].addr;
12742                 tmp_ptr += local_used;
12743
12744                 /* Use physical addresses when talking to ISC hardware */
12745                 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
12746                         /* XXX KDM use busdma */
12747 #if 0
12748                         rq->local = vtophys(tmp_ptr);
12749 #endif
12750                 } else
12751                         rq->local = tmp_ptr;
12752
12753                 tmp_ptr = (uint8_t *)remote_sglist[j].addr;
12754                 tmp_ptr += remote_used;
12755                 rq->remote = tmp_ptr;
12756
12757                 rq->callback = NULL;
12758
12759                 local_used += cur_len;
12760                 if (local_used >= local_sglist[i].len) {
12761                         i++;
12762                         local_used = 0;
12763                 }
12764
12765                 remote_used += cur_len;
12766                 if (remote_used >= remote_sglist[j].len) {
12767                         j++;
12768                         remote_used = 0;
12769                 }
12770                 total_used += cur_len;
12771
12772                 if (total_used >= io->scsiio.kern_data_len)
12773                         rq->callback = callback;
12774
12775                 if ((rq->size & 0x7) != 0) {
12776                         printf("%s: warning: size %d is not on 8b boundary\n",
12777                                __func__, rq->size);
12778                 }
12779                 if (((uintptr_t)rq->local & 0x7) != 0) {
12780                         printf("%s: warning: local %p not on 8b boundary\n",
12781                                __func__, rq->local);
12782                 }
12783                 if (((uintptr_t)rq->remote & 0x7) != 0) {
12784                         printf("%s: warning: remote %p not on 8b boundary\n",
12785                                __func__, rq->local);
12786                 }
12787 #if 0
12788                 printf("%s: %s: local %#x remote %#x size %d\n", __func__,
12789                        (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
12790                        rq->local, rq->remote, rq->size);
12791 #endif
12792
12793                 isc_ret = ctl_dt_single(rq);
12794                 if (isc_ret == CTL_HA_STATUS_WAIT)
12795                         continue;
12796
12797                 if (isc_ret == CTL_HA_STATUS_DISCONNECT) {
12798                         rq->ret = CTL_HA_STATUS_SUCCESS;
12799                 } else {
12800                         rq->ret = isc_ret;
12801                 }
12802                 callback(rq);
12803                 goto bailout;
12804         }
12805
12806 bailout:
12807         return (retval);
12808
12809 }
12810
12811 static void
12812 ctl_datamove_remote_read(union ctl_io *io)
12813 {
12814         int retval;
12815         int i;
12816
12817         /*
12818          * This will send an error to the other controller in the case of a
12819          * failure.
12820          */
12821         retval = ctl_datamove_remote_sgl_setup(io);
12822         if (retval != 0)
12823                 return;
12824
12825         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
12826                                           ctl_datamove_remote_read_cb);
12827         if ((retval != 0)
12828          && ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0)) {
12829                 /*
12830                  * Make sure we free memory if there was an error..  The
12831                  * ctl_datamove_remote_xfer() function will send the
12832                  * datamove done message, or call the callback with an
12833                  * error if there is a problem.
12834                  */
12835                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12836                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12837         }
12838
12839         return;
12840 }
12841
12842 /*
12843  * Process a datamove request from the other controller.  This is used for
12844  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
12845  * first.  Once that is complete, the data gets DMAed into the remote
12846  * controller's memory.  For reads, we DMA from the remote controller's
12847  * memory into our memory first, and then move it out to the FETD.
12848  */
12849 static void
12850 ctl_datamove_remote(union ctl_io *io)
12851 {
12852         struct ctl_softc *softc;
12853
12854         softc = control_softc;
12855
12856         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
12857
12858         /*
12859          * Note that we look for an aborted I/O here, but don't do some of
12860          * the other checks that ctl_datamove() normally does.  We don't
12861          * need to run the task queue, because this I/O is on the ISC
12862          * queue, which is executed by the work thread after the task queue.
12863          * We don't need to run the datamove delay code, since that should
12864          * have been done if need be on the other controller.
12865          */
12866         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12867
12868                 printf("%s: tag 0x%04x on (%d:%d:%d:%d) aborted\n", __func__,
12869                        io->scsiio.tag_num, io->io_hdr.nexus.initid.id,
12870                        io->io_hdr.nexus.targ_port,
12871                        io->io_hdr.nexus.targ_target.id,
12872                        io->io_hdr.nexus.targ_lun);
12873                 io->io_hdr.status = CTL_CMD_ABORTED;
12874                 io->io_hdr.port_status = 31338;
12875
12876                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12877
12878                 return;
12879         }
12880
12881         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT) {
12882                 ctl_datamove_remote_write(io);
12883         } else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN){
12884                 ctl_datamove_remote_read(io);
12885         } else {
12886                 union ctl_ha_msg msg;
12887                 struct scsi_sense_data *sense;
12888                 uint8_t sks[3];
12889                 int retry_count;
12890
12891                 memset(&msg, 0, sizeof(msg));
12892
12893                 msg.hdr.msg_type = CTL_MSG_BAD_JUJU;
12894                 msg.hdr.status = CTL_SCSI_ERROR;
12895                 msg.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
12896
12897                 retry_count = 4243;
12898
12899                 sense = &msg.scsi.sense_data;
12900                 sks[0] = SSD_SCS_VALID;
12901                 sks[1] = (retry_count >> 8) & 0xff;
12902                 sks[2] = retry_count & 0xff;
12903
12904                 /* "Internal target failure" */
12905                 scsi_set_sense_data(sense,
12906                                     /*sense_format*/ SSD_TYPE_NONE,
12907                                     /*current_error*/ 1,
12908                                     /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
12909                                     /*asc*/ 0x44,
12910                                     /*ascq*/ 0x00,
12911                                     /*type*/ SSD_ELEM_SKS,
12912                                     /*size*/ sizeof(sks),
12913                                     /*data*/ sks,
12914                                     SSD_ELEM_NONE);
12915
12916                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12917                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12918                         ctl_failover_io(io, /*have_lock*/ 1);
12919                         return;
12920                 }
12921
12922                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0) >
12923                     CTL_HA_STATUS_SUCCESS) {
12924                         /* XXX KDM what to do if this fails? */
12925                 }
12926                 return;
12927         }
12928
12929 }
12930
12931 static int
12932 ctl_process_done(union ctl_io *io)
12933 {
12934         struct ctl_lun *lun;
12935         struct ctl_softc *ctl_softc;
12936         void (*fe_done)(union ctl_io *io);
12937         uint32_t targ_port = ctl_port_idx(io->io_hdr.nexus.targ_port);
12938
12939         CTL_DEBUG_PRINT(("ctl_process_done\n"));
12940
12941         fe_done =
12942             control_softc->ctl_ports[targ_port]->fe_done;
12943
12944 #ifdef CTL_TIME_IO
12945         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12946                 char str[256];
12947                 char path_str[64];
12948                 struct sbuf sb;
12949
12950                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12951                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12952
12953                 sbuf_cat(&sb, path_str);
12954                 switch (io->io_hdr.io_type) {
12955                 case CTL_IO_SCSI:
12956                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12957                         sbuf_printf(&sb, "\n");
12958                         sbuf_cat(&sb, path_str);
12959                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12960                                     io->scsiio.tag_num, io->scsiio.tag_type);
12961                         break;
12962                 case CTL_IO_TASK:
12963                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12964                                     "Tag Type: %d\n", io->taskio.task_action,
12965                                     io->taskio.tag_num, io->taskio.tag_type);
12966                         break;
12967                 default:
12968                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12969                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12970                         break;
12971                 }
12972                 sbuf_cat(&sb, path_str);
12973                 sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
12974                             (intmax_t)time_uptime - io->io_hdr.start_time);
12975                 sbuf_finish(&sb);
12976                 printf("%s", sbuf_data(&sb));
12977         }
12978 #endif /* CTL_TIME_IO */
12979
12980         switch (io->io_hdr.io_type) {
12981         case CTL_IO_SCSI:
12982                 break;
12983         case CTL_IO_TASK:
12984                 if (bootverbose || verbose > 0)
12985                         ctl_io_error_print(io, NULL);
12986                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
12987                         ctl_free_io(io);
12988                 else
12989                         fe_done(io);
12990                 return (CTL_RETVAL_COMPLETE);
12991                 break;
12992         default:
12993                 printf("ctl_process_done: invalid io type %d\n",
12994                        io->io_hdr.io_type);
12995                 panic("ctl_process_done: invalid io type %d\n",
12996                       io->io_hdr.io_type);
12997                 break; /* NOTREACHED */
12998         }
12999
13000         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13001         if (lun == NULL) {
13002                 CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
13003                                  io->io_hdr.nexus.targ_mapped_lun));
13004                 fe_done(io);
13005                 goto bailout;
13006         }
13007         ctl_softc = lun->ctl_softc;
13008
13009         mtx_lock(&lun->lun_lock);
13010
13011         /*
13012          * Check to see if we have any errors to inject here.  We only
13013          * inject errors for commands that don't already have errors set.
13014          */
13015         if ((STAILQ_FIRST(&lun->error_list) != NULL)
13016          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))
13017                 ctl_inject_error(lun, io);
13018
13019         /*
13020          * XXX KDM how do we treat commands that aren't completed
13021          * successfully?
13022          *
13023          * XXX KDM should we also track I/O latency?
13024          */
13025         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
13026             io->io_hdr.io_type == CTL_IO_SCSI) {
13027 #ifdef CTL_TIME_IO
13028                 struct bintime cur_bt;
13029 #endif
13030                 int type;
13031
13032                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13033                     CTL_FLAG_DATA_IN)
13034                         type = CTL_STATS_READ;
13035                 else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13036                     CTL_FLAG_DATA_OUT)
13037                         type = CTL_STATS_WRITE;
13038                 else
13039                         type = CTL_STATS_NO_IO;
13040
13041                 lun->stats.ports[targ_port].bytes[type] +=
13042                     io->scsiio.kern_total_len;
13043                 lun->stats.ports[targ_port].operations[type]++;
13044 #ifdef CTL_TIME_IO
13045                 bintime_add(&lun->stats.ports[targ_port].dma_time[type],
13046                    &io->io_hdr.dma_bt);
13047                 lun->stats.ports[targ_port].num_dmas[type] +=
13048                     io->io_hdr.num_dmas;
13049                 getbintime(&cur_bt);
13050                 bintime_sub(&cur_bt, &io->io_hdr.start_bt);
13051                 bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
13052 #endif
13053         }
13054
13055         /*
13056          * Remove this from the OOA queue.
13057          */
13058         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
13059
13060         /*
13061          * Run through the blocked queue on this LUN and see if anything
13062          * has become unblocked, now that this transaction is done.
13063          */
13064         ctl_check_blocked(lun);
13065
13066         /*
13067          * If the LUN has been invalidated, free it if there is nothing
13068          * left on its OOA queue.
13069          */
13070         if ((lun->flags & CTL_LUN_INVALID)
13071          && TAILQ_EMPTY(&lun->ooa_queue)) {
13072                 mtx_unlock(&lun->lun_lock);
13073                 mtx_lock(&ctl_softc->ctl_lock);
13074                 ctl_free_lun(lun);
13075                 mtx_unlock(&ctl_softc->ctl_lock);
13076         } else
13077                 mtx_unlock(&lun->lun_lock);
13078
13079         /*
13080          * If this command has been aborted, make sure we set the status
13081          * properly.  The FETD is responsible for freeing the I/O and doing
13082          * whatever it needs to do to clean up its state.
13083          */
13084         if (io->io_hdr.flags & CTL_FLAG_ABORT)
13085                 io->io_hdr.status = CTL_CMD_ABORTED;
13086
13087         /*
13088          * We print out status for every task management command.  For SCSI
13089          * commands, we filter out any unit attention errors; they happen
13090          * on every boot, and would clutter up the log.  Note:  task
13091          * management commands aren't printed here, they are printed above,
13092          * since they should never even make it down here.
13093          */
13094         switch (io->io_hdr.io_type) {
13095         case CTL_IO_SCSI: {
13096                 int error_code, sense_key, asc, ascq;
13097
13098                 sense_key = 0;
13099
13100                 if (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR)
13101                  && (io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)) {
13102                         /*
13103                          * Since this is just for printing, no need to
13104                          * show errors here.
13105                          */
13106                         scsi_extract_sense_len(&io->scsiio.sense_data,
13107                                                io->scsiio.sense_len,
13108                                                &error_code,
13109                                                &sense_key,
13110                                                &asc,
13111                                                &ascq,
13112                                                /*show_errors*/ 0);
13113                 }
13114
13115                 if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
13116                  && (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SCSI_ERROR)
13117                   || (io->scsiio.scsi_status != SCSI_STATUS_CHECK_COND)
13118                   || (sense_key != SSD_KEY_UNIT_ATTENTION))) {
13119
13120                         if ((time_uptime - ctl_softc->last_print_jiffies) <= 0){
13121                                 ctl_softc->skipped_prints++;
13122                         } else {
13123                                 uint32_t skipped_prints;
13124
13125                                 skipped_prints = ctl_softc->skipped_prints;
13126
13127                                 ctl_softc->skipped_prints = 0;
13128                                 ctl_softc->last_print_jiffies = time_uptime;
13129
13130                                 if (skipped_prints > 0) {
13131 #ifdef NEEDTOPORT
13132                                         csevent_log(CSC_CTL | CSC_SHELF_SW |
13133                                             CTL_ERROR_REPORT,
13134                                             csevent_LogType_Trace,
13135                                             csevent_Severity_Information,
13136                                             csevent_AlertLevel_Green,
13137                                             csevent_FRU_Firmware,
13138                                             csevent_FRU_Unknown,
13139                                             "High CTL error volume, %d prints "
13140                                             "skipped", skipped_prints);
13141 #endif
13142                                 }
13143                                 if (bootverbose || verbose > 0)
13144                                         ctl_io_error_print(io, NULL);
13145                         }
13146                 }
13147                 break;
13148         }
13149         case CTL_IO_TASK:
13150                 if (bootverbose || verbose > 0)
13151                         ctl_io_error_print(io, NULL);
13152                 break;
13153         default:
13154                 break;
13155         }
13156
13157         /*
13158          * Tell the FETD or the other shelf controller we're done with this
13159          * command.  Note that only SCSI commands get to this point.  Task
13160          * management commands are completed above.
13161          *
13162          * We only send status to the other controller if we're in XFER
13163          * mode.  In SER_ONLY mode, the I/O is done on the controller that
13164          * received the I/O (from CTL's perspective), and so the status is
13165          * generated there.
13166          *
13167          * XXX KDM if we hold the lock here, we could cause a deadlock
13168          * if the frontend comes back in in this context to queue
13169          * something.
13170          */
13171         if ((ctl_softc->ha_mode == CTL_HA_MODE_XFER)
13172          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
13173                 union ctl_ha_msg msg;
13174
13175                 memset(&msg, 0, sizeof(msg));
13176                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
13177                 msg.hdr.original_sc = io->io_hdr.original_sc;
13178                 msg.hdr.nexus = io->io_hdr.nexus;
13179                 msg.hdr.status = io->io_hdr.status;
13180                 msg.scsi.scsi_status = io->scsiio.scsi_status;
13181                 msg.scsi.tag_num = io->scsiio.tag_num;
13182                 msg.scsi.tag_type = io->scsiio.tag_type;
13183                 msg.scsi.sense_len = io->scsiio.sense_len;
13184                 msg.scsi.sense_residual = io->scsiio.sense_residual;
13185                 msg.scsi.residual = io->scsiio.residual;
13186                 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
13187                        sizeof(io->scsiio.sense_data));
13188                 /*
13189                  * We copy this whether or not this is an I/O-related
13190                  * command.  Otherwise, we'd have to go and check to see
13191                  * whether it's a read/write command, and it really isn't
13192                  * worth it.
13193                  */
13194                 memcpy(&msg.scsi.lbalen,
13195                        &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
13196                        sizeof(msg.scsi.lbalen));
13197
13198                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13199                                 sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
13200                         /* XXX do something here */
13201                 }
13202
13203                 ctl_free_io(io);
13204         } else
13205                 fe_done(io);
13206
13207 bailout:
13208
13209         return (CTL_RETVAL_COMPLETE);
13210 }
13211
13212 /*
13213  * Front end should call this if it doesn't do autosense.  When the request
13214  * sense comes back in from the initiator, we'll dequeue this and send it.
13215  */
13216 int
13217 ctl_queue_sense(union ctl_io *io)
13218 {
13219         struct ctl_lun *lun;
13220         struct ctl_softc *ctl_softc;
13221         uint32_t initidx, targ_lun;
13222
13223         ctl_softc = control_softc;
13224
13225         CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
13226
13227         /*
13228          * LUN lookup will likely move to the ctl_work_thread() once we
13229          * have our new queueing infrastructure (that doesn't put things on
13230          * a per-LUN queue initially).  That is so that we can handle
13231          * things like an INQUIRY to a LUN that we don't have enabled.  We
13232          * can't deal with that right now.
13233          */
13234         mtx_lock(&ctl_softc->ctl_lock);
13235
13236         /*
13237          * If we don't have a LUN for this, just toss the sense
13238          * information.
13239          */
13240         targ_lun = io->io_hdr.nexus.targ_lun;
13241         if (io->io_hdr.nexus.lun_map_fn != NULL)
13242                 targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
13243         if ((targ_lun < CTL_MAX_LUNS)
13244          && (ctl_softc->ctl_luns[targ_lun] != NULL))
13245                 lun = ctl_softc->ctl_luns[targ_lun];
13246         else
13247                 goto bailout;
13248
13249         initidx = ctl_get_initindex(&io->io_hdr.nexus);
13250
13251         mtx_lock(&lun->lun_lock);
13252         /*
13253          * Already have CA set for this LUN...toss the sense information.
13254          */
13255         if (ctl_is_set(lun->have_ca, initidx)) {
13256                 mtx_unlock(&lun->lun_lock);
13257                 goto bailout;
13258         }
13259
13260         memcpy(&lun->pending_sense[initidx].sense, &io->scsiio.sense_data,
13261                ctl_min(sizeof(lun->pending_sense[initidx].sense),
13262                sizeof(io->scsiio.sense_data)));
13263         ctl_set_mask(lun->have_ca, initidx);
13264         mtx_unlock(&lun->lun_lock);
13265
13266 bailout:
13267         mtx_unlock(&ctl_softc->ctl_lock);
13268
13269         ctl_free_io(io);
13270
13271         return (CTL_RETVAL_COMPLETE);
13272 }
13273
13274 /*
13275  * Primary command inlet from frontend ports.  All SCSI and task I/O
13276  * requests must go through this function.
13277  */
13278 int
13279 ctl_queue(union ctl_io *io)
13280 {
13281         struct ctl_softc *ctl_softc;
13282
13283         CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
13284
13285         ctl_softc = control_softc;
13286
13287 #ifdef CTL_TIME_IO
13288         io->io_hdr.start_time = time_uptime;
13289         getbintime(&io->io_hdr.start_bt);
13290 #endif /* CTL_TIME_IO */
13291
13292         /* Map FE-specific LUN ID into global one. */
13293         if (io->io_hdr.nexus.lun_map_fn != NULL)
13294                 io->io_hdr.nexus.targ_mapped_lun = io->io_hdr.nexus.lun_map_fn(
13295                     io->io_hdr.nexus.lun_map_arg, io->io_hdr.nexus.targ_lun);
13296         else
13297                 io->io_hdr.nexus.targ_mapped_lun = io->io_hdr.nexus.targ_lun;
13298
13299         switch (io->io_hdr.io_type) {
13300         case CTL_IO_SCSI:
13301         case CTL_IO_TASK:
13302                 ctl_enqueue_incoming(io);
13303                 break;
13304         default:
13305                 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
13306                 return (EINVAL);
13307         }
13308
13309         return (CTL_RETVAL_COMPLETE);
13310 }
13311
13312 #ifdef CTL_IO_DELAY
13313 static void
13314 ctl_done_timer_wakeup(void *arg)
13315 {
13316         union ctl_io *io;
13317
13318         io = (union ctl_io *)arg;
13319         ctl_done(io);
13320 }
13321 #endif /* CTL_IO_DELAY */
13322
13323 void
13324 ctl_done(union ctl_io *io)
13325 {
13326         struct ctl_softc *ctl_softc;
13327
13328         ctl_softc = control_softc;
13329
13330         /*
13331          * Enable this to catch duplicate completion issues.
13332          */
13333 #if 0
13334         if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
13335                 printf("%s: type %d msg %d cdb %x iptl: "
13336                        "%d:%d:%d:%d tag 0x%04x "
13337                        "flag %#x status %x\n",
13338                         __func__,
13339                         io->io_hdr.io_type,
13340                         io->io_hdr.msg_type,
13341                         io->scsiio.cdb[0],
13342                         io->io_hdr.nexus.initid.id,
13343                         io->io_hdr.nexus.targ_port,
13344                         io->io_hdr.nexus.targ_target.id,
13345                         io->io_hdr.nexus.targ_lun,
13346                         (io->io_hdr.io_type ==
13347                         CTL_IO_TASK) ?
13348                         io->taskio.tag_num :
13349                         io->scsiio.tag_num,
13350                         io->io_hdr.flags,
13351                         io->io_hdr.status);
13352         } else
13353                 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
13354 #endif
13355
13356         /*
13357          * This is an internal copy of an I/O, and should not go through
13358          * the normal done processing logic.
13359          */
13360         if (io->io_hdr.flags & CTL_FLAG_INT_COPY)
13361                 return;
13362
13363         /*
13364          * We need to send a msg to the serializing shelf to finish the IO
13365          * as well.  We don't send a finish message to the other shelf if
13366          * this is a task management command.  Task management commands
13367          * aren't serialized in the OOA queue, but rather just executed on
13368          * both shelf controllers for commands that originated on that
13369          * controller.
13370          */
13371         if ((io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)
13372          && (io->io_hdr.io_type != CTL_IO_TASK)) {
13373                 union ctl_ha_msg msg_io;
13374
13375                 msg_io.hdr.msg_type = CTL_MSG_FINISH_IO;
13376                 msg_io.hdr.serializing_sc = io->io_hdr.serializing_sc;
13377                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_io,
13378                     sizeof(msg_io), 0 ) != CTL_HA_STATUS_SUCCESS) {
13379                 }
13380                 /* continue on to finish IO */
13381         }
13382 #ifdef CTL_IO_DELAY
13383         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
13384                 struct ctl_lun *lun;
13385
13386                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13387
13388                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
13389         } else {
13390                 struct ctl_lun *lun;
13391
13392                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13393
13394                 if ((lun != NULL)
13395                  && (lun->delay_info.done_delay > 0)) {
13396                         struct callout *callout;
13397
13398                         callout = (struct callout *)&io->io_hdr.timer_bytes;
13399                         callout_init(callout, /*mpsafe*/ 1);
13400                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
13401                         callout_reset(callout,
13402                                       lun->delay_info.done_delay * hz,
13403                                       ctl_done_timer_wakeup, io);
13404                         if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
13405                                 lun->delay_info.done_delay = 0;
13406                         return;
13407                 }
13408         }
13409 #endif /* CTL_IO_DELAY */
13410
13411         ctl_enqueue_done(io);
13412 }
13413
13414 int
13415 ctl_isc(struct ctl_scsiio *ctsio)
13416 {
13417         struct ctl_lun *lun;
13418         int retval;
13419
13420         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13421
13422         CTL_DEBUG_PRINT(("ctl_isc: command: %02x\n", ctsio->cdb[0]));
13423
13424         CTL_DEBUG_PRINT(("ctl_isc: calling data_submit()\n"));
13425
13426         retval = lun->backend->data_submit((union ctl_io *)ctsio);
13427
13428         return (retval);
13429 }
13430
13431
13432 static void
13433 ctl_work_thread(void *arg)
13434 {
13435         struct ctl_thread *thr = (struct ctl_thread *)arg;
13436         struct ctl_softc *softc = thr->ctl_softc;
13437         union ctl_io *io;
13438         int retval;
13439
13440         CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
13441
13442         for (;;) {
13443                 retval = 0;
13444
13445                 /*
13446                  * We handle the queues in this order:
13447                  * - ISC
13448                  * - done queue (to free up resources, unblock other commands)
13449                  * - RtR queue
13450                  * - incoming queue
13451                  *
13452                  * If those queues are empty, we break out of the loop and
13453                  * go to sleep.
13454                  */
13455                 mtx_lock(&thr->queue_lock);
13456                 io = (union ctl_io *)STAILQ_FIRST(&thr->isc_queue);
13457                 if (io != NULL) {
13458                         STAILQ_REMOVE_HEAD(&thr->isc_queue, links);
13459                         mtx_unlock(&thr->queue_lock);
13460                         ctl_handle_isc(io);
13461                         continue;
13462                 }
13463                 io = (union ctl_io *)STAILQ_FIRST(&thr->done_queue);
13464                 if (io != NULL) {
13465                         STAILQ_REMOVE_HEAD(&thr->done_queue, links);
13466                         /* clear any blocked commands, call fe_done */
13467                         mtx_unlock(&thr->queue_lock);
13468                         retval = ctl_process_done(io);
13469                         continue;
13470                 }
13471                 io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
13472                 if (io != NULL) {
13473                         STAILQ_REMOVE_HEAD(&thr->incoming_queue, links);
13474                         mtx_unlock(&thr->queue_lock);
13475                         if (io->io_hdr.io_type == CTL_IO_TASK)
13476                                 ctl_run_task(io);
13477                         else
13478                                 ctl_scsiio_precheck(softc, &io->scsiio);
13479                         continue;
13480                 }
13481                 if (!ctl_pause_rtr) {
13482                         io = (union ctl_io *)STAILQ_FIRST(&thr->rtr_queue);
13483                         if (io != NULL) {
13484                                 STAILQ_REMOVE_HEAD(&thr->rtr_queue, links);
13485                                 mtx_unlock(&thr->queue_lock);
13486                                 retval = ctl_scsiio(&io->scsiio);
13487                                 if (retval != CTL_RETVAL_COMPLETE)
13488                                         CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
13489                                 continue;
13490                         }
13491                 }
13492
13493                 /* Sleep until we have something to do. */
13494                 mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0);
13495         }
13496 }
13497
13498 static void
13499 ctl_lun_thread(void *arg)
13500 {
13501         struct ctl_softc *softc = (struct ctl_softc *)arg;
13502         struct ctl_be_lun *be_lun;
13503         int retval;
13504
13505         CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
13506
13507         for (;;) {
13508                 retval = 0;
13509                 mtx_lock(&softc->ctl_lock);
13510                 be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
13511                 if (be_lun != NULL) {
13512                         STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
13513                         mtx_unlock(&softc->ctl_lock);
13514                         ctl_create_lun(be_lun);
13515                         continue;
13516                 }
13517
13518                 /* Sleep until we have something to do. */
13519                 mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock,
13520                     PDROP | PRIBIO, "-", 0);
13521         }
13522 }
13523
13524 static void
13525 ctl_enqueue_incoming(union ctl_io *io)
13526 {
13527         struct ctl_softc *softc = control_softc;
13528         struct ctl_thread *thr;
13529         u_int idx;
13530
13531         idx = (io->io_hdr.nexus.targ_port * 127 +
13532                io->io_hdr.nexus.initid.id) % worker_threads;
13533         thr = &softc->threads[idx];
13534         mtx_lock(&thr->queue_lock);
13535         STAILQ_INSERT_TAIL(&thr->incoming_queue, &io->io_hdr, links);
13536         mtx_unlock(&thr->queue_lock);
13537         wakeup(thr);
13538 }
13539
13540 static void
13541 ctl_enqueue_rtr(union ctl_io *io)
13542 {
13543         struct ctl_softc *softc = control_softc;
13544         struct ctl_thread *thr;
13545
13546         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13547         mtx_lock(&thr->queue_lock);
13548         STAILQ_INSERT_TAIL(&thr->rtr_queue, &io->io_hdr, links);
13549         mtx_unlock(&thr->queue_lock);
13550         wakeup(thr);
13551 }
13552
13553 static void
13554 ctl_enqueue_done(union ctl_io *io)
13555 {
13556         struct ctl_softc *softc = control_softc;
13557         struct ctl_thread *thr;
13558
13559         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13560         mtx_lock(&thr->queue_lock);
13561         STAILQ_INSERT_TAIL(&thr->done_queue, &io->io_hdr, links);
13562         mtx_unlock(&thr->queue_lock);
13563         wakeup(thr);
13564 }
13565
13566 static void
13567 ctl_enqueue_isc(union ctl_io *io)
13568 {
13569         struct ctl_softc *softc = control_softc;
13570         struct ctl_thread *thr;
13571
13572         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13573         mtx_lock(&thr->queue_lock);
13574         STAILQ_INSERT_TAIL(&thr->isc_queue, &io->io_hdr, links);
13575         mtx_unlock(&thr->queue_lock);
13576         wakeup(thr);
13577 }
13578
13579 /* Initialization and failover */
13580
13581 void
13582 ctl_init_isc_msg(void)
13583 {
13584         printf("CTL: Still calling this thing\n");
13585 }
13586
13587 /*
13588  * Init component
13589  *      Initializes component into configuration defined by bootMode
13590  *      (see hasc-sv.c)
13591  *      returns hasc_Status:
13592  *              OK
13593  *              ERROR - fatal error
13594  */
13595 static ctl_ha_comp_status
13596 ctl_isc_init(struct ctl_ha_component *c)
13597 {
13598         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13599
13600         c->status = ret;
13601         return ret;
13602 }
13603
13604 /* Start component
13605  *      Starts component in state requested. If component starts successfully,
13606  *      it must set its own state to the requestrd state
13607  *      When requested state is HASC_STATE_HA, the component may refine it
13608  *      by adding _SLAVE or _MASTER flags.
13609  *      Currently allowed state transitions are:
13610  *      UNKNOWN->HA             - initial startup
13611  *      UNKNOWN->SINGLE - initial startup when no parter detected
13612  *      HA->SINGLE              - failover
13613  * returns ctl_ha_comp_status:
13614  *              OK      - component successfully started in requested state
13615  *              FAILED  - could not start the requested state, failover may
13616  *                        be possible
13617  *              ERROR   - fatal error detected, no future startup possible
13618  */
13619 static ctl_ha_comp_status
13620 ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
13621 {
13622         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13623
13624         printf("%s: go\n", __func__);
13625
13626         // UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
13627         if (c->state == CTL_HA_STATE_UNKNOWN ) {
13628                 ctl_is_single = 0;
13629                 if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
13630                     != CTL_HA_STATUS_SUCCESS) {
13631                         printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
13632                         ret = CTL_HA_COMP_STATUS_ERROR;
13633                 }
13634         } else if (CTL_HA_STATE_IS_HA(c->state)
13635                 && CTL_HA_STATE_IS_SINGLE(state)){
13636                 // HA->SINGLE transition
13637                 ctl_failover();
13638                 ctl_is_single = 1;
13639         } else {
13640                 printf("ctl_isc_start:Invalid state transition %X->%X\n",
13641                        c->state, state);
13642                 ret = CTL_HA_COMP_STATUS_ERROR;
13643         }
13644         if (CTL_HA_STATE_IS_SINGLE(state))
13645                 ctl_is_single = 1;
13646
13647         c->state = state;
13648         c->status = ret;
13649         return ret;
13650 }
13651
13652 /*
13653  * Quiesce component
13654  * The component must clear any error conditions (set status to OK) and
13655  * prepare itself to another Start call
13656  * returns ctl_ha_comp_status:
13657  *      OK
13658  *      ERROR
13659  */
13660 static ctl_ha_comp_status
13661 ctl_isc_quiesce(struct ctl_ha_component *c)
13662 {
13663         int ret = CTL_HA_COMP_STATUS_OK;
13664
13665         ctl_pause_rtr = 1;
13666         c->status = ret;
13667         return ret;
13668 }
13669
13670 struct ctl_ha_component ctl_ha_component_ctlisc =
13671 {
13672         .name = "CTL ISC",
13673         .state = CTL_HA_STATE_UNKNOWN,
13674         .init = ctl_isc_init,
13675         .start = ctl_isc_start,
13676         .quiesce = ctl_isc_quiesce
13677 };
13678
13679 /*
13680  *  vim: ts=8
13681  */