sys/cam/ctl/ctl.c

   1 /*-
   2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
   3  * Copyright (c) 2012 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Edward Tomasz Napierala
   7  * under sponsorship from the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  16  *    substantially similar to the "NO WARRANTY" disclaimer below
  17  *    ("Disclaimer") and any redistribution must be conditioned upon
  18  *    including a substantially similar Disclaimer requirement for further
  19  *    binary redistribution.
  20  *
  21  * NO WARRANTY
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  30  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  31  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGES.
  33  *
  34  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl.c#8 $
  35  */
  36 /*
  37  * CAM Target Layer, a SCSI device emulation subsystem.
  38  *
  39  * Author: Ken Merry <ken@FreeBSD.org>
  40  */
  41
  42 #define _CTL_C
  43
  44 #include <sys/cdefs.h>
  45 __FBSDID("$FreeBSD$");
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/types.h>
  51 #include <sys/kthread.h>
  52 #include <sys/bio.h>
  53 #include <sys/fcntl.h>
  54 #include <sys/lock.h>
  55 #include <sys/module.h>
  56 #include <sys/mutex.h>
  57 #include <sys/condvar.h>
  58 #include <sys/malloc.h>
  59 #include <sys/conf.h>
  60 #include <sys/ioccom.h>
  61 #include <sys/queue.h>
  62 #include <sys/sbuf.h>
  63 #include <sys/smp.h>
  64 #include <sys/endian.h>
  65 #include <sys/sysctl.h>
  66
  67 #include <cam/cam.h>
  68 #include <cam/scsi/scsi_all.h>
  69 #include <cam/scsi/scsi_da.h>
  70 #include <cam/ctl/ctl_io.h>
  71 #include <cam/ctl/ctl.h>
  72 #include <cam/ctl/ctl_frontend.h>
  73 #include <cam/ctl/ctl_frontend_internal.h>
  74 #include <cam/ctl/ctl_util.h>
  75 #include <cam/ctl/ctl_backend.h>
  76 #include <cam/ctl/ctl_ioctl.h>
  77 #include <cam/ctl/ctl_ha.h>
  78 #include <cam/ctl/ctl_private.h>
  79 #include <cam/ctl/ctl_debug.h>
  80 #include <cam/ctl/ctl_scsi_all.h>
  81 #include <cam/ctl/ctl_error.h>
  82
  83 struct ctl_softc *control_softc = NULL;
  84
  85 /*
  86  * Size and alignment macros needed for Copan-specific HA hardware.  These
  87  * can go away when the HA code is re-written, and uses busdma for any
  88  * hardware.
  89  */
  90 #define CTL_ALIGN_8B(target, source, type)                              \
  91         if (((uint32_t)source & 0x7) != 0)                              \
  92                 target = (type)(source + (0x8 - ((uint32_t)source & 0x7)));\
  93         else                                                            \
  94                 target = (type)source;
  95
  96 #define CTL_SIZE_8B(target, size)                                       \
  97         if ((size & 0x7) != 0)                                          \
  98                 target = size + (0x8 - (size & 0x7));                   \
  99         else                                                            \
 100                 target = size;
 101
 102 #define CTL_ALIGN_8B_MARGIN     16
 103
 104 /*
 105  * Template mode pages.
 106  */
 107
 108 /*
 109  * Note that these are default values only.  The actual values will be
 110  * filled in when the user does a mode sense.
 111  */
 112 static struct copan_power_subpage power_page_default = {
 113         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 114         /*subpage*/ PWR_SUBPAGE_CODE,
 115         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 116                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 117         /*page_version*/ PWR_VERSION,
 118         /* total_luns */ 26,
 119         /* max_active_luns*/ PWR_DFLT_MAX_LUNS,
 120         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 121                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 122                       0, 0, 0, 0, 0, 0}
 123 };
 124
 125 static struct copan_power_subpage power_page_changeable = {
 126         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 127         /*subpage*/ PWR_SUBPAGE_CODE,
 128         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 129                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 130         /*page_version*/ 0,
 131         /* total_luns */ 0,
 132         /* max_active_luns*/ 0,
 133         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 134                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 135                       0, 0, 0, 0, 0, 0}
 136 };
 137
 138 static struct copan_aps_subpage aps_page_default = {
 139         APS_PAGE_CODE | SMPH_SPF, //page_code
 140         APS_SUBPAGE_CODE, //subpage
 141         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 142          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 143         APS_VERSION, //page_version
 144         0, //lock_active
 145         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 146         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 147         0, 0, 0, 0, 0} //reserved
 148 };
 149
 150 static struct copan_aps_subpage aps_page_changeable = {
 151         APS_PAGE_CODE | SMPH_SPF, //page_code
 152         APS_SUBPAGE_CODE, //subpage
 153         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 154          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 155         0, //page_version
 156         0, //lock_active
 157         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 158         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 159         0, 0, 0, 0, 0} //reserved
 160 };
 161
 162 static struct copan_debugconf_subpage debugconf_page_default = {
 163         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 164         DBGCNF_SUBPAGE_CODE,            /* subpage */
 165         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 166          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 167         DBGCNF_VERSION,                 /* page_version */
 168         {CTL_TIME_IO_DEFAULT_SECS>>8,
 169          CTL_TIME_IO_DEFAULT_SECS>>0},  /* ctl_time_io_secs */
 170 };
 171
 172 static struct copan_debugconf_subpage debugconf_page_changeable = {
 173         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 174         DBGCNF_SUBPAGE_CODE,            /* subpage */
 175         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 176          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 177         0,                              /* page_version */
 178         {0xff,0xff},                    /* ctl_time_io_secs */
 179 };
 180
 181 static struct scsi_format_page format_page_default = {
 182         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 183         /*page_length*/sizeof(struct scsi_format_page) - 2,
 184         /*tracks_per_zone*/ {0, 0},
 185         /*alt_sectors_per_zone*/ {0, 0},
 186         /*alt_tracks_per_zone*/ {0, 0},
 187         /*alt_tracks_per_lun*/ {0, 0},
 188         /*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
 189                                 CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
 190         /*bytes_per_sector*/ {0, 0},
 191         /*interleave*/ {0, 0},
 192         /*track_skew*/ {0, 0},
 193         /*cylinder_skew*/ {0, 0},
 194         /*flags*/ SFP_HSEC,
 195         /*reserved*/ {0, 0, 0}
 196 };
 197
 198 static struct scsi_format_page format_page_changeable = {
 199         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 200         /*page_length*/sizeof(struct scsi_format_page) - 2,
 201         /*tracks_per_zone*/ {0, 0},
 202         /*alt_sectors_per_zone*/ {0, 0},
 203         /*alt_tracks_per_zone*/ {0, 0},
 204         /*alt_tracks_per_lun*/ {0, 0},
 205         /*sectors_per_track*/ {0, 0},
 206         /*bytes_per_sector*/ {0, 0},
 207         /*interleave*/ {0, 0},
 208         /*track_skew*/ {0, 0},
 209         /*cylinder_skew*/ {0, 0},
 210         /*flags*/ 0,
 211         /*reserved*/ {0, 0, 0}
 212 };
 213
 214 static struct scsi_rigid_disk_page rigid_disk_page_default = {
 215         /*page_code*/SMS_RIGID_DISK_PAGE,
 216         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 217         /*cylinders*/ {0, 0, 0},
 218         /*heads*/ CTL_DEFAULT_HEADS,
 219         /*start_write_precomp*/ {0, 0, 0},
 220         /*start_reduced_current*/ {0, 0, 0},
 221         /*step_rate*/ {0, 0},
 222         /*landing_zone_cylinder*/ {0, 0, 0},
 223         /*rpl*/ SRDP_RPL_DISABLED,
 224         /*rotational_offset*/ 0,
 225         /*reserved1*/ 0,
 226         /*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
 227                            CTL_DEFAULT_ROTATION_RATE & 0xff},
 228         /*reserved2*/ {0, 0}
 229 };
 230
 231 static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
 232         /*page_code*/SMS_RIGID_DISK_PAGE,
 233         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 234         /*cylinders*/ {0, 0, 0},
 235         /*heads*/ 0,
 236         /*start_write_precomp*/ {0, 0, 0},
 237         /*start_reduced_current*/ {0, 0, 0},
 238         /*step_rate*/ {0, 0},
 239         /*landing_zone_cylinder*/ {0, 0, 0},
 240         /*rpl*/ 0,
 241         /*rotational_offset*/ 0,
 242         /*reserved1*/ 0,
 243         /*rotation_rate*/ {0, 0},
 244         /*reserved2*/ {0, 0}
 245 };
 246
 247 static struct scsi_caching_page caching_page_default = {
 248         /*page_code*/SMS_CACHING_PAGE,
 249         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 250         /*flags1*/ SCP_DISC | SCP_WCE,
 251         /*ret_priority*/ 0,
 252         /*disable_pf_transfer_len*/ {0xff, 0xff},
 253         /*min_prefetch*/ {0, 0},
 254         /*max_prefetch*/ {0xff, 0xff},
 255         /*max_pf_ceiling*/ {0xff, 0xff},
 256         /*flags2*/ 0,
 257         /*cache_segments*/ 0,
 258         /*cache_seg_size*/ {0, 0},
 259         /*reserved*/ 0,
 260         /*non_cache_seg_size*/ {0, 0, 0}
 261 };
 262
 263 static struct scsi_caching_page caching_page_changeable = {
 264         /*page_code*/SMS_CACHING_PAGE,
 265         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 266         /*flags1*/ SCP_WCE | SCP_RCD,
 267         /*ret_priority*/ 0,
 268         /*disable_pf_transfer_len*/ {0, 0},
 269         /*min_prefetch*/ {0, 0},
 270         /*max_prefetch*/ {0, 0},
 271         /*max_pf_ceiling*/ {0, 0},
 272         /*flags2*/ 0,
 273         /*cache_segments*/ 0,
 274         /*cache_seg_size*/ {0, 0},
 275         /*reserved*/ 0,
 276         /*non_cache_seg_size*/ {0, 0, 0}
 277 };
 278
 279 static struct scsi_control_page control_page_default = {
 280         /*page_code*/SMS_CONTROL_MODE_PAGE,
 281         /*page_length*/sizeof(struct scsi_control_page) - 2,
 282         /*rlec*/0,
 283         /*queue_flags*/SCP_QUEUE_ALG_RESTRICTED,
 284         /*eca_and_aen*/0,
 285         /*flags4*/SCP_TAS,
 286         /*aen_holdoff_period*/{0, 0},
 287         /*busy_timeout_period*/{0, 0},
 288         /*extended_selftest_completion_time*/{0, 0}
 289 };
 290
 291 static struct scsi_control_page control_page_changeable = {
 292         /*page_code*/SMS_CONTROL_MODE_PAGE,
 293         /*page_length*/sizeof(struct scsi_control_page) - 2,
 294         /*rlec*/SCP_DSENSE,
 295         /*queue_flags*/SCP_QUEUE_ALG_MASK,
 296         /*eca_and_aen*/SCP_SWP,
 297         /*flags4*/0,
 298         /*aen_holdoff_period*/{0, 0},
 299         /*busy_timeout_period*/{0, 0},
 300         /*extended_selftest_completion_time*/{0, 0}
 301 };
 302
 303
 304 /*
 305  * XXX KDM move these into the softc.
 306  */
 307 static int rcv_sync_msg;
 308 static int persis_offset;
 309 static uint8_t ctl_pause_rtr;
 310 static int     ctl_is_single = 1;
 311 static int     index_to_aps_page;
 312
 313 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 314 static int worker_threads = -1;
 315 TUNABLE_INT("kern.cam.ctl.worker_threads", &worker_threads);
 316 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
 317     &worker_threads, 1, "Number of worker threads");
 318 static int verbose = 0;
 319 TUNABLE_INT("kern.cam.ctl.verbose", &verbose);
 320 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, verbose, CTLFLAG_RWTUN,
 321     &verbose, 0, "Show SCSI errors returned to initiator");
 322
 323 /*
 324  * Supported pages (0x00), Serial number (0x80), Device ID (0x83),
 325  * Extended INQUIRY Data (0x86), Mode Page Policy (0x87),
 326  * SCSI Ports (0x88), Third-party Copy (0x8F), Block limits (0xB0),
 327  * Block Device Characteristics (0xB1) and Logical Block Provisioning (0xB2)
 328  */
 329 #define SCSI_EVPD_NUM_SUPPORTED_PAGES   10
 330
 331 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 332                                   int param);
 333 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 334 static int ctl_init(void);
 335 void ctl_shutdown(void);
 336 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 337 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 338 static void ctl_ioctl_online(void *arg);
 339 static void ctl_ioctl_offline(void *arg);
 340 static int ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id);
 341 static int ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id);
 342 static int ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio);
 343 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
 344 static int ctl_ioctl_submit_wait(union ctl_io *io);
 345 static void ctl_ioctl_datamove(union ctl_io *io);
 346 static void ctl_ioctl_done(union ctl_io *io);
 347 static void ctl_ioctl_hard_startstop_callback(void *arg,
 348                                               struct cfi_metatask *metatask);
 349 static void ctl_ioctl_bbrread_callback(void *arg,struct cfi_metatask *metatask);
 350 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 351                               struct ctl_ooa *ooa_hdr,
 352                               struct ctl_ooa_entry *kern_entries);
 353 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 354                      struct thread *td);
 355 static uint32_t ctl_map_lun(int port_num, uint32_t lun);
 356 static uint32_t ctl_map_lun_back(int port_num, uint32_t lun);
 357 #ifdef unused
 358 static union ctl_io *ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port,
 359                                    uint32_t targ_target, uint32_t targ_lun,
 360                                    int can_wait);
 361 static void ctl_kfree_io(union ctl_io *io);
 362 #endif /* unused */
 363 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
 364                          struct ctl_be_lun *be_lun, struct ctl_id target_id);
 365 static int ctl_free_lun(struct ctl_lun *lun);
 366 static void ctl_create_lun(struct ctl_be_lun *be_lun);
 367 /**
 368 static void ctl_failover_change_pages(struct ctl_softc *softc,
 369                                       struct ctl_scsiio *ctsio, int master);
 370 **/
 371
 372 static int ctl_do_mode_select(union ctl_io *io);
 373 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
 374                            uint64_t res_key, uint64_t sa_res_key,
 375                            uint8_t type, uint32_t residx,
 376                            struct ctl_scsiio *ctsio,
 377                            struct scsi_per_res_out *cdb,
 378                            struct scsi_per_res_out_parms* param);
 379 static void ctl_pro_preempt_other(struct ctl_lun *lun,
 380                                   union ctl_ha_msg *msg);
 381 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
 382 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
 383 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
 384 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
 385 static int ctl_inquiry_evpd_eid(struct ctl_scsiio *ctsio, int alloc_len);
 386 static int ctl_inquiry_evpd_mpp(struct ctl_scsiio *ctsio, int alloc_len);
 387 static int ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio,
 388                                          int alloc_len);
 389 static int ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio,
 390                                          int alloc_len);
 391 static int ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len);
 392 static int ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len);
 393 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
 394 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
 395 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len);
 396 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2);
 397 static ctl_action ctl_check_for_blockage(struct ctl_lun *lun,
 398     union ctl_io *pending_io, union ctl_io *ooa_io);
 399 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
 400                                 union ctl_io *starting_io);
 401 static int ctl_check_blocked(struct ctl_lun *lun);
 402 static int ctl_scsiio_lun_check(struct ctl_softc *ctl_softc,
 403                                 struct ctl_lun *lun,
 404                                 const struct ctl_cmd_entry *entry,
 405                                 struct ctl_scsiio *ctsio);
 406 //static int ctl_check_rtr(union ctl_io *pending_io, struct ctl_softc *softc);
 407 static void ctl_failover(void);
 408 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 409                                struct ctl_scsiio *ctsio);
 410 static int ctl_scsiio(struct ctl_scsiio *ctsio);
 411
 412 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 413 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 414                             ctl_ua_type ua_type);
 415 static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 416                          ctl_ua_type ua_type);
 417 static int ctl_abort_task(union ctl_io *io);
 418 static int ctl_abort_task_set(union ctl_io *io);
 419 static int ctl_i_t_nexus_reset(union ctl_io *io);
 420 static void ctl_run_task(union ctl_io *io);
 421 #ifdef CTL_IO_DELAY
 422 static void ctl_datamove_timer_wakeup(void *arg);
 423 static void ctl_done_timer_wakeup(void *arg);
 424 #endif /* CTL_IO_DELAY */
 425
 426 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 427 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
 428 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
 429 static void ctl_datamove_remote_write(union ctl_io *io);
 430 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
 431 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 432 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 433 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 434                                     ctl_ha_dt_cb callback);
 435 static void ctl_datamove_remote_read(union ctl_io *io);
 436 static void ctl_datamove_remote(union ctl_io *io);
 437 static int ctl_process_done(union ctl_io *io);
 438 static void ctl_lun_thread(void *arg);
 439 static void ctl_work_thread(void *arg);
 440 static void ctl_enqueue_incoming(union ctl_io *io);
 441 static void ctl_enqueue_rtr(union ctl_io *io);
 442 static void ctl_enqueue_done(union ctl_io *io);
 443 static void ctl_enqueue_isc(union ctl_io *io);
 444 static const struct ctl_cmd_entry *
 445     ctl_get_cmd_entry(struct ctl_scsiio *ctsio, int *sa);
 446 static const struct ctl_cmd_entry *
 447     ctl_validate_command(struct ctl_scsiio *ctsio);
 448 static int ctl_cmd_applicable(uint8_t lun_type,
 449     const struct ctl_cmd_entry *entry);
 450
 451 /*
 452  * Load the serialization table.  This isn't very pretty, but is probably
 453  * the easiest way to do it.
 454  */
 455 #include "ctl_ser_table.c"
 456
 457 /*
 458  * We only need to define open, close and ioctl routines for this driver.
 459  */
 460 static struct cdevsw ctl_cdevsw = {
 461         .d_version =    D_VERSION,
 462         .d_flags =      0,
 463         .d_open =       ctl_open,
 464         .d_close =      ctl_close,
 465         .d_ioctl =      ctl_ioctl,
 466         .d_name =       "ctl",
 467 };
 468
 469
 470 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 471 MALLOC_DEFINE(M_CTLIO, "ctlio", "Memory used for CTL requests");
 472
 473 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 474
 475 static moduledata_t ctl_moduledata = {
 476         "ctl",
 477         ctl_module_event_handler,
 478         NULL
 479 };
 480
 481 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
 482 MODULE_VERSION(ctl, 1);
 483
 484 static struct ctl_frontend ioctl_frontend =
 485 {
 486         .name = "ioctl",
 487 };
 488
 489 static void
 490 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 491                             union ctl_ha_msg *msg_info)
 492 {
 493         struct ctl_scsiio *ctsio;
 494
 495         if (msg_info->hdr.original_sc == NULL) {
 496                 printf("%s: original_sc == NULL!\n", __func__);
 497                 /* XXX KDM now what? */
 498                 return;
 499         }
 500
 501         ctsio = &msg_info->hdr.original_sc->scsiio;
 502         ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 503         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 504         ctsio->io_hdr.status = msg_info->hdr.status;
 505         ctsio->scsi_status = msg_info->scsi.scsi_status;
 506         ctsio->sense_len = msg_info->scsi.sense_len;
 507         ctsio->sense_residual = msg_info->scsi.sense_residual;
 508         ctsio->residual = msg_info->scsi.residual;
 509         memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 510                sizeof(ctsio->sense_data));
 511         memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
 512                &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));
 513         ctl_enqueue_isc((union ctl_io *)ctsio);
 514 }
 515
 516 static void
 517 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
 518                                 union ctl_ha_msg *msg_info)
 519 {
 520         struct ctl_scsiio *ctsio;
 521
 522         if (msg_info->hdr.serializing_sc == NULL) {
 523                 printf("%s: serializing_sc == NULL!\n", __func__);
 524                 /* XXX KDM now what? */
 525                 return;
 526         }
 527
 528         ctsio = &msg_info->hdr.serializing_sc->scsiio;
 529 #if 0
 530         /*
 531          * Attempt to catch the situation where an I/O has
 532          * been freed, and we're using it again.
 533          */
 534         if (ctsio->io_hdr.io_type == 0xff) {
 535                 union ctl_io *tmp_io;
 536                 tmp_io = (union ctl_io *)ctsio;
 537                 printf("%s: %p use after free!\n", __func__,
 538                        ctsio);
 539                 printf("%s: type %d msg %d cdb %x iptl: "
 540                        "%d:%d:%d:%d tag 0x%04x "
 541                        "flag %#x status %x\n",
 542                         __func__,
 543                         tmp_io->io_hdr.io_type,
 544                         tmp_io->io_hdr.msg_type,
 545                         tmp_io->scsiio.cdb[0],
 546                         tmp_io->io_hdr.nexus.initid.id,
 547                         tmp_io->io_hdr.nexus.targ_port,
 548                         tmp_io->io_hdr.nexus.targ_target.id,
 549                         tmp_io->io_hdr.nexus.targ_lun,
 550                         (tmp_io->io_hdr.io_type ==
 551                         CTL_IO_TASK) ?
 552                         tmp_io->taskio.tag_num :
 553                         tmp_io->scsiio.tag_num,
 554                         tmp_io->io_hdr.flags,
 555                         tmp_io->io_hdr.status);
 556         }
 557 #endif
 558         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 559         ctl_enqueue_isc((union ctl_io *)ctsio);
 560 }
 561
 562 /*
 563  * ISC (Inter Shelf Communication) event handler.  Events from the HA
 564  * subsystem come in here.
 565  */
 566 static void
 567 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 568 {
 569         struct ctl_softc *ctl_softc;
 570         union ctl_io *io;
 571         struct ctl_prio *presio;
 572         ctl_ha_status isc_status;
 573
 574         ctl_softc = control_softc;
 575         io = NULL;
 576
 577
 578 #if 0
 579         printf("CTL: Isc Msg event %d\n", event);
 580 #endif
 581         if (event == CTL_HA_EVT_MSG_RECV) {
 582                 union ctl_ha_msg msg_info;
 583
 584                 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
 585                                              sizeof(msg_info), /*wait*/ 0);
 586 #if 0
 587                 printf("CTL: msg_type %d\n", msg_info.msg_type);
 588 #endif
 589                 if (isc_status != 0) {
 590                         printf("Error receiving message, status = %d\n",
 591                                isc_status);
 592                         return;
 593                 }
 594
 595                 switch (msg_info.hdr.msg_type) {
 596                 case CTL_MSG_SERIALIZE:
 597 #if 0
 598                         printf("Serialize\n");
 599 #endif
 600                         io = ctl_alloc_io((void *)ctl_softc->othersc_pool);
 601                         if (io == NULL) {
 602                                 printf("ctl_isc_event_handler: can't allocate "
 603                                        "ctl_io!\n");
 604                                 /* Bad Juju */
 605                                 /* Need to set busy and send msg back */
 606                                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
 607                                 msg_info.hdr.status = CTL_SCSI_ERROR;
 608                                 msg_info.scsi.scsi_status = SCSI_STATUS_BUSY;
 609                                 msg_info.scsi.sense_len = 0;
 610                                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
 611                                     sizeof(msg_info), 0) > CTL_HA_STATUS_SUCCESS){
 612                                 }
 613                                 goto bailout;
 614                         }
 615                         ctl_zero_io(io);
 616                         // populate ctsio from msg_info
 617                         io->io_hdr.io_type = CTL_IO_SCSI;
 618                         io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
 619                         io->io_hdr.original_sc = msg_info.hdr.original_sc;
 620 #if 0
 621                         printf("pOrig %x\n", (int)msg_info.original_sc);
 622 #endif
 623                         io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
 624                                             CTL_FLAG_IO_ACTIVE;
 625                         /*
 626                          * If we're in serialization-only mode, we don't
 627                          * want to go through full done processing.  Thus
 628                          * the COPY flag.
 629                          *
 630                          * XXX KDM add another flag that is more specific.
 631                          */
 632                         if (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)
 633                                 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
 634                         io->io_hdr.nexus = msg_info.hdr.nexus;
 635 #if 0
 636                         printf("targ %d, port %d, iid %d, lun %d\n",
 637                                io->io_hdr.nexus.targ_target.id,
 638                                io->io_hdr.nexus.targ_port,
 639                                io->io_hdr.nexus.initid.id,
 640                                io->io_hdr.nexus.targ_lun);
 641 #endif
 642                         io->scsiio.tag_num = msg_info.scsi.tag_num;
 643                         io->scsiio.tag_type = msg_info.scsi.tag_type;
 644                         memcpy(io->scsiio.cdb, msg_info.scsi.cdb,
 645                                CTL_MAX_CDBLEN);
 646                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
 647                                 const struct ctl_cmd_entry *entry;
 648
 649                                 entry = ctl_get_cmd_entry(&io->scsiio, NULL);
 650                                 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
 651                                 io->io_hdr.flags |=
 652                                         entry->flags & CTL_FLAG_DATA_MASK;
 653                         }
 654                         ctl_enqueue_isc(io);
 655                         break;
 656
 657                 /* Performed on the Originating SC, XFER mode only */
 658                 case CTL_MSG_DATAMOVE: {
 659                         struct ctl_sg_entry *sgl;
 660                         int i, j;
 661
 662                         io = msg_info.hdr.original_sc;
 663                         if (io == NULL) {
 664                                 printf("%s: original_sc == NULL!\n", __func__);
 665                                 /* XXX KDM do something here */
 666                                 break;
 667                         }
 668                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
 669                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 670                         /*
 671                          * Keep track of this, we need to send it back over
 672                          * when the datamove is complete.
 673                          */
 674                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 675
 676                         if (msg_info.dt.sg_sequence == 0) {
 677                                 /*
 678                                  * XXX KDM we use the preallocated S/G list
 679                                  * here, but we'll need to change this to
 680                                  * dynamic allocation if we need larger S/G
 681                                  * lists.
 682                                  */
 683                                 if (msg_info.dt.kern_sg_entries >
 684                                     sizeof(io->io_hdr.remote_sglist) /
 685                                     sizeof(io->io_hdr.remote_sglist[0])) {
 686                                         printf("%s: number of S/G entries "
 687                                             "needed %u > allocated num %zd\n",
 688                                             __func__,
 689                                             msg_info.dt.kern_sg_entries,
 690                                             sizeof(io->io_hdr.remote_sglist)/
 691                                             sizeof(io->io_hdr.remote_sglist[0]));
 692
 693                                         /*
 694                                          * XXX KDM send a message back to
 695                                          * the other side to shut down the
 696                                          * DMA.  The error will come back
 697                                          * through via the normal channel.
 698                                          */
 699                                         break;
 700                                 }
 701                                 sgl = io->io_hdr.remote_sglist;
 702                                 memset(sgl, 0,
 703                                        sizeof(io->io_hdr.remote_sglist));
 704
 705                                 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
 706
 707                                 io->scsiio.kern_sg_entries =
 708                                         msg_info.dt.kern_sg_entries;
 709                                 io->scsiio.rem_sg_entries =
 710                                         msg_info.dt.kern_sg_entries;
 711                                 io->scsiio.kern_data_len =
 712                                         msg_info.dt.kern_data_len;
 713                                 io->scsiio.kern_total_len =
 714                                         msg_info.dt.kern_total_len;
 715                                 io->scsiio.kern_data_resid =
 716                                         msg_info.dt.kern_data_resid;
 717                                 io->scsiio.kern_rel_offset =
 718                                         msg_info.dt.kern_rel_offset;
 719                                 /*
 720                                  * Clear out per-DMA flags.
 721                                  */
 722                                 io->io_hdr.flags &= ~CTL_FLAG_RDMA_MASK;
 723                                 /*
 724                                  * Add per-DMA flags that are set for this
 725                                  * particular DMA request.
 726                                  */
 727                                 io->io_hdr.flags |= msg_info.dt.flags &
 728                                                     CTL_FLAG_RDMA_MASK;
 729                         } else
 730                                 sgl = (struct ctl_sg_entry *)
 731                                         io->scsiio.kern_data_ptr;
 732
 733                         for (i = msg_info.dt.sent_sg_entries, j = 0;
 734                              i < (msg_info.dt.sent_sg_entries +
 735                              msg_info.dt.cur_sg_entries); i++, j++) {
 736                                 sgl[i].addr = msg_info.dt.sg_list[j].addr;
 737                                 sgl[i].len = msg_info.dt.sg_list[j].len;
 738
 739 #if 0
 740                                 printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
 741                                        __func__,
 742                                        msg_info.dt.sg_list[j].addr,
 743                                        msg_info.dt.sg_list[j].len,
 744                                        sgl[i].addr, sgl[i].len, j, i);
 745 #endif
 746                         }
 747 #if 0
 748                         memcpy(&sgl[msg_info.dt.sent_sg_entries],
 749                                msg_info.dt.sg_list,
 750                                sizeof(*sgl) * msg_info.dt.cur_sg_entries);
 751 #endif
 752
 753                         /*
 754                          * If this is the last piece of the I/O, we've got
 755                          * the full S/G list.  Queue processing in the thread.
 756                          * Otherwise wait for the next piece.
 757                          */
 758                         if (msg_info.dt.sg_last != 0)
 759                                 ctl_enqueue_isc(io);
 760                         break;
 761                 }
 762                 /* Performed on the Serializing (primary) SC, XFER mode only */
 763                 case CTL_MSG_DATAMOVE_DONE: {
 764                         if (msg_info.hdr.serializing_sc == NULL) {
 765                                 printf("%s: serializing_sc == NULL!\n",
 766                                        __func__);
 767                                 /* XXX KDM now what? */
 768                                 break;
 769                         }
 770                         /*
 771                          * We grab the sense information here in case
 772                          * there was a failure, so we can return status
 773                          * back to the initiator.
 774                          */
 775                         io = msg_info.hdr.serializing_sc;
 776                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
 777                         io->io_hdr.status = msg_info.hdr.status;
 778                         io->scsiio.scsi_status = msg_info.scsi.scsi_status;
 779                         io->scsiio.sense_len = msg_info.scsi.sense_len;
 780                         io->scsiio.sense_residual =msg_info.scsi.sense_residual;
 781                         io->io_hdr.port_status = msg_info.scsi.fetd_status;
 782                         io->scsiio.residual = msg_info.scsi.residual;
 783                         memcpy(&io->scsiio.sense_data,&msg_info.scsi.sense_data,
 784                                sizeof(io->scsiio.sense_data));
 785                         ctl_enqueue_isc(io);
 786                         break;
 787                 }
 788
 789                 /* Preformed on Originating SC, SER_ONLY mode */
 790                 case CTL_MSG_R2R:
 791                         io = msg_info.hdr.original_sc;
 792                         if (io == NULL) {
 793                                 printf("%s: Major Bummer\n", __func__);
 794                                 return;
 795                         } else {
 796 #if 0
 797                                 printf("pOrig %x\n",(int) ctsio);
 798 #endif
 799                         }
 800                         io->io_hdr.msg_type = CTL_MSG_R2R;
 801                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 802                         ctl_enqueue_isc(io);
 803                         break;
 804
 805                 /*
 806                  * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
 807                  * mode.
 808                  * Performed on the Originating (i.e. secondary) SC in XFER
 809                  * mode
 810                  */
 811                 case CTL_MSG_FINISH_IO:
 812                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER)
 813                                 ctl_isc_handler_finish_xfer(ctl_softc,
 814                                                             &msg_info);
 815                         else
 816                                 ctl_isc_handler_finish_ser_only(ctl_softc,
 817                                                                 &msg_info);
 818                         break;
 819
 820                 /* Preformed on Originating SC */
 821                 case CTL_MSG_BAD_JUJU:
 822                         io = msg_info.hdr.original_sc;
 823                         if (io == NULL) {
 824                                 printf("%s: Bad JUJU!, original_sc is NULL!\n",
 825                                        __func__);
 826                                 break;
 827                         }
 828                         ctl_copy_sense_data(&msg_info, io);
 829                         /*
 830                          * IO should have already been cleaned up on other
 831                          * SC so clear this flag so we won't send a message
 832                          * back to finish the IO there.
 833                          */
 834                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
 835                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 836
 837                         /* io = msg_info.hdr.serializing_sc; */
 838                         io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
 839                         ctl_enqueue_isc(io);
 840                         break;
 841
 842                 /* Handle resets sent from the other side */
 843                 case CTL_MSG_MANAGE_TASKS: {
 844                         struct ctl_taskio *taskio;
 845                         taskio = (struct ctl_taskio *)ctl_alloc_io(
 846                                 (void *)ctl_softc->othersc_pool);
 847                         if (taskio == NULL) {
 848                                 printf("ctl_isc_event_handler: can't allocate "
 849                                        "ctl_io!\n");
 850                                 /* Bad Juju */
 851                                 /* should I just call the proper reset func
 852                                    here??? */
 853                                 goto bailout;
 854                         }
 855                         ctl_zero_io((union ctl_io *)taskio);
 856                         taskio->io_hdr.io_type = CTL_IO_TASK;
 857                         taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
 858                         taskio->io_hdr.nexus = msg_info.hdr.nexus;
 859                         taskio->task_action = msg_info.task.task_action;
 860                         taskio->tag_num = msg_info.task.tag_num;
 861                         taskio->tag_type = msg_info.task.tag_type;
 862 #ifdef CTL_TIME_IO
 863                         taskio->io_hdr.start_time = time_uptime;
 864                         getbintime(&taskio->io_hdr.start_bt);
 865 #if 0
 866                         cs_prof_gettime(&taskio->io_hdr.start_ticks);
 867 #endif
 868 #endif /* CTL_TIME_IO */
 869                         ctl_run_task((union ctl_io *)taskio);
 870                         break;
 871                 }
 872                 /* Persistent Reserve action which needs attention */
 873                 case CTL_MSG_PERS_ACTION:
 874                         presio = (struct ctl_prio *)ctl_alloc_io(
 875                                 (void *)ctl_softc->othersc_pool);
 876                         if (presio == NULL) {
 877                                 printf("ctl_isc_event_handler: can't allocate "
 878                                        "ctl_io!\n");
 879                                 /* Bad Juju */
 880                                 /* Need to set busy and send msg back */
 881                                 goto bailout;
 882                         }
 883                         ctl_zero_io((union ctl_io *)presio);
 884                         presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
 885                         presio->pr_msg = msg_info.pr;
 886                         ctl_enqueue_isc((union ctl_io *)presio);
 887                         break;
 888                 case CTL_MSG_SYNC_FE:
 889                         rcv_sync_msg = 1;
 890                         break;
 891                 case CTL_MSG_APS_LOCK: {
 892                         // It's quicker to execute this then to
 893                         // queue it.
 894                         struct ctl_lun *lun;
 895                         struct ctl_page_index *page_index;
 896                         struct copan_aps_subpage *current_sp;
 897                         uint32_t targ_lun;
 898
 899                         targ_lun = msg_info.hdr.nexus.targ_mapped_lun;
 900                         lun = ctl_softc->ctl_luns[targ_lun];
 901                         mtx_lock(&lun->lun_lock);
 902                         page_index = &lun->mode_pages.index[index_to_aps_page];
 903                         current_sp = (struct copan_aps_subpage *)
 904                                      (page_index->page_data +
 905                                      (page_index->page_len * CTL_PAGE_CURRENT));
 906
 907                         current_sp->lock_active = msg_info.aps.lock_flag;
 908                         mtx_unlock(&lun->lun_lock);
 909                         break;
 910                 }
 911                 default:
 912                         printf("How did I get here?\n");
 913                 }
 914         } else if (event == CTL_HA_EVT_MSG_SENT) {
 915                 if (param != CTL_HA_STATUS_SUCCESS) {
 916                         printf("Bad status from ctl_ha_msg_send status %d\n",
 917                                param);
 918                 }
 919                 return;
 920         } else if (event == CTL_HA_EVT_DISCONNECT) {
 921                 printf("CTL: Got a disconnect from Isc\n");
 922                 return;
 923         } else {
 924                 printf("ctl_isc_event_handler: Unknown event %d\n", event);
 925                 return;
 926         }
 927
 928 bailout:
 929         return;
 930 }
 931
 932 static void
 933 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
 934 {
 935         struct scsi_sense_data *sense;
 936
 937         sense = &dest->scsiio.sense_data;
 938         bcopy(&src->scsi.sense_data, sense, sizeof(*sense));
 939         dest->scsiio.scsi_status = src->scsi.scsi_status;
 940         dest->scsiio.sense_len = src->scsi.sense_len;
 941         dest->io_hdr.status = src->hdr.status;
 942 }
 943
 944 static int
 945 ctl_init(void)
 946 {
 947         struct ctl_softc *softc;
 948         struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
 949         struct ctl_port *port;
 950         uint8_t sc_id =0;
 951         int i, error, retval;
 952         //int isc_retval;
 953
 954         retval = 0;
 955         ctl_pause_rtr = 0;
 956         rcv_sync_msg = 0;
 957
 958         control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
 959                                M_WAITOK | M_ZERO);
 960         softc = control_softc;
 961
 962         softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
 963                               "cam/ctl");
 964
 965         softc->dev->si_drv1 = softc;
 966
 967         /*
 968          * By default, return a "bad LUN" peripheral qualifier for unknown
 969          * LUNs.  The user can override this default using the tunable or
 970          * sysctl.  See the comment in ctl_inquiry_std() for more details.
 971          */
 972         softc->inquiry_pq_no_lun = 1;
 973         TUNABLE_INT_FETCH("kern.cam.ctl.inquiry_pq_no_lun",
 974                           &softc->inquiry_pq_no_lun);
 975         sysctl_ctx_init(&softc->sysctl_ctx);
 976         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
 977                 SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
 978                 CTLFLAG_RD, 0, "CAM Target Layer");
 979
 980         if (softc->sysctl_tree == NULL) {
 981                 printf("%s: unable to allocate sysctl tree\n", __func__);
 982                 destroy_dev(softc->dev);
 983                 free(control_softc, M_DEVBUF);
 984                 control_softc = NULL;
 985                 return (ENOMEM);
 986         }
 987
 988         SYSCTL_ADD_INT(&softc->sysctl_ctx,
 989                        SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
 990                        "inquiry_pq_no_lun", CTLFLAG_RW,
 991                        &softc->inquiry_pq_no_lun, 0,
 992                        "Report no lun possible for invalid LUNs");
 993
 994         mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
 995         mtx_init(&softc->pool_lock, "CTL pool mutex", NULL, MTX_DEF);
 996         softc->open_count = 0;
 997
 998         /*
 999          * Default to actually sending a SYNCHRONIZE CACHE command down to
1000          * the drive.
1001          */
1002         softc->flags = CTL_FLAG_REAL_SYNC;
1003
1004         /*
1005          * In Copan's HA scheme, the "master" and "slave" roles are
1006          * figured out through the slot the controller is in.  Although it
1007          * is an active/active system, someone has to be in charge.
1008          */
1009 #ifdef NEEDTOPORT
1010         scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
1011 #endif
1012
1013         if (sc_id == 0) {
1014                 softc->flags |= CTL_FLAG_MASTER_SHELF;
1015                 persis_offset = 0;
1016         } else
1017                 persis_offset = CTL_MAX_INITIATORS;
1018
1019         /*
1020          * XXX KDM need to figure out where we want to get our target ID
1021          * and WWID.  Is it different on each port?
1022          */
1023         softc->target.id = 0;
1024         softc->target.wwid[0] = 0x12345678;
1025         softc->target.wwid[1] = 0x87654321;
1026         STAILQ_INIT(&softc->lun_list);
1027         STAILQ_INIT(&softc->pending_lun_queue);
1028         STAILQ_INIT(&softc->fe_list);
1029         STAILQ_INIT(&softc->port_list);
1030         STAILQ_INIT(&softc->be_list);
1031         STAILQ_INIT(&softc->io_pools);
1032         ctl_tpc_init(softc);
1033
1034         if (ctl_pool_create(softc, CTL_POOL_INTERNAL, CTL_POOL_ENTRIES_INTERNAL,
1035                             &internal_pool)!= 0){
1036                 printf("ctl: can't allocate %d entry internal pool, "
1037                        "exiting\n", CTL_POOL_ENTRIES_INTERNAL);
1038                 return (ENOMEM);
1039         }
1040
1041         if (ctl_pool_create(softc, CTL_POOL_EMERGENCY,
1042                             CTL_POOL_ENTRIES_EMERGENCY, &emergency_pool) != 0) {
1043                 printf("ctl: can't allocate %d entry emergency pool, "
1044                        "exiting\n", CTL_POOL_ENTRIES_EMERGENCY);
1045                 ctl_pool_free(internal_pool);
1046                 return (ENOMEM);
1047         }
1048
1049         if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC,
1050                             &other_pool) != 0)
1051         {
1052                 printf("ctl: can't allocate %d entry other SC pool, "
1053                        "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1054                 ctl_pool_free(internal_pool);
1055                 ctl_pool_free(emergency_pool);
1056                 return (ENOMEM);
1057         }
1058
1059         softc->internal_pool = internal_pool;
1060         softc->emergency_pool = emergency_pool;
1061         softc->othersc_pool = other_pool;
1062
1063         if (worker_threads <= 0)
1064                 worker_threads = max(1, mp_ncpus / 4);
1065         if (worker_threads > CTL_MAX_THREADS)
1066                 worker_threads = CTL_MAX_THREADS;
1067
1068         for (i = 0; i < worker_threads; i++) {
1069                 struct ctl_thread *thr = &softc->threads[i];
1070
1071                 mtx_init(&thr->queue_lock, "CTL queue mutex", NULL, MTX_DEF);
1072                 thr->ctl_softc = softc;
1073                 STAILQ_INIT(&thr->incoming_queue);
1074                 STAILQ_INIT(&thr->rtr_queue);
1075                 STAILQ_INIT(&thr->done_queue);
1076                 STAILQ_INIT(&thr->isc_queue);
1077
1078                 error = kproc_kthread_add(ctl_work_thread, thr,
1079                     &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i);
1080                 if (error != 0) {
1081                         printf("error creating CTL work thread!\n");
1082                         ctl_pool_free(internal_pool);
1083                         ctl_pool_free(emergency_pool);
1084                         ctl_pool_free(other_pool);
1085                         return (error);
1086                 }
1087         }
1088         error = kproc_kthread_add(ctl_lun_thread, softc,
1089             &softc->ctl_proc, NULL, 0, 0, "ctl", "lun");
1090         if (error != 0) {
1091                 printf("error creating CTL lun thread!\n");
1092                 ctl_pool_free(internal_pool);
1093                 ctl_pool_free(emergency_pool);
1094                 ctl_pool_free(other_pool);
1095                 return (error);
1096         }
1097         if (bootverbose)
1098                 printf("ctl: CAM Target Layer loaded\n");
1099
1100         /*
1101          * Initialize the ioctl front end.
1102          */
1103         ctl_frontend_register(&ioctl_frontend);
1104         port = &softc->ioctl_info.port;
1105         port->frontend = &ioctl_frontend;
1106         sprintf(softc->ioctl_info.port_name, "ioctl");
1107         port->port_type = CTL_PORT_IOCTL;
1108         port->num_requested_ctl_io = 100;
1109         port->port_name = softc->ioctl_info.port_name;
1110         port->port_online = ctl_ioctl_online;
1111         port->port_offline = ctl_ioctl_offline;
1112         port->onoff_arg = &softc->ioctl_info;
1113         port->lun_enable = ctl_ioctl_lun_enable;
1114         port->lun_disable = ctl_ioctl_lun_disable;
1115         port->targ_lun_arg = &softc->ioctl_info;
1116         port->fe_datamove = ctl_ioctl_datamove;
1117         port->fe_done = ctl_ioctl_done;
1118         port->max_targets = 15;
1119         port->max_target_id = 15;
1120
1121         if (ctl_port_register(&softc->ioctl_info.port,
1122                           (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
1123                 printf("ctl: ioctl front end registration failed, will "
1124                        "continue anyway\n");
1125         }
1126
1127 #ifdef CTL_IO_DELAY
1128         if (sizeof(struct callout) > CTL_TIMER_BYTES) {
1129                 printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
1130                        sizeof(struct callout), CTL_TIMER_BYTES);
1131                 return (EINVAL);
1132         }
1133 #endif /* CTL_IO_DELAY */
1134
1135         return (0);
1136 }
1137
1138 void
1139 ctl_shutdown(void)
1140 {
1141         struct ctl_softc *softc;
1142         struct ctl_lun *lun, *next_lun;
1143         struct ctl_io_pool *pool;
1144
1145         softc = (struct ctl_softc *)control_softc;
1146
1147         if (ctl_port_deregister(&softc->ioctl_info.port) != 0)
1148                 printf("ctl: ioctl front end deregistration failed\n");
1149
1150         mtx_lock(&softc->ctl_lock);
1151
1152         /*
1153          * Free up each LUN.
1154          */
1155         for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1156                 next_lun = STAILQ_NEXT(lun, links);
1157                 ctl_free_lun(lun);
1158         }
1159
1160         mtx_unlock(&softc->ctl_lock);
1161
1162         ctl_frontend_deregister(&ioctl_frontend);
1163
1164         /*
1165          * This will rip the rug out from under any FETDs or anyone else
1166          * that has a pool allocated.  Since we increment our module
1167          * refcount any time someone outside the main CTL module allocates
1168          * a pool, we shouldn't have any problems here.  The user won't be
1169          * able to unload the CTL module until client modules have
1170          * successfully unloaded.
1171          */
1172         while ((pool = STAILQ_FIRST(&softc->io_pools)) != NULL)
1173                 ctl_pool_free(pool);
1174
1175 #if 0
1176         ctl_shutdown_thread(softc->work_thread);
1177         mtx_destroy(&softc->queue_lock);
1178 #endif
1179
1180         ctl_tpc_shutdown(softc);
1181         mtx_destroy(&softc->pool_lock);
1182         mtx_destroy(&softc->ctl_lock);
1183
1184         destroy_dev(softc->dev);
1185
1186         sysctl_ctx_free(&softc->sysctl_ctx);
1187
1188         free(control_softc, M_DEVBUF);
1189         control_softc = NULL;
1190
1191         if (bootverbose)
1192                 printf("ctl: CAM Target Layer unloaded\n");
1193 }
1194
1195 static int
1196 ctl_module_event_handler(module_t mod, int what, void *arg)
1197 {
1198
1199         switch (what) {
1200         case MOD_LOAD:
1201                 return (ctl_init());
1202         case MOD_UNLOAD:
1203                 return (EBUSY);
1204         default:
1205                 return (EOPNOTSUPP);
1206         }
1207 }
1208
1209 /*
1210  * XXX KDM should we do some access checks here?  Bump a reference count to
1211  * prevent a CTL module from being unloaded while someone has it open?
1212  */
1213 static int
1214 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1215 {
1216         return (0);
1217 }
1218
1219 static int
1220 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1221 {
1222         return (0);
1223 }
1224
1225 int
1226 ctl_port_enable(ctl_port_type port_type)
1227 {
1228         struct ctl_softc *softc;
1229         struct ctl_port *port;
1230
1231         if (ctl_is_single == 0) {
1232                 union ctl_ha_msg msg_info;
1233                 int isc_retval;
1234
1235 #if 0
1236                 printf("%s: HA mode, synchronizing frontend enable\n",
1237                         __func__);
1238 #endif
1239                 msg_info.hdr.msg_type = CTL_MSG_SYNC_FE;
1240                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1241                         sizeof(msg_info), 1 )) > CTL_HA_STATUS_SUCCESS) {
1242                         printf("Sync msg send error retval %d\n", isc_retval);
1243                 }
1244                 if (!rcv_sync_msg) {
1245                         isc_retval=ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
1246                                 sizeof(msg_info), 1);
1247                 }
1248 #if 0
1249                 printf("CTL:Frontend Enable\n");
1250         } else {
1251                 printf("%s: single mode, skipping frontend synchronization\n",
1252                         __func__);
1253 #endif
1254         }
1255
1256         softc = control_softc;
1257
1258         STAILQ_FOREACH(port, &softc->port_list, links) {
1259                 if (port_type & port->port_type)
1260                 {
1261 #if 0
1262                         printf("port %d\n", port->targ_port);
1263 #endif
1264                         ctl_port_online(port);
1265                 }
1266         }
1267
1268         return (0);
1269 }
1270
1271 int
1272 ctl_port_disable(ctl_port_type port_type)
1273 {
1274         struct ctl_softc *softc;
1275         struct ctl_port *port;
1276
1277         softc = control_softc;
1278
1279         STAILQ_FOREACH(port, &softc->port_list, links) {
1280                 if (port_type & port->port_type)
1281                         ctl_port_offline(port);
1282         }
1283
1284         return (0);
1285 }
1286
1287 /*
1288  * Returns 0 for success, 1 for failure.
1289  * Currently the only failure mode is if there aren't enough entries
1290  * allocated.  So, in case of a failure, look at num_entries_dropped,
1291  * reallocate and try again.
1292  */
1293 int
1294 ctl_port_list(struct ctl_port_entry *entries, int num_entries_alloced,
1295               int *num_entries_filled, int *num_entries_dropped,
1296               ctl_port_type port_type, int no_virtual)
1297 {
1298         struct ctl_softc *softc;
1299         struct ctl_port *port;
1300         int entries_dropped, entries_filled;
1301         int retval;
1302         int i;
1303
1304         softc = control_softc;
1305
1306         retval = 0;
1307         entries_filled = 0;
1308         entries_dropped = 0;
1309
1310         i = 0;
1311         mtx_lock(&softc->ctl_lock);
1312         STAILQ_FOREACH(port, &softc->port_list, links) {
1313                 struct ctl_port_entry *entry;
1314
1315                 if ((port->port_type & port_type) == 0)
1316                         continue;
1317
1318                 if ((no_virtual != 0)
1319                  && (port->virtual_port != 0))
1320                         continue;
1321
1322                 if (entries_filled >= num_entries_alloced) {
1323                         entries_dropped++;
1324                         continue;
1325                 }
1326                 entry = &entries[i];
1327
1328                 entry->port_type = port->port_type;
1329                 strlcpy(entry->port_name, port->port_name,
1330                         sizeof(entry->port_name));
1331                 entry->physical_port = port->physical_port;
1332                 entry->virtual_port = port->virtual_port;
1333                 entry->wwnn = port->wwnn;
1334                 entry->wwpn = port->wwpn;
1335
1336                 i++;
1337                 entries_filled++;
1338         }
1339
1340         mtx_unlock(&softc->ctl_lock);
1341
1342         if (entries_dropped > 0)
1343                 retval = 1;
1344
1345         *num_entries_dropped = entries_dropped;
1346         *num_entries_filled = entries_filled;
1347
1348         return (retval);
1349 }
1350
1351 static void
1352 ctl_ioctl_online(void *arg)
1353 {
1354         struct ctl_ioctl_info *ioctl_info;
1355
1356         ioctl_info = (struct ctl_ioctl_info *)arg;
1357
1358         ioctl_info->flags |= CTL_IOCTL_FLAG_ENABLED;
1359 }
1360
1361 static void
1362 ctl_ioctl_offline(void *arg)
1363 {
1364         struct ctl_ioctl_info *ioctl_info;
1365
1366         ioctl_info = (struct ctl_ioctl_info *)arg;
1367
1368         ioctl_info->flags &= ~CTL_IOCTL_FLAG_ENABLED;
1369 }
1370
1371 /*
1372  * Remove an initiator by port number and initiator ID.
1373  * Returns 0 for success, -1 for failure.
1374  */
1375 int
1376 ctl_remove_initiator(struct ctl_port *port, int iid)
1377 {
1378         struct ctl_softc *softc = control_softc;
1379
1380         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1381
1382         if (iid > CTL_MAX_INIT_PER_PORT) {
1383                 printf("%s: initiator ID %u > maximun %u!\n",
1384                        __func__, iid, CTL_MAX_INIT_PER_PORT);
1385                 return (-1);
1386         }
1387
1388         mtx_lock(&softc->ctl_lock);
1389         port->wwpn_iid[iid].in_use--;
1390         port->wwpn_iid[iid].last_use = time_uptime;
1391         mtx_unlock(&softc->ctl_lock);
1392
1393         return (0);
1394 }
1395
1396 /*
1397  * Add an initiator to the initiator map.
1398  * Returns iid for success, < 0 for failure.
1399  */
1400 int
1401 ctl_add_initiator(struct ctl_port *port, int iid, uint64_t wwpn, char *name)
1402 {
1403         struct ctl_softc *softc = control_softc;
1404         time_t best_time;
1405         int i, best;
1406
1407         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1408
1409         if (iid >= CTL_MAX_INIT_PER_PORT) {
1410                 printf("%s: WWPN %#jx initiator ID %u > maximum %u!\n",
1411                        __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1412                 free(name, M_CTL);
1413                 return (-1);
1414         }
1415
1416         mtx_lock(&softc->ctl_lock);
1417
1418         if (iid < 0 && (wwpn != 0 || name != NULL)) {
1419                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1420                         if (wwpn != 0 && wwpn == port->wwpn_iid[i].wwpn) {
1421                                 iid = i;
1422                                 break;
1423                         }
1424                         if (name != NULL && port->wwpn_iid[i].name != NULL &&
1425                             strcmp(name, port->wwpn_iid[i].name) == 0) {
1426                                 iid = i;
1427                                 break;
1428                         }
1429                 }
1430         }
1431
1432         if (iid < 0) {
1433                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1434                         if (port->wwpn_iid[i].in_use == 0 &&
1435                             port->wwpn_iid[i].wwpn == 0 &&
1436                             port->wwpn_iid[i].name == NULL) {
1437                                 iid = i;
1438                                 break;
1439                         }
1440                 }
1441         }
1442
1443         if (iid < 0) {
1444                 best = -1;
1445                 best_time = INT32_MAX;
1446                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1447                         if (port->wwpn_iid[i].in_use == 0) {
1448                                 if (port->wwpn_iid[i].last_use < best_time) {
1449                                         best = i;
1450                                         best_time = port->wwpn_iid[i].last_use;
1451                                 }
1452                         }
1453                 }
1454                 iid = best;
1455         }
1456
1457         if (iid < 0) {
1458                 mtx_unlock(&softc->ctl_lock);
1459                 free(name, M_CTL);
1460                 return (-2);
1461         }
1462
1463         if (port->wwpn_iid[iid].in_use > 0 && (wwpn != 0 || name != NULL)) {
1464                 /*
1465                  * This is not an error yet.
1466                  */
1467                 if (wwpn != 0 && wwpn == port->wwpn_iid[iid].wwpn) {
1468 #if 0
1469                         printf("%s: port %d iid %u WWPN %#jx arrived"
1470                             " again\n", __func__, port->targ_port,
1471                             iid, (uintmax_t)wwpn);
1472 #endif
1473                         goto take;
1474                 }
1475                 if (name != NULL && port->wwpn_iid[iid].name != NULL &&
1476                     strcmp(name, port->wwpn_iid[iid].name) == 0) {
1477 #if 0
1478                         printf("%s: port %d iid %u name '%s' arrived"
1479                             " again\n", __func__, port->targ_port,
1480                             iid, name);
1481 #endif
1482                         goto take;
1483                 }
1484
1485                 /*
1486                  * This is an error, but what do we do about it?  The
1487                  * driver is telling us we have a new WWPN for this
1488                  * initiator ID, so we pretty much need to use it.
1489                  */
1490                 printf("%s: port %d iid %u WWPN %#jx '%s' arrived,"
1491                     " but WWPN %#jx '%s' is still at that address\n",
1492                     __func__, port->targ_port, iid, wwpn, name,
1493                     (uintmax_t)port->wwpn_iid[iid].wwpn,
1494                     port->wwpn_iid[iid].name);
1495
1496                 /*
1497                  * XXX KDM clear have_ca and ua_pending on each LUN for
1498                  * this initiator.
1499                  */
1500         }
1501 take:
1502         free(port->wwpn_iid[iid].name, M_CTL);
1503         port->wwpn_iid[iid].name = name;
1504         port->wwpn_iid[iid].wwpn = wwpn;
1505         port->wwpn_iid[iid].in_use++;
1506         mtx_unlock(&softc->ctl_lock);
1507
1508         return (iid);
1509 }
1510
1511 static int
1512 ctl_create_iid(struct ctl_port *port, int iid, uint8_t *buf)
1513 {
1514         int len;
1515
1516         switch (port->port_type) {
1517         case CTL_PORT_FC:
1518         {
1519                 struct scsi_transportid_fcp *id =
1520                     (struct scsi_transportid_fcp *)buf;
1521                 if (port->wwpn_iid[iid].wwpn == 0)
1522                         return (0);
1523                 memset(id, 0, sizeof(*id));
1524                 id->format_protocol = SCSI_PROTO_FC;
1525                 scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->n_port_name);
1526                 return (sizeof(*id));
1527         }
1528         case CTL_PORT_ISCSI:
1529         {
1530                 struct scsi_transportid_iscsi_port *id =
1531                     (struct scsi_transportid_iscsi_port *)buf;
1532                 if (port->wwpn_iid[iid].name == NULL)
1533                         return (0);
1534                 memset(id, 0, 256);
1535                 id->format_protocol = SCSI_TRN_ISCSI_FORMAT_PORT |
1536                     SCSI_PROTO_ISCSI;
1537                 len = strlcpy(id->iscsi_name, port->wwpn_iid[iid].name, 252) + 1;
1538                 len = roundup2(min(len, 252), 4);
1539                 scsi_ulto2b(len, id->additional_length);
1540                 return (sizeof(*id) + len);
1541         }
1542         case CTL_PORT_SAS:
1543         {
1544                 struct scsi_transportid_sas *id =
1545                     (struct scsi_transportid_sas *)buf;
1546                 if (port->wwpn_iid[iid].wwpn == 0)
1547                         return (0);
1548                 memset(id, 0, sizeof(*id));
1549                 id->format_protocol = SCSI_PROTO_SAS;
1550                 scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->sas_address);
1551                 return (sizeof(*id));
1552         }
1553         default:
1554         {
1555                 struct scsi_transportid_spi *id =
1556                     (struct scsi_transportid_spi *)buf;
1557                 memset(id, 0, sizeof(*id));
1558                 id->format_protocol = SCSI_PROTO_SPI;
1559                 scsi_ulto2b(iid, id->scsi_addr);
1560                 scsi_ulto2b(port->targ_port, id->rel_trgt_port_id);
1561                 return (sizeof(*id));
1562         }
1563         }
1564 }
1565
1566 static int
1567 ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
1568 {
1569         return (0);
1570 }
1571
1572 static int
1573 ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
1574 {
1575         return (0);
1576 }
1577
1578 /*
1579  * Data movement routine for the CTL ioctl frontend port.
1580  */
1581 static int
1582 ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
1583 {
1584         struct ctl_sg_entry *ext_sglist, *kern_sglist;
1585         struct ctl_sg_entry ext_entry, kern_entry;
1586         int ext_sglen, ext_sg_entries, kern_sg_entries;
1587         int ext_sg_start, ext_offset;
1588         int len_to_copy, len_copied;
1589         int kern_watermark, ext_watermark;
1590         int ext_sglist_malloced;
1591         int i, j;
1592
1593         ext_sglist_malloced = 0;
1594         ext_sg_start = 0;
1595         ext_offset = 0;
1596
1597         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove\n"));
1598
1599         /*
1600          * If this flag is set, fake the data transfer.
1601          */
1602         if (ctsio->io_hdr.flags & CTL_FLAG_NO_DATAMOVE) {
1603                 ctsio->ext_data_filled = ctsio->ext_data_len;
1604                 goto bailout;
1605         }
1606
1607         /*
1608          * To simplify things here, if we have a single buffer, stick it in
1609          * a S/G entry and just make it a single entry S/G list.
1610          */
1611         if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
1612                 int len_seen;
1613
1614                 ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
1615
1616                 ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
1617                                                            M_WAITOK);
1618                 ext_sglist_malloced = 1;
1619                 if (copyin(ctsio->ext_data_ptr, ext_sglist,
1620                                    ext_sglen) != 0) {
1621                         ctl_set_internal_failure(ctsio,
1622                                                  /*sks_valid*/ 0,
1623                                                  /*retry_count*/ 0);
1624                         goto bailout;
1625                 }
1626                 ext_sg_entries = ctsio->ext_sg_entries;
1627                 len_seen = 0;
1628                 for (i = 0; i < ext_sg_entries; i++) {
1629                         if ((len_seen + ext_sglist[i].len) >=
1630                              ctsio->ext_data_filled) {
1631                                 ext_sg_start = i;
1632                                 ext_offset = ctsio->ext_data_filled - len_seen;
1633                                 break;
1634                         }
1635                         len_seen += ext_sglist[i].len;
1636                 }
1637         } else {
1638                 ext_sglist = &ext_entry;
1639                 ext_sglist->addr = ctsio->ext_data_ptr;
1640                 ext_sglist->len = ctsio->ext_data_len;
1641                 ext_sg_entries = 1;
1642                 ext_sg_start = 0;
1643                 ext_offset = ctsio->ext_data_filled;
1644         }
1645
1646         if (ctsio->kern_sg_entries > 0) {
1647                 kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
1648                 kern_sg_entries = ctsio->kern_sg_entries;
1649         } else {
1650                 kern_sglist = &kern_entry;
1651                 kern_sglist->addr = ctsio->kern_data_ptr;
1652                 kern_sglist->len = ctsio->kern_data_len;
1653                 kern_sg_entries = 1;
1654         }
1655
1656
1657         kern_watermark = 0;
1658         ext_watermark = ext_offset;
1659         len_copied = 0;
1660         for (i = ext_sg_start, j = 0;
1661              i < ext_sg_entries && j < kern_sg_entries;) {
1662                 uint8_t *ext_ptr, *kern_ptr;
1663
1664                 len_to_copy = ctl_min(ext_sglist[i].len - ext_watermark,
1665                                       kern_sglist[j].len - kern_watermark);
1666
1667                 ext_ptr = (uint8_t *)ext_sglist[i].addr;
1668                 ext_ptr = ext_ptr + ext_watermark;
1669                 if (ctsio->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
1670                         /*
1671                          * XXX KDM fix this!
1672                          */
1673                         panic("need to implement bus address support");
1674 #if 0
1675                         kern_ptr = bus_to_virt(kern_sglist[j].addr);
1676 #endif
1677                 } else
1678                         kern_ptr = (uint8_t *)kern_sglist[j].addr;
1679                 kern_ptr = kern_ptr + kern_watermark;
1680
1681                 kern_watermark += len_to_copy;
1682                 ext_watermark += len_to_copy;
1683
1684                 if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
1685                      CTL_FLAG_DATA_IN) {
1686                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1687                                          "bytes to user\n", len_to_copy));
1688                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1689                                          "to %p\n", kern_ptr, ext_ptr));
1690                         if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
1691                                 ctl_set_internal_failure(ctsio,
1692                                                          /*sks_valid*/ 0,
1693                                                          /*retry_count*/ 0);
1694                                 goto bailout;
1695                         }
1696                 } else {
1697                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1698                                          "bytes from user\n", len_to_copy));
1699                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1700                                          "to %p\n", ext_ptr, kern_ptr));
1701                         if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
1702                                 ctl_set_internal_failure(ctsio,
1703                                                          /*sks_valid*/ 0,
1704                                                          /*retry_count*/0);
1705                                 goto bailout;
1706                         }
1707                 }
1708
1709                 len_copied += len_to_copy;
1710
1711                 if (ext_sglist[i].len == ext_watermark) {
1712                         i++;
1713                         ext_watermark = 0;
1714                 }
1715
1716                 if (kern_sglist[j].len == kern_watermark) {
1717                         j++;
1718                         kern_watermark = 0;
1719                 }
1720         }
1721
1722         ctsio->ext_data_filled += len_copied;
1723
1724         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_sg_entries: %d, "
1725                          "kern_sg_entries: %d\n", ext_sg_entries,
1726                          kern_sg_entries));
1727         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_data_len = %d, "
1728                          "kern_data_len = %d\n", ctsio->ext_data_len,
1729                          ctsio->kern_data_len));
1730
1731
1732         /* XXX KDM set residual?? */
1733 bailout:
1734
1735         if (ext_sglist_malloced != 0)
1736                 free(ext_sglist, M_CTL);
1737
1738         return (CTL_RETVAL_COMPLETE);
1739 }
1740
1741 /*
1742  * Serialize a command that went down the "wrong" side, and so was sent to
1743  * this controller for execution.  The logic is a little different than the
1744  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1745  * sent back to the other side, but in the success case, we execute the
1746  * command on this side (XFER mode) or tell the other side to execute it
1747  * (SER_ONLY mode).
1748  */
1749 static int
1750 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
1751 {
1752         struct ctl_softc *ctl_softc;
1753         union ctl_ha_msg msg_info;
1754         struct ctl_lun *lun;
1755         int retval = 0;
1756         uint32_t targ_lun;
1757
1758         ctl_softc = control_softc;
1759
1760         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
1761         lun = ctl_softc->ctl_luns[targ_lun];
1762         if (lun==NULL)
1763         {
1764                 /*
1765                  * Why isn't LUN defined? The other side wouldn't
1766                  * send a cmd if the LUN is undefined.
1767                  */
1768                 printf("%s: Bad JUJU!, LUN is NULL!\n", __func__);
1769
1770                 /* "Logical unit not supported" */
1771                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1772                                    lun,
1773                                    /*sense_format*/SSD_TYPE_NONE,
1774                                    /*current_error*/ 1,
1775                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1776                                    /*asc*/ 0x25,
1777                                    /*ascq*/ 0x00,
1778                                    SSD_ELEM_NONE);
1779
1780                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1781                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1782                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1783                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1784                 msg_info.hdr.serializing_sc = NULL;
1785                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1786                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1787                                 sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1788                 }
1789                 return(1);
1790
1791         }
1792
1793         mtx_lock(&lun->lun_lock);
1794         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1795
1796         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1797                 (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1798                  ooa_links))) {
1799         case CTL_ACTION_BLOCK:
1800                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
1801                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
1802                                   blocked_links);
1803                 break;
1804         case CTL_ACTION_PASS:
1805         case CTL_ACTION_SKIP:
1806                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
1807                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
1808                         ctl_enqueue_rtr((union ctl_io *)ctsio);
1809                 } else {
1810
1811                         /* send msg back to other side */
1812                         msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1813                         msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
1814                         msg_info.hdr.msg_type = CTL_MSG_R2R;
1815 #if 0
1816                         printf("2. pOrig %x\n", (int)msg_info.hdr.original_sc);
1817 #endif
1818                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1819                             sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1820                         }
1821                 }
1822                 break;
1823         case CTL_ACTION_OVERLAP:
1824                 /* OVERLAPPED COMMANDS ATTEMPTED */
1825                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1826                                    lun,
1827                                    /*sense_format*/SSD_TYPE_NONE,
1828                                    /*current_error*/ 1,
1829                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1830                                    /*asc*/ 0x4E,
1831                                    /*ascq*/ 0x00,
1832                                    SSD_ELEM_NONE);
1833
1834                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1835                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1836                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1837                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1838                 msg_info.hdr.serializing_sc = NULL;
1839                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1840 #if 0
1841                 printf("BAD JUJU:Major Bummer Overlap\n");
1842 #endif
1843                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1844                 retval = 1;
1845                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1846                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1847                 }
1848                 break;
1849         case CTL_ACTION_OVERLAP_TAG:
1850                 /* TAGGED OVERLAPPED COMMANDS (NN = QUEUE TAG) */
1851                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1852                                    lun,
1853                                    /*sense_format*/SSD_TYPE_NONE,
1854                                    /*current_error*/ 1,
1855                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1856                                    /*asc*/ 0x4D,
1857                                    /*ascq*/ ctsio->tag_num & 0xff,
1858                                    SSD_ELEM_NONE);
1859
1860                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1861                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1862                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1863                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1864                 msg_info.hdr.serializing_sc = NULL;
1865                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1866 #if 0
1867                 printf("BAD JUJU:Major Bummer Overlap Tag\n");
1868 #endif
1869                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1870                 retval = 1;
1871                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1872                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1873                 }
1874                 break;
1875         case CTL_ACTION_ERROR:
1876         default:
1877                 /* "Internal target failure" */
1878                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1879                                    lun,
1880                                    /*sense_format*/SSD_TYPE_NONE,
1881                                    /*current_error*/ 1,
1882                                    /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
1883                                    /*asc*/ 0x44,
1884                                    /*ascq*/ 0x00,
1885                                    SSD_ELEM_NONE);
1886
1887                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1888                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1889                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1890                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1891                 msg_info.hdr.serializing_sc = NULL;
1892                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1893 #if 0
1894                 printf("BAD JUJU:Major Bummer HW Error\n");
1895 #endif
1896                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1897                 retval = 1;
1898                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1899                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1900                 }
1901                 break;
1902         }
1903         mtx_unlock(&lun->lun_lock);
1904         return (retval);
1905 }
1906
1907 static int
1908 ctl_ioctl_submit_wait(union ctl_io *io)
1909 {
1910         struct ctl_fe_ioctl_params params;
1911         ctl_fe_ioctl_state last_state;
1912         int done, retval;
1913
1914         retval = 0;
1915
1916         bzero(&params, sizeof(params));
1917
1918         mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
1919         cv_init(&params.sem, "ctlioccv");
1920         params.state = CTL_IOCTL_INPROG;
1921         last_state = params.state;
1922
1923         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = &params;
1924
1925         CTL_DEBUG_PRINT(("ctl_ioctl_submit_wait\n"));
1926
1927         /* This shouldn't happen */
1928         if ((retval = ctl_queue(io)) != CTL_RETVAL_COMPLETE)
1929                 return (retval);
1930
1931         done = 0;
1932
1933         do {
1934                 mtx_lock(&params.ioctl_mtx);
1935                 /*
1936                  * Check the state here, and don't sleep if the state has
1937                  * already changed (i.e. wakeup has already occured, but we
1938                  * weren't waiting yet).
1939                  */
1940                 if (params.state == last_state) {
1941                         /* XXX KDM cv_wait_sig instead? */
1942                         cv_wait(&params.sem, &params.ioctl_mtx);
1943                 }
1944                 last_state = params.state;
1945
1946                 switch (params.state) {
1947                 case CTL_IOCTL_INPROG:
1948                         /* Why did we wake up? */
1949                         /* XXX KDM error here? */
1950                         mtx_unlock(&params.ioctl_mtx);
1951                         break;
1952                 case CTL_IOCTL_DATAMOVE:
1953                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DATAMOVE\n"));
1954
1955                         /*
1956                          * change last_state back to INPROG to avoid
1957                          * deadlock on subsequent data moves.
1958                          */
1959                         params.state = last_state = CTL_IOCTL_INPROG;
1960
1961                         mtx_unlock(&params.ioctl_mtx);
1962                         ctl_ioctl_do_datamove(&io->scsiio);
1963                         /*
1964                          * Note that in some cases, most notably writes,
1965                          * this will queue the I/O and call us back later.
1966                          * In other cases, generally reads, this routine
1967                          * will immediately call back and wake us up,
1968                          * probably using our own context.
1969                          */
1970                         io->scsiio.be_move_done(io);
1971                         break;
1972                 case CTL_IOCTL_DONE:
1973                         mtx_unlock(&params.ioctl_mtx);
1974                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DONE\n"));
1975                         done = 1;
1976                         break;
1977                 default:
1978                         mtx_unlock(&params.ioctl_mtx);
1979                         /* XXX KDM error here? */
1980                         break;
1981                 }
1982         } while (done == 0);
1983
1984         mtx_destroy(&params.ioctl_mtx);
1985         cv_destroy(&params.sem);
1986
1987         return (CTL_RETVAL_COMPLETE);
1988 }
1989
1990 static void
1991 ctl_ioctl_datamove(union ctl_io *io)
1992 {
1993         struct ctl_fe_ioctl_params *params;
1994
1995         params = (struct ctl_fe_ioctl_params *)
1996                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1997
1998         mtx_lock(&params->ioctl_mtx);
1999         params->state = CTL_IOCTL_DATAMOVE;
2000         cv_broadcast(&params->sem);
2001         mtx_unlock(&params->ioctl_mtx);
2002 }
2003
2004 static void
2005 ctl_ioctl_done(union ctl_io *io)
2006 {
2007         struct ctl_fe_ioctl_params *params;
2008
2009         params = (struct ctl_fe_ioctl_params *)
2010                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2011
2012         mtx_lock(&params->ioctl_mtx);
2013         params->state = CTL_IOCTL_DONE;
2014         cv_broadcast(&params->sem);
2015         mtx_unlock(&params->ioctl_mtx);
2016 }
2017
2018 static void
2019 ctl_ioctl_hard_startstop_callback(void *arg, struct cfi_metatask *metatask)
2020 {
2021         struct ctl_fe_ioctl_startstop_info *sd_info;
2022
2023         sd_info = (struct ctl_fe_ioctl_startstop_info *)arg;
2024
2025         sd_info->hs_info.status = metatask->status;
2026         sd_info->hs_info.total_luns = metatask->taskinfo.startstop.total_luns;
2027         sd_info->hs_info.luns_complete =
2028                 metatask->taskinfo.startstop.luns_complete;
2029         sd_info->hs_info.luns_failed = metatask->taskinfo.startstop.luns_failed;
2030
2031         cv_broadcast(&sd_info->sem);
2032 }
2033
2034 static void
2035 ctl_ioctl_bbrread_callback(void *arg, struct cfi_metatask *metatask)
2036 {
2037         struct ctl_fe_ioctl_bbrread_info *fe_bbr_info;
2038
2039         fe_bbr_info = (struct ctl_fe_ioctl_bbrread_info *)arg;
2040
2041         mtx_lock(fe_bbr_info->lock);
2042         fe_bbr_info->bbr_info->status = metatask->status;
2043         fe_bbr_info->bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2044         fe_bbr_info->wakeup_done = 1;
2045         mtx_unlock(fe_bbr_info->lock);
2046
2047         cv_broadcast(&fe_bbr_info->sem);
2048 }
2049
2050 /*
2051  * Returns 0 for success, errno for failure.
2052  */
2053 static int
2054 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
2055                    struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
2056 {
2057         union ctl_io *io;
2058         int retval;
2059
2060         retval = 0;
2061
2062         mtx_lock(&lun->lun_lock);
2063         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
2064              (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2065              ooa_links)) {
2066                 struct ctl_ooa_entry *entry;
2067
2068                 /*
2069                  * If we've got more than we can fit, just count the
2070                  * remaining entries.
2071                  */
2072                 if (*cur_fill_num >= ooa_hdr->alloc_num)
2073                         continue;
2074
2075                 entry = &kern_entries[*cur_fill_num];
2076
2077                 entry->tag_num = io->scsiio.tag_num;
2078                 entry->lun_num = lun->lun;
2079 #ifdef CTL_TIME_IO
2080                 entry->start_bt = io->io_hdr.start_bt;
2081 #endif
2082                 bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
2083                 entry->cdb_len = io->scsiio.cdb_len;
2084                 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
2085                         entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
2086
2087                 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
2088                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
2089
2090                 if (io->io_hdr.flags & CTL_FLAG_ABORT)
2091                         entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
2092
2093                 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
2094                         entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
2095
2096                 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
2097                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
2098         }
2099         mtx_unlock(&lun->lun_lock);
2100
2101         return (retval);
2102 }
2103
2104 static void *
2105 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2106                  size_t error_str_len)
2107 {
2108         void *kptr;
2109
2110         kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2111
2112         if (copyin(user_addr, kptr, len) != 0) {
2113                 snprintf(error_str, error_str_len, "Error copying %d bytes "
2114                          "from user address %p to kernel address %p", len,
2115                          user_addr, kptr);
2116                 free(kptr, M_CTL);
2117                 return (NULL);
2118         }
2119
2120         return (kptr);
2121 }
2122
2123 static void
2124 ctl_free_args(int num_args, struct ctl_be_arg *args)
2125 {
2126         int i;
2127
2128         if (args == NULL)
2129                 return;
2130
2131         for (i = 0; i < num_args; i++) {
2132                 free(args[i].kname, M_CTL);
2133                 free(args[i].kvalue, M_CTL);
2134         }
2135
2136         free(args, M_CTL);
2137 }
2138
2139 static struct ctl_be_arg *
2140 ctl_copyin_args(int num_args, struct ctl_be_arg *uargs,
2141                 char *error_str, size_t error_str_len)
2142 {
2143         struct ctl_be_arg *args;
2144         int i;
2145
2146         args = ctl_copyin_alloc(uargs, num_args * sizeof(*args),
2147                                 error_str, error_str_len);
2148
2149         if (args == NULL)
2150                 goto bailout;
2151
2152         for (i = 0; i < num_args; i++) {
2153                 args[i].kname = NULL;
2154                 args[i].kvalue = NULL;
2155         }
2156
2157         for (i = 0; i < num_args; i++) {
2158                 uint8_t *tmpptr;
2159
2160                 args[i].kname = ctl_copyin_alloc(args[i].name,
2161                         args[i].namelen, error_str, error_str_len);
2162                 if (args[i].kname == NULL)
2163                         goto bailout;
2164
2165                 if (args[i].kname[args[i].namelen - 1] != '\0') {
2166                         snprintf(error_str, error_str_len, "Argument %d "
2167                                  "name is not NUL-terminated", i);
2168                         goto bailout;
2169                 }
2170
2171                 if (args[i].flags & CTL_BEARG_RD) {
2172                         tmpptr = ctl_copyin_alloc(args[i].value,
2173                                 args[i].vallen, error_str, error_str_len);
2174                         if (tmpptr == NULL)
2175                                 goto bailout;
2176                         if ((args[i].flags & CTL_BEARG_ASCII)
2177                          && (tmpptr[args[i].vallen - 1] != '\0')) {
2178                                 snprintf(error_str, error_str_len, "Argument "
2179                                     "%d value is not NUL-terminated", i);
2180                                 goto bailout;
2181                         }
2182                         args[i].kvalue = tmpptr;
2183                 } else {
2184                         args[i].kvalue = malloc(args[i].vallen,
2185                             M_CTL, M_WAITOK | M_ZERO);
2186                 }
2187         }
2188
2189         return (args);
2190 bailout:
2191
2192         ctl_free_args(num_args, args);
2193
2194         return (NULL);
2195 }
2196
2197 static void
2198 ctl_copyout_args(int num_args, struct ctl_be_arg *args)
2199 {
2200         int i;
2201
2202         for (i = 0; i < num_args; i++) {
2203                 if (args[i].flags & CTL_BEARG_WR)
2204                         copyout(args[i].kvalue, args[i].value, args[i].vallen);
2205         }
2206 }
2207
2208 /*
2209  * Escape characters that are illegal or not recommended in XML.
2210  */
2211 int
2212 ctl_sbuf_printf_esc(struct sbuf *sb, char *str)
2213 {
2214         int retval;
2215
2216         retval = 0;
2217
2218         for (; *str; str++) {
2219                 switch (*str) {
2220                 case '&':
2221                         retval = sbuf_printf(sb, "&amp;");
2222                         break;
2223                 case '>':
2224                         retval = sbuf_printf(sb, "&gt;");
2225                         break;
2226                 case '<':
2227                         retval = sbuf_printf(sb, "&lt;");
2228                         break;
2229                 default:
2230                         retval = sbuf_putc(sb, *str);
2231                         break;
2232                 }
2233
2234                 if (retval != 0)
2235                         break;
2236
2237         }
2238
2239         return (retval);
2240 }
2241
2242 static void
2243 ctl_id_sbuf(struct ctl_devid *id, struct sbuf *sb)
2244 {
2245         struct scsi_vpd_id_descriptor *desc;
2246         int i;
2247
2248         if (id == NULL || id->len < 4)
2249                 return;
2250         desc = (struct scsi_vpd_id_descriptor *)id->data;
2251         switch (desc->id_type & SVPD_ID_TYPE_MASK) {
2252         case SVPD_ID_TYPE_T10:
2253                 sbuf_printf(sb, "t10.");
2254                 break;
2255         case SVPD_ID_TYPE_EUI64:
2256                 sbuf_printf(sb, "eui.");
2257                 break;
2258         case SVPD_ID_TYPE_NAA:
2259                 sbuf_printf(sb, "naa.");
2260                 break;
2261         case SVPD_ID_TYPE_SCSI_NAME:
2262                 break;
2263         }
2264         switch (desc->proto_codeset & SVPD_ID_CODESET_MASK) {
2265         case SVPD_ID_CODESET_BINARY:
2266                 for (i = 0; i < desc->length; i++)
2267                         sbuf_printf(sb, "%02x", desc->identifier[i]);
2268                 break;
2269         case SVPD_ID_CODESET_ASCII:
2270                 sbuf_printf(sb, "%.*s", (int)desc->length,
2271                     (char *)desc->identifier);
2272                 break;
2273         case SVPD_ID_CODESET_UTF8:
2274                 sbuf_printf(sb, "%s", (char *)desc->identifier);
2275                 break;
2276         }
2277 }
2278
2279 static int
2280 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2281           struct thread *td)
2282 {
2283         struct ctl_softc *softc;
2284         int retval;
2285
2286         softc = control_softc;
2287
2288         retval = 0;
2289
2290         switch (cmd) {
2291         case CTL_IO: {
2292                 union ctl_io *io;
2293                 void *pool_tmp;
2294
2295                 /*
2296                  * If we haven't been "enabled", don't allow any SCSI I/O
2297                  * to this FETD.
2298                  */
2299                 if ((softc->ioctl_info.flags & CTL_IOCTL_FLAG_ENABLED) == 0) {
2300                         retval = EPERM;
2301                         break;
2302                 }
2303
2304                 io = ctl_alloc_io(softc->ioctl_info.port.ctl_pool_ref);
2305                 if (io == NULL) {
2306                         printf("ctl_ioctl: can't allocate ctl_io!\n");
2307                         retval = ENOSPC;
2308                         break;
2309                 }
2310
2311                 /*
2312                  * Need to save the pool reference so it doesn't get
2313                  * spammed by the user's ctl_io.
2314                  */
2315                 pool_tmp = io->io_hdr.pool;
2316
2317                 memcpy(io, (void *)addr, sizeof(*io));
2318
2319                 io->io_hdr.pool = pool_tmp;
2320                 /*
2321                  * No status yet, so make sure the status is set properly.
2322                  */
2323                 io->io_hdr.status = CTL_STATUS_NONE;
2324
2325                 /*
2326                  * The user sets the initiator ID, target and LUN IDs.
2327                  */
2328                 io->io_hdr.nexus.targ_port = softc->ioctl_info.port.targ_port;
2329                 io->io_hdr.flags |= CTL_FLAG_USER_REQ;
2330                 if ((io->io_hdr.io_type == CTL_IO_SCSI)
2331                  && (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
2332                         io->scsiio.tag_num = softc->ioctl_info.cur_tag_num++;
2333
2334                 retval = ctl_ioctl_submit_wait(io);
2335
2336                 if (retval != 0) {
2337                         ctl_free_io(io);
2338                         break;
2339                 }
2340
2341                 memcpy((void *)addr, io, sizeof(*io));
2342
2343                 /* return this to our pool */
2344                 ctl_free_io(io);
2345
2346                 break;
2347         }
2348         case CTL_ENABLE_PORT:
2349         case CTL_DISABLE_PORT:
2350         case CTL_SET_PORT_WWNS: {
2351                 struct ctl_port *port;
2352                 struct ctl_port_entry *entry;
2353
2354                 entry = (struct ctl_port_entry *)addr;
2355
2356                 mtx_lock(&softc->ctl_lock);
2357                 STAILQ_FOREACH(port, &softc->port_list, links) {
2358                         int action, done;
2359
2360                         action = 0;
2361                         done = 0;
2362
2363                         if ((entry->port_type == CTL_PORT_NONE)
2364                          && (entry->targ_port == port->targ_port)) {
2365                                 /*
2366                                  * If the user only wants to enable or
2367                                  * disable or set WWNs on a specific port,
2368                                  * do the operation and we're done.
2369                                  */
2370                                 action = 1;
2371                                 done = 1;
2372                         } else if (entry->port_type & port->port_type) {
2373                                 /*
2374                                  * Compare the user's type mask with the
2375                                  * particular frontend type to see if we
2376                                  * have a match.
2377                                  */
2378                                 action = 1;
2379                                 done = 0;
2380
2381                                 /*
2382                                  * Make sure the user isn't trying to set
2383                                  * WWNs on multiple ports at the same time.
2384                                  */
2385                                 if (cmd == CTL_SET_PORT_WWNS) {
2386                                         printf("%s: Can't set WWNs on "
2387                                                "multiple ports\n", __func__);
2388                                         retval = EINVAL;
2389                                         break;
2390                                 }
2391                         }
2392                         if (action != 0) {
2393                                 /*
2394                                  * XXX KDM we have to drop the lock here,
2395                                  * because the online/offline operations
2396                                  * can potentially block.  We need to
2397                                  * reference count the frontends so they
2398                                  * can't go away,
2399                                  */
2400                                 mtx_unlock(&softc->ctl_lock);
2401
2402                                 if (cmd == CTL_ENABLE_PORT) {
2403                                         struct ctl_lun *lun;
2404
2405                                         STAILQ_FOREACH(lun, &softc->lun_list,
2406                                                        links) {
2407                                                 port->lun_enable(port->targ_lun_arg,
2408                                                     lun->target,
2409                                                     lun->lun);
2410                                         }
2411
2412                                         ctl_port_online(port);
2413                                 } else if (cmd == CTL_DISABLE_PORT) {
2414                                         struct ctl_lun *lun;
2415
2416                                         ctl_port_offline(port);
2417
2418                                         STAILQ_FOREACH(lun, &softc->lun_list,
2419                                                        links) {
2420                                                 port->lun_disable(
2421                                                     port->targ_lun_arg,
2422                                                     lun->target,
2423                                                     lun->lun);
2424                                         }
2425                                 }
2426
2427                                 mtx_lock(&softc->ctl_lock);
2428
2429                                 if (cmd == CTL_SET_PORT_WWNS)
2430                                         ctl_port_set_wwns(port,
2431                                             (entry->flags & CTL_PORT_WWNN_VALID) ?
2432                                             1 : 0, entry->wwnn,
2433                                             (entry->flags & CTL_PORT_WWPN_VALID) ?
2434                                             1 : 0, entry->wwpn);
2435                         }
2436                         if (done != 0)
2437                                 break;
2438                 }
2439                 mtx_unlock(&softc->ctl_lock);
2440                 break;
2441         }
2442         case CTL_GET_PORT_LIST: {
2443                 struct ctl_port *port;
2444                 struct ctl_port_list *list;
2445                 int i;
2446
2447                 list = (struct ctl_port_list *)addr;
2448
2449                 if (list->alloc_len != (list->alloc_num *
2450                     sizeof(struct ctl_port_entry))) {
2451                         printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2452                                "alloc_num %u * sizeof(struct ctl_port_entry) "
2453                                "%zu\n", __func__, list->alloc_len,
2454                                list->alloc_num, sizeof(struct ctl_port_entry));
2455                         retval = EINVAL;
2456                         break;
2457                 }
2458                 list->fill_len = 0;
2459                 list->fill_num = 0;
2460                 list->dropped_num = 0;
2461                 i = 0;
2462                 mtx_lock(&softc->ctl_lock);
2463                 STAILQ_FOREACH(port, &softc->port_list, links) {
2464                         struct ctl_port_entry entry, *list_entry;
2465
2466                         if (list->fill_num >= list->alloc_num) {
2467                                 list->dropped_num++;
2468                                 continue;
2469                         }
2470
2471                         entry.port_type = port->port_type;
2472                         strlcpy(entry.port_name, port->port_name,
2473                                 sizeof(entry.port_name));
2474                         entry.targ_port = port->targ_port;
2475                         entry.physical_port = port->physical_port;
2476                         entry.virtual_port = port->virtual_port;
2477                         entry.wwnn = port->wwnn;
2478                         entry.wwpn = port->wwpn;
2479                         if (port->status & CTL_PORT_STATUS_ONLINE)
2480                                 entry.online = 1;
2481                         else
2482                                 entry.online = 0;
2483
2484                         list_entry = &list->entries[i];
2485
2486                         retval = copyout(&entry, list_entry, sizeof(entry));
2487                         if (retval != 0) {
2488                                 printf("%s: CTL_GET_PORT_LIST: copyout "
2489                                        "returned %d\n", __func__, retval);
2490                                 break;
2491                         }
2492                         i++;
2493                         list->fill_num++;
2494                         list->fill_len += sizeof(entry);
2495                 }
2496                 mtx_unlock(&softc->ctl_lock);
2497
2498                 /*
2499                  * If this is non-zero, we had a copyout fault, so there's
2500                  * probably no point in attempting to set the status inside
2501                  * the structure.
2502                  */
2503                 if (retval != 0)
2504                         break;
2505
2506                 if (list->dropped_num > 0)
2507                         list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2508                 else
2509                         list->status = CTL_PORT_LIST_OK;
2510                 break;
2511         }
2512         case CTL_DUMP_OOA: {
2513                 struct ctl_lun *lun;
2514                 union ctl_io *io;
2515                 char printbuf[128];
2516                 struct sbuf sb;
2517
2518                 mtx_lock(&softc->ctl_lock);
2519                 printf("Dumping OOA queues:\n");
2520                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2521                         mtx_lock(&lun->lun_lock);
2522                         for (io = (union ctl_io *)TAILQ_FIRST(
2523                              &lun->ooa_queue); io != NULL;
2524                              io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2525                              ooa_links)) {
2526                                 sbuf_new(&sb, printbuf, sizeof(printbuf),
2527                                          SBUF_FIXEDLEN);
2528                                 sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2529                                             (intmax_t)lun->lun,
2530                                             io->scsiio.tag_num,
2531                                             (io->io_hdr.flags &
2532                                             CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2533                                             (io->io_hdr.flags &
2534                                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2535                                             (io->io_hdr.flags &
2536                                             CTL_FLAG_ABORT) ? " ABORT" : "",
2537                                             (io->io_hdr.flags &
2538                                         CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2539                                 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2540                                 sbuf_finish(&sb);
2541                                 printf("%s\n", sbuf_data(&sb));
2542                         }
2543                         mtx_unlock(&lun->lun_lock);
2544                 }
2545                 printf("OOA queues dump done\n");
2546                 mtx_unlock(&softc->ctl_lock);
2547                 break;
2548         }
2549         case CTL_GET_OOA: {
2550                 struct ctl_lun *lun;
2551                 struct ctl_ooa *ooa_hdr;
2552                 struct ctl_ooa_entry *entries;
2553                 uint32_t cur_fill_num;
2554
2555                 ooa_hdr = (struct ctl_ooa *)addr;
2556
2557                 if ((ooa_hdr->alloc_len == 0)
2558                  || (ooa_hdr->alloc_num == 0)) {
2559                         printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2560                                "must be non-zero\n", __func__,
2561                                ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2562                         retval = EINVAL;
2563                         break;
2564                 }
2565
2566                 if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2567                     sizeof(struct ctl_ooa_entry))) {
2568                         printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2569                                "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2570                                __func__, ooa_hdr->alloc_len,
2571                                ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2572                         retval = EINVAL;
2573                         break;
2574                 }
2575
2576                 entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2577                 if (entries == NULL) {
2578                         printf("%s: could not allocate %d bytes for OOA "
2579                                "dump\n", __func__, ooa_hdr->alloc_len);
2580                         retval = ENOMEM;
2581                         break;
2582                 }
2583
2584                 mtx_lock(&softc->ctl_lock);
2585                 if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2586                  && ((ooa_hdr->lun_num >= CTL_MAX_LUNS)
2587                   || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2588                         mtx_unlock(&softc->ctl_lock);
2589                         free(entries, M_CTL);
2590                         printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2591                                __func__, (uintmax_t)ooa_hdr->lun_num);
2592                         retval = EINVAL;
2593                         break;
2594                 }
2595
2596                 cur_fill_num = 0;
2597
2598                 if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2599                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
2600                                 retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2601                                         ooa_hdr, entries);
2602                                 if (retval != 0)
2603                                         break;
2604                         }
2605                         if (retval != 0) {
2606                                 mtx_unlock(&softc->ctl_lock);
2607                                 free(entries, M_CTL);
2608                                 break;
2609                         }
2610                 } else {
2611                         lun = softc->ctl_luns[ooa_hdr->lun_num];
2612
2613                         retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
2614                                                     entries);
2615                 }
2616                 mtx_unlock(&softc->ctl_lock);
2617
2618                 ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2619                 ooa_hdr->fill_len = ooa_hdr->fill_num *
2620                         sizeof(struct ctl_ooa_entry);
2621                 retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2622                 if (retval != 0) {
2623                         printf("%s: error copying out %d bytes for OOA dump\n",
2624                                __func__, ooa_hdr->fill_len);
2625                 }
2626
2627                 getbintime(&ooa_hdr->cur_bt);
2628
2629                 if (cur_fill_num > ooa_hdr->alloc_num) {
2630                         ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2631                         ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2632                 } else {
2633                         ooa_hdr->dropped_num = 0;
2634                         ooa_hdr->status = CTL_OOA_OK;
2635                 }
2636
2637                 free(entries, M_CTL);
2638                 break;
2639         }
2640         case CTL_CHECK_OOA: {
2641                 union ctl_io *io;
2642                 struct ctl_lun *lun;
2643                 struct ctl_ooa_info *ooa_info;
2644
2645
2646                 ooa_info = (struct ctl_ooa_info *)addr;
2647
2648                 if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2649                         ooa_info->status = CTL_OOA_INVALID_LUN;
2650                         break;
2651                 }
2652                 mtx_lock(&softc->ctl_lock);
2653                 lun = softc->ctl_luns[ooa_info->lun_id];
2654                 if (lun == NULL) {
2655                         mtx_unlock(&softc->ctl_lock);
2656                         ooa_info->status = CTL_OOA_INVALID_LUN;
2657                         break;
2658                 }
2659                 mtx_lock(&lun->lun_lock);
2660                 mtx_unlock(&softc->ctl_lock);
2661                 ooa_info->num_entries = 0;
2662                 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2663                      io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2664                      &io->io_hdr, ooa_links)) {
2665                         ooa_info->num_entries++;
2666                 }
2667                 mtx_unlock(&lun->lun_lock);
2668
2669                 ooa_info->status = CTL_OOA_SUCCESS;
2670
2671                 break;
2672         }
2673         case CTL_HARD_START:
2674         case CTL_HARD_STOP: {
2675                 struct ctl_fe_ioctl_startstop_info ss_info;
2676                 struct cfi_metatask *metatask;
2677                 struct mtx hs_mtx;
2678
2679                 mtx_init(&hs_mtx, "HS Mutex", NULL, MTX_DEF);
2680
2681                 cv_init(&ss_info.sem, "hard start/stop cv" );
2682
2683                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2684                 if (metatask == NULL) {
2685                         retval = ENOMEM;
2686                         mtx_destroy(&hs_mtx);
2687                         break;
2688                 }
2689
2690                 if (cmd == CTL_HARD_START)
2691                         metatask->tasktype = CFI_TASK_STARTUP;
2692                 else
2693                         metatask->tasktype = CFI_TASK_SHUTDOWN;
2694
2695                 metatask->callback = ctl_ioctl_hard_startstop_callback;
2696                 metatask->callback_arg = &ss_info;
2697
2698                 cfi_action(metatask);
2699
2700                 /* Wait for the callback */
2701                 mtx_lock(&hs_mtx);
2702                 cv_wait_sig(&ss_info.sem, &hs_mtx);
2703                 mtx_unlock(&hs_mtx);
2704
2705                 /*
2706                  * All information has been copied from the metatask by the
2707                  * time cv_broadcast() is called, so we free the metatask here.
2708                  */
2709                 cfi_free_metatask(metatask);
2710
2711                 memcpy((void *)addr, &ss_info.hs_info, sizeof(ss_info.hs_info));
2712
2713                 mtx_destroy(&hs_mtx);
2714                 break;
2715         }
2716         case CTL_BBRREAD: {
2717                 struct ctl_bbrread_info *bbr_info;
2718                 struct ctl_fe_ioctl_bbrread_info fe_bbr_info;
2719                 struct mtx bbr_mtx;
2720                 struct cfi_metatask *metatask;
2721
2722                 bbr_info = (struct ctl_bbrread_info *)addr;
2723
2724                 bzero(&fe_bbr_info, sizeof(fe_bbr_info));
2725
2726                 bzero(&bbr_mtx, sizeof(bbr_mtx));
2727                 mtx_init(&bbr_mtx, "BBR Mutex", NULL, MTX_DEF);
2728
2729                 fe_bbr_info.bbr_info = bbr_info;
2730                 fe_bbr_info.lock = &bbr_mtx;
2731
2732                 cv_init(&fe_bbr_info.sem, "BBR read cv");
2733                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2734
2735                 if (metatask == NULL) {
2736                         mtx_destroy(&bbr_mtx);
2737                         cv_destroy(&fe_bbr_info.sem);
2738                         retval = ENOMEM;
2739                         break;
2740                 }
2741                 metatask->tasktype = CFI_TASK_BBRREAD;
2742                 metatask->callback = ctl_ioctl_bbrread_callback;
2743                 metatask->callback_arg = &fe_bbr_info;
2744                 metatask->taskinfo.bbrread.lun_num = bbr_info->lun_num;
2745                 metatask->taskinfo.bbrread.lba = bbr_info->lba;
2746                 metatask->taskinfo.bbrread.len = bbr_info->len;
2747
2748                 cfi_action(metatask);
2749
2750                 mtx_lock(&bbr_mtx);
2751                 while (fe_bbr_info.wakeup_done == 0)
2752                         cv_wait_sig(&fe_bbr_info.sem, &bbr_mtx);
2753                 mtx_unlock(&bbr_mtx);
2754
2755                 bbr_info->status = metatask->status;
2756                 bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2757                 bbr_info->scsi_status = metatask->taskinfo.bbrread.scsi_status;
2758                 memcpy(&bbr_info->sense_data,
2759                        &metatask->taskinfo.bbrread.sense_data,
2760                        ctl_min(sizeof(bbr_info->sense_data),
2761                                sizeof(metatask->taskinfo.bbrread.sense_data)));
2762
2763                 cfi_free_metatask(metatask);
2764
2765                 mtx_destroy(&bbr_mtx);
2766                 cv_destroy(&fe_bbr_info.sem);
2767
2768                 break;
2769         }
2770         case CTL_DELAY_IO: {
2771                 struct ctl_io_delay_info *delay_info;
2772 #ifdef CTL_IO_DELAY
2773                 struct ctl_lun *lun;
2774 #endif /* CTL_IO_DELAY */
2775
2776                 delay_info = (struct ctl_io_delay_info *)addr;
2777
2778 #ifdef CTL_IO_DELAY
2779                 mtx_lock(&softc->ctl_lock);
2780
2781                 if ((delay_info->lun_id >= CTL_MAX_LUNS)
2782                  || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2783                         delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2784                 } else {
2785                         lun = softc->ctl_luns[delay_info->lun_id];
2786                         mtx_lock(&lun->lun_lock);
2787
2788                         delay_info->status = CTL_DELAY_STATUS_OK;
2789
2790                         switch (delay_info->delay_type) {
2791                         case CTL_DELAY_TYPE_CONT:
2792                                 break;
2793                         case CTL_DELAY_TYPE_ONESHOT:
2794                                 break;
2795                         default:
2796                                 delay_info->status =
2797                                         CTL_DELAY_STATUS_INVALID_TYPE;
2798                                 break;
2799                         }
2800
2801                         switch (delay_info->delay_loc) {
2802                         case CTL_DELAY_LOC_DATAMOVE:
2803                                 lun->delay_info.datamove_type =
2804                                         delay_info->delay_type;
2805                                 lun->delay_info.datamove_delay =
2806                                         delay_info->delay_secs;
2807                                 break;
2808                         case CTL_DELAY_LOC_DONE:
2809                                 lun->delay_info.done_type =
2810                                         delay_info->delay_type;
2811                                 lun->delay_info.done_delay =
2812                                         delay_info->delay_secs;
2813                                 break;
2814                         default:
2815                                 delay_info->status =
2816                                         CTL_DELAY_STATUS_INVALID_LOC;
2817                                 break;
2818                         }
2819                         mtx_unlock(&lun->lun_lock);
2820                 }
2821
2822                 mtx_unlock(&softc->ctl_lock);
2823 #else
2824                 delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2825 #endif /* CTL_IO_DELAY */
2826                 break;
2827         }
2828         case CTL_REALSYNC_SET: {
2829                 int *syncstate;
2830
2831                 syncstate = (int *)addr;
2832
2833                 mtx_lock(&softc->ctl_lock);
2834                 switch (*syncstate) {
2835                 case 0:
2836                         softc->flags &= ~CTL_FLAG_REAL_SYNC;
2837                         break;
2838                 case 1:
2839                         softc->flags |= CTL_FLAG_REAL_SYNC;
2840                         break;
2841                 default:
2842                         retval = EINVAL;
2843                         break;
2844                 }
2845                 mtx_unlock(&softc->ctl_lock);
2846                 break;
2847         }
2848         case CTL_REALSYNC_GET: {
2849                 int *syncstate;
2850
2851                 syncstate = (int*)addr;
2852
2853                 mtx_lock(&softc->ctl_lock);
2854                 if (softc->flags & CTL_FLAG_REAL_SYNC)
2855                         *syncstate = 1;
2856                 else
2857                         *syncstate = 0;
2858                 mtx_unlock(&softc->ctl_lock);
2859
2860                 break;
2861         }
2862         case CTL_SETSYNC:
2863         case CTL_GETSYNC: {
2864                 struct ctl_sync_info *sync_info;
2865                 struct ctl_lun *lun;
2866
2867                 sync_info = (struct ctl_sync_info *)addr;
2868
2869                 mtx_lock(&softc->ctl_lock);
2870                 lun = softc->ctl_luns[sync_info->lun_id];
2871                 if (lun == NULL) {
2872                         mtx_unlock(&softc->ctl_lock);
2873                         sync_info->status = CTL_GS_SYNC_NO_LUN;
2874                 }
2875                 /*
2876                  * Get or set the sync interval.  We're not bounds checking
2877                  * in the set case, hopefully the user won't do something
2878                  * silly.
2879                  */
2880                 mtx_lock(&lun->lun_lock);
2881                 mtx_unlock(&softc->ctl_lock);
2882                 if (cmd == CTL_GETSYNC)
2883                         sync_info->sync_interval = lun->sync_interval;
2884                 else
2885                         lun->sync_interval = sync_info->sync_interval;
2886                 mtx_unlock(&lun->lun_lock);
2887
2888                 sync_info->status = CTL_GS_SYNC_OK;
2889
2890                 break;
2891         }
2892         case CTL_GETSTATS: {
2893                 struct ctl_stats *stats;
2894                 struct ctl_lun *lun;
2895                 int i;
2896
2897                 stats = (struct ctl_stats *)addr;
2898
2899                 if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2900                      stats->alloc_len) {
2901                         stats->status = CTL_SS_NEED_MORE_SPACE;
2902                         stats->num_luns = softc->num_luns;
2903                         break;
2904                 }
2905                 /*
2906                  * XXX KDM no locking here.  If the LUN list changes,
2907                  * things can blow up.
2908                  */
2909                 for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2910                      i++, lun = STAILQ_NEXT(lun, links)) {
2911                         retval = copyout(&lun->stats, &stats->lun_stats[i],
2912                                          sizeof(lun->stats));
2913                         if (retval != 0)
2914                                 break;
2915                 }
2916                 stats->num_luns = softc->num_luns;
2917                 stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2918                                  softc->num_luns;
2919                 stats->status = CTL_SS_OK;
2920 #ifdef CTL_TIME_IO
2921                 stats->flags = CTL_STATS_FLAG_TIME_VALID;
2922 #else
2923                 stats->flags = CTL_STATS_FLAG_NONE;
2924 #endif
2925                 getnanouptime(&stats->timestamp);
2926                 break;
2927         }
2928         case CTL_ERROR_INJECT: {
2929                 struct ctl_error_desc *err_desc, *new_err_desc;
2930                 struct ctl_lun *lun;
2931
2932                 err_desc = (struct ctl_error_desc *)addr;
2933
2934                 new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2935                                       M_WAITOK | M_ZERO);
2936                 bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2937
2938                 mtx_lock(&softc->ctl_lock);
2939                 lun = softc->ctl_luns[err_desc->lun_id];
2940                 if (lun == NULL) {
2941                         mtx_unlock(&softc->ctl_lock);
2942                         free(new_err_desc, M_CTL);
2943                         printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2944                                __func__, (uintmax_t)err_desc->lun_id);
2945                         retval = EINVAL;
2946                         break;
2947                 }
2948                 mtx_lock(&lun->lun_lock);
2949                 mtx_unlock(&softc->ctl_lock);
2950
2951                 /*
2952                  * We could do some checking here to verify the validity
2953                  * of the request, but given the complexity of error
2954                  * injection requests, the checking logic would be fairly
2955                  * complex.
2956                  *
2957                  * For now, if the request is invalid, it just won't get
2958                  * executed and might get deleted.
2959                  */
2960                 STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2961
2962                 /*
2963                  * XXX KDM check to make sure the serial number is unique,
2964                  * in case we somehow manage to wrap.  That shouldn't
2965                  * happen for a very long time, but it's the right thing to
2966                  * do.
2967                  */
2968                 new_err_desc->serial = lun->error_serial;
2969                 err_desc->serial = lun->error_serial;
2970                 lun->error_serial++;
2971
2972                 mtx_unlock(&lun->lun_lock);
2973                 break;
2974         }
2975         case CTL_ERROR_INJECT_DELETE: {
2976                 struct ctl_error_desc *delete_desc, *desc, *desc2;
2977                 struct ctl_lun *lun;
2978                 int delete_done;
2979
2980                 delete_desc = (struct ctl_error_desc *)addr;
2981                 delete_done = 0;
2982
2983                 mtx_lock(&softc->ctl_lock);
2984                 lun = softc->ctl_luns[delete_desc->lun_id];
2985                 if (lun == NULL) {
2986                         mtx_unlock(&softc->ctl_lock);
2987                         printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
2988                                __func__, (uintmax_t)delete_desc->lun_id);
2989                         retval = EINVAL;
2990                         break;
2991                 }
2992                 mtx_lock(&lun->lun_lock);
2993                 mtx_unlock(&softc->ctl_lock);
2994                 STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
2995                         if (desc->serial != delete_desc->serial)
2996                                 continue;
2997
2998                         STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
2999                                       links);
3000                         free(desc, M_CTL);
3001                         delete_done = 1;
3002                 }
3003                 mtx_unlock(&lun->lun_lock);
3004                 if (delete_done == 0) {
3005                         printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
3006                                "error serial %ju on LUN %u\n", __func__,
3007                                delete_desc->serial, delete_desc->lun_id);
3008                         retval = EINVAL;
3009                         break;
3010                 }
3011                 break;
3012         }
3013         case CTL_DUMP_STRUCTS: {
3014                 int i, j, k, idx;
3015                 struct ctl_port *port;
3016                 struct ctl_frontend *fe;
3017
3018                 mtx_lock(&softc->ctl_lock);
3019                 printf("CTL Persistent Reservation information start:\n");
3020                 for (i = 0; i < CTL_MAX_LUNS; i++) {
3021                         struct ctl_lun *lun;
3022
3023                         lun = softc->ctl_luns[i];
3024
3025                         if ((lun == NULL)
3026                          || ((lun->flags & CTL_LUN_DISABLED) != 0))
3027                                 continue;
3028
3029                         for (j = 0; j < (CTL_MAX_PORTS * 2); j++) {
3030                                 for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
3031                                         idx = j * CTL_MAX_INIT_PER_PORT + k;
3032                                         if (lun->pr_keys[idx] == 0)
3033                                                 continue;
3034                                         printf("  LUN %d port %d iid %d key "
3035                                                "%#jx\n", i, j, k,
3036                                                (uintmax_t)lun->pr_keys[idx]);
3037                                 }
3038                         }
3039                 }
3040                 printf("CTL Persistent Reservation information end\n");
3041                 printf("CTL Ports:\n");
3042                 STAILQ_FOREACH(port, &softc->port_list, links) {
3043                         printf("  Port %d '%s' Frontend '%s' Type %u pp %d vp %d WWNN "
3044                                "%#jx WWPN %#jx\n", port->targ_port, port->port_name,
3045                                port->frontend->name, port->port_type,
3046                                port->physical_port, port->virtual_port,
3047                                (uintmax_t)port->wwnn, (uintmax_t)port->wwpn);
3048                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
3049                                 if (port->wwpn_iid[j].in_use == 0 &&
3050                                     port->wwpn_iid[j].wwpn == 0 &&
3051                                     port->wwpn_iid[j].name == NULL)
3052                                         continue;
3053
3054                                 printf("    iid %u use %d WWPN %#jx '%s'\n",
3055                                     j, port->wwpn_iid[j].in_use,
3056                                     (uintmax_t)port->wwpn_iid[j].wwpn,
3057                                     port->wwpn_iid[j].name);
3058                         }
3059                 }
3060                 printf("CTL Port information end\n");
3061                 mtx_unlock(&softc->ctl_lock);
3062                 /*
3063                  * XXX KDM calling this without a lock.  We'd likely want
3064                  * to drop the lock before calling the frontend's dump
3065                  * routine anyway.
3066                  */
3067                 printf("CTL Frontends:\n");
3068                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
3069                         printf("  Frontend '%s'\n", fe->name);
3070                         if (fe->fe_dump != NULL)
3071                                 fe->fe_dump();
3072                 }
3073                 printf("CTL Frontend information end\n");
3074                 break;
3075         }
3076         case CTL_LUN_REQ: {
3077                 struct ctl_lun_req *lun_req;
3078                 struct ctl_backend_driver *backend;
3079
3080                 lun_req = (struct ctl_lun_req *)addr;
3081
3082                 backend = ctl_backend_find(lun_req->backend);
3083                 if (backend == NULL) {
3084                         lun_req->status = CTL_LUN_ERROR;
3085                         snprintf(lun_req->error_str,
3086                                  sizeof(lun_req->error_str),
3087                                  "Backend \"%s\" not found.",
3088                                  lun_req->backend);
3089                         break;
3090                 }
3091                 if (lun_req->num_be_args > 0) {
3092                         lun_req->kern_be_args = ctl_copyin_args(
3093                                 lun_req->num_be_args,
3094                                 lun_req->be_args,
3095                                 lun_req->error_str,
3096                                 sizeof(lun_req->error_str));
3097                         if (lun_req->kern_be_args == NULL) {
3098                                 lun_req->status = CTL_LUN_ERROR;
3099                                 break;
3100                         }
3101                 }
3102
3103                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3104
3105                 if (lun_req->num_be_args > 0) {
3106                         ctl_copyout_args(lun_req->num_be_args,
3107                                       lun_req->kern_be_args);
3108                         ctl_free_args(lun_req->num_be_args,
3109                                       lun_req->kern_be_args);
3110                 }
3111                 break;
3112         }
3113         case CTL_LUN_LIST: {
3114                 struct sbuf *sb;
3115                 struct ctl_lun *lun;
3116                 struct ctl_lun_list *list;
3117                 struct ctl_option *opt;
3118
3119                 list = (struct ctl_lun_list *)addr;
3120
3121                 /*
3122                  * Allocate a fixed length sbuf here, based on the length
3123                  * of the user's buffer.  We could allocate an auto-extending
3124                  * buffer, and then tell the user how much larger our
3125                  * amount of data is than his buffer, but that presents
3126                  * some problems:
3127                  *
3128                  * 1.  The sbuf(9) routines use a blocking malloc, and so
3129                  *     we can't hold a lock while calling them with an
3130                  *     auto-extending buffer.
3131                  *
3132                  * 2.  There is not currently a LUN reference counting
3133                  *     mechanism, outside of outstanding transactions on
3134                  *     the LUN's OOA queue.  So a LUN could go away on us
3135                  *     while we're getting the LUN number, backend-specific
3136                  *     information, etc.  Thus, given the way things
3137                  *     currently work, we need to hold the CTL lock while
3138                  *     grabbing LUN information.
3139                  *
3140                  * So, from the user's standpoint, the best thing to do is
3141                  * allocate what he thinks is a reasonable buffer length,
3142                  * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
3143                  * double the buffer length and try again.  (And repeat
3144                  * that until he succeeds.)
3145                  */
3146                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3147                 if (sb == NULL) {
3148                         list->status = CTL_LUN_LIST_ERROR;
3149                         snprintf(list->error_str, sizeof(list->error_str),
3150                                  "Unable to allocate %d bytes for LUN list",
3151                                  list->alloc_len);
3152                         break;
3153                 }
3154
3155                 sbuf_printf(sb, "<ctllunlist>\n");
3156
3157                 mtx_lock(&softc->ctl_lock);
3158                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
3159                         mtx_lock(&lun->lun_lock);
3160                         retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3161                                              (uintmax_t)lun->lun);
3162
3163                         /*
3164                          * Bail out as soon as we see that we've overfilled
3165                          * the buffer.
3166                          */
3167                         if (retval != 0)
3168                                 break;
3169
3170                         retval = sbuf_printf(sb, "\t<backend_type>%s"
3171                                              "</backend_type>\n",
3172                                              (lun->backend == NULL) ?  "none" :
3173                                              lun->backend->name);
3174
3175                         if (retval != 0)
3176                                 break;
3177
3178                         retval = sbuf_printf(sb, "\t<lun_type>%d</lun_type>\n",
3179                                              lun->be_lun->lun_type);
3180
3181                         if (retval != 0)
3182                                 break;
3183
3184                         if (lun->backend == NULL) {
3185                                 retval = sbuf_printf(sb, "</lun>\n");
3186                                 if (retval != 0)
3187                                         break;
3188                                 continue;
3189                         }
3190
3191                         retval = sbuf_printf(sb, "\t<size>%ju</size>\n",
3192                                              (lun->be_lun->maxlba > 0) ?
3193                                              lun->be_lun->maxlba + 1 : 0);
3194
3195                         if (retval != 0)
3196                                 break;
3197
3198                         retval = sbuf_printf(sb, "\t<blocksize>%u</blocksize>\n",
3199                                              lun->be_lun->blocksize);
3200
3201                         if (retval != 0)
3202                                 break;
3203
3204                         retval = sbuf_printf(sb, "\t<serial_number>");
3205
3206                         if (retval != 0)
3207                                 break;
3208
3209                         retval = ctl_sbuf_printf_esc(sb,
3210                                                      lun->be_lun->serial_num);
3211
3212                         if (retval != 0)
3213                                 break;
3214
3215                         retval = sbuf_printf(sb, "</serial_number>\n");
3216
3217                         if (retval != 0)
3218                                 break;
3219
3220                         retval = sbuf_printf(sb, "\t<device_id>");
3221
3222                         if (retval != 0)
3223                                 break;
3224
3225                         retval = ctl_sbuf_printf_esc(sb,lun->be_lun->device_id);
3226
3227                         if (retval != 0)
3228                                 break;
3229
3230                         retval = sbuf_printf(sb, "</device_id>\n");
3231
3232                         if (retval != 0)
3233                                 break;
3234
3235                         if (lun->backend->lun_info != NULL) {
3236                                 retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3237                                 if (retval != 0)
3238                                         break;
3239                         }
3240                         STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3241                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3242                                     opt->name, opt->value, opt->name);
3243                                 if (retval != 0)
3244                                         break;
3245                         }
3246
3247                         retval = sbuf_printf(sb, "</lun>\n");
3248
3249                         if (retval != 0)
3250                                 break;
3251                         mtx_unlock(&lun->lun_lock);
3252                 }
3253                 if (lun != NULL)
3254                         mtx_unlock(&lun->lun_lock);
3255                 mtx_unlock(&softc->ctl_lock);
3256
3257                 if ((retval != 0)
3258                  || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3259                         retval = 0;
3260                         sbuf_delete(sb);
3261                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3262                         snprintf(list->error_str, sizeof(list->error_str),
3263                                  "Out of space, %d bytes is too small",
3264                                  list->alloc_len);
3265                         break;
3266                 }
3267
3268                 sbuf_finish(sb);
3269
3270                 retval = copyout(sbuf_data(sb), list->lun_xml,
3271                                  sbuf_len(sb) + 1);
3272
3273                 list->fill_len = sbuf_len(sb) + 1;
3274                 list->status = CTL_LUN_LIST_OK;
3275                 sbuf_delete(sb);
3276                 break;
3277         }
3278         case CTL_ISCSI: {
3279                 struct ctl_iscsi *ci;
3280                 struct ctl_frontend *fe;
3281
3282                 ci = (struct ctl_iscsi *)addr;
3283
3284                 fe = ctl_frontend_find("iscsi");
3285                 if (fe == NULL) {
3286                         ci->status = CTL_ISCSI_ERROR;
3287                         snprintf(ci->error_str, sizeof(ci->error_str),
3288                             "Frontend \"iscsi\" not found.");
3289                         break;
3290                 }
3291
3292                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3293                 break;
3294         }
3295         case CTL_PORT_REQ: {
3296                 struct ctl_req *req;
3297                 struct ctl_frontend *fe;
3298
3299                 req = (struct ctl_req *)addr;
3300
3301                 fe = ctl_frontend_find(req->driver);
3302                 if (fe == NULL) {
3303                         req->status = CTL_LUN_ERROR;
3304                         snprintf(req->error_str, sizeof(req->error_str),
3305                             "Frontend \"%s\" not found.", req->driver);
3306                         break;
3307                 }
3308                 if (req->num_args > 0) {
3309                         req->kern_args = ctl_copyin_args(req->num_args,
3310                             req->args, req->error_str, sizeof(req->error_str));
3311                         if (req->kern_args == NULL) {
3312                                 req->status = CTL_LUN_ERROR;
3313                                 break;
3314                         }
3315                 }
3316
3317                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3318
3319                 if (req->num_args > 0) {
3320                         ctl_copyout_args(req->num_args, req->kern_args);
3321                         ctl_free_args(req->num_args, req->kern_args);
3322                 }
3323                 break;
3324         }
3325         case CTL_PORT_LIST: {
3326                 struct sbuf *sb;
3327                 struct ctl_port *port;
3328                 struct ctl_lun_list *list;
3329                 struct ctl_option *opt;
3330                 int j;
3331
3332                 list = (struct ctl_lun_list *)addr;
3333
3334                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3335                 if (sb == NULL) {
3336                         list->status = CTL_LUN_LIST_ERROR;
3337                         snprintf(list->error_str, sizeof(list->error_str),
3338                                  "Unable to allocate %d bytes for LUN list",
3339                                  list->alloc_len);
3340                         break;
3341                 }
3342
3343                 sbuf_printf(sb, "<ctlportlist>\n");
3344
3345                 mtx_lock(&softc->ctl_lock);
3346                 STAILQ_FOREACH(port, &softc->port_list, links) {
3347                         retval = sbuf_printf(sb, "<targ_port id=\"%ju\">\n",
3348                                              (uintmax_t)port->targ_port);
3349
3350                         /*
3351                          * Bail out as soon as we see that we've overfilled
3352                          * the buffer.
3353                          */
3354                         if (retval != 0)
3355                                 break;
3356
3357                         retval = sbuf_printf(sb, "\t<frontend_type>%s"
3358                             "</frontend_type>\n", port->frontend->name);
3359                         if (retval != 0)
3360                                 break;
3361
3362                         retval = sbuf_printf(sb, "\t<port_type>%d</port_type>\n",
3363                                              port->port_type);
3364                         if (retval != 0)
3365                                 break;
3366
3367                         retval = sbuf_printf(sb, "\t<online>%s</online>\n",
3368                             (port->status & CTL_PORT_STATUS_ONLINE) ? "YES" : "NO");
3369                         if (retval != 0)
3370                                 break;
3371
3372                         retval = sbuf_printf(sb, "\t<port_name>%s</port_name>\n",
3373                             port->port_name);
3374                         if (retval != 0)
3375                                 break;
3376
3377                         retval = sbuf_printf(sb, "\t<physical_port>%d</physical_port>\n",
3378                             port->physical_port);
3379                         if (retval != 0)
3380                                 break;
3381
3382                         retval = sbuf_printf(sb, "\t<virtual_port>%d</virtual_port>\n",
3383                             port->virtual_port);
3384                         if (retval != 0)
3385                                 break;
3386
3387                         if (port->target_devid != NULL) {
3388                                 sbuf_printf(sb, "\t<target>");
3389                                 ctl_id_sbuf(port->target_devid, sb);
3390                                 sbuf_printf(sb, "</target>\n");
3391                         }
3392
3393                         if (port->port_devid != NULL) {
3394                                 sbuf_printf(sb, "\t<port>");
3395                                 ctl_id_sbuf(port->port_devid, sb);
3396                                 sbuf_printf(sb, "</port>\n");
3397                         }
3398
3399                         if (port->port_info != NULL) {
3400                                 retval = port->port_info(port->onoff_arg, sb);
3401                                 if (retval != 0)
3402                                         break;
3403                         }
3404                         STAILQ_FOREACH(opt, &port->options, links) {
3405                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3406                                     opt->name, opt->value, opt->name);
3407                                 if (retval != 0)
3408                                         break;
3409                         }
3410
3411                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
3412                                 if (port->wwpn_iid[j].in_use == 0 ||
3413                                     (port->wwpn_iid[j].wwpn == 0 &&
3414                                      port->wwpn_iid[j].name == NULL))
3415                                         continue;
3416
3417                                 if (port->wwpn_iid[j].name != NULL)
3418                                         retval = sbuf_printf(sb,
3419                                             "\t<initiator>%u %s</initiator>\n",
3420                                             j, port->wwpn_iid[j].name);
3421                                 else
3422                                         retval = sbuf_printf(sb,
3423                                             "\t<initiator>%u naa.%08jx</initiator>\n",
3424                                             j, port->wwpn_iid[j].wwpn);
3425                                 if (retval != 0)
3426                                         break;
3427                         }
3428                         if (retval != 0)
3429                                 break;
3430
3431                         retval = sbuf_printf(sb, "</targ_port>\n");
3432                         if (retval != 0)
3433                                 break;
3434                 }
3435                 mtx_unlock(&softc->ctl_lock);
3436
3437                 if ((retval != 0)
3438                  || ((retval = sbuf_printf(sb, "</ctlportlist>\n")) != 0)) {
3439                         retval = 0;
3440                         sbuf_delete(sb);
3441                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3442                         snprintf(list->error_str, sizeof(list->error_str),
3443                                  "Out of space, %d bytes is too small",
3444                                  list->alloc_len);
3445                         break;
3446                 }
3447
3448                 sbuf_finish(sb);
3449
3450                 retval = copyout(sbuf_data(sb), list->lun_xml,
3451                                  sbuf_len(sb) + 1);
3452
3453                 list->fill_len = sbuf_len(sb) + 1;
3454                 list->status = CTL_LUN_LIST_OK;
3455                 sbuf_delete(sb);
3456                 break;
3457         }
3458         default: {
3459                 /* XXX KDM should we fix this? */
3460 #if 0
3461                 struct ctl_backend_driver *backend;
3462                 unsigned int type;
3463                 int found;
3464
3465                 found = 0;
3466
3467                 /*
3468                  * We encode the backend type as the ioctl type for backend
3469                  * ioctls.  So parse it out here, and then search for a
3470                  * backend of this type.
3471                  */
3472                 type = _IOC_TYPE(cmd);
3473
3474                 STAILQ_FOREACH(backend, &softc->be_list, links) {
3475                         if (backend->type == type) {
3476                                 found = 1;
3477                                 break;
3478                         }
3479                 }
3480                 if (found == 0) {
3481                         printf("ctl: unknown ioctl command %#lx or backend "
3482                                "%d\n", cmd, type);
3483                         retval = EINVAL;
3484                         break;
3485                 }
3486                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3487 #endif
3488                 retval = ENOTTY;
3489                 break;
3490         }
3491         }
3492         return (retval);
3493 }
3494
3495 uint32_t
3496 ctl_get_initindex(struct ctl_nexus *nexus)
3497 {
3498         if (nexus->targ_port < CTL_MAX_PORTS)
3499                 return (nexus->initid.id +
3500                         (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3501         else
3502                 return (nexus->initid.id +
3503                        ((nexus->targ_port - CTL_MAX_PORTS) *
3504                         CTL_MAX_INIT_PER_PORT));
3505 }
3506
3507 uint32_t
3508 ctl_get_resindex(struct ctl_nexus *nexus)
3509 {
3510         return (nexus->initid.id + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3511 }
3512
3513 uint32_t
3514 ctl_port_idx(int port_num)
3515 {
3516         if (port_num < CTL_MAX_PORTS)
3517                 return(port_num);
3518         else
3519                 return(port_num - CTL_MAX_PORTS);
3520 }
3521
3522 static uint32_t
3523 ctl_map_lun(int port_num, uint32_t lun_id)
3524 {
3525         struct ctl_port *port;
3526
3527         port = control_softc->ctl_ports[ctl_port_idx(port_num)];
3528         if (port == NULL)
3529                 return (UINT32_MAX);
3530         if (port->lun_map == NULL)
3531                 return (lun_id);
3532         return (port->lun_map(port->targ_lun_arg, lun_id));
3533 }
3534
3535 static uint32_t
3536 ctl_map_lun_back(int port_num, uint32_t lun_id)
3537 {
3538         struct ctl_port *port;
3539         uint32_t i;
3540
3541         port = control_softc->ctl_ports[ctl_port_idx(port_num)];
3542         if (port->lun_map == NULL)
3543                 return (lun_id);
3544         for (i = 0; i < CTL_MAX_LUNS; i++) {
3545                 if (port->lun_map(port->targ_lun_arg, i) == lun_id)
3546                         return (i);
3547         }
3548         return (UINT32_MAX);
3549 }
3550
3551 /*
3552  * Note:  This only works for bitmask sizes that are at least 32 bits, and
3553  * that are a power of 2.
3554  */
3555 int
3556 ctl_ffz(uint32_t *mask, uint32_t size)
3557 {
3558         uint32_t num_chunks, num_pieces;
3559         int i, j;
3560
3561         num_chunks = (size >> 5);
3562         if (num_chunks == 0)
3563                 num_chunks++;
3564         num_pieces = ctl_min((sizeof(uint32_t) * 8), size);
3565
3566         for (i = 0; i < num_chunks; i++) {
3567                 for (j = 0; j < num_pieces; j++) {
3568                         if ((mask[i] & (1 << j)) == 0)
3569                                 return ((i << 5) + j);
3570                 }
3571         }
3572
3573         return (-1);
3574 }
3575
3576 int
3577 ctl_set_mask(uint32_t *mask, uint32_t bit)
3578 {
3579         uint32_t chunk, piece;
3580
3581         chunk = bit >> 5;
3582         piece = bit % (sizeof(uint32_t) * 8);
3583
3584         if ((mask[chunk] & (1 << piece)) != 0)
3585                 return (-1);
3586         else
3587                 mask[chunk] |= (1 << piece);
3588
3589         return (0);
3590 }
3591
3592 int
3593 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3594 {
3595         uint32_t chunk, piece;
3596
3597         chunk = bit >> 5;
3598         piece = bit % (sizeof(uint32_t) * 8);
3599
3600         if ((mask[chunk] & (1 << piece)) == 0)
3601                 return (-1);
3602         else
3603                 mask[chunk] &= ~(1 << piece);
3604
3605         return (0);
3606 }
3607
3608 int
3609 ctl_is_set(uint32_t *mask, uint32_t bit)
3610 {
3611         uint32_t chunk, piece;
3612
3613         chunk = bit >> 5;
3614         piece = bit % (sizeof(uint32_t) * 8);
3615
3616         if ((mask[chunk] & (1 << piece)) == 0)
3617                 return (0);
3618         else
3619                 return (1);
3620 }
3621
3622 #ifdef unused
3623 /*
3624  * The bus, target and lun are optional, they can be filled in later.
3625  * can_wait is used to determine whether we can wait on the malloc or not.
3626  */
3627 union ctl_io*
3628 ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port, uint32_t targ_target,
3629               uint32_t targ_lun, int can_wait)
3630 {
3631         union ctl_io *io;
3632
3633         if (can_wait)
3634                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_WAITOK);
3635         else
3636                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3637
3638         if (io != NULL) {
3639                 io->io_hdr.io_type = io_type;
3640                 io->io_hdr.targ_port = targ_port;
3641                 /*
3642                  * XXX KDM this needs to change/go away.  We need to move
3643                  * to a preallocated pool of ctl_scsiio structures.
3644                  */
3645                 io->io_hdr.nexus.targ_target.id = targ_target;
3646                 io->io_hdr.nexus.targ_lun = targ_lun;
3647         }
3648
3649         return (io);
3650 }
3651
3652 void
3653 ctl_kfree_io(union ctl_io *io)
3654 {
3655         free(io, M_CTL);
3656 }
3657 #endif /* unused */
3658
3659 /*
3660  * ctl_softc, pool_type, total_ctl_io are passed in.
3661  * npool is passed out.
3662  */
3663 int
3664 ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type,
3665                 uint32_t total_ctl_io, struct ctl_io_pool **npool)
3666 {
3667         uint32_t i;
3668         union ctl_io *cur_io, *next_io;
3669         struct ctl_io_pool *pool;
3670         int retval;
3671
3672         retval = 0;
3673
3674         pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3675                                             M_NOWAIT | M_ZERO);
3676         if (pool == NULL) {
3677                 retval = ENOMEM;
3678                 goto bailout;
3679         }
3680
3681         pool->type = pool_type;
3682         pool->ctl_softc = ctl_softc;
3683
3684         mtx_lock(&ctl_softc->pool_lock);
3685         pool->id = ctl_softc->cur_pool_id++;
3686         mtx_unlock(&ctl_softc->pool_lock);
3687
3688         pool->flags = CTL_POOL_FLAG_NONE;
3689         pool->refcount = 1;             /* Reference for validity. */
3690         STAILQ_INIT(&pool->free_queue);
3691
3692         /*
3693          * XXX KDM other options here:
3694          * - allocate a page at a time
3695          * - allocate one big chunk of memory.
3696          * Page allocation might work well, but would take a little more
3697          * tracking.
3698          */
3699         for (i = 0; i < total_ctl_io; i++) {
3700                 cur_io = (union ctl_io *)malloc(sizeof(*cur_io), M_CTLIO,
3701                                                 M_NOWAIT);
3702                 if (cur_io == NULL) {
3703                         retval = ENOMEM;
3704                         break;
3705                 }
3706                 cur_io->io_hdr.pool = pool;
3707                 STAILQ_INSERT_TAIL(&pool->free_queue, &cur_io->io_hdr, links);
3708                 pool->total_ctl_io++;
3709                 pool->free_ctl_io++;
3710         }
3711
3712         if (retval != 0) {
3713                 for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3714                      cur_io != NULL; cur_io = next_io) {
3715                         next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr,
3716                                                               links);
3717                         STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr,
3718                                       ctl_io_hdr, links);
3719                         free(cur_io, M_CTLIO);
3720                 }
3721
3722                 free(pool, M_CTL);
3723                 goto bailout;
3724         }
3725         mtx_lock(&ctl_softc->pool_lock);
3726         ctl_softc->num_pools++;
3727         STAILQ_INSERT_TAIL(&ctl_softc->io_pools, pool, links);
3728         /*
3729          * Increment our usage count if this is an external consumer, so we
3730          * can't get unloaded until the external consumer (most likely a
3731          * FETD) unloads and frees his pool.
3732          *
3733          * XXX KDM will this increment the caller's module use count, or
3734          * mine?
3735          */
3736 #if 0
3737         if ((pool_type != CTL_POOL_EMERGENCY)
3738          && (pool_type != CTL_POOL_INTERNAL)
3739          && (pool_type != CTL_POOL_4OTHERSC))
3740                 MOD_INC_USE_COUNT;
3741 #endif
3742
3743         mtx_unlock(&ctl_softc->pool_lock);
3744
3745         *npool = pool;
3746
3747 bailout:
3748
3749         return (retval);
3750 }
3751
3752 static int
3753 ctl_pool_acquire(struct ctl_io_pool *pool)
3754 {
3755
3756         mtx_assert(&pool->ctl_softc->pool_lock, MA_OWNED);
3757
3758         if (pool->flags & CTL_POOL_FLAG_INVALID)
3759                 return (EINVAL);
3760
3761         pool->refcount++;
3762
3763         return (0);
3764 }
3765
3766 static void
3767 ctl_pool_release(struct ctl_io_pool *pool)
3768 {
3769         struct ctl_softc *ctl_softc = pool->ctl_softc;
3770         union ctl_io *io;
3771
3772         mtx_assert(&ctl_softc->pool_lock, MA_OWNED);
3773
3774         if (--pool->refcount != 0)
3775                 return;
3776
3777         while ((io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue)) != NULL) {
3778                 STAILQ_REMOVE(&pool->free_queue, &io->io_hdr, ctl_io_hdr,
3779                               links);
3780                 free(io, M_CTLIO);
3781         }
3782
3783         STAILQ_REMOVE(&ctl_softc->io_pools, pool, ctl_io_pool, links);
3784         ctl_softc->num_pools--;
3785
3786         /*
3787          * XXX KDM will this decrement the caller's usage count or mine?
3788          */
3789 #if 0
3790         if ((pool->type != CTL_POOL_EMERGENCY)
3791          && (pool->type != CTL_POOL_INTERNAL)
3792          && (pool->type != CTL_POOL_4OTHERSC))
3793                 MOD_DEC_USE_COUNT;
3794 #endif
3795
3796         free(pool, M_CTL);
3797 }
3798
3799 void
3800 ctl_pool_free(struct ctl_io_pool *pool)
3801 {
3802         struct ctl_softc *ctl_softc;
3803
3804         if (pool == NULL)
3805                 return;
3806
3807         ctl_softc = pool->ctl_softc;
3808         mtx_lock(&ctl_softc->pool_lock);
3809         pool->flags |= CTL_POOL_FLAG_INVALID;
3810         ctl_pool_release(pool);
3811         mtx_unlock(&ctl_softc->pool_lock);
3812 }
3813
3814 /*
3815  * This routine does not block (except for spinlocks of course).
3816  * It tries to allocate a ctl_io union from the caller's pool as quickly as
3817  * possible.
3818  */
3819 union ctl_io *
3820 ctl_alloc_io(void *pool_ref)
3821 {
3822         union ctl_io *io;
3823         struct ctl_softc *ctl_softc;
3824         struct ctl_io_pool *pool, *npool;
3825         struct ctl_io_pool *emergency_pool;
3826
3827         pool = (struct ctl_io_pool *)pool_ref;
3828
3829         if (pool == NULL) {
3830                 printf("%s: pool is NULL\n", __func__);
3831                 return (NULL);
3832         }
3833
3834         emergency_pool = NULL;
3835
3836         ctl_softc = pool->ctl_softc;
3837
3838         mtx_lock(&ctl_softc->pool_lock);
3839         /*
3840          * First, try to get the io structure from the user's pool.
3841          */
3842         if (ctl_pool_acquire(pool) == 0) {
3843                 io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3844                 if (io != NULL) {
3845                         STAILQ_REMOVE_HEAD(&pool->free_queue, links);
3846                         pool->total_allocated++;
3847                         pool->free_ctl_io--;
3848                         mtx_unlock(&ctl_softc->pool_lock);
3849                         return (io);
3850                 } else
3851                         ctl_pool_release(pool);
3852         }
3853         /*
3854          * If he doesn't have any io structures left, search for an
3855          * emergency pool and grab one from there.
3856          */
3857         STAILQ_FOREACH(npool, &ctl_softc->io_pools, links) {
3858                 if (npool->type != CTL_POOL_EMERGENCY)
3859                         continue;
3860
3861                 if (ctl_pool_acquire(npool) != 0)
3862                         continue;
3863
3864                 emergency_pool = npool;
3865
3866                 io = (union ctl_io *)STAILQ_FIRST(&npool->free_queue);
3867                 if (io != NULL) {
3868                         STAILQ_REMOVE_HEAD(&npool->free_queue, links);
3869                         npool->total_allocated++;
3870                         npool->free_ctl_io--;
3871                         mtx_unlock(&ctl_softc->pool_lock);
3872                         return (io);
3873                 } else
3874                         ctl_pool_release(npool);
3875         }
3876
3877         /* Drop the spinlock before we malloc */
3878         mtx_unlock(&ctl_softc->pool_lock);
3879
3880         /*
3881          * The emergency pool (if it exists) didn't have one, so try an
3882          * atomic (i.e. nonblocking) malloc and see if we get lucky.
3883          */
3884         io = (union ctl_io *)malloc(sizeof(*io), M_CTLIO, M_NOWAIT);
3885         if (io != NULL) {
3886                 /*
3887                  * If the emergency pool exists but is empty, add this
3888                  * ctl_io to its list when it gets freed.
3889                  */
3890                 if (emergency_pool != NULL) {
3891                         mtx_lock(&ctl_softc->pool_lock);
3892                         if (ctl_pool_acquire(emergency_pool) == 0) {
3893                                 io->io_hdr.pool = emergency_pool;
3894                                 emergency_pool->total_ctl_io++;
3895                                 /*
3896                                  * Need to bump this, otherwise
3897                                  * total_allocated and total_freed won't
3898                                  * match when we no longer have anything
3899                                  * outstanding.
3900                                  */
3901                                 emergency_pool->total_allocated++;
3902                         }
3903                         mtx_unlock(&ctl_softc->pool_lock);
3904                 } else
3905                         io->io_hdr.pool = NULL;
3906         }
3907
3908         return (io);
3909 }
3910
3911 void
3912 ctl_free_io(union ctl_io *io)
3913 {
3914         if (io == NULL)
3915                 return;
3916
3917         /*
3918          * If this ctl_io has a pool, return it to that pool.
3919          */
3920         if (io->io_hdr.pool != NULL) {
3921                 struct ctl_io_pool *pool;
3922
3923                 pool = (struct ctl_io_pool *)io->io_hdr.pool;
3924                 mtx_lock(&pool->ctl_softc->pool_lock);
3925                 io->io_hdr.io_type = 0xff;
3926                 STAILQ_INSERT_TAIL(&pool->free_queue, &io->io_hdr, links);
3927                 pool->total_freed++;
3928                 pool->free_ctl_io++;
3929                 ctl_pool_release(pool);
3930                 mtx_unlock(&pool->ctl_softc->pool_lock);
3931         } else {
3932                 /*
3933                  * Otherwise, just free it.  We probably malloced it and
3934                  * the emergency pool wasn't available.
3935                  */
3936                 free(io, M_CTLIO);
3937         }
3938
3939 }
3940
3941 void
3942 ctl_zero_io(union ctl_io *io)
3943 {
3944         void *pool_ref;
3945
3946         if (io == NULL)
3947                 return;
3948
3949         /*
3950          * May need to preserve linked list pointers at some point too.
3951          */
3952         pool_ref = io->io_hdr.pool;
3953
3954         memset(io, 0, sizeof(*io));
3955
3956         io->io_hdr.pool = pool_ref;
3957 }
3958
3959 /*
3960  * This routine is currently used for internal copies of ctl_ios that need
3961  * to persist for some reason after we've already returned status to the
3962  * FETD.  (Thus the flag set.)
3963  *
3964  * XXX XXX
3965  * Note that this makes a blind copy of all fields in the ctl_io, except
3966  * for the pool reference.  This includes any memory that has been
3967  * allocated!  That memory will no longer be valid after done has been
3968  * called, so this would be VERY DANGEROUS for command that actually does
3969  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3970  * start and stop commands, which don't transfer any data, so this is not a
3971  * problem.  If it is used for anything else, the caller would also need to
3972  * allocate data buffer space and this routine would need to be modified to
3973  * copy the data buffer(s) as well.
3974  */
3975 void
3976 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3977 {
3978         void *pool_ref;
3979
3980         if ((src == NULL)
3981          || (dest == NULL))
3982                 return;
3983
3984         /*
3985          * May need to preserve linked list pointers at some point too.
3986          */
3987         pool_ref = dest->io_hdr.pool;
3988
3989         memcpy(dest, src, ctl_min(sizeof(*src), sizeof(*dest)));
3990
3991         dest->io_hdr.pool = pool_ref;
3992         /*
3993          * We need to know that this is an internal copy, and doesn't need
3994          * to get passed back to the FETD that allocated it.
3995          */
3996         dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3997 }
3998
3999 #ifdef NEEDTOPORT
4000 static void
4001 ctl_update_power_subpage(struct copan_power_subpage *page)
4002 {
4003         int num_luns, num_partitions, config_type;
4004         struct ctl_softc *softc;
4005         cs_BOOL_t aor_present, shelf_50pct_power;
4006         cs_raidset_personality_t rs_type;
4007         int max_active_luns;
4008
4009         softc = control_softc;
4010
4011         /* subtract out the processor LUN */
4012         num_luns = softc->num_luns - 1;
4013         /*
4014          * Default to 7 LUNs active, which was the only number we allowed
4015          * in the past.
4016          */
4017         max_active_luns = 7;
4018
4019         num_partitions = config_GetRsPartitionInfo();
4020         config_type = config_GetConfigType();
4021         shelf_50pct_power = config_GetShelfPowerMode();
4022         aor_present = config_IsAorRsPresent();
4023
4024         rs_type = ddb_GetRsRaidType(1);
4025         if ((rs_type != CS_RAIDSET_PERSONALITY_RAID5)
4026          && (rs_type != CS_RAIDSET_PERSONALITY_RAID1)) {
4027                 EPRINT(0, "Unsupported RS type %d!", rs_type);
4028         }
4029
4030
4031         page->total_luns = num_luns;
4032
4033         switch (config_type) {
4034         case 40:
4035                 /*
4036                  * In a 40 drive configuration, it doesn't matter what DC
4037                  * cards we have, whether we have AOR enabled or not,
4038                  * partitioning or not, or what type of RAIDset we have.
4039                  * In that scenario, we can power up every LUN we present
4040                  * to the user.
4041                  */
4042                 max_active_luns = num_luns;
4043
4044                 break;
4045         case 64:
4046                 if (shelf_50pct_power == CS_FALSE) {
4047                         /* 25% power */
4048                         if (aor_present == CS_TRUE) {
4049                                 if (rs_type ==
4050                                      CS_RAIDSET_PERSONALITY_RAID5) {
4051                                         max_active_luns = 7;
4052                                 } else if (rs_type ==
4053                                          CS_RAIDSET_PERSONALITY_RAID1){
4054                                         max_active_luns = 14;
4055                                 } else {
4056                                         /* XXX KDM now what?? */
4057                                 }
4058                         } else {
4059                                 if (rs_type ==
4060                                      CS_RAIDSET_PERSONALITY_RAID5) {
4061                                         max_active_luns = 8;
4062                                 } else if (rs_type ==
4063                                          CS_RAIDSET_PERSONALITY_RAID1){
4064                                         max_active_luns = 16;
4065                                 } else {
4066                                         /* XXX KDM now what?? */
4067                                 }
4068                         }
4069                 } else {
4070                         /* 50% power */
4071                         /*
4072                          * With 50% power in a 64 drive configuration, we
4073                          * can power all LUNs we present.
4074                          */
4075                         max_active_luns = num_luns;
4076                 }
4077                 break;
4078         case 112:
4079                 if (shelf_50pct_power == CS_FALSE) {
4080                         /* 25% power */
4081                         if (aor_present == CS_TRUE) {
4082                                 if (rs_type ==
4083                                      CS_RAIDSET_PERSONALITY_RAID5) {
4084                                         max_active_luns = 7;
4085                                 } else if (rs_type ==
4086                                          CS_RAIDSET_PERSONALITY_RAID1){
4087                                         max_active_luns = 14;
4088                                 } else {
4089                                         /* XXX KDM now what?? */
4090                                 }
4091                         } else {
4092                                 if (rs_type ==
4093                                      CS_RAIDSET_PERSONALITY_RAID5) {
4094                                         max_active_luns = 8;
4095                                 } else if (rs_type ==
4096                                          CS_RAIDSET_PERSONALITY_RAID1){
4097                                         max_active_luns = 16;
4098                                 } else {
4099                                         /* XXX KDM now what?? */
4100                                 }
4101                         }
4102                 } else {
4103                         /* 50% power */
4104                         if (aor_present == CS_TRUE) {
4105                                 if (rs_type ==
4106                                      CS_RAIDSET_PERSONALITY_RAID5) {
4107                                         max_active_luns = 14;
4108                                 } else if (rs_type ==
4109                                          CS_RAIDSET_PERSONALITY_RAID1){
4110                                         /*
4111                                          * We're assuming here that disk
4112                                          * caching is enabled, and so we're
4113                                          * able to power up half of each
4114                                          * LUN, and cache all writes.
4115                                          */
4116                                         max_active_luns = num_luns;
4117                                 } else {
4118                                         /* XXX KDM now what?? */
4119                                 }
4120                         } else {
4121                                 if (rs_type ==
4122                                      CS_RAIDSET_PERSONALITY_RAID5) {
4123                                         max_active_luns = 15;
4124                                 } else if (rs_type ==
4125                                          CS_RAIDSET_PERSONALITY_RAID1){
4126                                         max_active_luns = 30;
4127                                 } else {
4128                                         /* XXX KDM now what?? */
4129                                 }
4130                         }
4131                 }
4132                 break;
4133         default:
4134                 /*
4135                  * In this case, we have an unknown configuration, so we
4136                  * just use the default from above.
4137                  */
4138                 break;
4139         }
4140
4141         page->max_active_luns = max_active_luns;
4142 #if 0
4143         printk("%s: total_luns = %d, max_active_luns = %d\n", __func__,
4144                page->total_luns, page->max_active_luns);
4145 #endif
4146 }
4147 #endif /* NEEDTOPORT */
4148
4149 /*
4150  * This routine could be used in the future to load default and/or saved
4151  * mode page parameters for a particuar lun.
4152  */
4153 static int
4154 ctl_init_page_index(struct ctl_lun *lun)
4155 {
4156         int i;
4157         struct ctl_page_index *page_index;
4158         struct ctl_softc *softc;
4159         const char *value;
4160
4161         memcpy(&lun->mode_pages.index, page_index_template,
4162                sizeof(page_index_template));
4163
4164         softc = lun->ctl_softc;
4165
4166         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4167
4168                 page_index = &lun->mode_pages.index[i];
4169                 /*
4170                  * If this is a disk-only mode page, there's no point in
4171                  * setting it up.  For some pages, we have to have some
4172                  * basic information about the disk in order to calculate the
4173                  * mode page data.
4174                  */
4175                 if ((lun->be_lun->lun_type != T_DIRECT)
4176                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
4177                         continue;
4178
4179                 switch (page_index->page_code & SMPH_PC_MASK) {
4180                 case SMS_FORMAT_DEVICE_PAGE: {
4181                         struct scsi_format_page *format_page;
4182
4183                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4184                                 panic("subpage is incorrect!");
4185
4186                         /*
4187                          * Sectors per track are set above.  Bytes per
4188                          * sector need to be set here on a per-LUN basis.
4189                          */
4190                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
4191                                &format_page_default,
4192                                sizeof(format_page_default));
4193                         memcpy(&lun->mode_pages.format_page[
4194                                CTL_PAGE_CHANGEABLE], &format_page_changeable,
4195                                sizeof(format_page_changeable));
4196                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
4197                                &format_page_default,
4198                                sizeof(format_page_default));
4199                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
4200                                &format_page_default,
4201                                sizeof(format_page_default));
4202
4203                         format_page = &lun->mode_pages.format_page[
4204                                 CTL_PAGE_CURRENT];
4205                         scsi_ulto2b(lun->be_lun->blocksize,
4206                                     format_page->bytes_per_sector);
4207
4208                         format_page = &lun->mode_pages.format_page[
4209                                 CTL_PAGE_DEFAULT];
4210                         scsi_ulto2b(lun->be_lun->blocksize,
4211                                     format_page->bytes_per_sector);
4212
4213                         format_page = &lun->mode_pages.format_page[
4214                                 CTL_PAGE_SAVED];
4215                         scsi_ulto2b(lun->be_lun->blocksize,
4216                                     format_page->bytes_per_sector);
4217
4218                         page_index->page_data =
4219                                 (uint8_t *)lun->mode_pages.format_page;
4220                         break;
4221                 }
4222                 case SMS_RIGID_DISK_PAGE: {
4223                         struct scsi_rigid_disk_page *rigid_disk_page;
4224                         uint32_t sectors_per_cylinder;
4225                         uint64_t cylinders;
4226 #ifndef __XSCALE__
4227                         int shift;
4228 #endif /* !__XSCALE__ */
4229
4230                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4231                                 panic("invalid subpage value %d",
4232                                       page_index->subpage);
4233
4234                         /*
4235                          * Rotation rate and sectors per track are set
4236                          * above.  We calculate the cylinders here based on
4237                          * capacity.  Due to the number of heads and
4238                          * sectors per track we're using, smaller arrays
4239                          * may turn out to have 0 cylinders.  Linux and
4240                          * FreeBSD don't pay attention to these mode pages
4241                          * to figure out capacity, but Solaris does.  It
4242                          * seems to deal with 0 cylinders just fine, and
4243                          * works out a fake geometry based on the capacity.
4244                          */
4245                         memcpy(&lun->mode_pages.rigid_disk_page[
4246                                CTL_PAGE_CURRENT], &rigid_disk_page_default,
4247                                sizeof(rigid_disk_page_default));
4248                         memcpy(&lun->mode_pages.rigid_disk_page[
4249                                CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
4250                                sizeof(rigid_disk_page_changeable));
4251                         memcpy(&lun->mode_pages.rigid_disk_page[
4252                                CTL_PAGE_DEFAULT], &rigid_disk_page_default,
4253                                sizeof(rigid_disk_page_default));
4254                         memcpy(&lun->mode_pages.rigid_disk_page[
4255                                CTL_PAGE_SAVED], &rigid_disk_page_default,
4256                                sizeof(rigid_disk_page_default));
4257
4258                         sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
4259                                 CTL_DEFAULT_HEADS;
4260
4261                         /*
4262                          * The divide method here will be more accurate,
4263                          * probably, but results in floating point being
4264                          * used in the kernel on i386 (__udivdi3()).  On the
4265                          * XScale, though, __udivdi3() is implemented in
4266                          * software.
4267                          *
4268                          * The shift method for cylinder calculation is
4269                          * accurate if sectors_per_cylinder is a power of
4270                          * 2.  Otherwise it might be slightly off -- you
4271                          * might have a bit of a truncation problem.
4272                          */
4273 #ifdef  __XSCALE__
4274                         cylinders = (lun->be_lun->maxlba + 1) /
4275                                 sectors_per_cylinder;
4276 #else
4277                         for (shift = 31; shift > 0; shift--) {
4278                                 if (sectors_per_cylinder & (1 << shift))
4279                                         break;
4280                         }
4281                         cylinders = (lun->be_lun->maxlba + 1) >> shift;
4282 #endif
4283
4284                         /*
4285                          * We've basically got 3 bytes, or 24 bits for the
4286                          * cylinder size in the mode page.  If we're over,
4287                          * just round down to 2^24.
4288                          */
4289                         if (cylinders > 0xffffff)
4290                                 cylinders = 0xffffff;
4291
4292                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4293                                 CTL_PAGE_CURRENT];
4294                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4295
4296                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4297                                 CTL_PAGE_DEFAULT];
4298                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4299
4300                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4301                                 CTL_PAGE_SAVED];
4302                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4303
4304                         page_index->page_data =
4305                                 (uint8_t *)lun->mode_pages.rigid_disk_page;
4306                         break;
4307                 }
4308                 case SMS_CACHING_PAGE: {
4309                         struct scsi_caching_page *caching_page;
4310
4311                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4312                                 panic("invalid subpage value %d",
4313                                       page_index->subpage);
4314                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
4315                                &caching_page_default,
4316                                sizeof(caching_page_default));
4317                         memcpy(&lun->mode_pages.caching_page[
4318                                CTL_PAGE_CHANGEABLE], &caching_page_changeable,
4319                                sizeof(caching_page_changeable));
4320                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4321                                &caching_page_default,
4322                                sizeof(caching_page_default));
4323                         caching_page = &lun->mode_pages.caching_page[
4324                             CTL_PAGE_SAVED];
4325                         value = ctl_get_opt(&lun->be_lun->options, "writecache");
4326                         if (value != NULL && strcmp(value, "off") == 0)
4327                                 caching_page->flags1 &= ~SCP_WCE;
4328                         value = ctl_get_opt(&lun->be_lun->options, "readcache");
4329                         if (value != NULL && strcmp(value, "off") == 0)
4330                                 caching_page->flags1 |= SCP_RCD;
4331                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
4332                                &lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4333                                sizeof(caching_page_default));
4334                         page_index->page_data =
4335                                 (uint8_t *)lun->mode_pages.caching_page;
4336                         break;
4337                 }
4338                 case SMS_CONTROL_MODE_PAGE: {
4339                         struct scsi_control_page *control_page;
4340
4341                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4342                                 panic("invalid subpage value %d",
4343                                       page_index->subpage);
4344
4345                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
4346                                &control_page_default,
4347                                sizeof(control_page_default));
4348                         memcpy(&lun->mode_pages.control_page[
4349                                CTL_PAGE_CHANGEABLE], &control_page_changeable,
4350                                sizeof(control_page_changeable));
4351                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
4352                                &control_page_default,
4353                                sizeof(control_page_default));
4354                         control_page = &lun->mode_pages.control_page[
4355                             CTL_PAGE_SAVED];
4356                         value = ctl_get_opt(&lun->be_lun->options, "reordering");
4357                         if (value != NULL && strcmp(value, "unrestricted") == 0) {
4358                                 control_page->queue_flags &= ~SCP_QUEUE_ALG_MASK;
4359                                 control_page->queue_flags |= SCP_QUEUE_ALG_UNRESTRICTED;
4360                         }
4361                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
4362                                &lun->mode_pages.control_page[CTL_PAGE_SAVED],
4363                                sizeof(control_page_default));
4364                         page_index->page_data =
4365                                 (uint8_t *)lun->mode_pages.control_page;
4366                         break;
4367
4368                 }
4369                 case SMS_VENDOR_SPECIFIC_PAGE:{
4370                         switch (page_index->subpage) {
4371                         case PWR_SUBPAGE_CODE: {
4372                                 struct copan_power_subpage *current_page,
4373                                                            *saved_page;
4374
4375                                 memcpy(&lun->mode_pages.power_subpage[
4376                                        CTL_PAGE_CURRENT],
4377                                        &power_page_default,
4378                                        sizeof(power_page_default));
4379                                 memcpy(&lun->mode_pages.power_subpage[
4380                                        CTL_PAGE_CHANGEABLE],
4381                                        &power_page_changeable,
4382                                        sizeof(power_page_changeable));
4383                                 memcpy(&lun->mode_pages.power_subpage[
4384                                        CTL_PAGE_DEFAULT],
4385                                        &power_page_default,
4386                                        sizeof(power_page_default));
4387                                 memcpy(&lun->mode_pages.power_subpage[
4388                                        CTL_PAGE_SAVED],
4389                                        &power_page_default,
4390                                        sizeof(power_page_default));
4391                                 page_index->page_data =
4392                                     (uint8_t *)lun->mode_pages.power_subpage;
4393
4394                                 current_page = (struct copan_power_subpage *)
4395                                         (page_index->page_data +
4396                                          (page_index->page_len *
4397                                           CTL_PAGE_CURRENT));
4398                                 saved_page = (struct copan_power_subpage *)
4399                                         (page_index->page_data +
4400                                          (page_index->page_len *
4401                                           CTL_PAGE_SAVED));
4402                                 break;
4403                         }
4404                         case APS_SUBPAGE_CODE: {
4405                                 struct copan_aps_subpage *current_page,
4406                                                          *saved_page;
4407
4408                                 // This gets set multiple times but
4409                                 // it should always be the same. It's
4410                                 // only done during init so who cares.
4411                                 index_to_aps_page = i;
4412
4413                                 memcpy(&lun->mode_pages.aps_subpage[
4414                                        CTL_PAGE_CURRENT],
4415                                        &aps_page_default,
4416                                        sizeof(aps_page_default));
4417                                 memcpy(&lun->mode_pages.aps_subpage[
4418                                        CTL_PAGE_CHANGEABLE],
4419                                        &aps_page_changeable,
4420                                        sizeof(aps_page_changeable));
4421                                 memcpy(&lun->mode_pages.aps_subpage[
4422                                        CTL_PAGE_DEFAULT],
4423                                        &aps_page_default,
4424                                        sizeof(aps_page_default));
4425                                 memcpy(&lun->mode_pages.aps_subpage[
4426                                        CTL_PAGE_SAVED],
4427                                        &aps_page_default,
4428                                        sizeof(aps_page_default));
4429                                 page_index->page_data =
4430                                         (uint8_t *)lun->mode_pages.aps_subpage;
4431
4432                                 current_page = (struct copan_aps_subpage *)
4433                                         (page_index->page_data +
4434                                          (page_index->page_len *
4435                                           CTL_PAGE_CURRENT));
4436                                 saved_page = (struct copan_aps_subpage *)
4437                                         (page_index->page_data +
4438                                          (page_index->page_len *
4439                                           CTL_PAGE_SAVED));
4440                                 break;
4441                         }
4442                         case DBGCNF_SUBPAGE_CODE: {
4443                                 struct copan_debugconf_subpage *current_page,
4444                                                                *saved_page;
4445
4446                                 memcpy(&lun->mode_pages.debugconf_subpage[
4447                                        CTL_PAGE_CURRENT],
4448                                        &debugconf_page_default,
4449                                        sizeof(debugconf_page_default));
4450                                 memcpy(&lun->mode_pages.debugconf_subpage[
4451                                        CTL_PAGE_CHANGEABLE],
4452                                        &debugconf_page_changeable,
4453                                        sizeof(debugconf_page_changeable));
4454                                 memcpy(&lun->mode_pages.debugconf_subpage[
4455                                        CTL_PAGE_DEFAULT],
4456                                        &debugconf_page_default,
4457                                        sizeof(debugconf_page_default));
4458                                 memcpy(&lun->mode_pages.debugconf_subpage[
4459                                        CTL_PAGE_SAVED],
4460                                        &debugconf_page_default,
4461                                        sizeof(debugconf_page_default));
4462                                 page_index->page_data =
4463                                         (uint8_t *)lun->mode_pages.debugconf_subpage;
4464
4465                                 current_page = (struct copan_debugconf_subpage *)
4466                                         (page_index->page_data +
4467                                          (page_index->page_len *
4468                                           CTL_PAGE_CURRENT));
4469                                 saved_page = (struct copan_debugconf_subpage *)
4470                                         (page_index->page_data +
4471                                          (page_index->page_len *
4472                                           CTL_PAGE_SAVED));
4473                                 break;
4474                         }
4475                         default:
4476                                 panic("invalid subpage value %d",
4477                                       page_index->subpage);
4478                                 break;
4479                         }
4480                         break;
4481                 }
4482                 default:
4483                         panic("invalid page value %d",
4484                               page_index->page_code & SMPH_PC_MASK);
4485                         break;
4486         }
4487         }
4488
4489         return (CTL_RETVAL_COMPLETE);
4490 }
4491
4492 /*
4493  * LUN allocation.
4494  *
4495  * Requirements:
4496  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4497  *   wants us to allocate the LUN and he can block.
4498  * - ctl_softc is always set
4499  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4500  *
4501  * Returns 0 for success, non-zero (errno) for failure.
4502  */
4503 static int
4504 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4505               struct ctl_be_lun *const be_lun, struct ctl_id target_id)
4506 {
4507         struct ctl_lun *nlun, *lun;
4508         struct ctl_port *port;
4509         struct scsi_vpd_id_descriptor *desc;
4510         struct scsi_vpd_id_t10 *t10id;
4511         const char *eui, *naa, *scsiname, *vendor, *value;
4512         int lun_number, i, lun_malloced;
4513         int devidlen, idlen1, idlen2 = 0, len;
4514
4515         if (be_lun == NULL)
4516                 return (EINVAL);
4517
4518         /*
4519          * We currently only support Direct Access or Processor LUN types.
4520          */
4521         switch (be_lun->lun_type) {
4522         case T_DIRECT:
4523                 break;
4524         case T_PROCESSOR:
4525                 break;
4526         case T_SEQUENTIAL:
4527         case T_CHANGER:
4528         default:
4529                 be_lun->lun_config_status(be_lun->be_lun,
4530                                           CTL_LUN_CONFIG_FAILURE);
4531                 break;
4532         }
4533         if (ctl_lun == NULL) {
4534                 lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4535                 lun_malloced = 1;
4536         } else {
4537                 lun_malloced = 0;
4538                 lun = ctl_lun;
4539         }
4540
4541         memset(lun, 0, sizeof(*lun));
4542         if (lun_malloced)
4543                 lun->flags = CTL_LUN_MALLOCED;
4544
4545         /* Generate LUN ID. */
4546         devidlen = max(CTL_DEVID_MIN_LEN,
4547             strnlen(be_lun->device_id, CTL_DEVID_LEN));
4548         idlen1 = sizeof(*t10id) + devidlen;
4549         len = sizeof(struct scsi_vpd_id_descriptor) + idlen1;
4550         scsiname = ctl_get_opt(&be_lun->options, "scsiname");
4551         if (scsiname != NULL) {
4552                 idlen2 = roundup2(strlen(scsiname) + 1, 4);
4553                 len += sizeof(struct scsi_vpd_id_descriptor) + idlen2;
4554         }
4555         eui = ctl_get_opt(&be_lun->options, "eui");
4556         if (eui != NULL) {
4557                 len += sizeof(struct scsi_vpd_id_descriptor) + 8;
4558         }
4559         naa = ctl_get_opt(&be_lun->options, "naa");
4560         if (naa != NULL) {
4561                 len += sizeof(struct scsi_vpd_id_descriptor) + 8;
4562         }
4563         lun->lun_devid = malloc(sizeof(struct ctl_devid) + len,
4564             M_CTL, M_WAITOK | M_ZERO);
4565         lun->lun_devid->len = len;
4566         desc = (struct scsi_vpd_id_descriptor *)lun->lun_devid->data;
4567         desc->proto_codeset = SVPD_ID_CODESET_ASCII;
4568         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
4569         desc->length = idlen1;
4570         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
4571         memset(t10id->vendor, ' ', sizeof(t10id->vendor));
4572         if ((vendor = ctl_get_opt(&be_lun->options, "vendor")) == NULL) {
4573                 strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
4574         } else {
4575                 strncpy(t10id->vendor, vendor,
4576                     min(sizeof(t10id->vendor), strlen(vendor)));
4577         }
4578         strncpy((char *)t10id->vendor_spec_id,
4579             (char *)be_lun->device_id, devidlen);
4580         if (scsiname != NULL) {
4581                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4582                     desc->length);
4583                 desc->proto_codeset = SVPD_ID_CODESET_UTF8;
4584                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4585                     SVPD_ID_TYPE_SCSI_NAME;
4586                 desc->length = idlen2;
4587                 strlcpy(desc->identifier, scsiname, idlen2);
4588         }
4589         if (eui != NULL) {
4590                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4591                     desc->length);
4592                 desc->proto_codeset = SVPD_ID_CODESET_BINARY;
4593                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4594                     SVPD_ID_TYPE_EUI64;
4595                 desc->length = 8;
4596                 scsi_u64to8b(strtouq(eui, NULL, 0), desc->identifier);
4597         }
4598         if (naa != NULL) {
4599                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4600                     desc->length);
4601                 desc->proto_codeset = SVPD_ID_CODESET_BINARY;
4602                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4603                     SVPD_ID_TYPE_NAA;
4604                 desc->length = 8;
4605                 scsi_u64to8b(strtouq(naa, NULL, 0), desc->identifier);
4606         }
4607
4608         mtx_lock(&ctl_softc->ctl_lock);
4609         /*
4610          * See if the caller requested a particular LUN number.  If so, see
4611          * if it is available.  Otherwise, allocate the first available LUN.
4612          */
4613         if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4614                 if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4615                  || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4616                         mtx_unlock(&ctl_softc->ctl_lock);
4617                         if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4618                                 printf("ctl: requested LUN ID %d is higher "
4619                                        "than CTL_MAX_LUNS - 1 (%d)\n",
4620                                        be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4621                         } else {
4622                                 /*
4623                                  * XXX KDM return an error, or just assign
4624                                  * another LUN ID in this case??
4625                                  */
4626                                 printf("ctl: requested LUN ID %d is already "
4627                                        "in use\n", be_lun->req_lun_id);
4628                         }
4629                         if (lun->flags & CTL_LUN_MALLOCED)
4630                                 free(lun, M_CTL);
4631                         be_lun->lun_config_status(be_lun->be_lun,
4632                                                   CTL_LUN_CONFIG_FAILURE);
4633                         return (ENOSPC);
4634                 }
4635                 lun_number = be_lun->req_lun_id;
4636         } else {
4637                 lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, CTL_MAX_LUNS);
4638                 if (lun_number == -1) {
4639                         mtx_unlock(&ctl_softc->ctl_lock);
4640                         printf("ctl: can't allocate LUN on target %ju, out of "
4641                                "LUNs\n", (uintmax_t)target_id.id);
4642                         if (lun->flags & CTL_LUN_MALLOCED)
4643                                 free(lun, M_CTL);
4644                         be_lun->lun_config_status(be_lun->be_lun,
4645                                                   CTL_LUN_CONFIG_FAILURE);
4646                         return (ENOSPC);
4647                 }
4648         }
4649         ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4650
4651         mtx_init(&lun->lun_lock, "CTL LUN", NULL, MTX_DEF);
4652         lun->target = target_id;
4653         lun->lun = lun_number;
4654         lun->be_lun = be_lun;
4655         /*
4656          * The processor LUN is always enabled.  Disk LUNs come on line
4657          * disabled, and must be enabled by the backend.
4658          */
4659         lun->flags |= CTL_LUN_DISABLED;
4660         lun->backend = be_lun->be;
4661         be_lun->ctl_lun = lun;
4662         be_lun->lun_id = lun_number;
4663         atomic_add_int(&be_lun->be->num_luns, 1);
4664         if (be_lun->flags & CTL_LUN_FLAG_OFFLINE)
4665                 lun->flags |= CTL_LUN_OFFLINE;
4666
4667         if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4668                 lun->flags |= CTL_LUN_STOPPED;
4669
4670         if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4671                 lun->flags |= CTL_LUN_INOPERABLE;
4672
4673         if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4674                 lun->flags |= CTL_LUN_PRIMARY_SC;
4675
4676         value = ctl_get_opt(&be_lun->options, "readonly");
4677         if (value != NULL && strcmp(value, "on") == 0)
4678                 lun->flags |= CTL_LUN_READONLY;
4679
4680         lun->ctl_softc = ctl_softc;
4681         TAILQ_INIT(&lun->ooa_queue);
4682         TAILQ_INIT(&lun->blocked_queue);
4683         STAILQ_INIT(&lun->error_list);
4684         ctl_tpc_lun_init(lun);
4685
4686         /*
4687          * Initialize the mode page index.
4688          */
4689         ctl_init_page_index(lun);
4690
4691         /*
4692          * Set the poweron UA for all initiators on this LUN only.
4693          */
4694         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4695                 lun->pending_ua[i] = CTL_UA_POWERON;
4696
4697         /*
4698          * Now, before we insert this lun on the lun list, set the lun
4699          * inventory changed UA for all other luns.
4700          */
4701         STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4702                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4703                         nlun->pending_ua[i] |= CTL_UA_LUN_CHANGE;
4704                 }
4705         }
4706
4707         STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4708
4709         ctl_softc->ctl_luns[lun_number] = lun;
4710
4711         ctl_softc->num_luns++;
4712
4713         /* Setup statistics gathering */
4714         lun->stats.device_type = be_lun->lun_type;
4715         lun->stats.lun_number = lun_number;
4716         if (lun->stats.device_type == T_DIRECT)
4717                 lun->stats.blocksize = be_lun->blocksize;
4718         else
4719                 lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4720         for (i = 0;i < CTL_MAX_PORTS;i++)
4721                 lun->stats.ports[i].targ_port = i;
4722
4723         mtx_unlock(&ctl_softc->ctl_lock);
4724
4725         lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4726
4727         /*
4728          * Run through each registered FETD and bring it online if it isn't
4729          * already.  Enable the target ID if it hasn't been enabled, and
4730          * enable this particular LUN.
4731          */
4732         STAILQ_FOREACH(port, &ctl_softc->port_list, links) {
4733                 int retval;
4734
4735                 retval = port->lun_enable(port->targ_lun_arg, target_id,lun_number);
4736                 if (retval != 0) {
4737                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4738                                "%d for lun_enable on target %ju lun %d\n",
4739                                port->port_name, port->targ_port, retval,
4740                                (uintmax_t)target_id.id, lun_number);
4741                 } else
4742                         port->status |= CTL_PORT_STATUS_LUN_ONLINE;
4743         }
4744         return (0);
4745 }
4746
4747 /*
4748  * Delete a LUN.
4749  * Assumptions:
4750  * - LUN has already been marked invalid and any pending I/O has been taken
4751  *   care of.
4752  */
4753 static int
4754 ctl_free_lun(struct ctl_lun *lun)
4755 {
4756         struct ctl_softc *softc;
4757 #if 0
4758         struct ctl_port *port;
4759 #endif
4760         struct ctl_lun *nlun;
4761         int i;
4762
4763         softc = lun->ctl_softc;
4764
4765         mtx_assert(&softc->ctl_lock, MA_OWNED);
4766
4767         STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4768
4769         ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4770
4771         softc->ctl_luns[lun->lun] = NULL;
4772
4773         if (!TAILQ_EMPTY(&lun->ooa_queue))
4774                 panic("Freeing a LUN %p with outstanding I/O!!\n", lun);
4775
4776         softc->num_luns--;
4777
4778         /*
4779          * XXX KDM this scheme only works for a single target/multiple LUN
4780          * setup.  It needs to be revamped for a multiple target scheme.
4781          *
4782          * XXX KDM this results in port->lun_disable() getting called twice,
4783          * once when ctl_disable_lun() is called, and a second time here.
4784          * We really need to re-think the LUN disable semantics.  There
4785          * should probably be several steps/levels to LUN removal:
4786          *  - disable
4787          *  - invalidate
4788          *  - free
4789          *
4790          * Right now we only have a disable method when communicating to
4791          * the front end ports, at least for individual LUNs.
4792          */
4793 #if 0
4794         STAILQ_FOREACH(port, &softc->port_list, links) {
4795                 int retval;
4796
4797                 retval = port->lun_disable(port->targ_lun_arg, lun->target,
4798                                          lun->lun);
4799                 if (retval != 0) {
4800                         printf("ctl_free_lun: FETD %s port %d returned error "
4801                                "%d for lun_disable on target %ju lun %jd\n",
4802                                port->port_name, port->targ_port, retval,
4803                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4804                 }
4805
4806                 if (STAILQ_FIRST(&softc->lun_list) == NULL) {
4807                         port->status &= ~CTL_PORT_STATUS_LUN_ONLINE;
4808
4809                         retval = port->targ_disable(port->targ_lun_arg,lun->target);
4810                         if (retval != 0) {
4811                                 printf("ctl_free_lun: FETD %s port %d "
4812                                        "returned error %d for targ_disable on "
4813                                        "target %ju\n", port->port_name,
4814                                        port->targ_port, retval,
4815                                        (uintmax_t)lun->target.id);
4816                         } else
4817                                 port->status &= ~CTL_PORT_STATUS_TARG_ONLINE;
4818
4819                         if ((port->status & CTL_PORT_STATUS_TARG_ONLINE) != 0)
4820                                 continue;
4821
4822 #if 0
4823                         port->port_offline(port->onoff_arg);
4824                         port->status &= ~CTL_PORT_STATUS_ONLINE;
4825 #endif
4826                 }
4827         }
4828 #endif
4829
4830         /*
4831          * Tell the backend to free resources, if this LUN has a backend.
4832          */
4833         atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4834         lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4835
4836         ctl_tpc_lun_shutdown(lun);
4837         mtx_destroy(&lun->lun_lock);
4838         free(lun->lun_devid, M_CTL);
4839         if (lun->flags & CTL_LUN_MALLOCED)
4840                 free(lun, M_CTL);
4841
4842         STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4843                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4844                         nlun->pending_ua[i] |= CTL_UA_LUN_CHANGE;
4845                 }
4846         }
4847
4848         return (0);
4849 }
4850
4851 static void
4852 ctl_create_lun(struct ctl_be_lun *be_lun)
4853 {
4854         struct ctl_softc *ctl_softc;
4855
4856         ctl_softc = control_softc;
4857
4858         /*
4859          * ctl_alloc_lun() should handle all potential failure cases.
4860          */
4861         ctl_alloc_lun(ctl_softc, NULL, be_lun, ctl_softc->target);
4862 }
4863
4864 int
4865 ctl_add_lun(struct ctl_be_lun *be_lun)
4866 {
4867         struct ctl_softc *ctl_softc = control_softc;
4868
4869         mtx_lock(&ctl_softc->ctl_lock);
4870         STAILQ_INSERT_TAIL(&ctl_softc->pending_lun_queue, be_lun, links);
4871         mtx_unlock(&ctl_softc->ctl_lock);
4872         wakeup(&ctl_softc->pending_lun_queue);
4873
4874         return (0);
4875 }
4876
4877 int
4878 ctl_enable_lun(struct ctl_be_lun *be_lun)
4879 {
4880         struct ctl_softc *ctl_softc;
4881         struct ctl_port *port, *nport;
4882         struct ctl_lun *lun;
4883         int retval;
4884
4885         ctl_softc = control_softc;
4886
4887         lun = (struct ctl_lun *)be_lun->ctl_lun;
4888
4889         mtx_lock(&ctl_softc->ctl_lock);
4890         mtx_lock(&lun->lun_lock);
4891         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4892                 /*
4893                  * eh?  Why did we get called if the LUN is already
4894                  * enabled?
4895                  */
4896                 mtx_unlock(&lun->lun_lock);
4897                 mtx_unlock(&ctl_softc->ctl_lock);
4898                 return (0);
4899         }
4900         lun->flags &= ~CTL_LUN_DISABLED;
4901         mtx_unlock(&lun->lun_lock);
4902
4903         for (port = STAILQ_FIRST(&ctl_softc->port_list); port != NULL; port = nport) {
4904                 nport = STAILQ_NEXT(port, links);
4905
4906                 /*
4907                  * Drop the lock while we call the FETD's enable routine.
4908                  * This can lead to a callback into CTL (at least in the
4909                  * case of the internal initiator frontend.
4910                  */
4911                 mtx_unlock(&ctl_softc->ctl_lock);
4912                 retval = port->lun_enable(port->targ_lun_arg, lun->target,lun->lun);
4913                 mtx_lock(&ctl_softc->ctl_lock);
4914                 if (retval != 0) {
4915                         printf("%s: FETD %s port %d returned error "
4916                                "%d for lun_enable on target %ju lun %jd\n",
4917                                __func__, port->port_name, port->targ_port, retval,
4918                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4919                 }
4920 #if 0
4921                  else {
4922             /* NOTE:  TODO:  why does lun enable affect port status? */
4923                         port->status |= CTL_PORT_STATUS_LUN_ONLINE;
4924                 }
4925 #endif
4926         }
4927
4928         mtx_unlock(&ctl_softc->ctl_lock);
4929
4930         return (0);
4931 }
4932
4933 int
4934 ctl_disable_lun(struct ctl_be_lun *be_lun)
4935 {
4936         struct ctl_softc *ctl_softc;
4937         struct ctl_port *port;
4938         struct ctl_lun *lun;
4939         int retval;
4940
4941         ctl_softc = control_softc;
4942
4943         lun = (struct ctl_lun *)be_lun->ctl_lun;
4944
4945         mtx_lock(&ctl_softc->ctl_lock);
4946         mtx_lock(&lun->lun_lock);
4947         if (lun->flags & CTL_LUN_DISABLED) {
4948                 mtx_unlock(&lun->lun_lock);
4949                 mtx_unlock(&ctl_softc->ctl_lock);
4950                 return (0);
4951         }
4952         lun->flags |= CTL_LUN_DISABLED;
4953         mtx_unlock(&lun->lun_lock);
4954
4955         STAILQ_FOREACH(port, &ctl_softc->port_list, links) {
4956                 mtx_unlock(&ctl_softc->ctl_lock);
4957                 /*
4958                  * Drop the lock before we call the frontend's disable
4959                  * routine, to avoid lock order reversals.
4960                  *
4961                  * XXX KDM what happens if the frontend list changes while
4962                  * we're traversing it?  It's unlikely, but should be handled.
4963                  */
4964                 retval = port->lun_disable(port->targ_lun_arg, lun->target,
4965                                          lun->lun);
4966                 mtx_lock(&ctl_softc->ctl_lock);
4967                 if (retval != 0) {
4968                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4969                                "%d for lun_disable on target %ju lun %jd\n",
4970                                port->port_name, port->targ_port, retval,
4971                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4972                 }
4973         }
4974
4975         mtx_unlock(&ctl_softc->ctl_lock);
4976
4977         return (0);
4978 }
4979
4980 int
4981 ctl_start_lun(struct ctl_be_lun *be_lun)
4982 {
4983         struct ctl_softc *ctl_softc;
4984         struct ctl_lun *lun;
4985
4986         ctl_softc = control_softc;
4987
4988         lun = (struct ctl_lun *)be_lun->ctl_lun;
4989
4990         mtx_lock(&lun->lun_lock);
4991         lun->flags &= ~CTL_LUN_STOPPED;
4992         mtx_unlock(&lun->lun_lock);
4993
4994         return (0);
4995 }
4996
4997 int
4998 ctl_stop_lun(struct ctl_be_lun *be_lun)
4999 {
5000         struct ctl_softc *ctl_softc;
5001         struct ctl_lun *lun;
5002
5003         ctl_softc = control_softc;
5004
5005         lun = (struct ctl_lun *)be_lun->ctl_lun;
5006
5007         mtx_lock(&lun->lun_lock);
5008         lun->flags |= CTL_LUN_STOPPED;
5009         mtx_unlock(&lun->lun_lock);
5010
5011         return (0);
5012 }
5013
5014 int
5015 ctl_lun_offline(struct ctl_be_lun *be_lun)
5016 {
5017         struct ctl_softc *ctl_softc;
5018         struct ctl_lun *lun;
5019
5020         ctl_softc = control_softc;
5021
5022         lun = (struct ctl_lun *)be_lun->ctl_lun;
5023
5024         mtx_lock(&lun->lun_lock);
5025         lun->flags |= CTL_LUN_OFFLINE;
5026         mtx_unlock(&lun->lun_lock);
5027
5028         return (0);
5029 }
5030
5031 int
5032 ctl_lun_online(struct ctl_be_lun *be_lun)
5033 {
5034         struct ctl_softc *ctl_softc;
5035         struct ctl_lun *lun;
5036
5037         ctl_softc = control_softc;
5038
5039         lun = (struct ctl_lun *)be_lun->ctl_lun;
5040
5041         mtx_lock(&lun->lun_lock);
5042         lun->flags &= ~CTL_LUN_OFFLINE;
5043         mtx_unlock(&lun->lun_lock);
5044
5045         return (0);
5046 }
5047
5048 int
5049 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
5050 {
5051         struct ctl_softc *ctl_softc;
5052         struct ctl_lun *lun;
5053
5054         ctl_softc = control_softc;
5055
5056         lun = (struct ctl_lun *)be_lun->ctl_lun;
5057
5058         mtx_lock(&lun->lun_lock);
5059
5060         /*
5061          * The LUN needs to be disabled before it can be marked invalid.
5062          */
5063         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
5064                 mtx_unlock(&lun->lun_lock);
5065                 return (-1);
5066         }
5067         /*
5068          * Mark the LUN invalid.
5069          */
5070         lun->flags |= CTL_LUN_INVALID;
5071
5072         /*
5073          * If there is nothing in the OOA queue, go ahead and free the LUN.
5074          * If we have something in the OOA queue, we'll free it when the
5075          * last I/O completes.
5076          */
5077         if (TAILQ_EMPTY(&lun->ooa_queue)) {
5078                 mtx_unlock(&lun->lun_lock);
5079                 mtx_lock(&ctl_softc->ctl_lock);
5080                 ctl_free_lun(lun);
5081                 mtx_unlock(&ctl_softc->ctl_lock);
5082         } else
5083                 mtx_unlock(&lun->lun_lock);
5084
5085         return (0);
5086 }
5087
5088 int
5089 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
5090 {
5091         struct ctl_softc *ctl_softc;
5092         struct ctl_lun *lun;
5093
5094         ctl_softc = control_softc;
5095         lun = (struct ctl_lun *)be_lun->ctl_lun;
5096
5097         mtx_lock(&lun->lun_lock);
5098         lun->flags |= CTL_LUN_INOPERABLE;
5099         mtx_unlock(&lun->lun_lock);
5100
5101         return (0);
5102 }
5103
5104 int
5105 ctl_lun_operable(struct ctl_be_lun *be_lun)
5106 {
5107         struct ctl_softc *ctl_softc;
5108         struct ctl_lun *lun;
5109
5110         ctl_softc = control_softc;
5111         lun = (struct ctl_lun *)be_lun->ctl_lun;
5112
5113         mtx_lock(&lun->lun_lock);
5114         lun->flags &= ~CTL_LUN_INOPERABLE;
5115         mtx_unlock(&lun->lun_lock);
5116
5117         return (0);
5118 }
5119
5120 int
5121 ctl_lun_power_lock(struct ctl_be_lun *be_lun, struct ctl_nexus *nexus,
5122                    int lock)
5123 {
5124         struct ctl_softc *softc;
5125         struct ctl_lun *lun;
5126         struct copan_aps_subpage *current_sp;
5127         struct ctl_page_index *page_index;
5128         int i;
5129
5130         softc = control_softc;
5131
5132         mtx_lock(&softc->ctl_lock);
5133
5134         lun = (struct ctl_lun *)be_lun->ctl_lun;
5135         mtx_lock(&lun->lun_lock);
5136
5137         page_index = NULL;
5138         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
5139                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
5140                      APS_PAGE_CODE)
5141                         continue;
5142
5143                 if (lun->mode_pages.index[i].subpage != APS_SUBPAGE_CODE)
5144                         continue;
5145                 page_index = &lun->mode_pages.index[i];
5146         }
5147
5148         if (page_index == NULL) {
5149                 mtx_unlock(&lun->lun_lock);
5150                 mtx_unlock(&softc->ctl_lock);
5151                 printf("%s: APS subpage not found for lun %ju!\n", __func__,
5152                        (uintmax_t)lun->lun);
5153                 return (1);
5154         }
5155 #if 0
5156         if ((softc->aps_locked_lun != 0)
5157          && (softc->aps_locked_lun != lun->lun)) {
5158                 printf("%s: attempt to lock LUN %llu when %llu is already "
5159                        "locked\n");
5160                 mtx_unlock(&lun->lun_lock);
5161                 mtx_unlock(&softc->ctl_lock);
5162                 return (1);
5163         }
5164 #endif
5165
5166         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
5167                 (page_index->page_len * CTL_PAGE_CURRENT));
5168
5169         if (lock != 0) {
5170                 current_sp->lock_active = APS_LOCK_ACTIVE;
5171                 softc->aps_locked_lun = lun->lun;
5172         } else {
5173                 current_sp->lock_active = 0;
5174                 softc->aps_locked_lun = 0;
5175         }
5176
5177
5178         /*
5179          * If we're in HA mode, try to send the lock message to the other
5180          * side.
5181          */
5182         if (ctl_is_single == 0) {
5183                 int isc_retval;
5184                 union ctl_ha_msg lock_msg;
5185
5186                 lock_msg.hdr.nexus = *nexus;
5187                 lock_msg.hdr.msg_type = CTL_MSG_APS_LOCK;
5188                 if (lock != 0)
5189                         lock_msg.aps.lock_flag = 1;
5190                 else
5191                         lock_msg.aps.lock_flag = 0;
5192                 isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &lock_msg,
5193                                          sizeof(lock_msg), 0);
5194                 if (isc_retval > CTL_HA_STATUS_SUCCESS) {
5195                         printf("%s: APS (lock=%d) error returned from "
5196                                "ctl_ha_msg_send: %d\n", __func__, lock, isc_retval);
5197                         mtx_unlock(&lun->lun_lock);
5198                         mtx_unlock(&softc->ctl_lock);
5199                         return (1);
5200                 }
5201         }
5202
5203         mtx_unlock(&lun->lun_lock);
5204         mtx_unlock(&softc->ctl_lock);
5205
5206         return (0);
5207 }
5208
5209 void
5210 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
5211 {
5212         struct ctl_lun *lun;
5213         struct ctl_softc *softc;
5214         int i;
5215
5216         softc = control_softc;
5217
5218         lun = (struct ctl_lun *)be_lun->ctl_lun;
5219
5220         mtx_lock(&lun->lun_lock);
5221
5222         for (i = 0; i < CTL_MAX_INITIATORS; i++)
5223                 lun->pending_ua[i] |= CTL_UA_CAPACITY_CHANGED;
5224
5225         mtx_unlock(&lun->lun_lock);
5226 }
5227
5228 /*
5229  * Backend "memory move is complete" callback for requests that never
5230  * make it down to say RAIDCore's configuration code.
5231  */
5232 int
5233 ctl_config_move_done(union ctl_io *io)
5234 {
5235         int retval;
5236
5237         retval = CTL_RETVAL_COMPLETE;
5238
5239
5240         CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
5241         /*
5242          * XXX KDM this shouldn't happen, but what if it does?
5243          */
5244         if (io->io_hdr.io_type != CTL_IO_SCSI)
5245                 panic("I/O type isn't CTL_IO_SCSI!");
5246
5247         if ((io->io_hdr.port_status == 0)
5248          && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
5249          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
5250                 io->io_hdr.status = CTL_SUCCESS;
5251         else if ((io->io_hdr.port_status != 0)
5252               && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
5253               && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)){
5254                 /*
5255                  * For hardware error sense keys, the sense key
5256                  * specific value is defined to be a retry count,
5257                  * but we use it to pass back an internal FETD
5258                  * error code.  XXX KDM  Hopefully the FETD is only
5259                  * using 16 bits for an error code, since that's
5260                  * all the space we have in the sks field.
5261                  */
5262                 ctl_set_internal_failure(&io->scsiio,
5263                                          /*sks_valid*/ 1,
5264                                          /*retry_count*/
5265                                          io->io_hdr.port_status);
5266                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5267                         free(io->scsiio.kern_data_ptr, M_CTL);
5268                 ctl_done(io);
5269                 goto bailout;
5270         }
5271
5272         if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
5273          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
5274          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
5275                 /*
5276                  * XXX KDM just assuming a single pointer here, and not a
5277                  * S/G list.  If we start using S/G lists for config data,
5278                  * we'll need to know how to clean them up here as well.
5279                  */
5280                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5281                         free(io->scsiio.kern_data_ptr, M_CTL);
5282                 /* Hopefully the user has already set the status... */
5283                 ctl_done(io);
5284         } else {
5285                 /*
5286                  * XXX KDM now we need to continue data movement.  Some
5287                  * options:
5288                  * - call ctl_scsiio() again?  We don't do this for data
5289                  *   writes, because for those at least we know ahead of
5290                  *   time where the write will go and how long it is.  For
5291                  *   config writes, though, that information is largely
5292                  *   contained within the write itself, thus we need to
5293                  *   parse out the data again.
5294                  *
5295                  * - Call some other function once the data is in?
5296                  */
5297
5298                 /*
5299                  * XXX KDM call ctl_scsiio() again for now, and check flag
5300                  * bits to see whether we're allocated or not.
5301                  */
5302                 retval = ctl_scsiio(&io->scsiio);
5303         }
5304 bailout:
5305         return (retval);
5306 }
5307
5308 /*
5309  * This gets called by a backend driver when it is done with a
5310  * data_submit method.
5311  */
5312 void
5313 ctl_data_submit_done(union ctl_io *io)
5314 {
5315         /*
5316          * If the IO_CONT flag is set, we need to call the supplied
5317          * function to continue processing the I/O, instead of completing
5318          * the I/O just yet.
5319          *
5320          * If there is an error, though, we don't want to keep processing.
5321          * Instead, just send status back to the initiator.
5322          */
5323         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5324             (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5325             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5326              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5327                 io->scsiio.io_cont(io);
5328                 return;
5329         }
5330         ctl_done(io);
5331 }
5332
5333 /*
5334  * This gets called by a backend driver when it is done with a
5335  * configuration write.
5336  */
5337 void
5338 ctl_config_write_done(union ctl_io *io)
5339 {
5340         uint8_t *buf;
5341
5342         /*
5343          * If the IO_CONT flag is set, we need to call the supplied
5344          * function to continue processing the I/O, instead of completing
5345          * the I/O just yet.
5346          *
5347          * If there is an error, though, we don't want to keep processing.
5348          * Instead, just send status back to the initiator.
5349          */
5350         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5351             (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5352             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5353              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5354                 io->scsiio.io_cont(io);
5355                 return;
5356         }
5357         /*
5358          * Since a configuration write can be done for commands that actually
5359          * have data allocated, like write buffer, and commands that have
5360          * no data, like start/stop unit, we need to check here.
5361          */
5362         if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5363                 buf = io->scsiio.kern_data_ptr;
5364         else
5365                 buf = NULL;
5366         ctl_done(io);
5367         if (buf)
5368                 free(buf, M_CTL);
5369 }
5370
5371 /*
5372  * SCSI release command.
5373  */
5374 int
5375 ctl_scsi_release(struct ctl_scsiio *ctsio)
5376 {
5377         int length, longid, thirdparty_id, resv_id;
5378         struct ctl_softc *ctl_softc;
5379         struct ctl_lun *lun;
5380         uint32_t residx;
5381
5382         length = 0;
5383         resv_id = 0;
5384
5385         CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
5386
5387         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5388         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5389         ctl_softc = control_softc;
5390
5391         switch (ctsio->cdb[0]) {
5392         case RELEASE_10: {
5393                 struct scsi_release_10 *cdb;
5394
5395                 cdb = (struct scsi_release_10 *)ctsio->cdb;
5396
5397                 if (cdb->byte2 & SR10_LONGID)
5398                         longid = 1;
5399                 else
5400                         thirdparty_id = cdb->thirdparty_id;
5401
5402                 resv_id = cdb->resv_id;
5403                 length = scsi_2btoul(cdb->length);
5404                 break;
5405         }
5406         }
5407
5408
5409         /*
5410          * XXX KDM right now, we only support LUN reservation.  We don't
5411          * support 3rd party reservations, or extent reservations, which
5412          * might actually need the parameter list.  If we've gotten this
5413          * far, we've got a LUN reservation.  Anything else got kicked out
5414          * above.  So, according to SPC, ignore the length.
5415          */
5416         length = 0;
5417
5418         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5419          && (length > 0)) {
5420                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5421                 ctsio->kern_data_len = length;
5422                 ctsio->kern_total_len = length;
5423                 ctsio->kern_data_resid = 0;
5424                 ctsio->kern_rel_offset = 0;
5425                 ctsio->kern_sg_entries = 0;
5426                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5427                 ctsio->be_move_done = ctl_config_move_done;
5428                 ctl_datamove((union ctl_io *)ctsio);
5429
5430                 return (CTL_RETVAL_COMPLETE);
5431         }
5432
5433         if (length > 0)
5434                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5435
5436         mtx_lock(&lun->lun_lock);
5437
5438         /*
5439          * According to SPC, it is not an error for an intiator to attempt
5440          * to release a reservation on a LUN that isn't reserved, or that
5441          * is reserved by another initiator.  The reservation can only be
5442          * released, though, by the initiator who made it or by one of
5443          * several reset type events.
5444          */
5445         if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == residx))
5446                         lun->flags &= ~CTL_LUN_RESERVED;
5447
5448         mtx_unlock(&lun->lun_lock);
5449
5450         ctsio->scsi_status = SCSI_STATUS_OK;
5451         ctsio->io_hdr.status = CTL_SUCCESS;
5452
5453         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5454                 free(ctsio->kern_data_ptr, M_CTL);
5455                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5456         }
5457
5458         ctl_done((union ctl_io *)ctsio);
5459         return (CTL_RETVAL_COMPLETE);
5460 }
5461
5462 int
5463 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5464 {
5465         int extent, thirdparty, longid;
5466         int resv_id, length;
5467         uint64_t thirdparty_id;
5468         struct ctl_softc *ctl_softc;
5469         struct ctl_lun *lun;
5470         uint32_t residx;
5471
5472         extent = 0;
5473         thirdparty = 0;
5474         longid = 0;
5475         resv_id = 0;
5476         length = 0;
5477         thirdparty_id = 0;
5478
5479         CTL_DEBUG_PRINT(("ctl_reserve\n"));
5480
5481         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5482         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5483         ctl_softc = control_softc;
5484
5485         switch (ctsio->cdb[0]) {
5486         case RESERVE_10: {
5487                 struct scsi_reserve_10 *cdb;
5488
5489                 cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5490
5491                 if (cdb->byte2 & SR10_LONGID)
5492                         longid = 1;
5493                 else
5494                         thirdparty_id = cdb->thirdparty_id;
5495
5496                 resv_id = cdb->resv_id;
5497                 length = scsi_2btoul(cdb->length);
5498                 break;
5499         }
5500         }
5501
5502         /*
5503          * XXX KDM right now, we only support LUN reservation.  We don't
5504          * support 3rd party reservations, or extent reservations, which
5505          * might actually need the parameter list.  If we've gotten this
5506          * far, we've got a LUN reservation.  Anything else got kicked out
5507          * above.  So, according to SPC, ignore the length.
5508          */
5509         length = 0;
5510
5511         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5512          && (length > 0)) {
5513                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5514                 ctsio->kern_data_len = length;
5515                 ctsio->kern_total_len = length;
5516                 ctsio->kern_data_resid = 0;
5517                 ctsio->kern_rel_offset = 0;
5518                 ctsio->kern_sg_entries = 0;
5519                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5520                 ctsio->be_move_done = ctl_config_move_done;
5521                 ctl_datamove((union ctl_io *)ctsio);
5522
5523                 return (CTL_RETVAL_COMPLETE);
5524         }
5525
5526         if (length > 0)
5527                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5528
5529         mtx_lock(&lun->lun_lock);
5530         if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx != residx)) {
5531                 ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
5532                 ctsio->io_hdr.status = CTL_SCSI_ERROR;
5533                 goto bailout;
5534         }
5535
5536         lun->flags |= CTL_LUN_RESERVED;
5537         lun->res_idx = residx;
5538
5539         ctsio->scsi_status = SCSI_STATUS_OK;
5540         ctsio->io_hdr.status = CTL_SUCCESS;
5541
5542 bailout:
5543         mtx_unlock(&lun->lun_lock);
5544
5545         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5546                 free(ctsio->kern_data_ptr, M_CTL);
5547                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5548         }
5549
5550         ctl_done((union ctl_io *)ctsio);
5551         return (CTL_RETVAL_COMPLETE);
5552 }
5553
5554 int
5555 ctl_start_stop(struct ctl_scsiio *ctsio)
5556 {
5557         struct scsi_start_stop_unit *cdb;
5558         struct ctl_lun *lun;
5559         struct ctl_softc *ctl_softc;
5560         int retval;
5561
5562         CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5563
5564         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5565         ctl_softc = control_softc;
5566         retval = 0;
5567
5568         cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5569
5570         /*
5571          * XXX KDM
5572          * We don't support the immediate bit on a stop unit.  In order to
5573          * do that, we would need to code up a way to know that a stop is
5574          * pending, and hold off any new commands until it completes, one
5575          * way or another.  Then we could accept or reject those commands
5576          * depending on its status.  We would almost need to do the reverse
5577          * of what we do below for an immediate start -- return the copy of
5578          * the ctl_io to the FETD with status to send to the host (and to
5579          * free the copy!) and then free the original I/O once the stop
5580          * actually completes.  That way, the OOA queue mechanism can work
5581          * to block commands that shouldn't proceed.  Another alternative
5582          * would be to put the copy in the queue in place of the original,
5583          * and return the original back to the caller.  That could be
5584          * slightly safer..
5585          */
5586         if ((cdb->byte2 & SSS_IMMED)
5587          && ((cdb->how & SSS_START) == 0)) {
5588                 ctl_set_invalid_field(ctsio,
5589                                       /*sks_valid*/ 1,
5590                                       /*command*/ 1,
5591                                       /*field*/ 1,
5592                                       /*bit_valid*/ 1,
5593                                       /*bit*/ 0);
5594                 ctl_done((union ctl_io *)ctsio);
5595                 return (CTL_RETVAL_COMPLETE);
5596         }
5597
5598         if ((lun->flags & CTL_LUN_PR_RESERVED)
5599          && ((cdb->how & SSS_START)==0)) {
5600                 uint32_t residx;
5601
5602                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5603                 if (lun->pr_keys[residx] == 0
5604                  || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5605
5606                         ctl_set_reservation_conflict(ctsio);
5607                         ctl_done((union ctl_io *)ctsio);
5608                         return (CTL_RETVAL_COMPLETE);
5609                 }
5610         }
5611
5612         /*
5613          * If there is no backend on this device, we can't start or stop
5614          * it.  In theory we shouldn't get any start/stop commands in the
5615          * first place at this level if the LUN doesn't have a backend.
5616          * That should get stopped by the command decode code.
5617          */
5618         if (lun->backend == NULL) {
5619                 ctl_set_invalid_opcode(ctsio);
5620                 ctl_done((union ctl_io *)ctsio);
5621                 return (CTL_RETVAL_COMPLETE);
5622         }
5623
5624         /*
5625          * XXX KDM Copan-specific offline behavior.
5626          * Figure out a reasonable way to port this?
5627          */
5628 #ifdef NEEDTOPORT
5629         mtx_lock(&lun->lun_lock);
5630
5631         if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5632          && (lun->flags & CTL_LUN_OFFLINE)) {
5633                 /*
5634                  * If the LUN is offline, and the on/offline bit isn't set,
5635                  * reject the start or stop.  Otherwise, let it through.
5636                  */
5637                 mtx_unlock(&lun->lun_lock);
5638                 ctl_set_lun_not_ready(ctsio);
5639                 ctl_done((union ctl_io *)ctsio);
5640         } else {
5641                 mtx_unlock(&lun->lun_lock);
5642 #endif /* NEEDTOPORT */
5643                 /*
5644                  * This could be a start or a stop when we're online,
5645                  * or a stop/offline or start/online.  A start or stop when
5646                  * we're offline is covered in the case above.
5647                  */
5648                 /*
5649                  * In the non-immediate case, we send the request to
5650                  * the backend and return status to the user when
5651                  * it is done.
5652                  *
5653                  * In the immediate case, we allocate a new ctl_io
5654                  * to hold a copy of the request, and send that to
5655                  * the backend.  We then set good status on the
5656                  * user's request and return it immediately.
5657                  */
5658                 if (cdb->byte2 & SSS_IMMED) {
5659                         union ctl_io *new_io;
5660
5661                         new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5662                         if (new_io == NULL) {
5663                                 ctl_set_busy(ctsio);
5664                                 ctl_done((union ctl_io *)ctsio);
5665                         } else {
5666                                 ctl_copy_io((union ctl_io *)ctsio,
5667                                             new_io);
5668                                 retval = lun->backend->config_write(new_io);
5669                                 ctl_set_success(ctsio);
5670                                 ctl_done((union ctl_io *)ctsio);
5671                         }
5672                 } else {
5673                         retval = lun->backend->config_write(
5674                                 (union ctl_io *)ctsio);
5675                 }
5676 #ifdef NEEDTOPORT
5677         }
5678 #endif
5679         return (retval);
5680 }
5681
5682 /*
5683  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5684  * we don't really do anything with the LBA and length fields if the user
5685  * passes them in.  Instead we'll just flush out the cache for the entire
5686  * LUN.
5687  */
5688 int
5689 ctl_sync_cache(struct ctl_scsiio *ctsio)
5690 {
5691         struct ctl_lun *lun;
5692         struct ctl_softc *ctl_softc;
5693         uint64_t starting_lba;
5694         uint32_t block_count;
5695         int retval;
5696
5697         CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5698
5699         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5700         ctl_softc = control_softc;
5701         retval = 0;
5702
5703         switch (ctsio->cdb[0]) {
5704         case SYNCHRONIZE_CACHE: {
5705                 struct scsi_sync_cache *cdb;
5706                 cdb = (struct scsi_sync_cache *)ctsio->cdb;
5707
5708                 starting_lba = scsi_4btoul(cdb->begin_lba);
5709                 block_count = scsi_2btoul(cdb->lb_count);
5710                 break;
5711         }
5712         case SYNCHRONIZE_CACHE_16: {
5713                 struct scsi_sync_cache_16 *cdb;
5714                 cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5715
5716                 starting_lba = scsi_8btou64(cdb->begin_lba);
5717                 block_count = scsi_4btoul(cdb->lb_count);
5718                 break;
5719         }
5720         default:
5721                 ctl_set_invalid_opcode(ctsio);
5722                 ctl_done((union ctl_io *)ctsio);
5723                 goto bailout;
5724                 break; /* NOTREACHED */
5725         }
5726
5727         /*
5728          * We check the LBA and length, but don't do anything with them.
5729          * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5730          * get flushed.  This check will just help satisfy anyone who wants
5731          * to see an error for an out of range LBA.
5732          */
5733         if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5734                 ctl_set_lba_out_of_range(ctsio);
5735                 ctl_done((union ctl_io *)ctsio);
5736                 goto bailout;
5737         }
5738
5739         /*
5740          * If this LUN has no backend, we can't flush the cache anyway.
5741          */
5742         if (lun->backend == NULL) {
5743                 ctl_set_invalid_opcode(ctsio);
5744                 ctl_done((union ctl_io *)ctsio);
5745                 goto bailout;
5746         }
5747
5748         /*
5749          * Check to see whether we're configured to send the SYNCHRONIZE
5750          * CACHE command directly to the back end.
5751          */
5752         mtx_lock(&lun->lun_lock);
5753         if ((ctl_softc->flags & CTL_FLAG_REAL_SYNC)
5754          && (++(lun->sync_count) >= lun->sync_interval)) {
5755                 lun->sync_count = 0;
5756                 mtx_unlock(&lun->lun_lock);
5757                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5758         } else {
5759                 mtx_unlock(&lun->lun_lock);
5760                 ctl_set_success(ctsio);
5761                 ctl_done((union ctl_io *)ctsio);
5762         }
5763
5764 bailout:
5765
5766         return (retval);
5767 }
5768
5769 int
5770 ctl_format(struct ctl_scsiio *ctsio)
5771 {
5772         struct scsi_format *cdb;
5773         struct ctl_lun *lun;
5774         struct ctl_softc *ctl_softc;
5775         int length, defect_list_len;
5776
5777         CTL_DEBUG_PRINT(("ctl_format\n"));
5778
5779         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5780         ctl_softc = control_softc;
5781
5782         cdb = (struct scsi_format *)ctsio->cdb;
5783
5784         length = 0;
5785         if (cdb->byte2 & SF_FMTDATA) {
5786                 if (cdb->byte2 & SF_LONGLIST)
5787                         length = sizeof(struct scsi_format_header_long);
5788                 else
5789                         length = sizeof(struct scsi_format_header_short);
5790         }
5791
5792         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5793          && (length > 0)) {
5794                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5795                 ctsio->kern_data_len = length;
5796                 ctsio->kern_total_len = length;
5797                 ctsio->kern_data_resid = 0;
5798                 ctsio->kern_rel_offset = 0;
5799                 ctsio->kern_sg_entries = 0;
5800                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5801                 ctsio->be_move_done = ctl_config_move_done;
5802                 ctl_datamove((union ctl_io *)ctsio);
5803
5804                 return (CTL_RETVAL_COMPLETE);
5805         }
5806
5807         defect_list_len = 0;
5808
5809         if (cdb->byte2 & SF_FMTDATA) {
5810                 if (cdb->byte2 & SF_LONGLIST) {
5811                         struct scsi_format_header_long *header;
5812
5813                         header = (struct scsi_format_header_long *)
5814                                 ctsio->kern_data_ptr;
5815
5816                         defect_list_len = scsi_4btoul(header->defect_list_len);
5817                         if (defect_list_len != 0) {
5818                                 ctl_set_invalid_field(ctsio,
5819                                                       /*sks_valid*/ 1,
5820                                                       /*command*/ 0,
5821                                                       /*field*/ 2,
5822                                                       /*bit_valid*/ 0,
5823                                                       /*bit*/ 0);
5824                                 goto bailout;
5825                         }
5826                 } else {
5827                         struct scsi_format_header_short *header;
5828
5829                         header = (struct scsi_format_header_short *)
5830                                 ctsio->kern_data_ptr;
5831
5832                         defect_list_len = scsi_2btoul(header->defect_list_len);
5833                         if (defect_list_len != 0) {
5834                                 ctl_set_invalid_field(ctsio,
5835                                                       /*sks_valid*/ 1,
5836                                                       /*command*/ 0,
5837                                                       /*field*/ 2,
5838                                                       /*bit_valid*/ 0,
5839                                                       /*bit*/ 0);
5840                                 goto bailout;
5841                         }
5842                 }
5843         }
5844
5845         /*
5846          * The format command will clear out the "Medium format corrupted"
5847          * status if set by the configuration code.  That status is really
5848          * just a way to notify the host that we have lost the media, and
5849          * get them to issue a command that will basically make them think
5850          * they're blowing away the media.
5851          */
5852         mtx_lock(&lun->lun_lock);
5853         lun->flags &= ~CTL_LUN_INOPERABLE;
5854         mtx_unlock(&lun->lun_lock);
5855
5856         ctsio->scsi_status = SCSI_STATUS_OK;
5857         ctsio->io_hdr.status = CTL_SUCCESS;
5858 bailout:
5859
5860         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5861                 free(ctsio->kern_data_ptr, M_CTL);
5862                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5863         }
5864
5865         ctl_done((union ctl_io *)ctsio);
5866         return (CTL_RETVAL_COMPLETE);
5867 }
5868
5869 int
5870 ctl_read_buffer(struct ctl_scsiio *ctsio)
5871 {
5872         struct scsi_read_buffer *cdb;
5873         struct ctl_lun *lun;
5874         int buffer_offset, len;
5875         static uint8_t descr[4];
5876         static uint8_t echo_descr[4] = { 0 };
5877
5878         CTL_DEBUG_PRINT(("ctl_read_buffer\n"));
5879
5880         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5881         cdb = (struct scsi_read_buffer *)ctsio->cdb;
5882
5883         if (lun->flags & CTL_LUN_PR_RESERVED) {
5884                 uint32_t residx;
5885
5886                 /*
5887                  * XXX KDM need a lock here.
5888                  */
5889                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5890                 if ((lun->res_type == SPR_TYPE_EX_AC
5891                   && residx != lun->pr_res_idx)
5892                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
5893                    || lun->res_type == SPR_TYPE_EX_AC_AR)
5894                   && lun->pr_keys[residx] == 0)) {
5895                         ctl_set_reservation_conflict(ctsio);
5896                         ctl_done((union ctl_io *)ctsio);
5897                         return (CTL_RETVAL_COMPLETE);
5898                 }
5899         }
5900
5901         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA &&
5902             (cdb->byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
5903             (cdb->byte2 & RWB_MODE) != RWB_MODE_DESCR) {
5904                 ctl_set_invalid_field(ctsio,
5905                                       /*sks_valid*/ 1,
5906                                       /*command*/ 1,
5907                                       /*field*/ 1,
5908                                       /*bit_valid*/ 1,
5909                                       /*bit*/ 4);
5910                 ctl_done((union ctl_io *)ctsio);
5911                 return (CTL_RETVAL_COMPLETE);
5912         }
5913
5914         len = scsi_3btoul(cdb->length);
5915         buffer_offset = scsi_3btoul(cdb->offset);
5916
5917         if (buffer_offset + len > sizeof(lun->write_buffer)) {
5918                 ctl_set_invalid_field(ctsio,
5919                                       /*sks_valid*/ 1,
5920                                       /*command*/ 1,
5921                                       /*field*/ 6,
5922                                       /*bit_valid*/ 0,
5923                                       /*bit*/ 0);
5924                 ctl_done((union ctl_io *)ctsio);
5925                 return (CTL_RETVAL_COMPLETE);
5926         }
5927
5928         if ((cdb->byte2 & RWB_MODE) == RWB_MODE_DESCR) {
5929                 descr[0] = 0;
5930                 scsi_ulto3b(sizeof(lun->write_buffer), &descr[1]);
5931                 ctsio->kern_data_ptr = descr;
5932                 len = min(len, sizeof(descr));
5933         } else if ((cdb->byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
5934                 ctsio->kern_data_ptr = echo_descr;
5935                 len = min(len, sizeof(echo_descr));
5936         } else
5937                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5938         ctsio->kern_data_len = len;
5939         ctsio->kern_total_len = len;
5940         ctsio->kern_data_resid = 0;
5941         ctsio->kern_rel_offset = 0;
5942         ctsio->kern_sg_entries = 0;
5943         ctsio->be_move_done = ctl_config_move_done;
5944         ctl_datamove((union ctl_io *)ctsio);
5945
5946         return (CTL_RETVAL_COMPLETE);
5947 }
5948
5949 int
5950 ctl_write_buffer(struct ctl_scsiio *ctsio)
5951 {
5952         struct scsi_write_buffer *cdb;
5953         struct ctl_lun *lun;
5954         int buffer_offset, len;
5955
5956         CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5957
5958         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5959         cdb = (struct scsi_write_buffer *)ctsio->cdb;
5960
5961         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5962                 ctl_set_invalid_field(ctsio,
5963                                       /*sks_valid*/ 1,
5964                                       /*command*/ 1,
5965                                       /*field*/ 1,
5966                                       /*bit_valid*/ 1,
5967                                       /*bit*/ 4);
5968                 ctl_done((union ctl_io *)ctsio);
5969                 return (CTL_RETVAL_COMPLETE);
5970         }
5971
5972         len = scsi_3btoul(cdb->length);
5973         buffer_offset = scsi_3btoul(cdb->offset);
5974
5975         if (buffer_offset + len > sizeof(lun->write_buffer)) {
5976                 ctl_set_invalid_field(ctsio,
5977                                       /*sks_valid*/ 1,
5978                                       /*command*/ 1,
5979                                       /*field*/ 6,
5980                                       /*bit_valid*/ 0,
5981                                       /*bit*/ 0);
5982                 ctl_done((union ctl_io *)ctsio);
5983                 return (CTL_RETVAL_COMPLETE);
5984         }
5985
5986         /*
5987          * If we've got a kernel request that hasn't been malloced yet,
5988          * malloc it and tell the caller the data buffer is here.
5989          */
5990         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5991                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5992                 ctsio->kern_data_len = len;
5993                 ctsio->kern_total_len = len;
5994                 ctsio->kern_data_resid = 0;
5995                 ctsio->kern_rel_offset = 0;
5996                 ctsio->kern_sg_entries = 0;
5997                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5998                 ctsio->be_move_done = ctl_config_move_done;
5999                 ctl_datamove((union ctl_io *)ctsio);
6000
6001                 return (CTL_RETVAL_COMPLETE);
6002         }
6003
6004         ctl_done((union ctl_io *)ctsio);
6005
6006         return (CTL_RETVAL_COMPLETE);
6007 }
6008
6009 int
6010 ctl_write_same(struct ctl_scsiio *ctsio)
6011 {
6012         struct ctl_lun *lun;
6013         struct ctl_lba_len_flags *lbalen;
6014         uint64_t lba;
6015         uint32_t num_blocks;
6016         int len, retval;
6017         uint8_t byte2;
6018
6019         retval = CTL_RETVAL_COMPLETE;
6020
6021         CTL_DEBUG_PRINT(("ctl_write_same\n"));
6022
6023         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6024
6025         switch (ctsio->cdb[0]) {
6026         case WRITE_SAME_10: {
6027                 struct scsi_write_same_10 *cdb;
6028
6029                 cdb = (struct scsi_write_same_10 *)ctsio->cdb;
6030
6031                 lba = scsi_4btoul(cdb->addr);
6032                 num_blocks = scsi_2btoul(cdb->length);
6033                 byte2 = cdb->byte2;
6034                 break;
6035         }
6036         case WRITE_SAME_16: {
6037                 struct scsi_write_same_16 *cdb;
6038
6039                 cdb = (struct scsi_write_same_16 *)ctsio->cdb;
6040
6041                 lba = scsi_8btou64(cdb->addr);
6042                 num_blocks = scsi_4btoul(cdb->length);
6043                 byte2 = cdb->byte2;
6044                 break;
6045         }
6046         default:
6047                 /*
6048                  * We got a command we don't support.  This shouldn't
6049                  * happen, commands should be filtered out above us.
6050                  */
6051                 ctl_set_invalid_opcode(ctsio);
6052                 ctl_done((union ctl_io *)ctsio);
6053
6054                 return (CTL_RETVAL_COMPLETE);
6055                 break; /* NOTREACHED */
6056         }
6057
6058         /* NDOB and ANCHOR flags can be used only together with UNMAP */
6059         if ((byte2 & SWS_UNMAP) == 0 &&
6060             (byte2 & (SWS_NDOB | SWS_ANCHOR)) != 0) {
6061                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
6062                     /*command*/ 1, /*field*/ 1, /*bit_valid*/ 1, /*bit*/ 0);
6063                 ctl_done((union ctl_io *)ctsio);
6064                 return (CTL_RETVAL_COMPLETE);
6065         }
6066
6067         /*
6068          * The first check is to make sure we're in bounds, the second
6069          * check is to catch wrap-around problems.  If the lba + num blocks
6070          * is less than the lba, then we've wrapped around and the block
6071          * range is invalid anyway.
6072          */
6073         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
6074          || ((lba + num_blocks) < lba)) {
6075                 ctl_set_lba_out_of_range(ctsio);
6076                 ctl_done((union ctl_io *)ctsio);
6077                 return (CTL_RETVAL_COMPLETE);
6078         }
6079
6080         /* Zero number of blocks means "to the last logical block" */
6081         if (num_blocks == 0) {
6082                 if ((lun->be_lun->maxlba + 1) - lba > UINT32_MAX) {
6083                         ctl_set_invalid_field(ctsio,
6084                                               /*sks_valid*/ 0,
6085                                               /*command*/ 1,
6086                                               /*field*/ 0,
6087                                               /*bit_valid*/ 0,
6088                                               /*bit*/ 0);
6089                         ctl_done((union ctl_io *)ctsio);
6090                         return (CTL_RETVAL_COMPLETE);
6091                 }
6092                 num_blocks = (lun->be_lun->maxlba + 1) - lba;
6093         }
6094
6095         len = lun->be_lun->blocksize;
6096
6097         /*
6098          * If we've got a kernel request that hasn't been malloced yet,
6099          * malloc it and tell the caller the data buffer is here.
6100          */
6101         if ((byte2 & SWS_NDOB) == 0 &&
6102             (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6103                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
6104                 ctsio->kern_data_len = len;
6105                 ctsio->kern_total_len = len;
6106                 ctsio->kern_data_resid = 0;
6107                 ctsio->kern_rel_offset = 0;
6108                 ctsio->kern_sg_entries = 0;
6109                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6110                 ctsio->be_move_done = ctl_config_move_done;
6111                 ctl_datamove((union ctl_io *)ctsio);
6112
6113                 return (CTL_RETVAL_COMPLETE);
6114         }
6115
6116         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
6117         lbalen->lba = lba;
6118         lbalen->len = num_blocks;
6119         lbalen->flags = byte2;
6120         retval = lun->backend->config_write((union ctl_io *)ctsio);
6121
6122         return (retval);
6123 }
6124
6125 int
6126 ctl_unmap(struct ctl_scsiio *ctsio)
6127 {
6128         struct ctl_lun *lun;
6129         struct scsi_unmap *cdb;
6130         struct ctl_ptr_len_flags *ptrlen;
6131         struct scsi_unmap_header *hdr;
6132         struct scsi_unmap_desc *buf, *end, *endnz, *range;
6133         uint64_t lba;
6134         uint32_t num_blocks;
6135         int len, retval;
6136         uint8_t byte2;
6137
6138         retval = CTL_RETVAL_COMPLETE;
6139
6140         CTL_DEBUG_PRINT(("ctl_unmap\n"));
6141
6142         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6143         cdb = (struct scsi_unmap *)ctsio->cdb;
6144
6145         len = scsi_2btoul(cdb->length);
6146         byte2 = cdb->byte2;
6147
6148         /*
6149          * If we've got a kernel request that hasn't been malloced yet,
6150          * malloc it and tell the caller the data buffer is here.
6151          */
6152         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6153                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
6154                 ctsio->kern_data_len = len;
6155                 ctsio->kern_total_len = len;
6156                 ctsio->kern_data_resid = 0;
6157                 ctsio->kern_rel_offset = 0;
6158                 ctsio->kern_sg_entries = 0;
6159                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6160                 ctsio->be_move_done = ctl_config_move_done;
6161                 ctl_datamove((union ctl_io *)ctsio);
6162
6163                 return (CTL_RETVAL_COMPLETE);
6164         }
6165
6166         len = ctsio->kern_total_len - ctsio->kern_data_resid;
6167         hdr = (struct scsi_unmap_header *)ctsio->kern_data_ptr;
6168         if (len < sizeof (*hdr) ||
6169             len < (scsi_2btoul(hdr->length) + sizeof(hdr->length)) ||
6170             len < (scsi_2btoul(hdr->desc_length) + sizeof (*hdr)) ||
6171             scsi_2btoul(hdr->desc_length) % sizeof(*buf) != 0) {
6172                 ctl_set_invalid_field(ctsio,
6173                                       /*sks_valid*/ 0,
6174                                       /*command*/ 0,
6175                                       /*field*/ 0,
6176                                       /*bit_valid*/ 0,
6177                                       /*bit*/ 0);
6178                 ctl_done((union ctl_io *)ctsio);
6179                 return (CTL_RETVAL_COMPLETE);
6180         }
6181         len = scsi_2btoul(hdr->desc_length);
6182         buf = (struct scsi_unmap_desc *)(hdr + 1);
6183         end = buf + len / sizeof(*buf);
6184
6185         endnz = buf;
6186         for (range = buf; range < end; range++) {
6187                 lba = scsi_8btou64(range->lba);
6188                 num_blocks = scsi_4btoul(range->length);
6189                 if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
6190                  || ((lba + num_blocks) < lba)) {
6191                         ctl_set_lba_out_of_range(ctsio);
6192                         ctl_done((union ctl_io *)ctsio);
6193                         return (CTL_RETVAL_COMPLETE);
6194                 }
6195                 if (num_blocks != 0)
6196                         endnz = range + 1;
6197         }
6198
6199         /*
6200          * Block backend can not handle zero last range.
6201          * Filter it out and return if there is nothing left.
6202          */
6203         len = (uint8_t *)endnz - (uint8_t *)buf;
6204         if (len == 0) {
6205                 ctl_set_success(ctsio);
6206                 ctl_done((union ctl_io *)ctsio);
6207                 return (CTL_RETVAL_COMPLETE);
6208         }
6209
6210         mtx_lock(&lun->lun_lock);
6211         ptrlen = (struct ctl_ptr_len_flags *)
6212             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
6213         ptrlen->ptr = (void *)buf;
6214         ptrlen->len = len;
6215         ptrlen->flags = byte2;
6216         ctl_check_blocked(lun);
6217         mtx_unlock(&lun->lun_lock);
6218
6219         retval = lun->backend->config_write((union ctl_io *)ctsio);
6220         return (retval);
6221 }
6222
6223 /*
6224  * Note that this function currently doesn't actually do anything inside
6225  * CTL to enforce things if the DQue bit is turned on.
6226  *
6227  * Also note that this function can't be used in the default case, because
6228  * the DQue bit isn't set in the changeable mask for the control mode page
6229  * anyway.  This is just here as an example for how to implement a page
6230  * handler, and a placeholder in case we want to allow the user to turn
6231  * tagged queueing on and off.
6232  *
6233  * The D_SENSE bit handling is functional, however, and will turn
6234  * descriptor sense on and off for a given LUN.
6235  */
6236 int
6237 ctl_control_page_handler(struct ctl_scsiio *ctsio,
6238                          struct ctl_page_index *page_index, uint8_t *page_ptr)
6239 {
6240         struct scsi_control_page *current_cp, *saved_cp, *user_cp;
6241         struct ctl_lun *lun;
6242         struct ctl_softc *softc;
6243         int set_ua;
6244         uint32_t initidx;
6245
6246         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6247         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
6248         set_ua = 0;
6249
6250         user_cp = (struct scsi_control_page *)page_ptr;
6251         current_cp = (struct scsi_control_page *)
6252                 (page_index->page_data + (page_index->page_len *
6253                 CTL_PAGE_CURRENT));
6254         saved_cp = (struct scsi_control_page *)
6255                 (page_index->page_data + (page_index->page_len *
6256                 CTL_PAGE_SAVED));
6257
6258         softc = control_softc;
6259
6260         mtx_lock(&lun->lun_lock);
6261         if (((current_cp->rlec & SCP_DSENSE) == 0)
6262          && ((user_cp->rlec & SCP_DSENSE) != 0)) {
6263                 /*
6264                  * Descriptor sense is currently turned off and the user
6265                  * wants to turn it on.
6266                  */
6267                 current_cp->rlec |= SCP_DSENSE;
6268                 saved_cp->rlec |= SCP_DSENSE;
6269                 lun->flags |= CTL_LUN_SENSE_DESC;
6270                 set_ua = 1;
6271         } else if (((current_cp->rlec & SCP_DSENSE) != 0)
6272                 && ((user_cp->rlec & SCP_DSENSE) == 0)) {
6273                 /*
6274                  * Descriptor sense is currently turned on, and the user
6275                  * wants to turn it off.
6276                  */
6277                 current_cp->rlec &= ~SCP_DSENSE;
6278                 saved_cp->rlec &= ~SCP_DSENSE;
6279                 lun->flags &= ~CTL_LUN_SENSE_DESC;
6280                 set_ua = 1;
6281         }
6282         if ((current_cp->queue_flags & SCP_QUEUE_ALG_MASK) !=
6283             (user_cp->queue_flags & SCP_QUEUE_ALG_MASK)) {
6284                 current_cp->queue_flags &= ~SCP_QUEUE_ALG_MASK;
6285                 current_cp->queue_flags |= user_cp->queue_flags & SCP_QUEUE_ALG_MASK;
6286                 saved_cp->queue_flags &= ~SCP_QUEUE_ALG_MASK;
6287                 saved_cp->queue_flags |= user_cp->queue_flags & SCP_QUEUE_ALG_MASK;
6288                 set_ua = 1;
6289         }
6290         if ((current_cp->eca_and_aen & SCP_SWP) !=
6291             (user_cp->eca_and_aen & SCP_SWP)) {
6292                 current_cp->eca_and_aen &= ~SCP_SWP;
6293                 current_cp->eca_and_aen |= user_cp->eca_and_aen & SCP_SWP;
6294                 saved_cp->eca_and_aen &= ~SCP_SWP;
6295                 saved_cp->eca_and_aen |= user_cp->eca_and_aen & SCP_SWP;
6296                 set_ua = 1;
6297         }
6298         if (set_ua != 0) {
6299                 int i;
6300                 /*
6301                  * Let other initiators know that the mode
6302                  * parameters for this LUN have changed.
6303                  */
6304                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
6305                         if (i == initidx)
6306                                 continue;
6307
6308                         lun->pending_ua[i] |= CTL_UA_MODE_CHANGE;
6309                 }
6310         }
6311         mtx_unlock(&lun->lun_lock);
6312
6313         return (0);
6314 }
6315
6316 int
6317 ctl_caching_sp_handler(struct ctl_scsiio *ctsio,
6318                      struct ctl_page_index *page_index, uint8_t *page_ptr)
6319 {
6320         struct scsi_caching_page *current_cp, *saved_cp, *user_cp;
6321         struct ctl_lun *lun;
6322         int set_ua;
6323         uint32_t initidx;
6324
6325         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6326         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
6327         set_ua = 0;
6328
6329         user_cp = (struct scsi_caching_page *)page_ptr;
6330         current_cp = (struct scsi_caching_page *)
6331                 (page_index->page_data + (page_index->page_len *
6332                 CTL_PAGE_CURRENT));
6333         saved_cp = (struct scsi_caching_page *)
6334                 (page_index->page_data + (page_index->page_len *
6335                 CTL_PAGE_SAVED));
6336
6337         mtx_lock(&lun->lun_lock);
6338         if ((current_cp->flags1 & (SCP_WCE | SCP_RCD)) !=
6339             (user_cp->flags1 & (SCP_WCE | SCP_RCD))) {
6340                 current_cp->flags1 &= ~(SCP_WCE | SCP_RCD);
6341                 current_cp->flags1 |= user_cp->flags1 & (SCP_WCE | SCP_RCD);
6342                 saved_cp->flags1 &= ~(SCP_WCE | SCP_RCD);
6343                 saved_cp->flags1 |= user_cp->flags1 & (SCP_WCE | SCP_RCD);
6344                 set_ua = 1;
6345         }
6346         if (set_ua != 0) {
6347                 int i;
6348                 /*
6349                  * Let other initiators know that the mode
6350                  * parameters for this LUN have changed.
6351                  */
6352                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
6353                         if (i == initidx)
6354                                 continue;
6355
6356                         lun->pending_ua[i] |= CTL_UA_MODE_CHANGE;
6357                 }
6358         }
6359         mtx_unlock(&lun->lun_lock);
6360
6361         return (0);
6362 }
6363
6364 int
6365 ctl_power_sp_handler(struct ctl_scsiio *ctsio,
6366                      struct ctl_page_index *page_index, uint8_t *page_ptr)
6367 {
6368         return (0);
6369 }
6370
6371 int
6372 ctl_power_sp_sense_handler(struct ctl_scsiio *ctsio,
6373                            struct ctl_page_index *page_index, int pc)
6374 {
6375         struct copan_power_subpage *page;
6376
6377         page = (struct copan_power_subpage *)page_index->page_data +
6378                 (page_index->page_len * pc);
6379
6380         switch (pc) {
6381         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6382                 /*
6383                  * We don't update the changable bits for this page.
6384                  */
6385                 break;
6386         case SMS_PAGE_CTRL_CURRENT >> 6:
6387         case SMS_PAGE_CTRL_DEFAULT >> 6:
6388         case SMS_PAGE_CTRL_SAVED >> 6:
6389 #ifdef NEEDTOPORT
6390                 ctl_update_power_subpage(page);
6391 #endif
6392                 break;
6393         default:
6394 #ifdef NEEDTOPORT
6395                 EPRINT(0, "Invalid PC %d!!", pc);
6396 #endif
6397                 break;
6398         }
6399         return (0);
6400 }
6401
6402
6403 int
6404 ctl_aps_sp_handler(struct ctl_scsiio *ctsio,
6405                    struct ctl_page_index *page_index, uint8_t *page_ptr)
6406 {
6407         struct copan_aps_subpage *user_sp;
6408         struct copan_aps_subpage *current_sp;
6409         union ctl_modepage_info *modepage_info;
6410         struct ctl_softc *softc;
6411         struct ctl_lun *lun;
6412         int retval;
6413
6414         retval = CTL_RETVAL_COMPLETE;
6415         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
6416                      (page_index->page_len * CTL_PAGE_CURRENT));
6417         softc = control_softc;
6418         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6419
6420         user_sp = (struct copan_aps_subpage *)page_ptr;
6421
6422         modepage_info = (union ctl_modepage_info *)
6423                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6424
6425         modepage_info->header.page_code = page_index->page_code & SMPH_PC_MASK;
6426         modepage_info->header.subpage = page_index->subpage;
6427         modepage_info->aps.lock_active = user_sp->lock_active;
6428
6429         mtx_lock(&softc->ctl_lock);
6430
6431         /*
6432          * If there is a request to lock the LUN and another LUN is locked
6433          * this is an error. If the requested LUN is already locked ignore
6434          * the request. If no LUN is locked attempt to lock it.
6435          * if there is a request to unlock the LUN and the LUN is currently
6436          * locked attempt to unlock it. Otherwise ignore the request. i.e.
6437          * if another LUN is locked or no LUN is locked.
6438          */
6439         if (user_sp->lock_active & APS_LOCK_ACTIVE) {
6440                 if (softc->aps_locked_lun == lun->lun) {
6441                         /*
6442                          * This LUN is already locked, so we're done.
6443                          */
6444                         retval = CTL_RETVAL_COMPLETE;
6445                 } else if (softc->aps_locked_lun == 0) {
6446                         /*
6447                          * No one has the lock, pass the request to the
6448                          * backend.
6449                          */
6450                         retval = lun->backend->config_write(
6451                                 (union ctl_io *)ctsio);
6452                 } else {
6453                         /*
6454                          * Someone else has the lock, throw out the request.
6455                          */
6456                         ctl_set_already_locked(ctsio);
6457                         free(ctsio->kern_data_ptr, M_CTL);
6458                         ctl_done((union ctl_io *)ctsio);
6459
6460                         /*
6461                          * Set the return value so that ctl_do_mode_select()
6462                          * won't try to complete the command.  We already
6463                          * completed it here.
6464                          */
6465                         retval = CTL_RETVAL_ERROR;
6466                 }
6467         } else if (softc->aps_locked_lun == lun->lun) {
6468                 /*
6469                  * This LUN is locked, so pass the unlock request to the
6470                  * backend.
6471                  */
6472                 retval = lun->backend->config_write((union ctl_io *)ctsio);
6473         }
6474         mtx_unlock(&softc->ctl_lock);
6475
6476         return (retval);
6477 }
6478
6479 int
6480 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6481                                 struct ctl_page_index *page_index,
6482                                 uint8_t *page_ptr)
6483 {
6484         uint8_t *c;
6485         int i;
6486
6487         c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6488         ctl_time_io_secs =
6489                 (c[0] << 8) |
6490                 (c[1] << 0) |
6491                 0;
6492         CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6493         printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6494         printf("page data:");
6495         for (i=0; i<8; i++)
6496                 printf(" %.2x",page_ptr[i]);
6497         printf("\n");
6498         return (0);
6499 }
6500
6501 int
6502 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6503                                struct ctl_page_index *page_index,
6504                                int pc)
6505 {
6506         struct copan_debugconf_subpage *page;
6507
6508         page = (struct copan_debugconf_subpage *)page_index->page_data +
6509                 (page_index->page_len * pc);
6510
6511         switch (pc) {
6512         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6513         case SMS_PAGE_CTRL_DEFAULT >> 6:
6514         case SMS_PAGE_CTRL_SAVED >> 6:
6515                 /*
6516                  * We don't update the changable or default bits for this page.
6517                  */
6518                 break;
6519         case SMS_PAGE_CTRL_CURRENT >> 6:
6520                 page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6521                 page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6522                 break;
6523         default:
6524 #ifdef NEEDTOPORT
6525                 EPRINT(0, "Invalid PC %d!!", pc);
6526 #endif /* NEEDTOPORT */
6527                 break;
6528         }
6529         return (0);
6530 }
6531
6532
6533 static int
6534 ctl_do_mode_select(union ctl_io *io)
6535 {
6536         struct scsi_mode_page_header *page_header;
6537         struct ctl_page_index *page_index;
6538         struct ctl_scsiio *ctsio;
6539         int control_dev, page_len;
6540         int page_len_offset, page_len_size;
6541         union ctl_modepage_info *modepage_info;
6542         struct ctl_lun *lun;
6543         int *len_left, *len_used;
6544         int retval, i;
6545
6546         ctsio = &io->scsiio;
6547         page_index = NULL;
6548         page_len = 0;
6549         retval = CTL_RETVAL_COMPLETE;
6550
6551         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6552
6553         if (lun->be_lun->lun_type != T_DIRECT)
6554                 control_dev = 1;
6555         else
6556                 control_dev = 0;
6557
6558         modepage_info = (union ctl_modepage_info *)
6559                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6560         len_left = &modepage_info->header.len_left;
6561         len_used = &modepage_info->header.len_used;
6562
6563 do_next_page:
6564
6565         page_header = (struct scsi_mode_page_header *)
6566                 (ctsio->kern_data_ptr + *len_used);
6567
6568         if (*len_left == 0) {
6569                 free(ctsio->kern_data_ptr, M_CTL);
6570                 ctl_set_success(ctsio);
6571                 ctl_done((union ctl_io *)ctsio);
6572                 return (CTL_RETVAL_COMPLETE);
6573         } else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6574
6575                 free(ctsio->kern_data_ptr, M_CTL);
6576                 ctl_set_param_len_error(ctsio);
6577                 ctl_done((union ctl_io *)ctsio);
6578                 return (CTL_RETVAL_COMPLETE);
6579
6580         } else if ((page_header->page_code & SMPH_SPF)
6581                 && (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6582
6583                 free(ctsio->kern_data_ptr, M_CTL);
6584                 ctl_set_param_len_error(ctsio);
6585                 ctl_done((union ctl_io *)ctsio);
6586                 return (CTL_RETVAL_COMPLETE);
6587         }
6588
6589
6590         /*
6591          * XXX KDM should we do something with the block descriptor?
6592          */
6593         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6594
6595                 if ((control_dev != 0)
6596                  && (lun->mode_pages.index[i].page_flags &
6597                      CTL_PAGE_FLAG_DISK_ONLY))
6598                         continue;
6599
6600                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6601                     (page_header->page_code & SMPH_PC_MASK))
6602                         continue;
6603
6604                 /*
6605                  * If neither page has a subpage code, then we've got a
6606                  * match.
6607                  */
6608                 if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6609                  && ((page_header->page_code & SMPH_SPF) == 0)) {
6610                         page_index = &lun->mode_pages.index[i];
6611                         page_len = page_header->page_length;
6612                         break;
6613                 }
6614
6615                 /*
6616                  * If both pages have subpages, then the subpage numbers
6617                  * have to match.
6618                  */
6619                 if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6620                   && (page_header->page_code & SMPH_SPF)) {
6621                         struct scsi_mode_page_header_sp *sph;
6622
6623                         sph = (struct scsi_mode_page_header_sp *)page_header;
6624
6625                         if (lun->mode_pages.index[i].subpage ==
6626                             sph->subpage) {
6627                                 page_index = &lun->mode_pages.index[i];
6628                                 page_len = scsi_2btoul(sph->page_length);
6629                                 break;
6630                         }
6631                 }
6632         }
6633
6634         /*
6635          * If we couldn't find the page, or if we don't have a mode select
6636          * handler for it, send back an error to the user.
6637          */
6638         if ((page_index == NULL)
6639          || (page_index->select_handler == NULL)) {
6640                 ctl_set_invalid_field(ctsio,
6641                                       /*sks_valid*/ 1,
6642                                       /*command*/ 0,
6643                                       /*field*/ *len_used,
6644                                       /*bit_valid*/ 0,
6645                                       /*bit*/ 0);
6646                 free(ctsio->kern_data_ptr, M_CTL);
6647                 ctl_done((union ctl_io *)ctsio);
6648                 return (CTL_RETVAL_COMPLETE);
6649         }
6650
6651         if (page_index->page_code & SMPH_SPF) {
6652                 page_len_offset = 2;
6653                 page_len_size = 2;
6654         } else {
6655                 page_len_size = 1;
6656                 page_len_offset = 1;
6657         }
6658
6659         /*
6660          * If the length the initiator gives us isn't the one we specify in
6661          * the mode page header, or if they didn't specify enough data in
6662          * the CDB to avoid truncating this page, kick out the request.
6663          */
6664         if ((page_len != (page_index->page_len - page_len_offset -
6665                           page_len_size))
6666          || (*len_left < page_index->page_len)) {
6667
6668
6669                 ctl_set_invalid_field(ctsio,
6670                                       /*sks_valid*/ 1,
6671                                       /*command*/ 0,
6672                                       /*field*/ *len_used + page_len_offset,
6673                                       /*bit_valid*/ 0,
6674                                       /*bit*/ 0);
6675                 free(ctsio->kern_data_ptr, M_CTL);
6676                 ctl_done((union ctl_io *)ctsio);
6677                 return (CTL_RETVAL_COMPLETE);
6678         }
6679
6680         /*
6681          * Run through the mode page, checking to make sure that the bits
6682          * the user changed are actually legal for him to change.
6683          */
6684         for (i = 0; i < page_index->page_len; i++) {
6685                 uint8_t *user_byte, *change_mask, *current_byte;
6686                 int bad_bit;
6687                 int j;
6688
6689                 user_byte = (uint8_t *)page_header + i;
6690                 change_mask = page_index->page_data +
6691                               (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6692                 current_byte = page_index->page_data +
6693                                (page_index->page_len * CTL_PAGE_CURRENT) + i;
6694
6695                 /*
6696                  * Check to see whether the user set any bits in this byte
6697                  * that he is not allowed to set.
6698                  */
6699                 if ((*user_byte & ~(*change_mask)) ==
6700                     (*current_byte & ~(*change_mask)))
6701                         continue;
6702
6703                 /*
6704                  * Go through bit by bit to determine which one is illegal.
6705                  */
6706                 bad_bit = 0;
6707                 for (j = 7; j >= 0; j--) {
6708                         if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6709                             (((1 << i) & ~(*change_mask)) & *current_byte)) {
6710                                 bad_bit = i;
6711                                 break;
6712                         }
6713                 }
6714                 ctl_set_invalid_field(ctsio,
6715                                       /*sks_valid*/ 1,
6716                                       /*command*/ 0,
6717                                       /*field*/ *len_used + i,
6718                                       /*bit_valid*/ 1,
6719                                       /*bit*/ bad_bit);
6720                 free(ctsio->kern_data_ptr, M_CTL);
6721                 ctl_done((union ctl_io *)ctsio);
6722                 return (CTL_RETVAL_COMPLETE);
6723         }
6724
6725         /*
6726          * Decrement these before we call the page handler, since we may
6727          * end up getting called back one way or another before the handler
6728          * returns to this context.
6729          */
6730         *len_left -= page_index->page_len;
6731         *len_used += page_index->page_len;
6732
6733         retval = page_index->select_handler(ctsio, page_index,
6734                                             (uint8_t *)page_header);
6735
6736         /*
6737          * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6738          * wait until this queued command completes to finish processing
6739          * the mode page.  If it returns anything other than
6740          * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6741          * already set the sense information, freed the data pointer, and
6742          * completed the io for us.
6743          */
6744         if (retval != CTL_RETVAL_COMPLETE)
6745                 goto bailout_no_done;
6746
6747         /*
6748          * If the initiator sent us more than one page, parse the next one.
6749          */
6750         if (*len_left > 0)
6751                 goto do_next_page;
6752
6753         ctl_set_success(ctsio);
6754         free(ctsio->kern_data_ptr, M_CTL);
6755         ctl_done((union ctl_io *)ctsio);
6756
6757 bailout_no_done:
6758
6759         return (CTL_RETVAL_COMPLETE);
6760
6761 }
6762
6763 int
6764 ctl_mode_select(struct ctl_scsiio *ctsio)
6765 {
6766         int param_len, pf, sp;
6767         int header_size, bd_len;
6768         int len_left, len_used;
6769         struct ctl_page_index *page_index;
6770         struct ctl_lun *lun;
6771         int control_dev, page_len;
6772         union ctl_modepage_info *modepage_info;
6773         int retval;
6774
6775         pf = 0;
6776         sp = 0;
6777         page_len = 0;
6778         len_used = 0;
6779         len_left = 0;
6780         retval = 0;
6781         bd_len = 0;
6782         page_index = NULL;
6783
6784         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6785
6786         if (lun->be_lun->lun_type != T_DIRECT)
6787                 control_dev = 1;
6788         else
6789                 control_dev = 0;
6790
6791         switch (ctsio->cdb[0]) {
6792         case MODE_SELECT_6: {
6793                 struct scsi_mode_select_6 *cdb;
6794
6795                 cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6796
6797                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6798                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6799
6800                 param_len = cdb->length;
6801                 header_size = sizeof(struct scsi_mode_header_6);
6802                 break;
6803         }
6804         case MODE_SELECT_10: {
6805                 struct scsi_mode_select_10 *cdb;
6806
6807                 cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6808
6809                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6810                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6811
6812                 param_len = scsi_2btoul(cdb->length);
6813                 header_size = sizeof(struct scsi_mode_header_10);
6814                 break;
6815         }
6816         default:
6817                 ctl_set_invalid_opcode(ctsio);
6818                 ctl_done((union ctl_io *)ctsio);
6819                 return (CTL_RETVAL_COMPLETE);
6820                 break; /* NOTREACHED */
6821         }
6822
6823         /*
6824          * From SPC-3:
6825          * "A parameter list length of zero indicates that the Data-Out Buffer
6826          * shall be empty. This condition shall not be considered as an error."
6827          */
6828         if (param_len == 0) {
6829                 ctl_set_success(ctsio);
6830                 ctl_done((union ctl_io *)ctsio);
6831                 return (CTL_RETVAL_COMPLETE);
6832         }
6833
6834         /*
6835          * Since we'll hit this the first time through, prior to
6836          * allocation, we don't need to free a data buffer here.
6837          */
6838         if (param_len < header_size) {
6839                 ctl_set_param_len_error(ctsio);
6840                 ctl_done((union ctl_io *)ctsio);
6841                 return (CTL_RETVAL_COMPLETE);
6842         }
6843
6844         /*
6845          * Allocate the data buffer and grab the user's data.  In theory,
6846          * we shouldn't have to sanity check the parameter list length here
6847          * because the maximum size is 64K.  We should be able to malloc
6848          * that much without too many problems.
6849          */
6850         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6851                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6852                 ctsio->kern_data_len = param_len;
6853                 ctsio->kern_total_len = param_len;
6854                 ctsio->kern_data_resid = 0;
6855                 ctsio->kern_rel_offset = 0;
6856                 ctsio->kern_sg_entries = 0;
6857                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6858                 ctsio->be_move_done = ctl_config_move_done;
6859                 ctl_datamove((union ctl_io *)ctsio);
6860
6861                 return (CTL_RETVAL_COMPLETE);
6862         }
6863
6864         switch (ctsio->cdb[0]) {
6865         case MODE_SELECT_6: {
6866                 struct scsi_mode_header_6 *mh6;
6867
6868                 mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6869                 bd_len = mh6->blk_desc_len;
6870                 break;
6871         }
6872         case MODE_SELECT_10: {
6873                 struct scsi_mode_header_10 *mh10;
6874
6875                 mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6876                 bd_len = scsi_2btoul(mh10->blk_desc_len);
6877                 break;
6878         }
6879         default:
6880                 panic("Invalid CDB type %#x", ctsio->cdb[0]);
6881                 break;
6882         }
6883
6884         if (param_len < (header_size + bd_len)) {
6885                 free(ctsio->kern_data_ptr, M_CTL);
6886                 ctl_set_param_len_error(ctsio);
6887                 ctl_done((union ctl_io *)ctsio);
6888                 return (CTL_RETVAL_COMPLETE);
6889         }
6890
6891         /*
6892          * Set the IO_CONT flag, so that if this I/O gets passed to
6893          * ctl_config_write_done(), it'll get passed back to
6894          * ctl_do_mode_select() for further processing, or completion if
6895          * we're all done.
6896          */
6897         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6898         ctsio->io_cont = ctl_do_mode_select;
6899
6900         modepage_info = (union ctl_modepage_info *)
6901                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6902
6903         memset(modepage_info, 0, sizeof(*modepage_info));
6904
6905         len_left = param_len - header_size - bd_len;
6906         len_used = header_size + bd_len;
6907
6908         modepage_info->header.len_left = len_left;
6909         modepage_info->header.len_used = len_used;
6910
6911         return (ctl_do_mode_select((union ctl_io *)ctsio));
6912 }
6913
6914 int
6915 ctl_mode_sense(struct ctl_scsiio *ctsio)
6916 {
6917         struct ctl_lun *lun;
6918         int pc, page_code, dbd, llba, subpage;
6919         int alloc_len, page_len, header_len, total_len;
6920         struct scsi_mode_block_descr *block_desc;
6921         struct ctl_page_index *page_index;
6922         int control_dev;
6923
6924         dbd = 0;
6925         llba = 0;
6926         block_desc = NULL;
6927         page_index = NULL;
6928
6929         CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6930
6931         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6932
6933         if (lun->be_lun->lun_type != T_DIRECT)
6934                 control_dev = 1;
6935         else
6936                 control_dev = 0;
6937
6938         if (lun->flags & CTL_LUN_PR_RESERVED) {
6939                 uint32_t residx;
6940
6941                 /*
6942                  * XXX KDM need a lock here.
6943                  */
6944                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
6945                 if ((lun->res_type == SPR_TYPE_EX_AC
6946                   && residx != lun->pr_res_idx)
6947                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
6948                    || lun->res_type == SPR_TYPE_EX_AC_AR)
6949                   && lun->pr_keys[residx] == 0)) {
6950                         ctl_set_reservation_conflict(ctsio);
6951                         ctl_done((union ctl_io *)ctsio);
6952                         return (CTL_RETVAL_COMPLETE);
6953                 }
6954         }
6955
6956         switch (ctsio->cdb[0]) {
6957         case MODE_SENSE_6: {
6958                 struct scsi_mode_sense_6 *cdb;
6959
6960                 cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6961
6962                 header_len = sizeof(struct scsi_mode_hdr_6);
6963                 if (cdb->byte2 & SMS_DBD)
6964                         dbd = 1;
6965                 else
6966                         header_len += sizeof(struct scsi_mode_block_descr);
6967
6968                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6969                 page_code = cdb->page & SMS_PAGE_CODE;
6970                 subpage = cdb->subpage;
6971                 alloc_len = cdb->length;
6972                 break;
6973         }
6974         case MODE_SENSE_10: {
6975                 struct scsi_mode_sense_10 *cdb;
6976
6977                 cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6978
6979                 header_len = sizeof(struct scsi_mode_hdr_10);
6980
6981                 if (cdb->byte2 & SMS_DBD)
6982                         dbd = 1;
6983                 else
6984                         header_len += sizeof(struct scsi_mode_block_descr);
6985                 if (cdb->byte2 & SMS10_LLBAA)
6986                         llba = 1;
6987                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6988                 page_code = cdb->page & SMS_PAGE_CODE;
6989                 subpage = cdb->subpage;
6990                 alloc_len = scsi_2btoul(cdb->length);
6991                 break;
6992         }
6993         default:
6994                 ctl_set_invalid_opcode(ctsio);
6995                 ctl_done((union ctl_io *)ctsio);
6996                 return (CTL_RETVAL_COMPLETE);
6997                 break; /* NOTREACHED */
6998         }
6999
7000         /*
7001          * We have to make a first pass through to calculate the size of
7002          * the pages that match the user's query.  Then we allocate enough
7003          * memory to hold it, and actually copy the data into the buffer.
7004          */
7005         switch (page_code) {
7006         case SMS_ALL_PAGES_PAGE: {
7007                 int i;
7008
7009                 page_len = 0;
7010
7011                 /*
7012                  * At the moment, values other than 0 and 0xff here are
7013                  * reserved according to SPC-3.
7014                  */
7015                 if ((subpage != SMS_SUBPAGE_PAGE_0)
7016                  && (subpage != SMS_SUBPAGE_ALL)) {
7017                         ctl_set_invalid_field(ctsio,
7018                                               /*sks_valid*/ 1,
7019                                               /*command*/ 1,
7020                                               /*field*/ 3,
7021                                               /*bit_valid*/ 0,
7022                                               /*bit*/ 0);
7023                         ctl_done((union ctl_io *)ctsio);
7024                         return (CTL_RETVAL_COMPLETE);
7025                 }
7026
7027                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
7028                         if ((control_dev != 0)
7029                          && (lun->mode_pages.index[i].page_flags &
7030                              CTL_PAGE_FLAG_DISK_ONLY))
7031                                 continue;
7032
7033                         /*
7034                          * We don't use this subpage if the user didn't
7035                          * request all subpages.
7036                          */
7037                         if ((lun->mode_pages.index[i].subpage != 0)
7038                          && (subpage == SMS_SUBPAGE_PAGE_0))
7039                                 continue;
7040
7041 #if 0
7042                         printf("found page %#x len %d\n",
7043                                lun->mode_pages.index[i].page_code &
7044                                SMPH_PC_MASK,
7045                                lun->mode_pages.index[i].page_len);
7046 #endif
7047                         page_len += lun->mode_pages.index[i].page_len;
7048                 }
7049                 break;
7050         }
7051         default: {
7052                 int i;
7053
7054                 page_len = 0;
7055
7056                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
7057                         /* Look for the right page code */
7058                         if ((lun->mode_pages.index[i].page_code &
7059                              SMPH_PC_MASK) != page_code)
7060                                 continue;
7061
7062                         /* Look for the right subpage or the subpage wildcard*/
7063                         if ((lun->mode_pages.index[i].subpage != subpage)
7064                          && (subpage != SMS_SUBPAGE_ALL))
7065                                 continue;
7066
7067                         /* Make sure the page is supported for this dev type */
7068                         if ((control_dev != 0)
7069                          && (lun->mode_pages.index[i].page_flags &
7070                              CTL_PAGE_FLAG_DISK_ONLY))
7071                                 continue;
7072
7073 #if 0
7074                         printf("found page %#x len %d\n",
7075                                lun->mode_pages.index[i].page_code &
7076                                SMPH_PC_MASK,
7077                                lun->mode_pages.index[i].page_len);
7078 #endif
7079
7080                         page_len += lun->mode_pages.index[i].page_len;
7081                 }
7082
7083                 if (page_len == 0) {
7084                         ctl_set_invalid_field(ctsio,
7085                                               /*sks_valid*/ 1,
7086                                               /*command*/ 1,
7087                                               /*field*/ 2,
7088                                               /*bit_valid*/ 1,
7089                                               /*bit*/ 5);
7090                         ctl_done((union ctl_io *)ctsio);
7091                         return (CTL_RETVAL_COMPLETE);
7092                 }
7093                 break;
7094         }
7095         }
7096
7097         total_len = header_len + page_len;
7098 #if 0
7099         printf("header_len = %d, page_len = %d, total_len = %d\n",
7100                header_len, page_len, total_len);
7101 #endif
7102
7103         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7104         ctsio->kern_sg_entries = 0;
7105         ctsio->kern_data_resid = 0;
7106         ctsio->kern_rel_offset = 0;
7107         if (total_len < alloc_len) {
7108                 ctsio->residual = alloc_len - total_len;
7109                 ctsio->kern_data_len = total_len;
7110                 ctsio->kern_total_len = total_len;
7111         } else {
7112                 ctsio->residual = 0;
7113                 ctsio->kern_data_len = alloc_len;
7114                 ctsio->kern_total_len = alloc_len;
7115         }
7116
7117         switch (ctsio->cdb[0]) {
7118         case MODE_SENSE_6: {
7119                 struct scsi_mode_hdr_6 *header;
7120
7121                 header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
7122
7123                 header->datalen = ctl_min(total_len - 1, 254);
7124                 if (control_dev == 0) {
7125                         header->dev_specific = 0x10; /* DPOFUA */
7126                         if ((lun->flags & CTL_LUN_READONLY) ||
7127                             (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
7128                             .eca_and_aen & SCP_SWP) != 0)
7129                                     header->dev_specific |= 0x80; /* WP */
7130                 }
7131                 if (dbd)
7132                         header->block_descr_len = 0;
7133                 else
7134                         header->block_descr_len =
7135                                 sizeof(struct scsi_mode_block_descr);
7136                 block_desc = (struct scsi_mode_block_descr *)&header[1];
7137                 break;
7138         }
7139         case MODE_SENSE_10: {
7140                 struct scsi_mode_hdr_10 *header;
7141                 int datalen;
7142
7143                 header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
7144
7145                 datalen = ctl_min(total_len - 2, 65533);
7146                 scsi_ulto2b(datalen, header->datalen);
7147                 if (control_dev == 0) {
7148                         header->dev_specific = 0x10; /* DPOFUA */
7149                         if ((lun->flags & CTL_LUN_READONLY) ||
7150                             (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
7151                             .eca_and_aen & SCP_SWP) != 0)
7152                                     header->dev_specific |= 0x80; /* WP */
7153                 }
7154                 if (dbd)
7155                         scsi_ulto2b(0, header->block_descr_len);
7156                 else
7157                         scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
7158                                     header->block_descr_len);
7159                 block_desc = (struct scsi_mode_block_descr *)&header[1];
7160                 break;
7161         }
7162         default:
7163                 panic("invalid CDB type %#x", ctsio->cdb[0]);
7164                 break; /* NOTREACHED */
7165         }
7166
7167         /*
7168          * If we've got a disk, use its blocksize in the block
7169          * descriptor.  Otherwise, just set it to 0.
7170          */
7171         if (dbd == 0) {
7172                 if (control_dev == 0)
7173                         scsi_ulto3b(lun->be_lun->blocksize,
7174                                     block_desc->block_len);
7175                 else
7176                         scsi_ulto3b(0, block_desc->block_len);
7177         }
7178
7179         switch (page_code) {
7180         case SMS_ALL_PAGES_PAGE: {
7181                 int i, data_used;
7182
7183                 data_used = header_len;
7184                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
7185                         struct ctl_page_index *page_index;
7186
7187                         page_index = &lun->mode_pages.index[i];
7188
7189                         if ((control_dev != 0)
7190                          && (page_index->page_flags &
7191                             CTL_PAGE_FLAG_DISK_ONLY))
7192                                 continue;
7193
7194                         /*
7195                          * We don't use this subpage if the user didn't
7196                          * request all subpages.  We already checked (above)
7197                          * to make sure the user only specified a subpage
7198                          * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
7199                          */
7200                         if ((page_index->subpage != 0)
7201                          && (subpage == SMS_SUBPAGE_PAGE_0))
7202                                 continue;
7203
7204                         /*
7205                          * Call the handler, if it exists, to update the
7206                          * page to the latest values.
7207                          */
7208                         if (page_index->sense_handler != NULL)
7209                                 page_index->sense_handler(ctsio, page_index,pc);
7210
7211                         memcpy(ctsio->kern_data_ptr + data_used,
7212                                page_index->page_data +
7213                                (page_index->page_len * pc),
7214                                page_index->page_len);
7215                         data_used += page_index->page_len;
7216                 }
7217                 break;
7218         }
7219         default: {
7220                 int i, data_used;
7221
7222                 data_used = header_len;
7223
7224                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
7225                         struct ctl_page_index *page_index;
7226
7227                         page_index = &lun->mode_pages.index[i];
7228
7229                         /* Look for the right page code */
7230                         if ((page_index->page_code & SMPH_PC_MASK) != page_code)
7231                                 continue;
7232
7233                         /* Look for the right subpage or the subpage wildcard*/
7234                         if ((page_index->subpage != subpage)
7235                          && (subpage != SMS_SUBPAGE_ALL))
7236                                 continue;
7237
7238                         /* Make sure the page is supported for this dev type */
7239                         if ((control_dev != 0)
7240                          && (page_index->page_flags &
7241                              CTL_PAGE_FLAG_DISK_ONLY))
7242                                 continue;
7243
7244                         /*
7245                          * Call the handler, if it exists, to update the
7246                          * page to the latest values.
7247                          */
7248                         if (page_index->sense_handler != NULL)
7249                                 page_index->sense_handler(ctsio, page_index,pc);
7250
7251                         memcpy(ctsio->kern_data_ptr + data_used,
7252                                page_index->page_data +
7253                                (page_index->page_len * pc),
7254                                page_index->page_len);
7255                         data_used += page_index->page_len;
7256                 }
7257                 break;
7258         }
7259         }
7260
7261         ctsio->scsi_status = SCSI_STATUS_OK;
7262
7263         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7264         ctsio->be_move_done = ctl_config_move_done;
7265         ctl_datamove((union ctl_io *)ctsio);
7266
7267         return (CTL_RETVAL_COMPLETE);
7268 }
7269
7270 int
7271 ctl_read_capacity(struct ctl_scsiio *ctsio)
7272 {
7273         struct scsi_read_capacity *cdb;
7274         struct scsi_read_capacity_data *data;
7275         struct ctl_lun *lun;
7276         uint32_t lba;
7277
7278         CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
7279
7280         cdb = (struct scsi_read_capacity *)ctsio->cdb;
7281
7282         lba = scsi_4btoul(cdb->addr);
7283         if (((cdb->pmi & SRC_PMI) == 0)
7284          && (lba != 0)) {
7285                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7286                                       /*sks_valid*/ 1,
7287                                       /*command*/ 1,
7288                                       /*field*/ 2,
7289                                       /*bit_valid*/ 0,
7290                                       /*bit*/ 0);
7291                 ctl_done((union ctl_io *)ctsio);
7292                 return (CTL_RETVAL_COMPLETE);
7293         }
7294
7295         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7296
7297         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7298         data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
7299         ctsio->residual = 0;
7300         ctsio->kern_data_len = sizeof(*data);
7301         ctsio->kern_total_len = sizeof(*data);
7302         ctsio->kern_data_resid = 0;
7303         ctsio->kern_rel_offset = 0;
7304         ctsio->kern_sg_entries = 0;
7305
7306         /*
7307          * If the maximum LBA is greater than 0xfffffffe, the user must
7308          * issue a SERVICE ACTION IN (16) command, with the read capacity
7309          * serivce action set.
7310          */
7311         if (lun->be_lun->maxlba > 0xfffffffe)
7312                 scsi_ulto4b(0xffffffff, data->addr);
7313         else
7314                 scsi_ulto4b(lun->be_lun->maxlba, data->addr);
7315
7316         /*
7317          * XXX KDM this may not be 512 bytes...
7318          */
7319         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7320
7321         ctsio->scsi_status = SCSI_STATUS_OK;
7322
7323         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7324         ctsio->be_move_done = ctl_config_move_done;
7325         ctl_datamove((union ctl_io *)ctsio);
7326
7327         return (CTL_RETVAL_COMPLETE);
7328 }
7329
7330 int
7331 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
7332 {
7333         struct scsi_read_capacity_16 *cdb;
7334         struct scsi_read_capacity_data_long *data;
7335         struct ctl_lun *lun;
7336         uint64_t lba;
7337         uint32_t alloc_len;
7338
7339         CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
7340
7341         cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
7342
7343         alloc_len = scsi_4btoul(cdb->alloc_len);
7344         lba = scsi_8btou64(cdb->addr);
7345
7346         if ((cdb->reladr & SRC16_PMI)
7347          && (lba != 0)) {
7348                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7349                                       /*sks_valid*/ 1,
7350                                       /*command*/ 1,
7351                                       /*field*/ 2,
7352                                       /*bit_valid*/ 0,
7353                                       /*bit*/ 0);
7354                 ctl_done((union ctl_io *)ctsio);
7355                 return (CTL_RETVAL_COMPLETE);
7356         }
7357
7358         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7359
7360         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7361         data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
7362
7363         if (sizeof(*data) < alloc_len) {
7364                 ctsio->residual = alloc_len - sizeof(*data);
7365                 ctsio->kern_data_len = sizeof(*data);
7366                 ctsio->kern_total_len = sizeof(*data);
7367         } else {
7368                 ctsio->residual = 0;
7369                 ctsio->kern_data_len = alloc_len;
7370                 ctsio->kern_total_len = alloc_len;
7371         }
7372         ctsio->kern_data_resid = 0;
7373         ctsio->kern_rel_offset = 0;
7374         ctsio->kern_sg_entries = 0;
7375
7376         scsi_u64to8b(lun->be_lun->maxlba, data->addr);
7377         /* XXX KDM this may not be 512 bytes... */
7378         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7379         data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
7380         scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
7381         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
7382                 data->lalba_lbp[0] |= SRC16_LBPME | SRC16_LBPRZ;
7383
7384         ctsio->scsi_status = SCSI_STATUS_OK;
7385
7386         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7387         ctsio->be_move_done = ctl_config_move_done;
7388         ctl_datamove((union ctl_io *)ctsio);
7389
7390         return (CTL_RETVAL_COMPLETE);
7391 }
7392
7393 int
7394 ctl_read_defect(struct ctl_scsiio *ctsio)
7395 {
7396         struct scsi_read_defect_data_10 *ccb10;
7397         struct scsi_read_defect_data_12 *ccb12;
7398         struct scsi_read_defect_data_hdr_10 *data10;
7399         struct scsi_read_defect_data_hdr_12 *data12;
7400         struct ctl_lun *lun;
7401         uint32_t alloc_len, data_len;
7402         uint8_t format;
7403
7404         CTL_DEBUG_PRINT(("ctl_read_defect\n"));
7405
7406         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7407         if (lun->flags & CTL_LUN_PR_RESERVED) {
7408                 uint32_t residx;
7409
7410                 /*
7411                  * XXX KDM need a lock here.
7412                  */
7413                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
7414                 if ((lun->res_type == SPR_TYPE_EX_AC
7415                   && residx != lun->pr_res_idx)
7416                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
7417                    || lun->res_type == SPR_TYPE_EX_AC_AR)
7418                   && lun->pr_keys[residx] == 0)) {
7419                         ctl_set_reservation_conflict(ctsio);
7420                         ctl_done((union ctl_io *)ctsio);
7421                         return (CTL_RETVAL_COMPLETE);
7422                 }
7423         }
7424
7425         if (ctsio->cdb[0] == READ_DEFECT_DATA_10) {
7426                 ccb10 = (struct scsi_read_defect_data_10 *)&ctsio->cdb;
7427                 format = ccb10->format;
7428                 alloc_len = scsi_2btoul(ccb10->alloc_length);
7429                 data_len = sizeof(*data10);
7430         } else {
7431                 ccb12 = (struct scsi_read_defect_data_12 *)&ctsio->cdb;
7432                 format = ccb12->format;
7433                 alloc_len = scsi_4btoul(ccb12->alloc_length);
7434                 data_len = sizeof(*data12);
7435         }
7436         if (alloc_len == 0) {
7437                 ctl_set_success(ctsio);
7438                 ctl_done((union ctl_io *)ctsio);
7439                 return (CTL_RETVAL_COMPLETE);
7440         }
7441
7442         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
7443         if (data_len < alloc_len) {
7444                 ctsio->residual = alloc_len - data_len;
7445                 ctsio->kern_data_len = data_len;
7446                 ctsio->kern_total_len = data_len;
7447         } else {
7448                 ctsio->residual = 0;
7449                 ctsio->kern_data_len = alloc_len;
7450                 ctsio->kern_total_len = alloc_len;
7451         }
7452         ctsio->kern_data_resid = 0;
7453         ctsio->kern_rel_offset = 0;
7454         ctsio->kern_sg_entries = 0;
7455
7456         if (ctsio->cdb[0] == READ_DEFECT_DATA_10) {
7457                 data10 = (struct scsi_read_defect_data_hdr_10 *)
7458                     ctsio->kern_data_ptr;
7459                 data10->format = format;
7460                 scsi_ulto2b(0, data10->length);
7461         } else {
7462                 data12 = (struct scsi_read_defect_data_hdr_12 *)
7463                     ctsio->kern_data_ptr;
7464                 data12->format = format;
7465                 scsi_ulto2b(0, data12->generation);
7466                 scsi_ulto4b(0, data12->length);
7467         }
7468
7469         ctsio->scsi_status = SCSI_STATUS_OK;
7470         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7471         ctsio->be_move_done = ctl_config_move_done;
7472         ctl_datamove((union ctl_io *)ctsio);
7473         return (CTL_RETVAL_COMPLETE);
7474 }
7475
7476 int
7477 ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
7478 {
7479         struct scsi_maintenance_in *cdb;
7480         int retval;
7481         int alloc_len, ext, total_len = 0, g, p, pc, pg;
7482         int num_target_port_groups, num_target_ports, single;
7483         struct ctl_lun *lun;
7484         struct ctl_softc *softc;
7485         struct ctl_port *port;
7486         struct scsi_target_group_data *rtg_ptr;
7487         struct scsi_target_group_data_extended *rtg_ext_ptr;
7488         struct scsi_target_port_group_descriptor *tpg_desc;
7489
7490         CTL_DEBUG_PRINT(("ctl_report_tagret_port_groups\n"));
7491
7492         cdb = (struct scsi_maintenance_in *)ctsio->cdb;
7493         softc = control_softc;
7494         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7495
7496         retval = CTL_RETVAL_COMPLETE;
7497
7498         switch (cdb->byte2 & STG_PDF_MASK) {
7499         case STG_PDF_LENGTH:
7500                 ext = 0;
7501                 break;
7502         case STG_PDF_EXTENDED:
7503                 ext = 1;
7504                 break;
7505         default:
7506                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7507                                       /*sks_valid*/ 1,
7508                                       /*command*/ 1,
7509                                       /*field*/ 2,
7510                                       /*bit_valid*/ 1,
7511                                       /*bit*/ 5);
7512                 ctl_done((union ctl_io *)ctsio);
7513                 return(retval);
7514         }
7515
7516         single = ctl_is_single;
7517         if (single)
7518                 num_target_port_groups = 1;
7519         else
7520                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
7521         num_target_ports = 0;
7522         mtx_lock(&softc->ctl_lock);
7523         STAILQ_FOREACH(port, &softc->port_list, links) {
7524                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7525                         continue;
7526                 if (ctl_map_lun_back(port->targ_port, lun->lun) >= CTL_MAX_LUNS)
7527                         continue;
7528                 num_target_ports++;
7529         }
7530         mtx_unlock(&softc->ctl_lock);
7531
7532         if (ext)
7533                 total_len = sizeof(struct scsi_target_group_data_extended);
7534         else
7535                 total_len = sizeof(struct scsi_target_group_data);
7536         total_len += sizeof(struct scsi_target_port_group_descriptor) *
7537                 num_target_port_groups +
7538             sizeof(struct scsi_target_port_descriptor) *
7539                 num_target_ports * num_target_port_groups;
7540
7541         alloc_len = scsi_4btoul(cdb->length);
7542
7543         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7544
7545         ctsio->kern_sg_entries = 0;
7546
7547         if (total_len < alloc_len) {
7548                 ctsio->residual = alloc_len - total_len;
7549                 ctsio->kern_data_len = total_len;
7550                 ctsio->kern_total_len = total_len;
7551         } else {
7552                 ctsio->residual = 0;
7553                 ctsio->kern_data_len = alloc_len;
7554                 ctsio->kern_total_len = alloc_len;
7555         }
7556         ctsio->kern_data_resid = 0;
7557         ctsio->kern_rel_offset = 0;
7558
7559         if (ext) {
7560                 rtg_ext_ptr = (struct scsi_target_group_data_extended *)
7561                     ctsio->kern_data_ptr;
7562                 scsi_ulto4b(total_len - 4, rtg_ext_ptr->length);
7563                 rtg_ext_ptr->format_type = 0x10;
7564                 rtg_ext_ptr->implicit_transition_time = 0;
7565                 tpg_desc = &rtg_ext_ptr->groups[0];
7566         } else {
7567                 rtg_ptr = (struct scsi_target_group_data *)
7568                     ctsio->kern_data_ptr;
7569                 scsi_ulto4b(total_len - 4, rtg_ptr->length);
7570                 tpg_desc = &rtg_ptr->groups[0];
7571         }
7572
7573         pg = ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS;
7574         mtx_lock(&softc->ctl_lock);
7575         for (g = 0; g < num_target_port_groups; g++) {
7576                 if (g == pg)
7577                         tpg_desc->pref_state = TPG_PRIMARY |
7578                             TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7579                 else
7580                         tpg_desc->pref_state =
7581                             TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7582                 tpg_desc->support = TPG_AO_SUP;
7583                 if (!single)
7584                         tpg_desc->support |= TPG_AN_SUP;
7585                 scsi_ulto2b(g + 1, tpg_desc->target_port_group);
7586                 tpg_desc->status = TPG_IMPLICIT;
7587                 pc = 0;
7588                 STAILQ_FOREACH(port, &softc->port_list, links) {
7589                         if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7590                                 continue;
7591                         if (ctl_map_lun_back(port->targ_port, lun->lun) >=
7592                             CTL_MAX_LUNS)
7593                                 continue;
7594                         p = port->targ_port % CTL_MAX_PORTS + g * CTL_MAX_PORTS;
7595                         scsi_ulto2b(p, tpg_desc->descriptors[pc].
7596                             relative_target_port_identifier);
7597                         pc++;
7598                 }
7599                 tpg_desc->target_port_count = pc;
7600                 tpg_desc = (struct scsi_target_port_group_descriptor *)
7601                     &tpg_desc->descriptors[pc];
7602         }
7603         mtx_unlock(&softc->ctl_lock);
7604
7605         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7606         ctsio->be_move_done = ctl_config_move_done;
7607
7608         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7609                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7610                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7611                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7612                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7613
7614         ctl_datamove((union ctl_io *)ctsio);
7615         return(retval);
7616 }
7617
7618 int
7619 ctl_report_supported_opcodes(struct ctl_scsiio *ctsio)
7620 {
7621         struct ctl_lun *lun;
7622         struct scsi_report_supported_opcodes *cdb;
7623         const struct ctl_cmd_entry *entry, *sentry;
7624         struct scsi_report_supported_opcodes_all *all;
7625         struct scsi_report_supported_opcodes_descr *descr;
7626         struct scsi_report_supported_opcodes_one *one;
7627         int retval;
7628         int alloc_len, total_len;
7629         int opcode, service_action, i, j, num;
7630
7631         CTL_DEBUG_PRINT(("ctl_report_supported_opcodes\n"));
7632
7633         cdb = (struct scsi_report_supported_opcodes *)ctsio->cdb;
7634         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7635
7636         retval = CTL_RETVAL_COMPLETE;
7637
7638         opcode = cdb->requested_opcode;
7639         service_action = scsi_2btoul(cdb->requested_service_action);
7640         switch (cdb->options & RSO_OPTIONS_MASK) {
7641         case RSO_OPTIONS_ALL:
7642                 num = 0;
7643                 for (i = 0; i < 256; i++) {
7644                         entry = &ctl_cmd_table[i];
7645                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7646                                 for (j = 0; j < 32; j++) {
7647                                         sentry = &((const struct ctl_cmd_entry *)
7648                                             entry->execute)[j];
7649                                         if (ctl_cmd_applicable(
7650                                             lun->be_lun->lun_type, sentry))
7651                                                 num++;
7652                                 }
7653                         } else {
7654                                 if (ctl_cmd_applicable(lun->be_lun->lun_type,
7655                                     entry))
7656                                         num++;
7657                         }
7658                 }
7659                 total_len = sizeof(struct scsi_report_supported_opcodes_all) +
7660                     num * sizeof(struct scsi_report_supported_opcodes_descr);
7661                 break;
7662         case RSO_OPTIONS_OC:
7663                 if (ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) {
7664                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7665                                               /*sks_valid*/ 1,
7666                                               /*command*/ 1,
7667                                               /*field*/ 2,
7668                                               /*bit_valid*/ 1,
7669                                               /*bit*/ 2);
7670                         ctl_done((union ctl_io *)ctsio);
7671                         return (CTL_RETVAL_COMPLETE);
7672                 }
7673                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7674                 break;
7675         case RSO_OPTIONS_OC_SA:
7676                 if ((ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) == 0 ||
7677                     service_action >= 32) {
7678                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7679                                               /*sks_valid*/ 1,
7680                                               /*command*/ 1,
7681                                               /*field*/ 2,
7682                                               /*bit_valid*/ 1,
7683                                               /*bit*/ 2);
7684                         ctl_done((union ctl_io *)ctsio);
7685                         return (CTL_RETVAL_COMPLETE);
7686                 }
7687                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7688                 break;
7689         default:
7690                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7691                                       /*sks_valid*/ 1,
7692                                       /*command*/ 1,
7693                                       /*field*/ 2,
7694                                       /*bit_valid*/ 1,
7695                                       /*bit*/ 2);
7696                 ctl_done((union ctl_io *)ctsio);
7697                 return (CTL_RETVAL_COMPLETE);
7698         }
7699
7700         alloc_len = scsi_4btoul(cdb->length);
7701
7702         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7703
7704         ctsio->kern_sg_entries = 0;
7705
7706         if (total_len < alloc_len) {
7707                 ctsio->residual = alloc_len - total_len;
7708                 ctsio->kern_data_len = total_len;
7709                 ctsio->kern_total_len = total_len;
7710         } else {
7711                 ctsio->residual = 0;
7712                 ctsio->kern_data_len = alloc_len;
7713                 ctsio->kern_total_len = alloc_len;
7714         }
7715         ctsio->kern_data_resid = 0;
7716         ctsio->kern_rel_offset = 0;
7717
7718         switch (cdb->options & RSO_OPTIONS_MASK) {
7719         case RSO_OPTIONS_ALL:
7720                 all = (struct scsi_report_supported_opcodes_all *)
7721                     ctsio->kern_data_ptr;
7722                 num = 0;
7723                 for (i = 0; i < 256; i++) {
7724                         entry = &ctl_cmd_table[i];
7725                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7726                                 for (j = 0; j < 32; j++) {
7727                                         sentry = &((const struct ctl_cmd_entry *)
7728                                             entry->execute)[j];
7729                                         if (!ctl_cmd_applicable(
7730                                             lun->be_lun->lun_type, sentry))
7731                                                 continue;
7732                                         descr = &all->descr[num++];
7733                                         descr->opcode = i;
7734                                         scsi_ulto2b(j, descr->service_action);
7735                                         descr->flags = RSO_SERVACTV;
7736                                         scsi_ulto2b(sentry->length,
7737                                             descr->cdb_length);
7738                                 }
7739                         } else {
7740                                 if (!ctl_cmd_applicable(lun->be_lun->lun_type,
7741                                     entry))
7742                                         continue;
7743                                 descr = &all->descr[num++];
7744                                 descr->opcode = i;
7745                                 scsi_ulto2b(0, descr->service_action);
7746                                 descr->flags = 0;
7747                                 scsi_ulto2b(entry->length, descr->cdb_length);
7748                         }
7749                 }
7750                 scsi_ulto4b(
7751                     num * sizeof(struct scsi_report_supported_opcodes_descr),
7752                     all->length);
7753                 break;
7754         case RSO_OPTIONS_OC:
7755                 one = (struct scsi_report_supported_opcodes_one *)
7756                     ctsio->kern_data_ptr;
7757                 entry = &ctl_cmd_table[opcode];
7758                 goto fill_one;
7759         case RSO_OPTIONS_OC_SA:
7760                 one = (struct scsi_report_supported_opcodes_one *)
7761                     ctsio->kern_data_ptr;
7762                 entry = &ctl_cmd_table[opcode];
7763                 entry = &((const struct ctl_cmd_entry *)
7764                     entry->execute)[service_action];
7765 fill_one:
7766                 if (ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
7767                         one->support = 3;
7768                         scsi_ulto2b(entry->length, one->cdb_length);
7769                         one->cdb_usage[0] = opcode;
7770                         memcpy(&one->cdb_usage[1], entry->usage,
7771                             entry->length - 1);
7772                 } else
7773                         one->support = 1;
7774                 break;
7775         }
7776
7777         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7778         ctsio->be_move_done = ctl_config_move_done;
7779
7780         ctl_datamove((union ctl_io *)ctsio);
7781         return(retval);
7782 }
7783
7784 int
7785 ctl_report_supported_tmf(struct ctl_scsiio *ctsio)
7786 {
7787         struct ctl_lun *lun;
7788         struct scsi_report_supported_tmf *cdb;
7789         struct scsi_report_supported_tmf_data *data;
7790         int retval;
7791         int alloc_len, total_len;
7792
7793         CTL_DEBUG_PRINT(("ctl_report_supported_tmf\n"));
7794
7795         cdb = (struct scsi_report_supported_tmf *)ctsio->cdb;
7796         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7797
7798         retval = CTL_RETVAL_COMPLETE;
7799
7800         total_len = sizeof(struct scsi_report_supported_tmf_data);
7801         alloc_len = scsi_4btoul(cdb->length);
7802
7803         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7804
7805         ctsio->kern_sg_entries = 0;
7806
7807         if (total_len < alloc_len) {
7808                 ctsio->residual = alloc_len - total_len;
7809                 ctsio->kern_data_len = total_len;
7810                 ctsio->kern_total_len = total_len;
7811         } else {
7812                 ctsio->residual = 0;
7813                 ctsio->kern_data_len = alloc_len;
7814                 ctsio->kern_total_len = alloc_len;
7815         }
7816         ctsio->kern_data_resid = 0;
7817         ctsio->kern_rel_offset = 0;
7818
7819         data = (struct scsi_report_supported_tmf_data *)ctsio->kern_data_ptr;
7820         data->byte1 |= RST_ATS | RST_ATSS | RST_CTSS | RST_LURS | RST_TRS;
7821         data->byte2 |= RST_ITNRS;
7822
7823         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7824         ctsio->be_move_done = ctl_config_move_done;
7825
7826         ctl_datamove((union ctl_io *)ctsio);
7827         return (retval);
7828 }
7829
7830 int
7831 ctl_report_timestamp(struct ctl_scsiio *ctsio)
7832 {
7833         struct ctl_lun *lun;
7834         struct scsi_report_timestamp *cdb;
7835         struct scsi_report_timestamp_data *data;
7836         struct timeval tv;
7837         int64_t timestamp;
7838         int retval;
7839         int alloc_len, total_len;
7840
7841         CTL_DEBUG_PRINT(("ctl_report_timestamp\n"));
7842
7843         cdb = (struct scsi_report_timestamp *)ctsio->cdb;
7844         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7845
7846         retval = CTL_RETVAL_COMPLETE;
7847
7848         total_len = sizeof(struct scsi_report_timestamp_data);
7849         alloc_len = scsi_4btoul(cdb->length);
7850
7851         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7852
7853         ctsio->kern_sg_entries = 0;
7854
7855         if (total_len < alloc_len) {
7856                 ctsio->residual = alloc_len - total_len;
7857                 ctsio->kern_data_len = total_len;
7858                 ctsio->kern_total_len = total_len;
7859         } else {
7860                 ctsio->residual = 0;
7861                 ctsio->kern_data_len = alloc_len;
7862                 ctsio->kern_total_len = alloc_len;
7863         }
7864         ctsio->kern_data_resid = 0;
7865         ctsio->kern_rel_offset = 0;
7866
7867         data = (struct scsi_report_timestamp_data *)ctsio->kern_data_ptr;
7868         scsi_ulto2b(sizeof(*data) - 2, data->length);
7869         data->origin = RTS_ORIG_OUTSIDE;
7870         getmicrotime(&tv);
7871         timestamp = (int64_t)tv.tv_sec * 1000 + tv.tv_usec / 1000;
7872         scsi_ulto4b(timestamp >> 16, data->timestamp);
7873         scsi_ulto2b(timestamp & 0xffff, &data->timestamp[4]);
7874
7875         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7876         ctsio->be_move_done = ctl_config_move_done;
7877
7878         ctl_datamove((union ctl_io *)ctsio);
7879         return (retval);
7880 }
7881
7882 int
7883 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7884 {
7885         struct scsi_per_res_in *cdb;
7886         int alloc_len, total_len = 0;
7887         /* struct scsi_per_res_in_rsrv in_data; */
7888         struct ctl_lun *lun;
7889         struct ctl_softc *softc;
7890
7891         CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7892
7893         softc = control_softc;
7894
7895         cdb = (struct scsi_per_res_in *)ctsio->cdb;
7896
7897         alloc_len = scsi_2btoul(cdb->length);
7898
7899         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7900
7901 retry:
7902         mtx_lock(&lun->lun_lock);
7903         switch (cdb->action) {
7904         case SPRI_RK: /* read keys */
7905                 total_len = sizeof(struct scsi_per_res_in_keys) +
7906                         lun->pr_key_count *
7907                         sizeof(struct scsi_per_res_key);
7908                 break;
7909         case SPRI_RR: /* read reservation */
7910                 if (lun->flags & CTL_LUN_PR_RESERVED)
7911                         total_len = sizeof(struct scsi_per_res_in_rsrv);
7912                 else
7913                         total_len = sizeof(struct scsi_per_res_in_header);
7914                 break;
7915         case SPRI_RC: /* report capabilities */
7916                 total_len = sizeof(struct scsi_per_res_cap);
7917                 break;
7918         case SPRI_RS: /* read full status */
7919                 total_len = sizeof(struct scsi_per_res_in_header) +
7920                     (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7921                     lun->pr_key_count;
7922                 break;
7923         default:
7924                 panic("Invalid PR type %x", cdb->action);
7925         }
7926         mtx_unlock(&lun->lun_lock);
7927
7928         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7929
7930         if (total_len < alloc_len) {
7931                 ctsio->residual = alloc_len - total_len;
7932                 ctsio->kern_data_len = total_len;
7933                 ctsio->kern_total_len = total_len;
7934         } else {
7935                 ctsio->residual = 0;
7936                 ctsio->kern_data_len = alloc_len;
7937                 ctsio->kern_total_len = alloc_len;
7938         }
7939
7940         ctsio->kern_data_resid = 0;
7941         ctsio->kern_rel_offset = 0;
7942         ctsio->kern_sg_entries = 0;
7943
7944         mtx_lock(&lun->lun_lock);
7945         switch (cdb->action) {
7946         case SPRI_RK: { // read keys
7947         struct scsi_per_res_in_keys *res_keys;
7948                 int i, key_count;
7949
7950                 res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7951
7952                 /*
7953                  * We had to drop the lock to allocate our buffer, which
7954                  * leaves time for someone to come in with another
7955                  * persistent reservation.  (That is unlikely, though,
7956                  * since this should be the only persistent reservation
7957                  * command active right now.)
7958                  */
7959                 if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7960                     (lun->pr_key_count *
7961                      sizeof(struct scsi_per_res_key)))){
7962                         mtx_unlock(&lun->lun_lock);
7963                         free(ctsio->kern_data_ptr, M_CTL);
7964                         printf("%s: reservation length changed, retrying\n",
7965                                __func__);
7966                         goto retry;
7967                 }
7968
7969                 scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7970
7971                 scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7972                              lun->pr_key_count, res_keys->header.length);
7973
7974                 for (i = 0, key_count = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7975                         if (lun->pr_keys[i] == 0)
7976                                 continue;
7977
7978                         /*
7979                          * We used lun->pr_key_count to calculate the
7980                          * size to allocate.  If it turns out the number of
7981                          * initiators with the registered flag set is
7982                          * larger than that (i.e. they haven't been kept in
7983                          * sync), we've got a problem.
7984                          */
7985                         if (key_count >= lun->pr_key_count) {
7986 #ifdef NEEDTOPORT
7987                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
7988                                             CTL_PR_ERROR,
7989                                             csevent_LogType_Fault,
7990                                             csevent_AlertLevel_Yellow,
7991                                             csevent_FRU_ShelfController,
7992                                             csevent_FRU_Firmware,
7993                                         csevent_FRU_Unknown,
7994                                             "registered keys %d >= key "
7995                                             "count %d", key_count,
7996                                             lun->pr_key_count);
7997 #endif
7998                                 key_count++;
7999                                 continue;
8000                         }
8001                         scsi_u64to8b(lun->pr_keys[i],
8002                             res_keys->keys[key_count].key);
8003                         key_count++;
8004                 }
8005                 break;
8006         }
8007         case SPRI_RR: { // read reservation
8008                 struct scsi_per_res_in_rsrv *res;
8009                 int tmp_len, header_only;
8010
8011                 res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
8012
8013                 scsi_ulto4b(lun->PRGeneration, res->header.generation);
8014
8015                 if (lun->flags & CTL_LUN_PR_RESERVED)
8016                 {
8017                         tmp_len = sizeof(struct scsi_per_res_in_rsrv);
8018                         scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
8019                                     res->header.length);
8020                         header_only = 0;
8021                 } else {
8022                         tmp_len = sizeof(struct scsi_per_res_in_header);
8023                         scsi_ulto4b(0, res->header.length);
8024                         header_only = 1;
8025                 }
8026
8027                 /*
8028                  * We had to drop the lock to allocate our buffer, which
8029                  * leaves time for someone to come in with another
8030                  * persistent reservation.  (That is unlikely, though,
8031                  * since this should be the only persistent reservation
8032                  * command active right now.)
8033                  */
8034                 if (tmp_len != total_len) {
8035                         mtx_unlock(&lun->lun_lock);
8036                         free(ctsio->kern_data_ptr, M_CTL);
8037                         printf("%s: reservation status changed, retrying\n",
8038                                __func__);
8039                         goto retry;
8040                 }
8041
8042                 /*
8043                  * No reservation held, so we're done.
8044                  */
8045                 if (header_only != 0)
8046                         break;
8047
8048                 /*
8049                  * If the registration is an All Registrants type, the key
8050                  * is 0, since it doesn't really matter.
8051                  */
8052                 if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8053                         scsi_u64to8b(lun->pr_keys[lun->pr_res_idx],
8054                             res->data.reservation);
8055                 }
8056                 res->data.scopetype = lun->res_type;
8057                 break;
8058         }
8059         case SPRI_RC:     //report capabilities
8060         {
8061                 struct scsi_per_res_cap *res_cap;
8062                 uint16_t type_mask;
8063
8064                 res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
8065                 scsi_ulto2b(sizeof(*res_cap), res_cap->length);
8066                 res_cap->flags2 |= SPRI_TMV | SPRI_ALLOW_5;
8067                 type_mask = SPRI_TM_WR_EX_AR |
8068                             SPRI_TM_EX_AC_RO |
8069                             SPRI_TM_WR_EX_RO |
8070                             SPRI_TM_EX_AC |
8071                             SPRI_TM_WR_EX |
8072                             SPRI_TM_EX_AC_AR;
8073                 scsi_ulto2b(type_mask, res_cap->type_mask);
8074                 break;
8075         }
8076         case SPRI_RS: { // read full status
8077                 struct scsi_per_res_in_full *res_status;
8078                 struct scsi_per_res_in_full_desc *res_desc;
8079                 struct ctl_port *port;
8080                 int i, len;
8081
8082                 res_status = (struct scsi_per_res_in_full*)ctsio->kern_data_ptr;
8083
8084                 /*
8085                  * We had to drop the lock to allocate our buffer, which
8086                  * leaves time for someone to come in with another
8087                  * persistent reservation.  (That is unlikely, though,
8088                  * since this should be the only persistent reservation
8089                  * command active right now.)
8090                  */
8091                 if (total_len < (sizeof(struct scsi_per_res_in_header) +
8092                     (sizeof(struct scsi_per_res_in_full_desc) + 256) *
8093                      lun->pr_key_count)){
8094                         mtx_unlock(&lun->lun_lock);
8095                         free(ctsio->kern_data_ptr, M_CTL);
8096                         printf("%s: reservation length changed, retrying\n",
8097                                __func__);
8098                         goto retry;
8099                 }
8100
8101                 scsi_ulto4b(lun->PRGeneration, res_status->header.generation);
8102
8103                 res_desc = &res_status->desc[0];
8104                 for (i = 0; i < 2*CTL_MAX_INITIATORS; i++) {
8105                         if (lun->pr_keys[i] == 0)
8106                                 continue;
8107
8108                         scsi_u64to8b(lun->pr_keys[i], res_desc->res_key.key);
8109                         if ((lun->flags & CTL_LUN_PR_RESERVED) &&
8110                             (lun->pr_res_idx == i ||
8111                              lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS)) {
8112                                 res_desc->flags = SPRI_FULL_R_HOLDER;
8113                                 res_desc->scopetype = lun->res_type;
8114                         }
8115                         scsi_ulto2b(i / CTL_MAX_INIT_PER_PORT,
8116                             res_desc->rel_trgt_port_id);
8117                         len = 0;
8118                         port = softc->ctl_ports[
8119                             ctl_port_idx(i / CTL_MAX_INIT_PER_PORT)];
8120                         if (port != NULL)
8121                                 len = ctl_create_iid(port,
8122                                     i % CTL_MAX_INIT_PER_PORT,
8123                                     res_desc->transport_id);
8124                         scsi_ulto4b(len, res_desc->additional_length);
8125                         res_desc = (struct scsi_per_res_in_full_desc *)
8126                             &res_desc->transport_id[len];
8127                 }
8128                 scsi_ulto4b((uint8_t *)res_desc - (uint8_t *)&res_status->desc[0],
8129                     res_status->header.length);
8130                 break;
8131         }
8132         default:
8133                 /*
8134                  * This is a bug, because we just checked for this above,
8135                  * and should have returned an error.
8136                  */
8137                 panic("Invalid PR type %x", cdb->action);
8138                 break; /* NOTREACHED */
8139         }
8140         mtx_unlock(&lun->lun_lock);
8141
8142         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8143         ctsio->be_move_done = ctl_config_move_done;
8144
8145         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
8146                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
8147                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
8148                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
8149                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
8150
8151         ctl_datamove((union ctl_io *)ctsio);
8152
8153         return (CTL_RETVAL_COMPLETE);
8154 }
8155
8156 /*
8157  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
8158  * it should return.
8159  */
8160 static int
8161 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
8162                 uint64_t sa_res_key, uint8_t type, uint32_t residx,
8163                 struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
8164                 struct scsi_per_res_out_parms* param)
8165 {
8166         union ctl_ha_msg persis_io;
8167         int retval, i;
8168         int isc_retval;
8169
8170         retval = 0;
8171
8172         mtx_lock(&lun->lun_lock);
8173         if (sa_res_key == 0) {
8174                 if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8175                         /* validate scope and type */
8176                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
8177                              SPR_LU_SCOPE) {
8178                                 mtx_unlock(&lun->lun_lock);
8179                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8180                                                       /*sks_valid*/ 1,
8181                                                       /*command*/ 1,
8182                                                       /*field*/ 2,
8183                                                       /*bit_valid*/ 1,
8184                                                       /*bit*/ 4);
8185                                 ctl_done((union ctl_io *)ctsio);
8186                                 return (1);
8187                         }
8188
8189                         if (type>8 || type==2 || type==4 || type==0) {
8190                                 mtx_unlock(&lun->lun_lock);
8191                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8192                                                       /*sks_valid*/ 1,
8193                                                       /*command*/ 1,
8194                                                       /*field*/ 2,
8195                                                       /*bit_valid*/ 1,
8196                                                       /*bit*/ 0);
8197                                 ctl_done((union ctl_io *)ctsio);
8198                                 return (1);
8199                         }
8200
8201                         /*
8202                          * Unregister everybody else and build UA for
8203                          * them
8204                          */
8205                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8206                                 if (i == residx || lun->pr_keys[i] == 0)
8207                                         continue;
8208
8209                                 if (!persis_offset
8210                                  && i <CTL_MAX_INITIATORS)
8211                                         lun->pending_ua[i] |=
8212                                                 CTL_UA_REG_PREEMPT;
8213                                 else if (persis_offset
8214                                       && i >= persis_offset)
8215                                         lun->pending_ua[i-persis_offset] |=
8216                                                 CTL_UA_REG_PREEMPT;
8217                                 lun->pr_keys[i] = 0;
8218                         }
8219                         lun->pr_key_count = 1;
8220                         lun->res_type = type;
8221                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8222                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8223                                 lun->pr_res_idx = residx;
8224
8225                         /* send msg to other side */
8226                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8227                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8228                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8229                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8230                         persis_io.pr.pr_info.res_type = type;
8231                         memcpy(persis_io.pr.pr_info.sa_res_key,
8232                                param->serv_act_res_key,
8233                                sizeof(param->serv_act_res_key));
8234                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8235                              &persis_io, sizeof(persis_io), 0)) >
8236                              CTL_HA_STATUS_SUCCESS) {
8237                                 printf("CTL:Persis Out error returned "
8238                                        "from ctl_ha_msg_send %d\n",
8239                                        isc_retval);
8240                         }
8241                 } else {
8242                         /* not all registrants */
8243                         mtx_unlock(&lun->lun_lock);
8244                         free(ctsio->kern_data_ptr, M_CTL);
8245                         ctl_set_invalid_field(ctsio,
8246                                               /*sks_valid*/ 1,
8247                                               /*command*/ 0,
8248                                               /*field*/ 8,
8249                                               /*bit_valid*/ 0,
8250                                               /*bit*/ 0);
8251                         ctl_done((union ctl_io *)ctsio);
8252                         return (1);
8253                 }
8254         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8255                 || !(lun->flags & CTL_LUN_PR_RESERVED)) {
8256                 int found = 0;
8257
8258                 if (res_key == sa_res_key) {
8259                         /* special case */
8260                         /*
8261                          * The spec implies this is not good but doesn't
8262                          * say what to do. There are two choices either
8263                          * generate a res conflict or check condition
8264                          * with illegal field in parameter data. Since
8265                          * that is what is done when the sa_res_key is
8266                          * zero I'll take that approach since this has
8267                          * to do with the sa_res_key.
8268                          */
8269                         mtx_unlock(&lun->lun_lock);
8270                         free(ctsio->kern_data_ptr, M_CTL);
8271                         ctl_set_invalid_field(ctsio,
8272                                               /*sks_valid*/ 1,
8273                                               /*command*/ 0,
8274                                               /*field*/ 8,
8275                                               /*bit_valid*/ 0,
8276                                               /*bit*/ 0);
8277                         ctl_done((union ctl_io *)ctsio);
8278                         return (1);
8279                 }
8280
8281                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8282                         if (lun->pr_keys[i] != sa_res_key)
8283                                 continue;
8284
8285                         found = 1;
8286                         lun->pr_keys[i] = 0;
8287                         lun->pr_key_count--;
8288
8289                         if (!persis_offset && i < CTL_MAX_INITIATORS)
8290                                 lun->pending_ua[i] |= CTL_UA_REG_PREEMPT;
8291                         else if (persis_offset && i >= persis_offset)
8292                                 lun->pending_ua[i-persis_offset] |=
8293                                         CTL_UA_REG_PREEMPT;
8294                 }
8295                 if (!found) {
8296                         mtx_unlock(&lun->lun_lock);
8297                         free(ctsio->kern_data_ptr, M_CTL);
8298                         ctl_set_reservation_conflict(ctsio);
8299                         ctl_done((union ctl_io *)ctsio);
8300                         return (CTL_RETVAL_COMPLETE);
8301                 }
8302                 /* send msg to other side */
8303                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8304                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8305                 persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8306                 persis_io.pr.pr_info.residx = lun->pr_res_idx;
8307                 persis_io.pr.pr_info.res_type = type;
8308                 memcpy(persis_io.pr.pr_info.sa_res_key,
8309                        param->serv_act_res_key,
8310                        sizeof(param->serv_act_res_key));
8311                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8312                      &persis_io, sizeof(persis_io), 0)) >
8313                      CTL_HA_STATUS_SUCCESS) {
8314                         printf("CTL:Persis Out error returned from "
8315                                "ctl_ha_msg_send %d\n", isc_retval);
8316                 }
8317         } else {
8318                 /* Reserved but not all registrants */
8319                 /* sa_res_key is res holder */
8320                 if (sa_res_key == lun->pr_keys[lun->pr_res_idx]) {
8321                         /* validate scope and type */
8322                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
8323                              SPR_LU_SCOPE) {
8324                                 mtx_unlock(&lun->lun_lock);
8325                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8326                                                       /*sks_valid*/ 1,
8327                                                       /*command*/ 1,
8328                                                       /*field*/ 2,
8329                                                       /*bit_valid*/ 1,
8330                                                       /*bit*/ 4);
8331                                 ctl_done((union ctl_io *)ctsio);
8332                                 return (1);
8333                         }
8334
8335                         if (type>8 || type==2 || type==4 || type==0) {
8336                                 mtx_unlock(&lun->lun_lock);
8337                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8338                                                       /*sks_valid*/ 1,
8339                                                       /*command*/ 1,
8340                                                       /*field*/ 2,
8341                                                       /*bit_valid*/ 1,
8342                                                       /*bit*/ 0);
8343                                 ctl_done((union ctl_io *)ctsio);
8344                                 return (1);
8345                         }
8346
8347                         /*
8348                          * Do the following:
8349                          * if sa_res_key != res_key remove all
8350                          * registrants w/sa_res_key and generate UA
8351                          * for these registrants(Registrations
8352                          * Preempted) if it wasn't an exclusive
8353                          * reservation generate UA(Reservations
8354                          * Preempted) for all other registered nexuses
8355                          * if the type has changed. Establish the new
8356                          * reservation and holder. If res_key and
8357                          * sa_res_key are the same do the above
8358                          * except don't unregister the res holder.
8359                          */
8360
8361                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8362                                 if (i == residx || lun->pr_keys[i] == 0)
8363                                         continue;
8364
8365                                 if (sa_res_key == lun->pr_keys[i]) {
8366                                         lun->pr_keys[i] = 0;
8367                                         lun->pr_key_count--;
8368
8369                                         if (!persis_offset
8370                                          && i < CTL_MAX_INITIATORS)
8371                                                 lun->pending_ua[i] |=
8372                                                         CTL_UA_REG_PREEMPT;
8373                                         else if (persis_offset
8374                                               && i >= persis_offset)
8375                                                 lun->pending_ua[i-persis_offset] |=
8376                                                   CTL_UA_REG_PREEMPT;
8377                                 } else if (type != lun->res_type
8378                                         && (lun->res_type == SPR_TYPE_WR_EX_RO
8379                                          || lun->res_type ==SPR_TYPE_EX_AC_RO)){
8380                                                 if (!persis_offset
8381                                                  && i < CTL_MAX_INITIATORS)
8382                                                         lun->pending_ua[i] |=
8383                                                         CTL_UA_RES_RELEASE;
8384                                                 else if (persis_offset
8385                                                       && i >= persis_offset)
8386                                                         lun->pending_ua[
8387                                                         i-persis_offset] |=
8388                                                         CTL_UA_RES_RELEASE;
8389                                 }
8390                         }
8391                         lun->res_type = type;
8392                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8393                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8394                                 lun->pr_res_idx = residx;
8395                         else
8396                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8397
8398                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8399                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8400                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8401                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8402                         persis_io.pr.pr_info.res_type = type;
8403                         memcpy(persis_io.pr.pr_info.sa_res_key,
8404                                param->serv_act_res_key,
8405                                sizeof(param->serv_act_res_key));
8406                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8407                              &persis_io, sizeof(persis_io), 0)) >
8408                              CTL_HA_STATUS_SUCCESS) {
8409                                 printf("CTL:Persis Out error returned "
8410                                        "from ctl_ha_msg_send %d\n",
8411                                        isc_retval);
8412                         }
8413                 } else {
8414                         /*
8415                          * sa_res_key is not the res holder just
8416                          * remove registrants
8417                          */
8418                         int found=0;
8419
8420                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8421                                 if (sa_res_key != lun->pr_keys[i])
8422                                         continue;
8423
8424                                 found = 1;
8425                                 lun->pr_keys[i] = 0;
8426                                 lun->pr_key_count--;
8427
8428                                 if (!persis_offset
8429                                  && i < CTL_MAX_INITIATORS)
8430                                         lun->pending_ua[i] |=
8431                                                 CTL_UA_REG_PREEMPT;
8432                                 else if (persis_offset
8433                                       && i >= persis_offset)
8434                                         lun->pending_ua[i-persis_offset] |=
8435                                                 CTL_UA_REG_PREEMPT;
8436                         }
8437
8438                         if (!found) {
8439                                 mtx_unlock(&lun->lun_lock);
8440                                 free(ctsio->kern_data_ptr, M_CTL);
8441                                 ctl_set_reservation_conflict(ctsio);
8442                                 ctl_done((union ctl_io *)ctsio);
8443                                 return (1);
8444                         }
8445                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8446                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8447                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8448                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8449                         persis_io.pr.pr_info.res_type = type;
8450                         memcpy(persis_io.pr.pr_info.sa_res_key,
8451                                param->serv_act_res_key,
8452                                sizeof(param->serv_act_res_key));
8453                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8454                              &persis_io, sizeof(persis_io), 0)) >
8455                              CTL_HA_STATUS_SUCCESS) {
8456                                 printf("CTL:Persis Out error returned "
8457                                        "from ctl_ha_msg_send %d\n",
8458                                 isc_retval);
8459                         }
8460                 }
8461         }
8462
8463         lun->PRGeneration++;
8464         mtx_unlock(&lun->lun_lock);
8465
8466         return (retval);
8467 }
8468
8469 static void
8470 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
8471 {
8472         uint64_t sa_res_key;
8473         int i;
8474
8475         sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
8476
8477         if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8478          || lun->pr_res_idx == CTL_PR_NO_RESERVATION
8479          || sa_res_key != lun->pr_keys[lun->pr_res_idx]) {
8480                 if (sa_res_key == 0) {
8481                         /*
8482                          * Unregister everybody else and build UA for
8483                          * them
8484                          */
8485                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8486                                 if (i == msg->pr.pr_info.residx ||
8487                                     lun->pr_keys[i] == 0)
8488                                         continue;
8489
8490                                 if (!persis_offset
8491                                  && i < CTL_MAX_INITIATORS)
8492                                         lun->pending_ua[i] |=
8493                                                 CTL_UA_REG_PREEMPT;
8494                                 else if (persis_offset && i >= persis_offset)
8495                                         lun->pending_ua[i - persis_offset] |=
8496                                                 CTL_UA_REG_PREEMPT;
8497                                 lun->pr_keys[i] = 0;
8498                         }
8499
8500                         lun->pr_key_count = 1;
8501                         lun->res_type = msg->pr.pr_info.res_type;
8502                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8503                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8504                                 lun->pr_res_idx = msg->pr.pr_info.residx;
8505                 } else {
8506                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8507                                 if (sa_res_key == lun->pr_keys[i])
8508                                         continue;
8509
8510                                 lun->pr_keys[i] = 0;
8511                                 lun->pr_key_count--;
8512
8513                                 if (!persis_offset
8514                                  && i < persis_offset)
8515                                         lun->pending_ua[i] |=
8516                                                 CTL_UA_REG_PREEMPT;
8517                                 else if (persis_offset
8518                                       && i >= persis_offset)
8519                                         lun->pending_ua[i - persis_offset] |=
8520                                                 CTL_UA_REG_PREEMPT;
8521                         }
8522                 }
8523         } else {
8524                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8525                         if (i == msg->pr.pr_info.residx ||
8526                             lun->pr_keys[i] == 0)
8527                                 continue;
8528
8529                         if (sa_res_key == lun->pr_keys[i]) {
8530                                 lun->pr_keys[i] = 0;
8531                                 lun->pr_key_count--;
8532                                 if (!persis_offset
8533                                  && i < CTL_MAX_INITIATORS)
8534                                         lun->pending_ua[i] |=
8535                                                 CTL_UA_REG_PREEMPT;
8536                                 else if (persis_offset
8537                                       && i >= persis_offset)
8538                                         lun->pending_ua[i - persis_offset] |=
8539                                                 CTL_UA_REG_PREEMPT;
8540                         } else if (msg->pr.pr_info.res_type != lun->res_type
8541                                 && (lun->res_type == SPR_TYPE_WR_EX_RO
8542                                  || lun->res_type == SPR_TYPE_EX_AC_RO)) {
8543                                         if (!persis_offset
8544                                          && i < persis_offset)
8545                                                 lun->pending_ua[i] |=
8546                                                         CTL_UA_RES_RELEASE;
8547                                         else if (persis_offset
8548                                               && i >= persis_offset)
8549                                         lun->pending_ua[i - persis_offset] |=
8550                                                 CTL_UA_RES_RELEASE;
8551                         }
8552                 }
8553                 lun->res_type = msg->pr.pr_info.res_type;
8554                 if (lun->res_type != SPR_TYPE_WR_EX_AR
8555                  && lun->res_type != SPR_TYPE_EX_AC_AR)
8556                         lun->pr_res_idx = msg->pr.pr_info.residx;
8557                 else
8558                         lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8559         }
8560         lun->PRGeneration++;
8561
8562 }
8563
8564
8565 int
8566 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
8567 {
8568         int retval;
8569         int isc_retval;
8570         u_int32_t param_len;
8571         struct scsi_per_res_out *cdb;
8572         struct ctl_lun *lun;
8573         struct scsi_per_res_out_parms* param;
8574         struct ctl_softc *softc;
8575         uint32_t residx;
8576         uint64_t res_key, sa_res_key;
8577         uint8_t type;
8578         union ctl_ha_msg persis_io;
8579         int    i;
8580
8581         CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
8582
8583         retval = CTL_RETVAL_COMPLETE;
8584
8585         softc = control_softc;
8586
8587         cdb = (struct scsi_per_res_out *)ctsio->cdb;
8588         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8589
8590         /*
8591          * We only support whole-LUN scope.  The scope & type are ignored for
8592          * register, register and ignore existing key and clear.
8593          * We sometimes ignore scope and type on preempts too!!
8594          * Verify reservation type here as well.
8595          */
8596         type = cdb->scope_type & SPR_TYPE_MASK;
8597         if ((cdb->action == SPRO_RESERVE)
8598          || (cdb->action == SPRO_RELEASE)) {
8599                 if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
8600                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8601                                               /*sks_valid*/ 1,
8602                                               /*command*/ 1,
8603                                               /*field*/ 2,
8604                                               /*bit_valid*/ 1,
8605                                               /*bit*/ 4);
8606                         ctl_done((union ctl_io *)ctsio);
8607                         return (CTL_RETVAL_COMPLETE);
8608                 }
8609
8610                 if (type>8 || type==2 || type==4 || type==0) {
8611                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8612                                               /*sks_valid*/ 1,
8613                                               /*command*/ 1,
8614                                               /*field*/ 2,
8615                                               /*bit_valid*/ 1,
8616                                               /*bit*/ 0);
8617                         ctl_done((union ctl_io *)ctsio);
8618                         return (CTL_RETVAL_COMPLETE);
8619                 }
8620         }
8621
8622         param_len = scsi_4btoul(cdb->length);
8623
8624         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
8625                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
8626                 ctsio->kern_data_len = param_len;
8627                 ctsio->kern_total_len = param_len;
8628                 ctsio->kern_data_resid = 0;
8629                 ctsio->kern_rel_offset = 0;
8630                 ctsio->kern_sg_entries = 0;
8631                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8632                 ctsio->be_move_done = ctl_config_move_done;
8633                 ctl_datamove((union ctl_io *)ctsio);
8634
8635                 return (CTL_RETVAL_COMPLETE);
8636         }
8637
8638         param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
8639
8640         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8641         res_key = scsi_8btou64(param->res_key.key);
8642         sa_res_key = scsi_8btou64(param->serv_act_res_key);
8643
8644         /*
8645          * Validate the reservation key here except for SPRO_REG_IGNO
8646          * This must be done for all other service actions
8647          */
8648         if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
8649                 mtx_lock(&lun->lun_lock);
8650                 if (lun->pr_keys[residx] != 0) {
8651                     if (res_key != lun->pr_keys[residx]) {
8652                                 /*
8653                                  * The current key passed in doesn't match
8654                                  * the one the initiator previously
8655                                  * registered.
8656                                  */
8657                                 mtx_unlock(&lun->lun_lock);
8658                                 free(ctsio->kern_data_ptr, M_CTL);
8659                                 ctl_set_reservation_conflict(ctsio);
8660                                 ctl_done((union ctl_io *)ctsio);
8661                                 return (CTL_RETVAL_COMPLETE);
8662                         }
8663                 } else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
8664                         /*
8665                          * We are not registered
8666                          */
8667                         mtx_unlock(&lun->lun_lock);
8668                         free(ctsio->kern_data_ptr, M_CTL);
8669                         ctl_set_reservation_conflict(ctsio);
8670                         ctl_done((union ctl_io *)ctsio);
8671                         return (CTL_RETVAL_COMPLETE);
8672                 } else if (res_key != 0) {
8673                         /*
8674                          * We are not registered and trying to register but
8675                          * the register key isn't zero.
8676                          */
8677                         mtx_unlock(&lun->lun_lock);
8678                         free(ctsio->kern_data_ptr, M_CTL);
8679                         ctl_set_reservation_conflict(ctsio);
8680                         ctl_done((union ctl_io *)ctsio);
8681                         return (CTL_RETVAL_COMPLETE);
8682                 }
8683                 mtx_unlock(&lun->lun_lock);
8684         }
8685
8686         switch (cdb->action & SPRO_ACTION_MASK) {
8687         case SPRO_REGISTER:
8688         case SPRO_REG_IGNO: {
8689
8690 #if 0
8691                 printf("Registration received\n");
8692 #endif
8693
8694                 /*
8695                  * We don't support any of these options, as we report in
8696                  * the read capabilities request (see
8697                  * ctl_persistent_reserve_in(), above).
8698                  */
8699                 if ((param->flags & SPR_SPEC_I_PT)
8700                  || (param->flags & SPR_ALL_TG_PT)
8701                  || (param->flags & SPR_APTPL)) {
8702                         int bit_ptr;
8703
8704                         if (param->flags & SPR_APTPL)
8705                                 bit_ptr = 0;
8706                         else if (param->flags & SPR_ALL_TG_PT)
8707                                 bit_ptr = 2;
8708                         else /* SPR_SPEC_I_PT */
8709                                 bit_ptr = 3;
8710
8711                         free(ctsio->kern_data_ptr, M_CTL);
8712                         ctl_set_invalid_field(ctsio,
8713                                               /*sks_valid*/ 1,
8714                                               /*command*/ 0,
8715                                               /*field*/ 20,
8716                                               /*bit_valid*/ 1,
8717                                               /*bit*/ bit_ptr);
8718                         ctl_done((union ctl_io *)ctsio);
8719                         return (CTL_RETVAL_COMPLETE);
8720                 }
8721
8722                 mtx_lock(&lun->lun_lock);
8723
8724                 /*
8725                  * The initiator wants to clear the
8726                  * key/unregister.
8727                  */
8728                 if (sa_res_key == 0) {
8729                         if ((res_key == 0
8730                           && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
8731                          || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
8732                           && lun->pr_keys[residx] == 0)) {
8733                                 mtx_unlock(&lun->lun_lock);
8734                                 goto done;
8735                         }
8736
8737                         lun->pr_keys[residx] = 0;
8738                         lun->pr_key_count--;
8739
8740                         if (residx == lun->pr_res_idx) {
8741                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8742                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8743
8744                                 if ((lun->res_type == SPR_TYPE_WR_EX_RO
8745                                   || lun->res_type == SPR_TYPE_EX_AC_RO)
8746                                  && lun->pr_key_count) {
8747                                         /*
8748                                          * If the reservation is a registrants
8749                                          * only type we need to generate a UA
8750                                          * for other registered inits.  The
8751                                          * sense code should be RESERVATIONS
8752                                          * RELEASED
8753                                          */
8754
8755                                         for (i = 0; i < CTL_MAX_INITIATORS;i++){
8756                                                 if (lun->pr_keys[
8757                                                     i + persis_offset] == 0)
8758                                                         continue;
8759                                                 lun->pending_ua[i] |=
8760                                                         CTL_UA_RES_RELEASE;
8761                                         }
8762                                 }
8763                                 lun->res_type = 0;
8764                         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8765                                 if (lun->pr_key_count==0) {
8766                                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8767                                         lun->res_type = 0;
8768                                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8769                                 }
8770                         }
8771                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8772                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8773                         persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8774                         persis_io.pr.pr_info.residx = residx;
8775                         if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8776                              &persis_io, sizeof(persis_io), 0 )) >
8777                              CTL_HA_STATUS_SUCCESS) {
8778                                 printf("CTL:Persis Out error returned from "
8779                                        "ctl_ha_msg_send %d\n", isc_retval);
8780                         }
8781                 } else /* sa_res_key != 0 */ {
8782
8783                         /*
8784                          * If we aren't registered currently then increment
8785                          * the key count and set the registered flag.
8786                          */
8787                         if (lun->pr_keys[residx] == 0)
8788                                 lun->pr_key_count++;
8789                         lun->pr_keys[residx] = sa_res_key;
8790
8791                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8792                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8793                         persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8794                         persis_io.pr.pr_info.residx = residx;
8795                         memcpy(persis_io.pr.pr_info.sa_res_key,
8796                                param->serv_act_res_key,
8797                                sizeof(param->serv_act_res_key));
8798                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8799                              &persis_io, sizeof(persis_io), 0)) >
8800                              CTL_HA_STATUS_SUCCESS) {
8801                                 printf("CTL:Persis Out error returned from "
8802                                        "ctl_ha_msg_send %d\n", isc_retval);
8803                         }
8804                 }
8805                 lun->PRGeneration++;
8806                 mtx_unlock(&lun->lun_lock);
8807
8808                 break;
8809         }
8810         case SPRO_RESERVE:
8811 #if 0
8812                 printf("Reserve executed type %d\n", type);
8813 #endif
8814                 mtx_lock(&lun->lun_lock);
8815                 if (lun->flags & CTL_LUN_PR_RESERVED) {
8816                         /*
8817                          * if this isn't the reservation holder and it's
8818                          * not a "all registrants" type or if the type is
8819                          * different then we have a conflict
8820                          */
8821                         if ((lun->pr_res_idx != residx
8822                           && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8823                          || lun->res_type != type) {
8824                                 mtx_unlock(&lun->lun_lock);
8825                                 free(ctsio->kern_data_ptr, M_CTL);
8826                                 ctl_set_reservation_conflict(ctsio);
8827                                 ctl_done((union ctl_io *)ctsio);
8828                                 return (CTL_RETVAL_COMPLETE);
8829                         }
8830                         mtx_unlock(&lun->lun_lock);
8831                 } else /* create a reservation */ {
8832                         /*
8833                          * If it's not an "all registrants" type record
8834                          * reservation holder
8835                          */
8836                         if (type != SPR_TYPE_WR_EX_AR
8837                          && type != SPR_TYPE_EX_AC_AR)
8838                                 lun->pr_res_idx = residx; /* Res holder */
8839                         else
8840                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8841
8842                         lun->flags |= CTL_LUN_PR_RESERVED;
8843                         lun->res_type = type;
8844
8845                         mtx_unlock(&lun->lun_lock);
8846
8847                         /* send msg to other side */
8848                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8849                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8850                         persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8851                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8852                         persis_io.pr.pr_info.res_type = type;
8853                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8854                              &persis_io, sizeof(persis_io), 0)) >
8855                              CTL_HA_STATUS_SUCCESS) {
8856                                 printf("CTL:Persis Out error returned from "
8857                                        "ctl_ha_msg_send %d\n", isc_retval);
8858                         }
8859                 }
8860                 break;
8861
8862         case SPRO_RELEASE:
8863                 mtx_lock(&lun->lun_lock);
8864                 if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8865                         /* No reservation exists return good status */
8866                         mtx_unlock(&lun->lun_lock);
8867                         goto done;
8868                 }
8869                 /*
8870                  * Is this nexus a reservation holder?
8871                  */
8872                 if (lun->pr_res_idx != residx
8873                  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8874                         /*
8875                          * not a res holder return good status but
8876                          * do nothing
8877                          */
8878                         mtx_unlock(&lun->lun_lock);
8879                         goto done;
8880                 }
8881
8882                 if (lun->res_type != type) {
8883                         mtx_unlock(&lun->lun_lock);
8884                         free(ctsio->kern_data_ptr, M_CTL);
8885                         ctl_set_illegal_pr_release(ctsio);
8886                         ctl_done((union ctl_io *)ctsio);
8887                         return (CTL_RETVAL_COMPLETE);
8888                 }
8889
8890                 /* okay to release */
8891                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8892                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8893                 lun->res_type = 0;
8894
8895                 /*
8896                  * if this isn't an exclusive access
8897                  * res generate UA for all other
8898                  * registrants.
8899                  */
8900                 if (type != SPR_TYPE_EX_AC
8901                  && type != SPR_TYPE_WR_EX) {
8902                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8903                                 if (i == residx ||
8904                                     lun->pr_keys[i + persis_offset] == 0)
8905                                         continue;
8906                                 lun->pending_ua[i] |= CTL_UA_RES_RELEASE;
8907                         }
8908                 }
8909                 mtx_unlock(&lun->lun_lock);
8910                 /* Send msg to other side */
8911                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8912                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8913                 persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8914                 if ((isc_retval=ctl_ha_msg_send( CTL_HA_CHAN_CTL, &persis_io,
8915                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8916                         printf("CTL:Persis Out error returned from "
8917                                "ctl_ha_msg_send %d\n", isc_retval);
8918                 }
8919                 break;
8920
8921         case SPRO_CLEAR:
8922                 /* send msg to other side */
8923
8924                 mtx_lock(&lun->lun_lock);
8925                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8926                 lun->res_type = 0;
8927                 lun->pr_key_count = 0;
8928                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8929
8930                 lun->pr_keys[residx] = 0;
8931
8932                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++)
8933                         if (lun->pr_keys[i] != 0) {
8934                                 if (!persis_offset && i < CTL_MAX_INITIATORS)
8935                                         lun->pending_ua[i] |=
8936                                                 CTL_UA_RES_PREEMPT;
8937                                 else if (persis_offset && i >= persis_offset)
8938                                         lun->pending_ua[i-persis_offset] |=
8939                                             CTL_UA_RES_PREEMPT;
8940
8941                                 lun->pr_keys[i] = 0;
8942                         }
8943                 lun->PRGeneration++;
8944                 mtx_unlock(&lun->lun_lock);
8945                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8946                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8947                 persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8948                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8949                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8950                         printf("CTL:Persis Out error returned from "
8951                                "ctl_ha_msg_send %d\n", isc_retval);
8952                 }
8953                 break;
8954
8955         case SPRO_PREEMPT: {
8956                 int nretval;
8957
8958                 nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8959                                           residx, ctsio, cdb, param);
8960                 if (nretval != 0)
8961                         return (CTL_RETVAL_COMPLETE);
8962                 break;
8963         }
8964         default:
8965                 panic("Invalid PR type %x", cdb->action);
8966         }
8967
8968 done:
8969         free(ctsio->kern_data_ptr, M_CTL);
8970         ctl_set_success(ctsio);
8971         ctl_done((union ctl_io *)ctsio);
8972
8973         return (retval);
8974 }
8975
8976 /*
8977  * This routine is for handling a message from the other SC pertaining to
8978  * persistent reserve out. All the error checking will have been done
8979  * so only perorming the action need be done here to keep the two
8980  * in sync.
8981  */
8982 static void
8983 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8984 {
8985         struct ctl_lun *lun;
8986         struct ctl_softc *softc;
8987         int i;
8988         uint32_t targ_lun;
8989
8990         softc = control_softc;
8991
8992         targ_lun = msg->hdr.nexus.targ_mapped_lun;
8993         lun = softc->ctl_luns[targ_lun];
8994         mtx_lock(&lun->lun_lock);
8995         switch(msg->pr.pr_info.action) {
8996         case CTL_PR_REG_KEY:
8997                 if (lun->pr_keys[msg->pr.pr_info.residx] == 0)
8998                         lun->pr_key_count++;
8999                 lun->pr_keys[msg->pr.pr_info.residx] =
9000                     scsi_8btou64(msg->pr.pr_info.sa_res_key);
9001                 lun->PRGeneration++;
9002                 break;
9003
9004         case CTL_PR_UNREG_KEY:
9005                 lun->pr_keys[msg->pr.pr_info.residx] = 0;
9006                 lun->pr_key_count--;
9007
9008                 /* XXX Need to see if the reservation has been released */
9009                 /* if so do we need to generate UA? */
9010                 if (msg->pr.pr_info.residx == lun->pr_res_idx) {
9011                         lun->flags &= ~CTL_LUN_PR_RESERVED;
9012                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
9013
9014                         if ((lun->res_type == SPR_TYPE_WR_EX_RO
9015                           || lun->res_type == SPR_TYPE_EX_AC_RO)
9016                          && lun->pr_key_count) {
9017                                 /*
9018                                  * If the reservation is a registrants
9019                                  * only type we need to generate a UA
9020                                  * for other registered inits.  The
9021                                  * sense code should be RESERVATIONS
9022                                  * RELEASED
9023                                  */
9024
9025                                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
9026                                         if (lun->pr_keys[i+
9027                                             persis_offset] == 0)
9028                                                 continue;
9029
9030                                         lun->pending_ua[i] |=
9031                                                 CTL_UA_RES_RELEASE;
9032                                 }
9033                         }
9034                         lun->res_type = 0;
9035                 } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
9036                         if (lun->pr_key_count==0) {
9037                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
9038                                 lun->res_type = 0;
9039                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
9040                         }
9041                 }
9042                 lun->PRGeneration++;
9043                 break;
9044
9045         case CTL_PR_RESERVE:
9046                 lun->flags |= CTL_LUN_PR_RESERVED;
9047                 lun->res_type = msg->pr.pr_info.res_type;
9048                 lun->pr_res_idx = msg->pr.pr_info.residx;
9049
9050                 break;
9051
9052         case CTL_PR_RELEASE:
9053                 /*
9054                  * if this isn't an exclusive access res generate UA for all
9055                  * other registrants.
9056                  */
9057                 if (lun->res_type != SPR_TYPE_EX_AC
9058                  && lun->res_type != SPR_TYPE_WR_EX) {
9059                         for (i = 0; i < CTL_MAX_INITIATORS; i++)
9060                                 if (lun->pr_keys[i+persis_offset] != 0)
9061                                         lun->pending_ua[i] |=
9062                                                 CTL_UA_RES_RELEASE;
9063                 }
9064
9065                 lun->flags &= ~CTL_LUN_PR_RESERVED;
9066                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
9067                 lun->res_type = 0;
9068                 break;
9069
9070         case CTL_PR_PREEMPT:
9071                 ctl_pro_preempt_other(lun, msg);
9072                 break;
9073         case CTL_PR_CLEAR:
9074                 lun->flags &= ~CTL_LUN_PR_RESERVED;
9075                 lun->res_type = 0;
9076                 lun->pr_key_count = 0;
9077                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
9078
9079                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
9080                         if (lun->pr_keys[i] == 0)
9081                                 continue;
9082                         if (!persis_offset
9083                          && i < CTL_MAX_INITIATORS)
9084                                 lun->pending_ua[i] |= CTL_UA_RES_PREEMPT;
9085                         else if (persis_offset
9086                               && i >= persis_offset)
9087                                 lun->pending_ua[i-persis_offset] |=
9088                                         CTL_UA_RES_PREEMPT;
9089                         lun->pr_keys[i] = 0;
9090                 }
9091                 lun->PRGeneration++;
9092                 break;
9093         }
9094
9095         mtx_unlock(&lun->lun_lock);
9096 }
9097
9098 int
9099 ctl_read_write(struct ctl_scsiio *ctsio)
9100 {
9101         struct ctl_lun *lun;
9102         struct ctl_lba_len_flags *lbalen;
9103         uint64_t lba;
9104         uint32_t num_blocks;
9105         int flags, retval;
9106         int isread;
9107
9108         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9109
9110         CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
9111
9112         flags = 0;
9113         retval = CTL_RETVAL_COMPLETE;
9114
9115         isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
9116               || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
9117         if (lun->flags & CTL_LUN_PR_RESERVED && isread) {
9118                 uint32_t residx;
9119
9120                 /*
9121                  * XXX KDM need a lock here.
9122                  */
9123                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
9124                 if ((lun->res_type == SPR_TYPE_EX_AC
9125                   && residx != lun->pr_res_idx)
9126                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
9127                    || lun->res_type == SPR_TYPE_EX_AC_AR)
9128                   && lun->pr_keys[residx] == 0)) {
9129                         ctl_set_reservation_conflict(ctsio);
9130                         ctl_done((union ctl_io *)ctsio);
9131                         return (CTL_RETVAL_COMPLETE);
9132                 }
9133         }
9134
9135         switch (ctsio->cdb[0]) {
9136         case READ_6:
9137         case WRITE_6: {
9138                 struct scsi_rw_6 *cdb;
9139
9140                 cdb = (struct scsi_rw_6 *)ctsio->cdb;
9141
9142                 lba = scsi_3btoul(cdb->addr);
9143                 /* only 5 bits are valid in the most significant address byte */
9144                 lba &= 0x1fffff;
9145                 num_blocks = cdb->length;
9146                 /*
9147                  * This is correct according to SBC-2.
9148                  */
9149                 if (num_blocks == 0)
9150                         num_blocks = 256;
9151                 break;
9152         }
9153         case READ_10:
9154         case WRITE_10: {
9155                 struct scsi_rw_10 *cdb;
9156
9157                 cdb = (struct scsi_rw_10 *)ctsio->cdb;
9158                 if (cdb->byte2 & SRW10_FUA)
9159                         flags |= CTL_LLF_FUA;
9160                 if (cdb->byte2 & SRW10_DPO)
9161                         flags |= CTL_LLF_DPO;
9162                 lba = scsi_4btoul(cdb->addr);
9163                 num_blocks = scsi_2btoul(cdb->length);
9164                 break;
9165         }
9166         case WRITE_VERIFY_10: {
9167                 struct scsi_write_verify_10 *cdb;
9168
9169                 cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
9170                 flags |= CTL_LLF_FUA;
9171                 if (cdb->byte2 & SWV_DPO)
9172                         flags |= CTL_LLF_DPO;
9173                 lba = scsi_4btoul(cdb->addr);
9174                 num_blocks = scsi_2btoul(cdb->length);
9175                 break;
9176         }
9177         case READ_12:
9178         case WRITE_12: {
9179                 struct scsi_rw_12 *cdb;
9180
9181                 cdb = (struct scsi_rw_12 *)ctsio->cdb;
9182                 if (cdb->byte2 & SRW12_FUA)
9183                         flags |= CTL_LLF_FUA;
9184                 if (cdb->byte2 & SRW12_DPO)
9185                         flags |= CTL_LLF_DPO;
9186                 lba = scsi_4btoul(cdb->addr);
9187                 num_blocks = scsi_4btoul(cdb->length);
9188                 break;
9189         }
9190         case WRITE_VERIFY_12: {
9191                 struct scsi_write_verify_12 *cdb;
9192
9193                 cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
9194                 flags |= CTL_LLF_FUA;
9195                 if (cdb->byte2 & SWV_DPO)
9196                         flags |= CTL_LLF_DPO;
9197                 lba = scsi_4btoul(cdb->addr);
9198                 num_blocks = scsi_4btoul(cdb->length);
9199                 break;
9200         }
9201         case READ_16:
9202         case WRITE_16: {
9203                 struct scsi_rw_16 *cdb;
9204
9205                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
9206                 if (cdb->byte2 & SRW12_FUA)
9207                         flags |= CTL_LLF_FUA;
9208                 if (cdb->byte2 & SRW12_DPO)
9209                         flags |= CTL_LLF_DPO;
9210                 lba = scsi_8btou64(cdb->addr);
9211                 num_blocks = scsi_4btoul(cdb->length);
9212                 break;
9213         }
9214         case WRITE_ATOMIC_16: {
9215                 struct scsi_rw_16 *cdb;
9216
9217                 if (lun->be_lun->atomicblock == 0) {
9218                         ctl_set_invalid_opcode(ctsio);
9219                         ctl_done((union ctl_io *)ctsio);
9220                         return (CTL_RETVAL_COMPLETE);
9221                 }
9222
9223                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
9224                 if (cdb->byte2 & SRW12_FUA)
9225                         flags |= CTL_LLF_FUA;
9226                 if (cdb->byte2 & SRW12_DPO)
9227                         flags |= CTL_LLF_DPO;
9228                 lba = scsi_8btou64(cdb->addr);
9229                 num_blocks = scsi_4btoul(cdb->length);
9230                 if (num_blocks > lun->be_lun->atomicblock) {
9231                         ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
9232                             /*command*/ 1, /*field*/ 12, /*bit_valid*/ 0,
9233                             /*bit*/ 0);
9234                         ctl_done((union ctl_io *)ctsio);
9235                         return (CTL_RETVAL_COMPLETE);
9236                 }
9237                 break;
9238         }
9239         case WRITE_VERIFY_16: {
9240                 struct scsi_write_verify_16 *cdb;
9241
9242                 cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
9243                 flags |= CTL_LLF_FUA;
9244                 if (cdb->byte2 & SWV_DPO)
9245                         flags |= CTL_LLF_DPO;
9246                 lba = scsi_8btou64(cdb->addr);
9247                 num_blocks = scsi_4btoul(cdb->length);
9248                 break;
9249         }
9250         default:
9251                 /*
9252                  * We got a command we don't support.  This shouldn't
9253                  * happen, commands should be filtered out above us.
9254                  */
9255                 ctl_set_invalid_opcode(ctsio);
9256                 ctl_done((union ctl_io *)ctsio);
9257
9258                 return (CTL_RETVAL_COMPLETE);
9259                 break; /* NOTREACHED */
9260         }
9261
9262         /*
9263          * The first check is to make sure we're in bounds, the second
9264          * check is to catch wrap-around problems.  If the lba + num blocks
9265          * is less than the lba, then we've wrapped around and the block
9266          * range is invalid anyway.
9267          */
9268         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9269          || ((lba + num_blocks) < lba)) {
9270                 ctl_set_lba_out_of_range(ctsio);
9271                 ctl_done((union ctl_io *)ctsio);
9272                 return (CTL_RETVAL_COMPLETE);
9273         }
9274
9275         /*
9276          * According to SBC-3, a transfer length of 0 is not an error.
9277          * Note that this cannot happen with WRITE(6) or READ(6), since 0
9278          * translates to 256 blocks for those commands.
9279          */
9280         if (num_blocks == 0) {
9281                 ctl_set_success(ctsio);
9282                 ctl_done((union ctl_io *)ctsio);
9283                 return (CTL_RETVAL_COMPLETE);
9284         }
9285
9286         /* Set FUA and/or DPO if caches are disabled. */
9287         if (isread) {
9288                 if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
9289                     SCP_RCD) != 0)
9290                         flags |= CTL_LLF_FUA | CTL_LLF_DPO;
9291         } else {
9292                 if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
9293                     SCP_WCE) == 0)
9294                         flags |= CTL_LLF_FUA;
9295         }
9296
9297         lbalen = (struct ctl_lba_len_flags *)
9298             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9299         lbalen->lba = lba;
9300         lbalen->len = num_blocks;
9301         lbalen->flags = (isread ? CTL_LLF_READ : CTL_LLF_WRITE) | flags;
9302
9303         ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9304         ctsio->kern_rel_offset = 0;
9305
9306         CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
9307
9308         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9309
9310         return (retval);
9311 }
9312
9313 static int
9314 ctl_cnw_cont(union ctl_io *io)
9315 {
9316         struct ctl_scsiio *ctsio;
9317         struct ctl_lun *lun;
9318         struct ctl_lba_len_flags *lbalen;
9319         int retval;
9320
9321         ctsio = &io->scsiio;
9322         ctsio->io_hdr.status = CTL_STATUS_NONE;
9323         ctsio->io_hdr.flags &= ~CTL_FLAG_IO_CONT;
9324         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9325         lbalen = (struct ctl_lba_len_flags *)
9326             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9327         lbalen->flags &= ~CTL_LLF_COMPARE;
9328         lbalen->flags |= CTL_LLF_WRITE;
9329
9330         CTL_DEBUG_PRINT(("ctl_cnw_cont: calling data_submit()\n"));
9331         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9332         return (retval);
9333 }
9334
9335 int
9336 ctl_cnw(struct ctl_scsiio *ctsio)
9337 {
9338         struct ctl_lun *lun;
9339         struct ctl_lba_len_flags *lbalen;
9340         uint64_t lba;
9341         uint32_t num_blocks;
9342         int flags, retval;
9343
9344         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9345
9346         CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
9347
9348         flags = 0;
9349         retval = CTL_RETVAL_COMPLETE;
9350
9351         switch (ctsio->cdb[0]) {
9352         case COMPARE_AND_WRITE: {
9353                 struct scsi_compare_and_write *cdb;
9354
9355                 cdb = (struct scsi_compare_and_write *)ctsio->cdb;
9356                 if (cdb->byte2 & SRW10_FUA)
9357                         flags |= CTL_LLF_FUA;
9358                 if (cdb->byte2 & SRW10_DPO)
9359                         flags |= CTL_LLF_DPO;
9360                 lba = scsi_8btou64(cdb->addr);
9361                 num_blocks = cdb->length;
9362                 break;
9363         }
9364         default:
9365                 /*
9366                  * We got a command we don't support.  This shouldn't
9367                  * happen, commands should be filtered out above us.
9368                  */
9369                 ctl_set_invalid_opcode(ctsio);
9370                 ctl_done((union ctl_io *)ctsio);
9371
9372                 return (CTL_RETVAL_COMPLETE);
9373                 break; /* NOTREACHED */
9374         }
9375
9376         /*
9377          * The first check is to make sure we're in bounds, the second
9378          * check is to catch wrap-around problems.  If the lba + num blocks
9379          * is less than the lba, then we've wrapped around and the block
9380          * range is invalid anyway.
9381          */
9382         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9383          || ((lba + num_blocks) < lba)) {
9384                 ctl_set_lba_out_of_range(ctsio);
9385                 ctl_done((union ctl_io *)ctsio);
9386                 return (CTL_RETVAL_COMPLETE);
9387         }
9388
9389         /*
9390          * According to SBC-3, a transfer length of 0 is not an error.
9391          */
9392         if (num_blocks == 0) {
9393                 ctl_set_success(ctsio);
9394                 ctl_done((union ctl_io *)ctsio);
9395                 return (CTL_RETVAL_COMPLETE);
9396         }
9397
9398         /* Set FUA if write cache is disabled. */
9399         if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
9400             SCP_WCE) == 0)
9401                 flags |= CTL_LLF_FUA;
9402
9403         ctsio->kern_total_len = 2 * num_blocks * lun->be_lun->blocksize;
9404         ctsio->kern_rel_offset = 0;
9405
9406         /*
9407          * Set the IO_CONT flag, so that if this I/O gets passed to
9408          * ctl_data_submit_done(), it'll get passed back to
9409          * ctl_ctl_cnw_cont() for further processing.
9410          */
9411         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
9412         ctsio->io_cont = ctl_cnw_cont;
9413
9414         lbalen = (struct ctl_lba_len_flags *)
9415             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9416         lbalen->lba = lba;
9417         lbalen->len = num_blocks;
9418         lbalen->flags = CTL_LLF_COMPARE | flags;
9419
9420         CTL_DEBUG_PRINT(("ctl_cnw: calling data_submit()\n"));
9421         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9422         return (retval);
9423 }
9424
9425 int
9426 ctl_verify(struct ctl_scsiio *ctsio)
9427 {
9428         struct ctl_lun *lun;
9429         struct ctl_lba_len_flags *lbalen;
9430         uint64_t lba;
9431         uint32_t num_blocks;
9432         int bytchk, flags;
9433         int retval;
9434
9435         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9436
9437         CTL_DEBUG_PRINT(("ctl_verify: command: %#x\n", ctsio->cdb[0]));
9438
9439         bytchk = 0;
9440         flags = CTL_LLF_FUA;
9441         retval = CTL_RETVAL_COMPLETE;
9442
9443         switch (ctsio->cdb[0]) {
9444         case VERIFY_10: {
9445                 struct scsi_verify_10 *cdb;
9446
9447                 cdb = (struct scsi_verify_10 *)ctsio->cdb;
9448                 if (cdb->byte2 & SVFY_BYTCHK)
9449                         bytchk = 1;
9450                 if (cdb->byte2 & SVFY_DPO)
9451                         flags |= CTL_LLF_DPO;
9452                 lba = scsi_4btoul(cdb->addr);
9453                 num_blocks = scsi_2btoul(cdb->length);
9454                 break;
9455         }
9456         case VERIFY_12: {
9457                 struct scsi_verify_12 *cdb;
9458
9459                 cdb = (struct scsi_verify_12 *)ctsio->cdb;
9460                 if (cdb->byte2 & SVFY_BYTCHK)
9461                         bytchk = 1;
9462                 if (cdb->byte2 & SVFY_DPO)
9463                         flags |= CTL_LLF_DPO;
9464                 lba = scsi_4btoul(cdb->addr);
9465                 num_blocks = scsi_4btoul(cdb->length);
9466                 break;
9467         }
9468         case VERIFY_16: {
9469                 struct scsi_rw_16 *cdb;
9470
9471                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
9472                 if (cdb->byte2 & SVFY_BYTCHK)
9473                         bytchk = 1;
9474                 if (cdb->byte2 & SVFY_DPO)
9475                         flags |= CTL_LLF_DPO;
9476                 lba = scsi_8btou64(cdb->addr);
9477                 num_blocks = scsi_4btoul(cdb->length);
9478                 break;
9479         }
9480         default:
9481                 /*
9482                  * We got a command we don't support.  This shouldn't
9483                  * happen, commands should be filtered out above us.
9484                  */
9485                 ctl_set_invalid_opcode(ctsio);
9486                 ctl_done((union ctl_io *)ctsio);
9487                 return (CTL_RETVAL_COMPLETE);
9488         }
9489
9490         /*
9491          * The first check is to make sure we're in bounds, the second
9492          * check is to catch wrap-around problems.  If the lba + num blocks
9493          * is less than the lba, then we've wrapped around and the block
9494          * range is invalid anyway.
9495          */
9496         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9497          || ((lba + num_blocks) < lba)) {
9498                 ctl_set_lba_out_of_range(ctsio);
9499                 ctl_done((union ctl_io *)ctsio);
9500                 return (CTL_RETVAL_COMPLETE);
9501         }
9502
9503         /*
9504          * According to SBC-3, a transfer length of 0 is not an error.
9505          */
9506         if (num_blocks == 0) {
9507                 ctl_set_success(ctsio);
9508                 ctl_done((union ctl_io *)ctsio);
9509                 return (CTL_RETVAL_COMPLETE);
9510         }
9511
9512         lbalen = (struct ctl_lba_len_flags *)
9513             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9514         lbalen->lba = lba;
9515         lbalen->len = num_blocks;
9516         if (bytchk) {
9517                 lbalen->flags = CTL_LLF_COMPARE | flags;
9518                 ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9519         } else {
9520                 lbalen->flags = CTL_LLF_VERIFY | flags;
9521                 ctsio->kern_total_len = 0;
9522         }
9523         ctsio->kern_rel_offset = 0;
9524
9525         CTL_DEBUG_PRINT(("ctl_verify: calling data_submit()\n"));
9526         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9527         return (retval);
9528 }
9529
9530 int
9531 ctl_report_luns(struct ctl_scsiio *ctsio)
9532 {
9533         struct scsi_report_luns *cdb;
9534         struct scsi_report_luns_data *lun_data;
9535         struct ctl_lun *lun, *request_lun;
9536         int num_luns, retval;
9537         uint32_t alloc_len, lun_datalen;
9538         int num_filled, well_known;
9539         uint32_t initidx, targ_lun_id, lun_id;
9540
9541         retval = CTL_RETVAL_COMPLETE;
9542         well_known = 0;
9543
9544         cdb = (struct scsi_report_luns *)ctsio->cdb;
9545
9546         CTL_DEBUG_PRINT(("ctl_report_luns\n"));
9547
9548         mtx_lock(&control_softc->ctl_lock);
9549         num_luns = control_softc->num_luns;
9550         mtx_unlock(&control_softc->ctl_lock);
9551
9552         switch (cdb->select_report) {
9553         case RPL_REPORT_DEFAULT:
9554         case RPL_REPORT_ALL:
9555                 break;
9556         case RPL_REPORT_WELLKNOWN:
9557                 well_known = 1;
9558                 num_luns = 0;
9559                 break;
9560         default:
9561                 ctl_set_invalid_field(ctsio,
9562                                       /*sks_valid*/ 1,
9563                                       /*command*/ 1,
9564                                       /*field*/ 2,
9565                                       /*bit_valid*/ 0,
9566                                       /*bit*/ 0);
9567                 ctl_done((union ctl_io *)ctsio);
9568                 return (retval);
9569                 break; /* NOTREACHED */
9570         }
9571
9572         alloc_len = scsi_4btoul(cdb->length);
9573         /*
9574          * The initiator has to allocate at least 16 bytes for this request,
9575          * so he can at least get the header and the first LUN.  Otherwise
9576          * we reject the request (per SPC-3 rev 14, section 6.21).
9577          */
9578         if (alloc_len < (sizeof(struct scsi_report_luns_data) +
9579             sizeof(struct scsi_report_luns_lundata))) {
9580                 ctl_set_invalid_field(ctsio,
9581                                       /*sks_valid*/ 1,
9582                                       /*command*/ 1,
9583                                       /*field*/ 6,
9584                                       /*bit_valid*/ 0,
9585                                       /*bit*/ 0);
9586                 ctl_done((union ctl_io *)ctsio);
9587                 return (retval);
9588         }
9589
9590         request_lun = (struct ctl_lun *)
9591                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9592
9593         lun_datalen = sizeof(*lun_data) +
9594                 (num_luns * sizeof(struct scsi_report_luns_lundata));
9595
9596         ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
9597         lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
9598         ctsio->kern_sg_entries = 0;
9599
9600         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9601
9602         mtx_lock(&control_softc->ctl_lock);
9603         for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
9604                 lun_id = ctl_map_lun(ctsio->io_hdr.nexus.targ_port, targ_lun_id);
9605                 if (lun_id >= CTL_MAX_LUNS)
9606                         continue;
9607                 lun = control_softc->ctl_luns[lun_id];
9608                 if (lun == NULL)
9609                         continue;
9610
9611                 if (targ_lun_id <= 0xff) {
9612                         /*
9613                          * Peripheral addressing method, bus number 0.
9614                          */
9615                         lun_data->luns[num_filled].lundata[0] =
9616                                 RPL_LUNDATA_ATYP_PERIPH;
9617                         lun_data->luns[num_filled].lundata[1] = targ_lun_id;
9618                         num_filled++;
9619                 } else if (targ_lun_id <= 0x3fff) {
9620                         /*
9621                          * Flat addressing method.
9622                          */
9623                         lun_data->luns[num_filled].lundata[0] =
9624                                 RPL_LUNDATA_ATYP_FLAT |
9625                                 (targ_lun_id & RPL_LUNDATA_FLAT_LUN_MASK);
9626 #ifdef OLDCTLHEADERS
9627                                 (SRLD_ADDR_FLAT << SRLD_ADDR_SHIFT) |
9628                                 (targ_lun_id & SRLD_BUS_LUN_MASK);
9629 #endif
9630                         lun_data->luns[num_filled].lundata[1] =
9631 #ifdef OLDCTLHEADERS
9632                                 targ_lun_id >> SRLD_BUS_LUN_BITS;
9633 #endif
9634                                 targ_lun_id >> RPL_LUNDATA_FLAT_LUN_BITS;
9635                         num_filled++;
9636                 } else {
9637                         printf("ctl_report_luns: bogus LUN number %jd, "
9638                                "skipping\n", (intmax_t)targ_lun_id);
9639                 }
9640                 /*
9641                  * According to SPC-3, rev 14 section 6.21:
9642                  *
9643                  * "The execution of a REPORT LUNS command to any valid and
9644                  * installed logical unit shall clear the REPORTED LUNS DATA
9645                  * HAS CHANGED unit attention condition for all logical
9646                  * units of that target with respect to the requesting
9647                  * initiator. A valid and installed logical unit is one
9648                  * having a PERIPHERAL QUALIFIER of 000b in the standard
9649                  * INQUIRY data (see 6.4.2)."
9650                  *
9651                  * If request_lun is NULL, the LUN this report luns command
9652                  * was issued to is either disabled or doesn't exist. In that
9653                  * case, we shouldn't clear any pending lun change unit
9654                  * attention.
9655                  */
9656                 if (request_lun != NULL) {
9657                         mtx_lock(&lun->lun_lock);
9658                         lun->pending_ua[initidx] &= ~CTL_UA_LUN_CHANGE;
9659                         mtx_unlock(&lun->lun_lock);
9660                 }
9661         }
9662         mtx_unlock(&control_softc->ctl_lock);
9663
9664         /*
9665          * It's quite possible that we've returned fewer LUNs than we allocated
9666          * space for.  Trim it.
9667          */
9668         lun_datalen = sizeof(*lun_data) +
9669                 (num_filled * sizeof(struct scsi_report_luns_lundata));
9670
9671         if (lun_datalen < alloc_len) {
9672                 ctsio->residual = alloc_len - lun_datalen;
9673                 ctsio->kern_data_len = lun_datalen;
9674                 ctsio->kern_total_len = lun_datalen;
9675         } else {
9676                 ctsio->residual = 0;
9677                 ctsio->kern_data_len = alloc_len;
9678                 ctsio->kern_total_len = alloc_len;
9679         }
9680         ctsio->kern_data_resid = 0;
9681         ctsio->kern_rel_offset = 0;
9682         ctsio->kern_sg_entries = 0;
9683
9684         /*
9685          * We set this to the actual data length, regardless of how much
9686          * space we actually have to return results.  If the user looks at
9687          * this value, he'll know whether or not he allocated enough space
9688          * and reissue the command if necessary.  We don't support well
9689          * known logical units, so if the user asks for that, return none.
9690          */
9691         scsi_ulto4b(lun_datalen - 8, lun_data->length);
9692
9693         /*
9694          * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
9695          * this request.
9696          */
9697         ctsio->scsi_status = SCSI_STATUS_OK;
9698
9699         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9700         ctsio->be_move_done = ctl_config_move_done;
9701         ctl_datamove((union ctl_io *)ctsio);
9702
9703         return (retval);
9704 }
9705
9706 int
9707 ctl_request_sense(struct ctl_scsiio *ctsio)
9708 {
9709         struct scsi_request_sense *cdb;
9710         struct scsi_sense_data *sense_ptr;
9711         struct ctl_lun *lun;
9712         uint32_t initidx;
9713         int have_error;
9714         scsi_sense_data_type sense_format;
9715
9716         cdb = (struct scsi_request_sense *)ctsio->cdb;
9717
9718         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9719
9720         CTL_DEBUG_PRINT(("ctl_request_sense\n"));
9721
9722         /*
9723          * Determine which sense format the user wants.
9724          */
9725         if (cdb->byte2 & SRS_DESC)
9726                 sense_format = SSD_TYPE_DESC;
9727         else
9728                 sense_format = SSD_TYPE_FIXED;
9729
9730         ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
9731         sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
9732         ctsio->kern_sg_entries = 0;
9733
9734         /*
9735          * struct scsi_sense_data, which is currently set to 256 bytes, is
9736          * larger than the largest allowed value for the length field in the
9737          * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
9738          */
9739         ctsio->residual = 0;
9740         ctsio->kern_data_len = cdb->length;
9741         ctsio->kern_total_len = cdb->length;
9742
9743         ctsio->kern_data_resid = 0;
9744         ctsio->kern_rel_offset = 0;
9745         ctsio->kern_sg_entries = 0;
9746
9747         /*
9748          * If we don't have a LUN, we don't have any pending sense.
9749          */
9750         if (lun == NULL)
9751                 goto no_sense;
9752
9753         have_error = 0;
9754         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9755         /*
9756          * Check for pending sense, and then for pending unit attentions.
9757          * Pending sense gets returned first, then pending unit attentions.
9758          */
9759         mtx_lock(&lun->lun_lock);
9760 #ifdef CTL_WITH_CA
9761         if (ctl_is_set(lun->have_ca, initidx)) {
9762                 scsi_sense_data_type stored_format;
9763
9764                 /*
9765                  * Check to see which sense format was used for the stored
9766                  * sense data.
9767                  */
9768                 stored_format = scsi_sense_type(&lun->pending_sense[initidx]);
9769
9770                 /*
9771                  * If the user requested a different sense format than the
9772                  * one we stored, then we need to convert it to the other
9773                  * format.  If we're going from descriptor to fixed format
9774                  * sense data, we may lose things in translation, depending
9775                  * on what options were used.
9776                  *
9777                  * If the stored format is SSD_TYPE_NONE (i.e. invalid),
9778                  * for some reason we'll just copy it out as-is.
9779                  */
9780                 if ((stored_format == SSD_TYPE_FIXED)
9781                  && (sense_format == SSD_TYPE_DESC))
9782                         ctl_sense_to_desc((struct scsi_sense_data_fixed *)
9783                             &lun->pending_sense[initidx],
9784                             (struct scsi_sense_data_desc *)sense_ptr);
9785                 else if ((stored_format == SSD_TYPE_DESC)
9786                       && (sense_format == SSD_TYPE_FIXED))
9787                         ctl_sense_to_fixed((struct scsi_sense_data_desc *)
9788                             &lun->pending_sense[initidx],
9789                             (struct scsi_sense_data_fixed *)sense_ptr);
9790                 else
9791                         memcpy(sense_ptr, &lun->pending_sense[initidx],
9792                                ctl_min(sizeof(*sense_ptr),
9793                                sizeof(lun->pending_sense[initidx])));
9794
9795                 ctl_clear_mask(lun->have_ca, initidx);
9796                 have_error = 1;
9797         } else
9798 #endif
9799         if (lun->pending_ua[initidx] != CTL_UA_NONE) {
9800                 ctl_ua_type ua_type;
9801
9802                 ua_type = ctl_build_ua(&lun->pending_ua[initidx],
9803                                        sense_ptr, sense_format);
9804                 if (ua_type != CTL_UA_NONE)
9805                         have_error = 1;
9806         }
9807         mtx_unlock(&lun->lun_lock);
9808
9809         /*
9810          * We already have a pending error, return it.
9811          */
9812         if (have_error != 0) {
9813                 /*
9814                  * We report the SCSI status as OK, since the status of the
9815                  * request sense command itself is OK.
9816                  */
9817                 ctsio->scsi_status = SCSI_STATUS_OK;
9818
9819                 /*
9820                  * We report 0 for the sense length, because we aren't doing
9821                  * autosense in this case.  We're reporting sense as
9822                  * parameter data.
9823                  */
9824                 ctsio->sense_len = 0;
9825                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9826                 ctsio->be_move_done = ctl_config_move_done;
9827                 ctl_datamove((union ctl_io *)ctsio);
9828
9829                 return (CTL_RETVAL_COMPLETE);
9830         }
9831
9832 no_sense:
9833
9834         /*
9835          * No sense information to report, so we report that everything is
9836          * okay.
9837          */
9838         ctl_set_sense_data(sense_ptr,
9839                            lun,
9840                            sense_format,
9841                            /*current_error*/ 1,
9842                            /*sense_key*/ SSD_KEY_NO_SENSE,
9843                            /*asc*/ 0x00,
9844                            /*ascq*/ 0x00,
9845                            SSD_ELEM_NONE);
9846
9847         ctsio->scsi_status = SCSI_STATUS_OK;
9848
9849         /*
9850          * We report 0 for the sense length, because we aren't doing
9851          * autosense in this case.  We're reporting sense as parameter data.
9852          */
9853         ctsio->sense_len = 0;
9854         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9855         ctsio->be_move_done = ctl_config_move_done;
9856         ctl_datamove((union ctl_io *)ctsio);
9857
9858         return (CTL_RETVAL_COMPLETE);
9859 }
9860
9861 int
9862 ctl_tur(struct ctl_scsiio *ctsio)
9863 {
9864         struct ctl_lun *lun;
9865
9866         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9867
9868         CTL_DEBUG_PRINT(("ctl_tur\n"));
9869
9870         if (lun == NULL)
9871                 return (EINVAL);
9872
9873         ctsio->scsi_status = SCSI_STATUS_OK;
9874         ctsio->io_hdr.status = CTL_SUCCESS;
9875
9876         ctl_done((union ctl_io *)ctsio);
9877
9878         return (CTL_RETVAL_COMPLETE);
9879 }
9880
9881 #ifdef notyet
9882 static int
9883 ctl_cmddt_inquiry(struct ctl_scsiio *ctsio)
9884 {
9885
9886 }
9887 #endif
9888
9889 static int
9890 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
9891 {
9892         struct scsi_vpd_supported_pages *pages;
9893         int sup_page_size;
9894         struct ctl_lun *lun;
9895
9896         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9897
9898         sup_page_size = sizeof(struct scsi_vpd_supported_pages) *
9899             SCSI_EVPD_NUM_SUPPORTED_PAGES;
9900         ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9901         pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9902         ctsio->kern_sg_entries = 0;
9903
9904         if (sup_page_size < alloc_len) {
9905                 ctsio->residual = alloc_len - sup_page_size;
9906                 ctsio->kern_data_len = sup_page_size;
9907                 ctsio->kern_total_len = sup_page_size;
9908         } else {
9909                 ctsio->residual = 0;
9910                 ctsio->kern_data_len = alloc_len;
9911                 ctsio->kern_total_len = alloc_len;
9912         }
9913         ctsio->kern_data_resid = 0;
9914         ctsio->kern_rel_offset = 0;
9915         ctsio->kern_sg_entries = 0;
9916
9917         /*
9918          * The control device is always connected.  The disk device, on the
9919          * other hand, may not be online all the time.  Need to change this
9920          * to figure out whether the disk device is actually online or not.
9921          */
9922         if (lun != NULL)
9923                 pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9924                                 lun->be_lun->lun_type;
9925         else
9926                 pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9927
9928         pages->length = SCSI_EVPD_NUM_SUPPORTED_PAGES;
9929         /* Supported VPD pages */
9930         pages->page_list[0] = SVPD_SUPPORTED_PAGES;
9931         /* Serial Number */
9932         pages->page_list[1] = SVPD_UNIT_SERIAL_NUMBER;
9933         /* Device Identification */
9934         pages->page_list[2] = SVPD_DEVICE_ID;
9935         /* Extended INQUIRY Data */
9936         pages->page_list[3] = SVPD_EXTENDED_INQUIRY_DATA;
9937         /* Mode Page Policy */
9938         pages->page_list[4] = SVPD_MODE_PAGE_POLICY;
9939         /* SCSI Ports */
9940         pages->page_list[5] = SVPD_SCSI_PORTS;
9941         /* Third-party Copy */
9942         pages->page_list[6] = SVPD_SCSI_TPC;
9943         /* Block limits */
9944         pages->page_list[7] = SVPD_BLOCK_LIMITS;
9945         /* Block Device Characteristics */
9946         pages->page_list[8] = SVPD_BDC;
9947         /* Logical Block Provisioning */
9948         pages->page_list[9] = SVPD_LBP;
9949
9950         ctsio->scsi_status = SCSI_STATUS_OK;
9951
9952         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9953         ctsio->be_move_done = ctl_config_move_done;
9954         ctl_datamove((union ctl_io *)ctsio);
9955
9956         return (CTL_RETVAL_COMPLETE);
9957 }
9958
9959 static int
9960 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9961 {
9962         struct scsi_vpd_unit_serial_number *sn_ptr;
9963         struct ctl_lun *lun;
9964
9965         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9966
9967         ctsio->kern_data_ptr = malloc(sizeof(*sn_ptr), M_CTL, M_WAITOK | M_ZERO);
9968         sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9969         ctsio->kern_sg_entries = 0;
9970
9971         if (sizeof(*sn_ptr) < alloc_len) {
9972                 ctsio->residual = alloc_len - sizeof(*sn_ptr);
9973                 ctsio->kern_data_len = sizeof(*sn_ptr);
9974                 ctsio->kern_total_len = sizeof(*sn_ptr);
9975         } else {
9976                 ctsio->residual = 0;
9977                 ctsio->kern_data_len = alloc_len;
9978                 ctsio->kern_total_len = alloc_len;
9979         }
9980         ctsio->kern_data_resid = 0;
9981         ctsio->kern_rel_offset = 0;
9982         ctsio->kern_sg_entries = 0;
9983
9984         /*
9985          * The control device is always connected.  The disk device, on the
9986          * other hand, may not be online all the time.  Need to change this
9987          * to figure out whether the disk device is actually online or not.
9988          */
9989         if (lun != NULL)
9990                 sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9991                                   lun->be_lun->lun_type;
9992         else
9993                 sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9994
9995         sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9996         sn_ptr->length = ctl_min(sizeof(*sn_ptr) - 4, CTL_SN_LEN);
9997         /*
9998          * If we don't have a LUN, we just leave the serial number as
9999          * all spaces.
10000          */
10001         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
10002         if (lun != NULL) {
10003                 strncpy((char *)sn_ptr->serial_num,
10004                         (char *)lun->be_lun->serial_num, CTL_SN_LEN);
10005         }
10006         ctsio->scsi_status = SCSI_STATUS_OK;
10007
10008         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10009         ctsio->be_move_done = ctl_config_move_done;
10010         ctl_datamove((union ctl_io *)ctsio);
10011
10012         return (CTL_RETVAL_COMPLETE);
10013 }
10014
10015
10016 static int
10017 ctl_inquiry_evpd_eid(struct ctl_scsiio *ctsio, int alloc_len)
10018 {
10019         struct scsi_vpd_extended_inquiry_data *eid_ptr;
10020         struct ctl_lun *lun;
10021         int data_len;
10022
10023         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10024
10025         data_len = sizeof(struct scsi_vpd_extended_inquiry_data);
10026         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10027         eid_ptr = (struct scsi_vpd_extended_inquiry_data *)ctsio->kern_data_ptr;
10028         ctsio->kern_sg_entries = 0;
10029
10030         if (data_len < alloc_len) {
10031                 ctsio->residual = alloc_len - data_len;
10032                 ctsio->kern_data_len = data_len;
10033                 ctsio->kern_total_len = data_len;
10034         } else {
10035                 ctsio->residual = 0;
10036                 ctsio->kern_data_len = alloc_len;
10037                 ctsio->kern_total_len = alloc_len;
10038         }
10039         ctsio->kern_data_resid = 0;
10040         ctsio->kern_rel_offset = 0;
10041         ctsio->kern_sg_entries = 0;
10042
10043         /*
10044          * The control device is always connected.  The disk device, on the
10045          * other hand, may not be online all the time.
10046          */
10047         if (lun != NULL)
10048                 eid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10049                                      lun->be_lun->lun_type;
10050         else
10051                 eid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10052         eid_ptr->page_code = SVPD_EXTENDED_INQUIRY_DATA;
10053         eid_ptr->page_length = data_len - 4;
10054         eid_ptr->flags2 = SVPD_EID_HEADSUP | SVPD_EID_ORDSUP | SVPD_EID_SIMPSUP;
10055         eid_ptr->flags3 = SVPD_EID_V_SUP;
10056
10057         ctsio->scsi_status = SCSI_STATUS_OK;
10058         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10059         ctsio->be_move_done = ctl_config_move_done;
10060         ctl_datamove((union ctl_io *)ctsio);
10061
10062         return (CTL_RETVAL_COMPLETE);
10063 }
10064
10065 static int
10066 ctl_inquiry_evpd_mpp(struct ctl_scsiio *ctsio, int alloc_len)
10067 {
10068         struct scsi_vpd_mode_page_policy *mpp_ptr;
10069         struct ctl_lun *lun;
10070         int data_len;
10071
10072         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10073
10074         data_len = sizeof(struct scsi_vpd_mode_page_policy) +
10075             sizeof(struct scsi_vpd_mode_page_policy_descr);
10076
10077         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10078         mpp_ptr = (struct scsi_vpd_mode_page_policy *)ctsio->kern_data_ptr;
10079         ctsio->kern_sg_entries = 0;
10080
10081         if (data_len < alloc_len) {
10082                 ctsio->residual = alloc_len - data_len;
10083                 ctsio->kern_data_len = data_len;
10084                 ctsio->kern_total_len = data_len;
10085         } else {
10086                 ctsio->residual = 0;
10087                 ctsio->kern_data_len = alloc_len;
10088                 ctsio->kern_total_len = alloc_len;
10089         }
10090         ctsio->kern_data_resid = 0;
10091         ctsio->kern_rel_offset = 0;
10092         ctsio->kern_sg_entries = 0;
10093
10094         /*
10095          * The control device is always connected.  The disk device, on the
10096          * other hand, may not be online all the time.
10097          */
10098         if (lun != NULL)
10099                 mpp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10100                                      lun->be_lun->lun_type;
10101         else
10102                 mpp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10103         mpp_ptr->page_code = SVPD_MODE_PAGE_POLICY;
10104         scsi_ulto2b(data_len - 4, mpp_ptr->page_length);
10105         mpp_ptr->descr[0].page_code = 0x3f;
10106         mpp_ptr->descr[0].subpage_code = 0xff;
10107         mpp_ptr->descr[0].policy = SVPD_MPP_SHARED;
10108
10109         ctsio->scsi_status = SCSI_STATUS_OK;
10110         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10111         ctsio->be_move_done = ctl_config_move_done;
10112         ctl_datamove((union ctl_io *)ctsio);
10113
10114         return (CTL_RETVAL_COMPLETE);
10115 }
10116
10117 static int
10118 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
10119 {
10120         struct scsi_vpd_device_id *devid_ptr;
10121         struct scsi_vpd_id_descriptor *desc;
10122         struct ctl_softc *ctl_softc;
10123         struct ctl_lun *lun;
10124         struct ctl_port *port;
10125         int data_len;
10126         uint8_t proto;
10127
10128         ctl_softc = control_softc;
10129
10130         port = ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)];
10131         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10132
10133         data_len = sizeof(struct scsi_vpd_device_id) +
10134             sizeof(struct scsi_vpd_id_descriptor) +
10135                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
10136             sizeof(struct scsi_vpd_id_descriptor) +
10137                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
10138         if (lun && lun->lun_devid)
10139                 data_len += lun->lun_devid->len;
10140         if (port->port_devid)
10141                 data_len += port->port_devid->len;
10142         if (port->target_devid)
10143                 data_len += port->target_devid->len;
10144
10145         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10146         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
10147         ctsio->kern_sg_entries = 0;
10148
10149         if (data_len < alloc_len) {
10150                 ctsio->residual = alloc_len - data_len;
10151                 ctsio->kern_data_len = data_len;
10152                 ctsio->kern_total_len = data_len;
10153         } else {
10154                 ctsio->residual = 0;
10155                 ctsio->kern_data_len = alloc_len;
10156                 ctsio->kern_total_len = alloc_len;
10157         }
10158         ctsio->kern_data_resid = 0;
10159         ctsio->kern_rel_offset = 0;
10160         ctsio->kern_sg_entries = 0;
10161
10162         /*
10163          * The control device is always connected.  The disk device, on the
10164          * other hand, may not be online all the time.
10165          */
10166         if (lun != NULL)
10167                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10168                                      lun->be_lun->lun_type;
10169         else
10170                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10171         devid_ptr->page_code = SVPD_DEVICE_ID;
10172         scsi_ulto2b(data_len - 4, devid_ptr->length);
10173
10174         if (port->port_type == CTL_PORT_FC)
10175                 proto = SCSI_PROTO_FC << 4;
10176         else if (port->port_type == CTL_PORT_ISCSI)
10177                 proto = SCSI_PROTO_ISCSI << 4;
10178         else
10179                 proto = SCSI_PROTO_SPI << 4;
10180         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
10181
10182         /*
10183          * We're using a LUN association here.  i.e., this device ID is a
10184          * per-LUN identifier.
10185          */
10186         if (lun && lun->lun_devid) {
10187                 memcpy(desc, lun->lun_devid->data, lun->lun_devid->len);
10188                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
10189                     lun->lun_devid->len);
10190         }
10191
10192         /*
10193          * This is for the WWPN which is a port association.
10194          */
10195         if (port->port_devid) {
10196                 memcpy(desc, port->port_devid->data, port->port_devid->len);
10197                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
10198                     port->port_devid->len);
10199         }
10200
10201         /*
10202          * This is for the Relative Target Port(type 4h) identifier
10203          */
10204         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
10205         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
10206             SVPD_ID_TYPE_RELTARG;
10207         desc->length = 4;
10208         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port, &desc->identifier[2]);
10209         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
10210             sizeof(struct scsi_vpd_id_rel_trgt_port_id));
10211
10212         /*
10213          * This is for the Target Port Group(type 5h) identifier
10214          */
10215         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
10216         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
10217             SVPD_ID_TYPE_TPORTGRP;
10218         desc->length = 4;
10219         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS + 1,
10220             &desc->identifier[2]);
10221         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
10222             sizeof(struct scsi_vpd_id_trgt_port_grp_id));
10223
10224         /*
10225          * This is for the Target identifier
10226          */
10227         if (port->target_devid) {
10228                 memcpy(desc, port->target_devid->data, port->target_devid->len);
10229         }
10230
10231         ctsio->scsi_status = SCSI_STATUS_OK;
10232         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10233         ctsio->be_move_done = ctl_config_move_done;
10234         ctl_datamove((union ctl_io *)ctsio);
10235
10236         return (CTL_RETVAL_COMPLETE);
10237 }
10238
10239 static int
10240 ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len)
10241 {
10242         struct ctl_softc *softc = control_softc;
10243         struct scsi_vpd_scsi_ports *sp;
10244         struct scsi_vpd_port_designation *pd;
10245         struct scsi_vpd_port_designation_cont *pdc;
10246         struct ctl_lun *lun;
10247         struct ctl_port *port;
10248         int data_len, num_target_ports, iid_len, id_len, g, pg, p;
10249         int num_target_port_groups, single;
10250
10251         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10252
10253         single = ctl_is_single;
10254         if (single)
10255                 num_target_port_groups = 1;
10256         else
10257                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
10258         num_target_ports = 0;
10259         iid_len = 0;
10260         id_len = 0;
10261         mtx_lock(&softc->ctl_lock);
10262         STAILQ_FOREACH(port, &softc->port_list, links) {
10263                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
10264                         continue;
10265                 if (lun != NULL &&
10266                     ctl_map_lun_back(port->targ_port, lun->lun) >=
10267                     CTL_MAX_LUNS)
10268                         continue;
10269                 num_target_ports++;
10270                 if (port->init_devid)
10271                         iid_len += port->init_devid->len;
10272                 if (port->port_devid)
10273                         id_len += port->port_devid->len;
10274         }
10275         mtx_unlock(&softc->ctl_lock);
10276
10277         data_len = sizeof(struct scsi_vpd_scsi_ports) + num_target_port_groups *
10278             num_target_ports * (sizeof(struct scsi_vpd_port_designation) +
10279              sizeof(struct scsi_vpd_port_designation_cont)) + iid_len + id_len;
10280         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10281         sp = (struct scsi_vpd_scsi_ports *)ctsio->kern_data_ptr;
10282         ctsio->kern_sg_entries = 0;
10283
10284         if (data_len < alloc_len) {
10285                 ctsio->residual = alloc_len - data_len;
10286                 ctsio->kern_data_len = data_len;
10287                 ctsio->kern_total_len = data_len;
10288         } else {
10289                 ctsio->residual = 0;
10290                 ctsio->kern_data_len = alloc_len;
10291                 ctsio->kern_total_len = alloc_len;
10292         }
10293         ctsio->kern_data_resid = 0;
10294         ctsio->kern_rel_offset = 0;
10295         ctsio->kern_sg_entries = 0;
10296
10297         /*
10298          * The control device is always connected.  The disk device, on the
10299          * other hand, may not be online all the time.  Need to change this
10300          * to figure out whether the disk device is actually online or not.
10301          */
10302         if (lun != NULL)
10303                 sp->device = (SID_QUAL_LU_CONNECTED << 5) |
10304                                   lun->be_lun->lun_type;
10305         else
10306                 sp->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10307
10308         sp->page_code = SVPD_SCSI_PORTS;
10309         scsi_ulto2b(data_len - sizeof(struct scsi_vpd_scsi_ports),
10310             sp->page_length);
10311         pd = &sp->design[0];
10312
10313         mtx_lock(&softc->ctl_lock);
10314         if (softc->flags & CTL_FLAG_MASTER_SHELF)
10315                 pg = 0;
10316         else
10317                 pg = 1;
10318         for (g = 0; g < num_target_port_groups; g++) {
10319                 STAILQ_FOREACH(port, &softc->port_list, links) {
10320                         if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
10321                                 continue;
10322                         if (lun != NULL &&
10323                             ctl_map_lun_back(port->targ_port, lun->lun) >=
10324                             CTL_MAX_LUNS)
10325                                 continue;
10326                         p = port->targ_port % CTL_MAX_PORTS + g * CTL_MAX_PORTS;
10327                         scsi_ulto2b(p, pd->relative_port_id);
10328                         if (port->init_devid && g == pg) {
10329                                 iid_len = port->init_devid->len;
10330                                 memcpy(pd->initiator_transportid,
10331                                     port->init_devid->data, port->init_devid->len);
10332                         } else
10333                                 iid_len = 0;
10334                         scsi_ulto2b(iid_len, pd->initiator_transportid_length);
10335                         pdc = (struct scsi_vpd_port_designation_cont *)
10336                             (&pd->initiator_transportid[iid_len]);
10337                         if (port->port_devid && g == pg) {
10338                                 id_len = port->port_devid->len;
10339                                 memcpy(pdc->target_port_descriptors,
10340                                     port->port_devid->data, port->port_devid->len);
10341                         } else
10342                                 id_len = 0;
10343                         scsi_ulto2b(id_len, pdc->target_port_descriptors_length);
10344                         pd = (struct scsi_vpd_port_designation *)
10345                             ((uint8_t *)pdc->target_port_descriptors + id_len);
10346                 }
10347         }
10348         mtx_unlock(&softc->ctl_lock);
10349
10350         ctsio->scsi_status = SCSI_STATUS_OK;
10351         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10352         ctsio->be_move_done = ctl_config_move_done;
10353         ctl_datamove((union ctl_io *)ctsio);
10354
10355         return (CTL_RETVAL_COMPLETE);
10356 }
10357
10358 static int
10359 ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
10360 {
10361         struct scsi_vpd_block_limits *bl_ptr;
10362         struct ctl_lun *lun;
10363         int bs;
10364
10365         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10366
10367         ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
10368         bl_ptr = (struct scsi_vpd_block_limits *)ctsio->kern_data_ptr;
10369         ctsio->kern_sg_entries = 0;
10370
10371         if (sizeof(*bl_ptr) < alloc_len) {
10372                 ctsio->residual = alloc_len - sizeof(*bl_ptr);
10373                 ctsio->kern_data_len = sizeof(*bl_ptr);
10374                 ctsio->kern_total_len = sizeof(*bl_ptr);
10375         } else {
10376                 ctsio->residual = 0;
10377                 ctsio->kern_data_len = alloc_len;
10378                 ctsio->kern_total_len = alloc_len;
10379         }
10380         ctsio->kern_data_resid = 0;
10381         ctsio->kern_rel_offset = 0;
10382         ctsio->kern_sg_entries = 0;
10383
10384         /*
10385          * The control device is always connected.  The disk device, on the
10386          * other hand, may not be online all the time.  Need to change this
10387          * to figure out whether the disk device is actually online or not.
10388          */
10389         if (lun != NULL)
10390                 bl_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10391                                   lun->be_lun->lun_type;
10392         else
10393                 bl_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10394
10395         bl_ptr->page_code = SVPD_BLOCK_LIMITS;
10396         scsi_ulto2b(sizeof(*bl_ptr) - 4, bl_ptr->page_length);
10397         bl_ptr->max_cmp_write_len = 0xff;
10398         scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
10399         if (lun != NULL) {
10400                 bs = lun->be_lun->blocksize;
10401                 scsi_ulto4b(MAXPHYS / bs, bl_ptr->opt_txfer_len);
10402                 if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10403                         scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
10404                         scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_blk_cnt);
10405                         if (lun->be_lun->pblockexp != 0) {
10406                                 scsi_ulto4b((1 << lun->be_lun->pblockexp),
10407                                     bl_ptr->opt_unmap_grain);
10408                                 scsi_ulto4b(0x80000000 | lun->be_lun->pblockoff,
10409                                     bl_ptr->unmap_grain_align);
10410                         }
10411                 }
10412                 scsi_ulto4b(lun->be_lun->atomicblock,
10413                     bl_ptr->max_atomic_transfer_length);
10414                 scsi_ulto4b(0, bl_ptr->atomic_alignment);
10415                 scsi_ulto4b(0, bl_ptr->atomic_transfer_length_granularity);
10416         }
10417         scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
10418
10419         ctsio->scsi_status = SCSI_STATUS_OK;
10420         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10421         ctsio->be_move_done = ctl_config_move_done;
10422         ctl_datamove((union ctl_io *)ctsio);
10423
10424         return (CTL_RETVAL_COMPLETE);
10425 }
10426
10427 static int
10428 ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len)
10429 {
10430         struct scsi_vpd_block_device_characteristics *bdc_ptr;
10431         struct ctl_lun *lun;
10432
10433         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10434
10435         ctsio->kern_data_ptr = malloc(sizeof(*bdc_ptr), M_CTL, M_WAITOK | M_ZERO);
10436         bdc_ptr = (struct scsi_vpd_block_device_characteristics *)ctsio->kern_data_ptr;
10437         ctsio->kern_sg_entries = 0;
10438
10439         if (sizeof(*bdc_ptr) < alloc_len) {
10440                 ctsio->residual = alloc_len - sizeof(*bdc_ptr);
10441                 ctsio->kern_data_len = sizeof(*bdc_ptr);
10442                 ctsio->kern_total_len = sizeof(*bdc_ptr);
10443         } else {
10444                 ctsio->residual = 0;
10445                 ctsio->kern_data_len = alloc_len;
10446                 ctsio->kern_total_len = alloc_len;
10447         }
10448         ctsio->kern_data_resid = 0;
10449         ctsio->kern_rel_offset = 0;
10450         ctsio->kern_sg_entries = 0;
10451
10452         /*
10453          * The control device is always connected.  The disk device, on the
10454          * other hand, may not be online all the time.  Need to change this
10455          * to figure out whether the disk device is actually online or not.
10456          */
10457         if (lun != NULL)
10458                 bdc_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10459                                   lun->be_lun->lun_type;
10460         else
10461                 bdc_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10462         bdc_ptr->page_code = SVPD_BDC;
10463         scsi_ulto2b(sizeof(*bdc_ptr) - 4, bdc_ptr->page_length);
10464         scsi_ulto2b(SVPD_NON_ROTATING, bdc_ptr->medium_rotation_rate);
10465         bdc_ptr->flags = SVPD_FUAB | SVPD_VBULS;
10466
10467         ctsio->scsi_status = SCSI_STATUS_OK;
10468         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10469         ctsio->be_move_done = ctl_config_move_done;
10470         ctl_datamove((union ctl_io *)ctsio);
10471
10472         return (CTL_RETVAL_COMPLETE);
10473 }
10474
10475 static int
10476 ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
10477 {
10478         struct scsi_vpd_logical_block_prov *lbp_ptr;
10479         struct ctl_lun *lun;
10480
10481         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10482
10483         ctsio->kern_data_ptr = malloc(sizeof(*lbp_ptr), M_CTL, M_WAITOK | M_ZERO);
10484         lbp_ptr = (struct scsi_vpd_logical_block_prov *)ctsio->kern_data_ptr;
10485         ctsio->kern_sg_entries = 0;
10486
10487         if (sizeof(*lbp_ptr) < alloc_len) {
10488                 ctsio->residual = alloc_len - sizeof(*lbp_ptr);
10489                 ctsio->kern_data_len = sizeof(*lbp_ptr);
10490                 ctsio->kern_total_len = sizeof(*lbp_ptr);
10491         } else {
10492                 ctsio->residual = 0;
10493                 ctsio->kern_data_len = alloc_len;
10494                 ctsio->kern_total_len = alloc_len;
10495         }
10496         ctsio->kern_data_resid = 0;
10497         ctsio->kern_rel_offset = 0;
10498         ctsio->kern_sg_entries = 0;
10499
10500         /*
10501          * The control device is always connected.  The disk device, on the
10502          * other hand, may not be online all the time.  Need to change this
10503          * to figure out whether the disk device is actually online or not.
10504          */
10505         if (lun != NULL)
10506                 lbp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10507                                   lun->be_lun->lun_type;
10508         else
10509                 lbp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10510
10511         lbp_ptr->page_code = SVPD_LBP;
10512         scsi_ulto2b(sizeof(*lbp_ptr) - 4, lbp_ptr->page_length);
10513         if (lun != NULL && lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10514                 lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 |
10515                     SVPD_LBP_WS10 | SVPD_LBP_RZ | SVPD_LBP_ANC_SUP;
10516                 lbp_ptr->prov_type = SVPD_LBP_RESOURCE;
10517         }
10518
10519         ctsio->scsi_status = SCSI_STATUS_OK;
10520         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10521         ctsio->be_move_done = ctl_config_move_done;
10522         ctl_datamove((union ctl_io *)ctsio);
10523
10524         return (CTL_RETVAL_COMPLETE);
10525 }
10526
10527 static int
10528 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
10529 {
10530         struct scsi_inquiry *cdb;
10531         struct ctl_lun *lun;
10532         int alloc_len, retval;
10533
10534         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10535         cdb = (struct scsi_inquiry *)ctsio->cdb;
10536
10537         retval = CTL_RETVAL_COMPLETE;
10538
10539         alloc_len = scsi_2btoul(cdb->length);
10540
10541         switch (cdb->page_code) {
10542         case SVPD_SUPPORTED_PAGES:
10543                 retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
10544                 break;
10545         case SVPD_UNIT_SERIAL_NUMBER:
10546                 retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
10547                 break;
10548         case SVPD_DEVICE_ID:
10549                 retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
10550                 break;
10551         case SVPD_EXTENDED_INQUIRY_DATA:
10552                 retval = ctl_inquiry_evpd_eid(ctsio, alloc_len);
10553                 break;
10554         case SVPD_MODE_PAGE_POLICY:
10555                 retval = ctl_inquiry_evpd_mpp(ctsio, alloc_len);
10556                 break;
10557         case SVPD_SCSI_PORTS:
10558                 retval = ctl_inquiry_evpd_scsi_ports(ctsio, alloc_len);
10559                 break;
10560         case SVPD_SCSI_TPC:
10561                 retval = ctl_inquiry_evpd_tpc(ctsio, alloc_len);
10562                 break;
10563         case SVPD_BLOCK_LIMITS:
10564                 retval = ctl_inquiry_evpd_block_limits(ctsio, alloc_len);
10565                 break;
10566         case SVPD_BDC:
10567                 retval = ctl_inquiry_evpd_bdc(ctsio, alloc_len);
10568                 break;
10569         case SVPD_LBP:
10570                 retval = ctl_inquiry_evpd_lbp(ctsio, alloc_len);
10571                 break;
10572         default:
10573                 ctl_set_invalid_field(ctsio,
10574                                       /*sks_valid*/ 1,
10575                                       /*command*/ 1,
10576                                       /*field*/ 2,
10577                                       /*bit_valid*/ 0,
10578                                       /*bit*/ 0);
10579                 ctl_done((union ctl_io *)ctsio);
10580                 retval = CTL_RETVAL_COMPLETE;
10581                 break;
10582         }
10583
10584         return (retval);
10585 }
10586
10587 static int
10588 ctl_inquiry_std(struct ctl_scsiio *ctsio)
10589 {
10590         struct scsi_inquiry_data *inq_ptr;
10591         struct scsi_inquiry *cdb;
10592         struct ctl_softc *ctl_softc;
10593         struct ctl_lun *lun;
10594         char *val;
10595         uint32_t alloc_len, data_len;
10596         ctl_port_type port_type;
10597
10598         ctl_softc = control_softc;
10599
10600         /*
10601          * Figure out whether we're talking to a Fibre Channel port or not.
10602          * We treat the ioctl front end, and any SCSI adapters, as packetized
10603          * SCSI front ends.
10604          */
10605         port_type = ctl_softc->ctl_ports[
10606             ctl_port_idx(ctsio->io_hdr.nexus.targ_port)]->port_type;
10607         if (port_type == CTL_PORT_IOCTL || port_type == CTL_PORT_INTERNAL)
10608                 port_type = CTL_PORT_SCSI;
10609
10610         lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10611         cdb = (struct scsi_inquiry *)ctsio->cdb;
10612         alloc_len = scsi_2btoul(cdb->length);
10613
10614         /*
10615          * We malloc the full inquiry data size here and fill it
10616          * in.  If the user only asks for less, we'll give him
10617          * that much.
10618          */
10619         data_len = offsetof(struct scsi_inquiry_data, vendor_specific1);
10620         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10621         inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
10622         ctsio->kern_sg_entries = 0;
10623         ctsio->kern_data_resid = 0;
10624         ctsio->kern_rel_offset = 0;
10625
10626         if (data_len < alloc_len) {
10627                 ctsio->residual = alloc_len - data_len;
10628                 ctsio->kern_data_len = data_len;
10629                 ctsio->kern_total_len = data_len;
10630         } else {
10631                 ctsio->residual = 0;
10632                 ctsio->kern_data_len = alloc_len;
10633                 ctsio->kern_total_len = alloc_len;
10634         }
10635
10636         /*
10637          * If we have a LUN configured, report it as connected.  Otherwise,
10638          * report that it is offline or no device is supported, depending
10639          * on the value of inquiry_pq_no_lun.
10640          *
10641          * According to the spec (SPC-4 r34), the peripheral qualifier
10642          * SID_QUAL_LU_OFFLINE (001b) is used in the following scenario:
10643          *
10644          * "A peripheral device having the specified peripheral device type
10645          * is not connected to this logical unit. However, the device
10646          * server is capable of supporting the specified peripheral device
10647          * type on this logical unit."
10648          *
10649          * According to the same spec, the peripheral qualifier
10650          * SID_QUAL_BAD_LU (011b) is used in this scenario:
10651          *
10652          * "The device server is not capable of supporting a peripheral
10653          * device on this logical unit. For this peripheral qualifier the
10654          * peripheral device type shall be set to 1Fh. All other peripheral
10655          * device type values are reserved for this peripheral qualifier."
10656          *
10657          * Given the text, it would seem that we probably want to report that
10658          * the LUN is offline here.  There is no LUN connected, but we can
10659          * support a LUN at the given LUN number.
10660          *
10661          * In the real world, though, it sounds like things are a little
10662          * different:
10663          *
10664          * - Linux, when presented with a LUN with the offline peripheral
10665          *   qualifier, will create an sg driver instance for it.  So when
10666          *   you attach it to CTL, you wind up with a ton of sg driver
10667          *   instances.  (One for every LUN that Linux bothered to probe.)
10668          *   Linux does this despite the fact that it issues a REPORT LUNs
10669          *   to LUN 0 to get the inventory of supported LUNs.
10670          *
10671          * - There is other anecdotal evidence (from Emulex folks) about
10672          *   arrays that use the offline peripheral qualifier for LUNs that
10673          *   are on the "passive" path in an active/passive array.
10674          *
10675          * So the solution is provide a hopefully reasonable default
10676          * (return bad/no LUN) and allow the user to change the behavior
10677          * with a tunable/sysctl variable.
10678          */
10679         if (lun != NULL)
10680                 inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10681                                   lun->be_lun->lun_type;
10682         else if (ctl_softc->inquiry_pq_no_lun == 0)
10683                 inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10684         else
10685                 inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
10686
10687         /* RMB in byte 2 is 0 */
10688         inq_ptr->version = SCSI_REV_SPC4;
10689
10690         /*
10691          * According to SAM-3, even if a device only supports a single
10692          * level of LUN addressing, it should still set the HISUP bit:
10693          *
10694          * 4.9.1 Logical unit numbers overview
10695          *
10696          * All logical unit number formats described in this standard are
10697          * hierarchical in structure even when only a single level in that
10698          * hierarchy is used. The HISUP bit shall be set to one in the
10699          * standard INQUIRY data (see SPC-2) when any logical unit number
10700          * format described in this standard is used.  Non-hierarchical
10701          * formats are outside the scope of this standard.
10702          *
10703          * Therefore we set the HiSup bit here.
10704          *
10705          * The reponse format is 2, per SPC-3.
10706          */
10707         inq_ptr->response_format = SID_HiSup | 2;
10708
10709         inq_ptr->additional_length = data_len -
10710             (offsetof(struct scsi_inquiry_data, additional_length) + 1);
10711         CTL_DEBUG_PRINT(("additional_length = %d\n",
10712                          inq_ptr->additional_length));
10713
10714         inq_ptr->spc3_flags = SPC3_SID_3PC | SPC3_SID_TPGS_IMPLICIT;
10715         /* 16 bit addressing */
10716         if (port_type == CTL_PORT_SCSI)
10717                 inq_ptr->spc2_flags = SPC2_SID_ADDR16;
10718         /* XXX set the SID_MultiP bit here if we're actually going to
10719            respond on multiple ports */
10720         inq_ptr->spc2_flags |= SPC2_SID_MultiP;
10721
10722         /* 16 bit data bus, synchronous transfers */
10723         if (port_type == CTL_PORT_SCSI)
10724                 inq_ptr->flags = SID_WBus16 | SID_Sync;
10725         /*
10726          * XXX KDM do we want to support tagged queueing on the control
10727          * device at all?
10728          */
10729         if ((lun == NULL)
10730          || (lun->be_lun->lun_type != T_PROCESSOR))
10731                 inq_ptr->flags |= SID_CmdQue;
10732         /*
10733          * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
10734          * We have 8 bytes for the vendor name, and 16 bytes for the device
10735          * name and 4 bytes for the revision.
10736          */
10737         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10738             "vendor")) == NULL) {
10739                 strncpy(inq_ptr->vendor, CTL_VENDOR, sizeof(inq_ptr->vendor));
10740         } else {
10741                 memset(inq_ptr->vendor, ' ', sizeof(inq_ptr->vendor));
10742                 strncpy(inq_ptr->vendor, val,
10743                     min(sizeof(inq_ptr->vendor), strlen(val)));
10744         }
10745         if (lun == NULL) {
10746                 strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
10747                     sizeof(inq_ptr->product));
10748         } else if ((val = ctl_get_opt(&lun->be_lun->options, "product")) == NULL) {
10749                 switch (lun->be_lun->lun_type) {
10750                 case T_DIRECT:
10751                         strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
10752                             sizeof(inq_ptr->product));
10753                         break;
10754                 case T_PROCESSOR:
10755                         strncpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT,
10756                             sizeof(inq_ptr->product));
10757                         break;
10758                 default:
10759                         strncpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT,
10760                             sizeof(inq_ptr->product));
10761                         break;
10762                 }
10763         } else {
10764                 memset(inq_ptr->product, ' ', sizeof(inq_ptr->product));
10765                 strncpy(inq_ptr->product, val,
10766                     min(sizeof(inq_ptr->product), strlen(val)));
10767         }
10768
10769         /*
10770          * XXX make this a macro somewhere so it automatically gets
10771          * incremented when we make changes.
10772          */
10773         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10774             "revision")) == NULL) {
10775                 strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
10776         } else {
10777                 memset(inq_ptr->revision, ' ', sizeof(inq_ptr->revision));
10778                 strncpy(inq_ptr->revision, val,
10779                     min(sizeof(inq_ptr->revision), strlen(val)));
10780         }
10781
10782         /*
10783          * For parallel SCSI, we support double transition and single
10784          * transition clocking.  We also support QAS (Quick Arbitration
10785          * and Selection) and Information Unit transfers on both the
10786          * control and array devices.
10787          */
10788         if (port_type == CTL_PORT_SCSI)
10789                 inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
10790                                     SID_SPI_IUS;
10791
10792         /* SAM-5 (no version claimed) */
10793         scsi_ulto2b(0x00A0, inq_ptr->version1);
10794         /* SPC-4 (no version claimed) */
10795         scsi_ulto2b(0x0460, inq_ptr->version2);
10796         if (port_type == CTL_PORT_FC) {
10797                 /* FCP-2 ANSI INCITS.350:2003 */
10798                 scsi_ulto2b(0x0917, inq_ptr->version3);
10799         } else if (port_type == CTL_PORT_SCSI) {
10800                 /* SPI-4 ANSI INCITS.362:200x */
10801                 scsi_ulto2b(0x0B56, inq_ptr->version3);
10802         } else if (port_type == CTL_PORT_ISCSI) {
10803                 /* iSCSI (no version claimed) */
10804                 scsi_ulto2b(0x0960, inq_ptr->version3);
10805         } else if (port_type == CTL_PORT_SAS) {
10806                 /* SAS (no version claimed) */
10807                 scsi_ulto2b(0x0BE0, inq_ptr->version3);
10808         }
10809
10810         if (lun == NULL) {
10811                 /* SBC-4 (no version claimed) */
10812                 scsi_ulto2b(0x0600, inq_ptr->version4);
10813         } else {
10814                 switch (lun->be_lun->lun_type) {
10815                 case T_DIRECT:
10816                         /* SBC-4 (no version claimed) */
10817                         scsi_ulto2b(0x0600, inq_ptr->version4);
10818                         break;
10819                 case T_PROCESSOR:
10820                 default:
10821                         break;
10822                 }
10823         }
10824
10825         ctsio->scsi_status = SCSI_STATUS_OK;
10826         if (ctsio->kern_data_len > 0) {
10827                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10828                 ctsio->be_move_done = ctl_config_move_done;
10829                 ctl_datamove((union ctl_io *)ctsio);
10830         } else {
10831                 ctsio->io_hdr.status = CTL_SUCCESS;
10832                 ctl_done((union ctl_io *)ctsio);
10833         }
10834
10835         return (CTL_RETVAL_COMPLETE);
10836 }
10837
10838 int
10839 ctl_inquiry(struct ctl_scsiio *ctsio)
10840 {
10841         struct scsi_inquiry *cdb;
10842         int retval;
10843
10844         CTL_DEBUG_PRINT(("ctl_inquiry\n"));
10845
10846         cdb = (struct scsi_inquiry *)ctsio->cdb;
10847         if (cdb->byte2 & SI_EVPD)
10848                 retval = ctl_inquiry_evpd(ctsio);
10849         else if (cdb->page_code == 0)
10850                 retval = ctl_inquiry_std(ctsio);
10851         else {
10852                 ctl_set_invalid_field(ctsio,
10853                                       /*sks_valid*/ 1,
10854                                       /*command*/ 1,
10855                                       /*field*/ 2,
10856                                       /*bit_valid*/ 0,
10857                                       /*bit*/ 0);
10858                 ctl_done((union ctl_io *)ctsio);
10859                 return (CTL_RETVAL_COMPLETE);
10860         }
10861
10862         return (retval);
10863 }
10864
10865 /*
10866  * For known CDB types, parse the LBA and length.
10867  */
10868 static int
10869 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len)
10870 {
10871         if (io->io_hdr.io_type != CTL_IO_SCSI)
10872                 return (1);
10873
10874         switch (io->scsiio.cdb[0]) {
10875         case COMPARE_AND_WRITE: {
10876                 struct scsi_compare_and_write *cdb;
10877
10878                 cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
10879
10880                 *lba = scsi_8btou64(cdb->addr);
10881                 *len = cdb->length;
10882                 break;
10883         }
10884         case READ_6:
10885         case WRITE_6: {
10886                 struct scsi_rw_6 *cdb;
10887
10888                 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
10889
10890                 *lba = scsi_3btoul(cdb->addr);
10891                 /* only 5 bits are valid in the most significant address byte */
10892                 *lba &= 0x1fffff;
10893                 *len = cdb->length;
10894                 break;
10895         }
10896         case READ_10:
10897         case WRITE_10: {
10898                 struct scsi_rw_10 *cdb;
10899
10900                 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
10901
10902                 *lba = scsi_4btoul(cdb->addr);
10903                 *len = scsi_2btoul(cdb->length);
10904                 break;
10905         }
10906         case WRITE_VERIFY_10: {
10907                 struct scsi_write_verify_10 *cdb;
10908
10909                 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
10910
10911                 *lba = scsi_4btoul(cdb->addr);
10912                 *len = scsi_2btoul(cdb->length);
10913                 break;
10914         }
10915         case READ_12:
10916         case WRITE_12: {
10917                 struct scsi_rw_12 *cdb;
10918
10919                 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
10920
10921                 *lba = scsi_4btoul(cdb->addr);
10922                 *len = scsi_4btoul(cdb->length);
10923                 break;
10924         }
10925         case WRITE_VERIFY_12: {
10926                 struct scsi_write_verify_12 *cdb;
10927
10928                 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
10929
10930                 *lba = scsi_4btoul(cdb->addr);
10931                 *len = scsi_4btoul(cdb->length);
10932                 break;
10933         }
10934         case READ_16:
10935         case WRITE_16:
10936         case WRITE_ATOMIC_16: {
10937                 struct scsi_rw_16 *cdb;
10938
10939                 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
10940
10941                 *lba = scsi_8btou64(cdb->addr);
10942                 *len = scsi_4btoul(cdb->length);
10943                 break;
10944         }
10945         case WRITE_VERIFY_16: {
10946                 struct scsi_write_verify_16 *cdb;
10947
10948                 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
10949
10950                 *lba = scsi_8btou64(cdb->addr);
10951                 *len = scsi_4btoul(cdb->length);
10952                 break;
10953         }
10954         case WRITE_SAME_10: {
10955                 struct scsi_write_same_10 *cdb;
10956
10957                 cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10958
10959                 *lba = scsi_4btoul(cdb->addr);
10960                 *len = scsi_2btoul(cdb->length);
10961                 break;
10962         }
10963         case WRITE_SAME_16: {
10964                 struct scsi_write_same_16 *cdb;
10965
10966                 cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10967
10968                 *lba = scsi_8btou64(cdb->addr);
10969                 *len = scsi_4btoul(cdb->length);
10970                 break;
10971         }
10972         case VERIFY_10: {
10973                 struct scsi_verify_10 *cdb;
10974
10975                 cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
10976
10977                 *lba = scsi_4btoul(cdb->addr);
10978                 *len = scsi_2btoul(cdb->length);
10979                 break;
10980         }
10981         case VERIFY_12: {
10982                 struct scsi_verify_12 *cdb;
10983
10984                 cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
10985
10986                 *lba = scsi_4btoul(cdb->addr);
10987                 *len = scsi_4btoul(cdb->length);
10988                 break;
10989         }
10990         case VERIFY_16: {
10991                 struct scsi_verify_16 *cdb;
10992
10993                 cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
10994
10995                 *lba = scsi_8btou64(cdb->addr);
10996                 *len = scsi_4btoul(cdb->length);
10997                 break;
10998         }
10999         case UNMAP: {
11000                 *lba = 0;
11001                 *len = UINT64_MAX;
11002                 break;
11003         }
11004         default:
11005                 return (1);
11006                 break; /* NOTREACHED */
11007         }
11008
11009         return (0);
11010 }
11011
11012 static ctl_action
11013 ctl_extent_check_lba(uint64_t lba1, uint64_t len1, uint64_t lba2, uint64_t len2)
11014 {
11015         uint64_t endlba1, endlba2;
11016
11017         endlba1 = lba1 + len1 - 1;
11018         endlba2 = lba2 + len2 - 1;
11019
11020         if ((endlba1 < lba2)
11021          || (endlba2 < lba1))
11022                 return (CTL_ACTION_PASS);
11023         else
11024                 return (CTL_ACTION_BLOCK);
11025 }
11026
11027 static int
11028 ctl_extent_check_unmap(union ctl_io *io, uint64_t lba2, uint64_t len2)
11029 {
11030         struct ctl_ptr_len_flags *ptrlen;
11031         struct scsi_unmap_desc *buf, *end, *range;
11032         uint64_t lba;
11033         uint32_t len;
11034
11035         /* If not UNMAP -- go other way. */
11036         if (io->io_hdr.io_type != CTL_IO_SCSI ||
11037             io->scsiio.cdb[0] != UNMAP)
11038                 return (CTL_ACTION_ERROR);
11039
11040         /* If UNMAP without data -- block and wait for data. */
11041         ptrlen = (struct ctl_ptr_len_flags *)
11042             &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
11043         if ((io->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0 ||
11044             ptrlen->ptr == NULL)
11045                 return (CTL_ACTION_BLOCK);
11046
11047         /* UNMAP with data -- check for collision. */
11048         buf = (struct scsi_unmap_desc *)ptrlen->ptr;
11049         end = buf + ptrlen->len / sizeof(*buf);
11050         for (range = buf; range < end; range++) {
11051                 lba = scsi_8btou64(range->lba);
11052                 len = scsi_4btoul(range->length);
11053                 if ((lba < lba2 + len2) && (lba + len > lba2))
11054                         return (CTL_ACTION_BLOCK);
11055         }
11056         return (CTL_ACTION_PASS);
11057 }
11058
11059 static ctl_action
11060 ctl_extent_check(union ctl_io *io1, union ctl_io *io2)
11061 {
11062         uint64_t lba1, lba2;
11063         uint64_t len1, len2;
11064         int retval;
11065
11066         if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
11067                 return (CTL_ACTION_ERROR);
11068
11069         retval = ctl_extent_check_unmap(io2, lba1, len1);
11070         if (retval != CTL_ACTION_ERROR)
11071                 return (retval);
11072
11073         if (ctl_get_lba_len(io2, &lba2, &len2) != 0)
11074                 return (CTL_ACTION_ERROR);
11075
11076         return (ctl_extent_check_lba(lba1, len1, lba2, len2));
11077 }
11078
11079 static ctl_action
11080 ctl_check_for_blockage(struct ctl_lun *lun, union ctl_io *pending_io,
11081     union ctl_io *ooa_io)
11082 {
11083         const struct ctl_cmd_entry *pending_entry, *ooa_entry;
11084         ctl_serialize_action *serialize_row;
11085
11086         /*
11087          * The initiator attempted multiple untagged commands at the same
11088          * time.  Can't do that.
11089          */
11090         if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
11091          && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
11092          && ((pending_io->io_hdr.nexus.targ_port ==
11093               ooa_io->io_hdr.nexus.targ_port)
11094           && (pending_io->io_hdr.nexus.initid.id ==
11095               ooa_io->io_hdr.nexus.initid.id))
11096          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
11097                 return (CTL_ACTION_OVERLAP);
11098
11099         /*
11100          * The initiator attempted to send multiple tagged commands with
11101          * the same ID.  (It's fine if different initiators have the same
11102          * tag ID.)
11103          *
11104          * Even if all of those conditions are true, we don't kill the I/O
11105          * if the command ahead of us has been aborted.  We won't end up
11106          * sending it to the FETD, and it's perfectly legal to resend a
11107          * command with the same tag number as long as the previous
11108          * instance of this tag number has been aborted somehow.
11109          */
11110         if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
11111          && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
11112          && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
11113          && ((pending_io->io_hdr.nexus.targ_port ==
11114               ooa_io->io_hdr.nexus.targ_port)
11115           && (pending_io->io_hdr.nexus.initid.id ==
11116               ooa_io->io_hdr.nexus.initid.id))
11117          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
11118                 return (CTL_ACTION_OVERLAP_TAG);
11119
11120         /*
11121          * If we get a head of queue tag, SAM-3 says that we should
11122          * immediately execute it.
11123          *
11124          * What happens if this command would normally block for some other
11125          * reason?  e.g. a request sense with a head of queue tag
11126          * immediately after a write.  Normally that would block, but this
11127          * will result in its getting executed immediately...
11128          *
11129          * We currently return "pass" instead of "skip", so we'll end up
11130          * going through the rest of the queue to check for overlapped tags.
11131          *
11132          * XXX KDM check for other types of blockage first??
11133          */
11134         if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
11135                 return (CTL_ACTION_PASS);
11136
11137         /*
11138          * Ordered tags have to block until all items ahead of them
11139          * have completed.  If we get called with an ordered tag, we always
11140          * block, if something else is ahead of us in the queue.
11141          */
11142         if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
11143                 return (CTL_ACTION_BLOCK);
11144
11145         /*
11146          * Simple tags get blocked until all head of queue and ordered tags
11147          * ahead of them have completed.  I'm lumping untagged commands in
11148          * with simple tags here.  XXX KDM is that the right thing to do?
11149          */
11150         if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
11151           || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
11152          && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
11153           || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
11154                 return (CTL_ACTION_BLOCK);
11155
11156         pending_entry = ctl_get_cmd_entry(&pending_io->scsiio, NULL);
11157         ooa_entry = ctl_get_cmd_entry(&ooa_io->scsiio, NULL);
11158
11159         serialize_row = ctl_serialize_table[ooa_entry->seridx];
11160
11161         switch (serialize_row[pending_entry->seridx]) {
11162         case CTL_SER_BLOCK:
11163                 return (CTL_ACTION_BLOCK);
11164         case CTL_SER_EXTENT:
11165                 return (ctl_extent_check(pending_io, ooa_io));
11166         case CTL_SER_EXTENTOPT:
11167                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT].queue_flags
11168                     & SCP_QUEUE_ALG_MASK) != SCP_QUEUE_ALG_UNRESTRICTED)
11169                         return (ctl_extent_check(pending_io, ooa_io));
11170                 /* FALLTHROUGH */
11171         case CTL_SER_PASS:
11172                 return (CTL_ACTION_PASS);
11173         case CTL_SER_BLOCKOPT:
11174                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT].queue_flags
11175                     & SCP_QUEUE_ALG_MASK) != SCP_QUEUE_ALG_UNRESTRICTED)
11176                         return (CTL_ACTION_BLOCK);
11177                 return (CTL_ACTION_PASS);
11178         case CTL_SER_SKIP:
11179                 return (CTL_ACTION_SKIP);
11180         default:
11181                 panic("invalid serialization value %d",
11182                       serialize_row[pending_entry->seridx]);
11183         }
11184
11185         return (CTL_ACTION_ERROR);
11186 }
11187
11188 /*
11189  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
11190  * Assumptions:
11191  * - pending_io is generally either incoming, or on the blocked queue
11192  * - starting I/O is the I/O we want to start the check with.
11193  */
11194 static ctl_action
11195 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
11196               union ctl_io *starting_io)
11197 {
11198         union ctl_io *ooa_io;
11199         ctl_action action;
11200
11201         mtx_assert(&lun->lun_lock, MA_OWNED);
11202
11203         /*
11204          * Run back along the OOA queue, starting with the current
11205          * blocked I/O and going through every I/O before it on the
11206          * queue.  If starting_io is NULL, we'll just end up returning
11207          * CTL_ACTION_PASS.
11208          */
11209         for (ooa_io = starting_io; ooa_io != NULL;
11210              ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
11211              ooa_links)){
11212
11213                 /*
11214                  * This routine just checks to see whether
11215                  * cur_blocked is blocked by ooa_io, which is ahead
11216                  * of it in the queue.  It doesn't queue/dequeue
11217                  * cur_blocked.
11218                  */
11219                 action = ctl_check_for_blockage(lun, pending_io, ooa_io);
11220                 switch (action) {
11221                 case CTL_ACTION_BLOCK:
11222                 case CTL_ACTION_OVERLAP:
11223                 case CTL_ACTION_OVERLAP_TAG:
11224                 case CTL_ACTION_SKIP:
11225                 case CTL_ACTION_ERROR:
11226                         return (action);
11227                         break; /* NOTREACHED */
11228                 case CTL_ACTION_PASS:
11229                         break;
11230                 default:
11231                         panic("invalid action %d", action);
11232                         break;  /* NOTREACHED */
11233                 }
11234         }
11235
11236         return (CTL_ACTION_PASS);
11237 }
11238
11239 /*
11240  * Assumptions:
11241  * - An I/O has just completed, and has been removed from the per-LUN OOA
11242  *   queue, so some items on the blocked queue may now be unblocked.
11243  */
11244 static int
11245 ctl_check_blocked(struct ctl_lun *lun)
11246 {
11247         union ctl_io *cur_blocked, *next_blocked;
11248
11249         mtx_assert(&lun->lun_lock, MA_OWNED);
11250
11251         /*
11252          * Run forward from the head of the blocked queue, checking each
11253          * entry against the I/Os prior to it on the OOA queue to see if
11254          * there is still any blockage.
11255          *
11256          * We cannot use the TAILQ_FOREACH() macro, because it can't deal
11257          * with our removing a variable on it while it is traversing the
11258          * list.
11259          */
11260         for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
11261              cur_blocked != NULL; cur_blocked = next_blocked) {
11262                 union ctl_io *prev_ooa;
11263                 ctl_action action;
11264
11265                 next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
11266                                                           blocked_links);
11267
11268                 prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
11269                                                       ctl_ooaq, ooa_links);
11270
11271                 /*
11272                  * If cur_blocked happens to be the first item in the OOA
11273                  * queue now, prev_ooa will be NULL, and the action
11274                  * returned will just be CTL_ACTION_PASS.
11275                  */
11276                 action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
11277
11278                 switch (action) {
11279                 case CTL_ACTION_BLOCK:
11280                         /* Nothing to do here, still blocked */
11281                         break;
11282                 case CTL_ACTION_OVERLAP:
11283                 case CTL_ACTION_OVERLAP_TAG:
11284                         /*
11285                          * This shouldn't happen!  In theory we've already
11286                          * checked this command for overlap...
11287                          */
11288                         break;
11289                 case CTL_ACTION_PASS:
11290                 case CTL_ACTION_SKIP: {
11291                         struct ctl_softc *softc;
11292                         const struct ctl_cmd_entry *entry;
11293                         uint32_t initidx;
11294                         int isc_retval;
11295
11296                         /*
11297                          * The skip case shouldn't happen, this transaction
11298                          * should have never made it onto the blocked queue.
11299                          */
11300                         /*
11301                          * This I/O is no longer blocked, we can remove it
11302                          * from the blocked queue.  Since this is a TAILQ
11303                          * (doubly linked list), we can do O(1) removals
11304                          * from any place on the list.
11305                          */
11306                         TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
11307                                      blocked_links);
11308                         cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
11309
11310                         if (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC){
11311                                 /*
11312                                  * Need to send IO back to original side to
11313                                  * run
11314                                  */
11315                                 union ctl_ha_msg msg_info;
11316
11317                                 msg_info.hdr.original_sc =
11318                                         cur_blocked->io_hdr.original_sc;
11319                                 msg_info.hdr.serializing_sc = cur_blocked;
11320                                 msg_info.hdr.msg_type = CTL_MSG_R2R;
11321                                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11322                                      &msg_info, sizeof(msg_info), 0)) >
11323                                      CTL_HA_STATUS_SUCCESS) {
11324                                         printf("CTL:Check Blocked error from "
11325                                                "ctl_ha_msg_send %d\n",
11326                                                isc_retval);
11327                                 }
11328                                 break;
11329                         }
11330                         entry = ctl_get_cmd_entry(&cur_blocked->scsiio, NULL);
11331                         softc = control_softc;
11332
11333                         initidx = ctl_get_initindex(&cur_blocked->io_hdr.nexus);
11334
11335                         /*
11336                          * Check this I/O for LUN state changes that may
11337                          * have happened while this command was blocked.
11338                          * The LUN state may have been changed by a command
11339                          * ahead of us in the queue, so we need to re-check
11340                          * for any states that can be caused by SCSI
11341                          * commands.
11342                          */
11343                         if (ctl_scsiio_lun_check(softc, lun, entry,
11344                                                  &cur_blocked->scsiio) == 0) {
11345                                 cur_blocked->io_hdr.flags |=
11346                                                       CTL_FLAG_IS_WAS_ON_RTR;
11347                                 ctl_enqueue_rtr(cur_blocked);
11348                         } else
11349                                 ctl_done(cur_blocked);
11350                         break;
11351                 }
11352                 default:
11353                         /*
11354                          * This probably shouldn't happen -- we shouldn't
11355                          * get CTL_ACTION_ERROR, or anything else.
11356                          */
11357                         break;
11358                 }
11359         }
11360
11361         return (CTL_RETVAL_COMPLETE);
11362 }
11363
11364 /*
11365  * This routine (with one exception) checks LUN flags that can be set by
11366  * commands ahead of us in the OOA queue.  These flags have to be checked
11367  * when a command initially comes in, and when we pull a command off the
11368  * blocked queue and are preparing to execute it.  The reason we have to
11369  * check these flags for commands on the blocked queue is that the LUN
11370  * state may have been changed by a command ahead of us while we're on the
11371  * blocked queue.
11372  *
11373  * Ordering is somewhat important with these checks, so please pay
11374  * careful attention to the placement of any new checks.
11375  */
11376 static int
11377 ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
11378     const struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
11379 {
11380         int retval;
11381         uint32_t residx;
11382
11383         retval = 0;
11384
11385         mtx_assert(&lun->lun_lock, MA_OWNED);
11386
11387         /*
11388          * If this shelf is a secondary shelf controller, we have to reject
11389          * any media access commands.
11390          */
11391 #if 0
11392         /* No longer needed for HA */
11393         if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
11394          && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
11395                 ctl_set_lun_standby(ctsio);
11396                 retval = 1;
11397                 goto bailout;
11398         }
11399 #endif
11400
11401         if (entry->pattern & CTL_LUN_PAT_WRITE) {
11402                 if (lun->flags & CTL_LUN_READONLY) {
11403                         ctl_set_sense(ctsio, /*current_error*/ 1,
11404                             /*sense_key*/ SSD_KEY_DATA_PROTECT,
11405                             /*asc*/ 0x27, /*ascq*/ 0x01, SSD_ELEM_NONE);
11406                         retval = 1;
11407                         goto bailout;
11408                 }
11409                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT]
11410                     .eca_and_aen & SCP_SWP) != 0) {
11411                         ctl_set_sense(ctsio, /*current_error*/ 1,
11412                             /*sense_key*/ SSD_KEY_DATA_PROTECT,
11413                             /*asc*/ 0x27, /*ascq*/ 0x02, SSD_ELEM_NONE);
11414                         retval = 1;
11415                         goto bailout;
11416                 }
11417         }
11418
11419         /*
11420          * Check for a reservation conflict.  If this command isn't allowed
11421          * even on reserved LUNs, and if this initiator isn't the one who
11422          * reserved us, reject the command with a reservation conflict.
11423          */
11424         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
11425         if ((lun->flags & CTL_LUN_RESERVED)
11426          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
11427                 if (lun->res_idx != residx) {
11428                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
11429                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
11430                         retval = 1;
11431                         goto bailout;
11432                 }
11433         }
11434
11435         if ((lun->flags & CTL_LUN_PR_RESERVED)
11436          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV) == 0)) {
11437                 /*
11438                  * if we aren't registered or it's a res holder type
11439                  * reservation and this isn't the res holder then set a
11440                  * conflict.
11441                  * NOTE: Commands which might be allowed on write exclusive
11442                  * type reservations are checked in the particular command
11443                  * for a conflict. Read and SSU are the only ones.
11444                  */
11445                 if (lun->pr_keys[residx] == 0
11446                  || (residx != lun->pr_res_idx && lun->res_type < 4)) {
11447                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
11448                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
11449                         retval = 1;
11450                         goto bailout;
11451                 }
11452
11453         }
11454
11455         if ((lun->flags & CTL_LUN_OFFLINE)
11456          && ((entry->flags & CTL_CMD_FLAG_OK_ON_OFFLINE) == 0)) {
11457                 ctl_set_lun_not_ready(ctsio);
11458                 retval = 1;
11459                 goto bailout;
11460         }
11461
11462         /*
11463          * If the LUN is stopped, see if this particular command is allowed
11464          * for a stopped lun.  Otherwise, reject it with 0x04,0x02.
11465          */
11466         if ((lun->flags & CTL_LUN_STOPPED)
11467          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
11468                 /* "Logical unit not ready, initializing cmd. required" */
11469                 ctl_set_lun_stopped(ctsio);
11470                 retval = 1;
11471                 goto bailout;
11472         }
11473
11474         if ((lun->flags & CTL_LUN_INOPERABLE)
11475          && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
11476                 /* "Medium format corrupted" */
11477                 ctl_set_medium_format_corrupted(ctsio);
11478                 retval = 1;
11479                 goto bailout;
11480         }
11481
11482 bailout:
11483         return (retval);
11484
11485 }
11486
11487 static void
11488 ctl_failover_io(union ctl_io *io, int have_lock)
11489 {
11490         ctl_set_busy(&io->scsiio);
11491         ctl_done(io);
11492 }
11493
11494 static void
11495 ctl_failover(void)
11496 {
11497         struct ctl_lun *lun;
11498         struct ctl_softc *ctl_softc;
11499         union ctl_io *next_io, *pending_io;
11500         union ctl_io *io;
11501         int lun_idx;
11502         int i;
11503
11504         ctl_softc = control_softc;
11505
11506         mtx_lock(&ctl_softc->ctl_lock);
11507         /*
11508          * Remove any cmds from the other SC from the rtr queue.  These
11509          * will obviously only be for LUNs for which we're the primary.
11510          * We can't send status or get/send data for these commands.
11511          * Since they haven't been executed yet, we can just remove them.
11512          * We'll either abort them or delete them below, depending on
11513          * which HA mode we're in.
11514          */
11515 #ifdef notyet
11516         mtx_lock(&ctl_softc->queue_lock);
11517         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->rtr_queue);
11518              io != NULL; io = next_io) {
11519                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
11520                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
11521                         STAILQ_REMOVE(&ctl_softc->rtr_queue, &io->io_hdr,
11522                                       ctl_io_hdr, links);
11523         }
11524         mtx_unlock(&ctl_softc->queue_lock);
11525 #endif
11526
11527         for (lun_idx=0; lun_idx < ctl_softc->num_luns; lun_idx++) {
11528                 lun = ctl_softc->ctl_luns[lun_idx];
11529                 if (lun==NULL)
11530                         continue;
11531
11532                 /*
11533                  * Processor LUNs are primary on both sides.
11534                  * XXX will this always be true?
11535                  */
11536                 if (lun->be_lun->lun_type == T_PROCESSOR)
11537                         continue;
11538
11539                 if ((lun->flags & CTL_LUN_PRIMARY_SC)
11540                  && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
11541                         printf("FAILOVER: primary lun %d\n", lun_idx);
11542                         /*
11543                          * Remove all commands from the other SC. First from the
11544                          * blocked queue then from the ooa queue. Once we have
11545                          * removed them. Call ctl_check_blocked to see if there
11546                          * is anything that can run.
11547                          */
11548                         for (io = (union ctl_io *)TAILQ_FIRST(
11549                              &lun->blocked_queue); io != NULL; io = next_io) {
11550
11551                                 next_io = (union ctl_io *)TAILQ_NEXT(
11552                                     &io->io_hdr, blocked_links);
11553
11554                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
11555                                         TAILQ_REMOVE(&lun->blocked_queue,
11556                                                      &io->io_hdr,blocked_links);
11557                                         io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
11558                                         TAILQ_REMOVE(&lun->ooa_queue,
11559                                                      &io->io_hdr, ooa_links);
11560
11561                                         ctl_free_io(io);
11562                                 }
11563                         }
11564
11565                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
11566                              io != NULL; io = next_io) {
11567
11568                                 next_io = (union ctl_io *)TAILQ_NEXT(
11569                                     &io->io_hdr, ooa_links);
11570
11571                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
11572
11573                                         TAILQ_REMOVE(&lun->ooa_queue,
11574                                                 &io->io_hdr,
11575                                                 ooa_links);
11576
11577                                         ctl_free_io(io);
11578                                 }
11579                         }
11580                         ctl_check_blocked(lun);
11581                 } else if ((lun->flags & CTL_LUN_PRIMARY_SC)
11582                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
11583
11584                         printf("FAILOVER: primary lun %d\n", lun_idx);
11585                         /*
11586                          * Abort all commands from the other SC.  We can't
11587                          * send status back for them now.  These should get
11588                          * cleaned up when they are completed or come out
11589                          * for a datamove operation.
11590                          */
11591                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
11592                              io != NULL; io = next_io) {
11593                                 next_io = (union ctl_io *)TAILQ_NEXT(
11594                                         &io->io_hdr, ooa_links);
11595
11596                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
11597                                         io->io_hdr.flags |= CTL_FLAG_ABORT;
11598                         }
11599                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
11600                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
11601
11602                         printf("FAILOVER: secondary lun %d\n", lun_idx);
11603
11604                         lun->flags |= CTL_LUN_PRIMARY_SC;
11605
11606                         /*
11607                          * We send all I/O that was sent to this controller
11608                          * and redirected to the other side back with
11609                          * busy status, and have the initiator retry it.
11610                          * Figuring out how much data has been transferred,
11611                          * etc. and picking up where we left off would be
11612                          * very tricky.
11613                          *
11614                          * XXX KDM need to remove I/O from the blocked
11615                          * queue as well!
11616                          */
11617                         for (pending_io = (union ctl_io *)TAILQ_FIRST(
11618                              &lun->ooa_queue); pending_io != NULL;
11619                              pending_io = next_io) {
11620
11621                                 next_io =  (union ctl_io *)TAILQ_NEXT(
11622                                         &pending_io->io_hdr, ooa_links);
11623
11624                                 pending_io->io_hdr.flags &=
11625                                         ~CTL_FLAG_SENT_2OTHER_SC;
11626
11627                                 if (pending_io->io_hdr.flags &
11628                                     CTL_FLAG_IO_ACTIVE) {
11629                                         pending_io->io_hdr.flags |=
11630                                                 CTL_FLAG_FAILOVER;
11631                                 } else {
11632                                         ctl_set_busy(&pending_io->scsiio);
11633                                         ctl_done(pending_io);
11634                                 }
11635                         }
11636
11637                         /*
11638                          * Build Unit Attention
11639                          */
11640                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11641                                 lun->pending_ua[i] |=
11642                                                      CTL_UA_ASYM_ACC_CHANGE;
11643                         }
11644                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
11645                         && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
11646                         printf("FAILOVER: secondary lun %d\n", lun_idx);
11647                         /*
11648                          * if the first io on the OOA is not on the RtR queue
11649                          * add it.
11650                          */
11651                         lun->flags |= CTL_LUN_PRIMARY_SC;
11652
11653                         pending_io = (union ctl_io *)TAILQ_FIRST(
11654                             &lun->ooa_queue);
11655                         if (pending_io==NULL) {
11656                                 printf("Nothing on OOA queue\n");
11657                                 continue;
11658                         }
11659
11660                         pending_io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11661                         if ((pending_io->io_hdr.flags &
11662                              CTL_FLAG_IS_WAS_ON_RTR) == 0) {
11663                                 pending_io->io_hdr.flags |=
11664                                     CTL_FLAG_IS_WAS_ON_RTR;
11665                                 ctl_enqueue_rtr(pending_io);
11666                         }
11667 #if 0
11668                         else
11669                         {
11670                                 printf("Tag 0x%04x is running\n",
11671                                       pending_io->scsiio.tag_num);
11672                         }
11673 #endif
11674
11675                         next_io = (union ctl_io *)TAILQ_NEXT(
11676                             &pending_io->io_hdr, ooa_links);
11677                         for (pending_io=next_io; pending_io != NULL;
11678                              pending_io = next_io) {
11679                                 pending_io->io_hdr.flags &=
11680                                     ~CTL_FLAG_SENT_2OTHER_SC;
11681                                 next_io = (union ctl_io *)TAILQ_NEXT(
11682                                         &pending_io->io_hdr, ooa_links);
11683                                 if (pending_io->io_hdr.flags &
11684                                     CTL_FLAG_IS_WAS_ON_RTR) {
11685 #if 0
11686                                         printf("Tag 0x%04x is running\n",
11687                                                 pending_io->scsiio.tag_num);
11688 #endif
11689                                         continue;
11690                                 }
11691
11692                                 switch (ctl_check_ooa(lun, pending_io,
11693                                     (union ctl_io *)TAILQ_PREV(
11694                                     &pending_io->io_hdr, ctl_ooaq,
11695                                     ooa_links))) {
11696
11697                                 case CTL_ACTION_BLOCK:
11698                                         TAILQ_INSERT_TAIL(&lun->blocked_queue,
11699                                                           &pending_io->io_hdr,
11700                                                           blocked_links);
11701                                         pending_io->io_hdr.flags |=
11702                                             CTL_FLAG_BLOCKED;
11703                                         break;
11704                                 case CTL_ACTION_PASS:
11705                                 case CTL_ACTION_SKIP:
11706                                         pending_io->io_hdr.flags |=
11707                                             CTL_FLAG_IS_WAS_ON_RTR;
11708                                         ctl_enqueue_rtr(pending_io);
11709                                         break;
11710                                 case CTL_ACTION_OVERLAP:
11711                                         ctl_set_overlapped_cmd(
11712                                             (struct ctl_scsiio *)pending_io);
11713                                         ctl_done(pending_io);
11714                                         break;
11715                                 case CTL_ACTION_OVERLAP_TAG:
11716                                         ctl_set_overlapped_tag(
11717                                             (struct ctl_scsiio *)pending_io,
11718                                             pending_io->scsiio.tag_num & 0xff);
11719                                         ctl_done(pending_io);
11720                                         break;
11721                                 case CTL_ACTION_ERROR:
11722                                 default:
11723                                         ctl_set_internal_failure(
11724                                                 (struct ctl_scsiio *)pending_io,
11725                                                 0,  // sks_valid
11726                                                 0); //retry count
11727                                         ctl_done(pending_io);
11728                                         break;
11729                                 }
11730                         }
11731
11732                         /*
11733                          * Build Unit Attention
11734                          */
11735                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11736                                 lun->pending_ua[i] |=
11737                                                      CTL_UA_ASYM_ACC_CHANGE;
11738                         }
11739                 } else {
11740                         panic("Unhandled HA mode failover, LUN flags = %#x, "
11741                               "ha_mode = #%x", lun->flags, ctl_softc->ha_mode);
11742                 }
11743         }
11744         ctl_pause_rtr = 0;
11745         mtx_unlock(&ctl_softc->ctl_lock);
11746 }
11747
11748 static int
11749 ctl_scsiio_precheck(struct ctl_softc *ctl_softc, struct ctl_scsiio *ctsio)
11750 {
11751         struct ctl_lun *lun;
11752         const struct ctl_cmd_entry *entry;
11753         uint32_t initidx, targ_lun;
11754         int retval;
11755
11756         retval = 0;
11757
11758         lun = NULL;
11759
11760         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
11761         if ((targ_lun < CTL_MAX_LUNS)
11762          && (ctl_softc->ctl_luns[targ_lun] != NULL)) {
11763                 lun = ctl_softc->ctl_luns[targ_lun];
11764                 /*
11765                  * If the LUN is invalid, pretend that it doesn't exist.
11766                  * It will go away as soon as all pending I/O has been
11767                  * completed.
11768                  */
11769                 if (lun->flags & CTL_LUN_DISABLED) {
11770                         lun = NULL;
11771                 } else {
11772                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
11773                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
11774                                 lun->be_lun;
11775                         if (lun->be_lun->lun_type == T_PROCESSOR) {
11776                                 ctsio->io_hdr.flags |= CTL_FLAG_CONTROL_DEV;
11777                         }
11778
11779                         /*
11780                          * Every I/O goes into the OOA queue for a
11781                          * particular LUN, and stays there until completion.
11782                          */
11783                         mtx_lock(&lun->lun_lock);
11784                         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr,
11785                             ooa_links);
11786                 }
11787         } else {
11788                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11789                 ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11790         }
11791
11792         /* Get command entry and return error if it is unsuppotyed. */
11793         entry = ctl_validate_command(ctsio);
11794         if (entry == NULL) {
11795                 if (lun)
11796                         mtx_unlock(&lun->lun_lock);
11797                 return (retval);
11798         }
11799
11800         ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
11801         ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
11802
11803         /*
11804          * Check to see whether we can send this command to LUNs that don't
11805          * exist.  This should pretty much only be the case for inquiry
11806          * and request sense.  Further checks, below, really require having
11807          * a LUN, so we can't really check the command anymore.  Just put
11808          * it on the rtr queue.
11809          */
11810         if (lun == NULL) {
11811                 if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) {
11812                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11813                         ctl_enqueue_rtr((union ctl_io *)ctsio);
11814                         return (retval);
11815                 }
11816
11817                 ctl_set_unsupported_lun(ctsio);
11818                 ctl_done((union ctl_io *)ctsio);
11819                 CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
11820                 return (retval);
11821         } else {
11822                 /*
11823                  * Make sure we support this particular command on this LUN.
11824                  * e.g., we don't support writes to the control LUN.
11825                  */
11826                 if (!ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
11827                         mtx_unlock(&lun->lun_lock);
11828                         ctl_set_invalid_opcode(ctsio);
11829                         ctl_done((union ctl_io *)ctsio);
11830                         return (retval);
11831                 }
11832         }
11833
11834         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11835
11836 #ifdef CTL_WITH_CA
11837         /*
11838          * If we've got a request sense, it'll clear the contingent
11839          * allegiance condition.  Otherwise, if we have a CA condition for
11840          * this initiator, clear it, because it sent down a command other
11841          * than request sense.
11842          */
11843         if ((ctsio->cdb[0] != REQUEST_SENSE)
11844          && (ctl_is_set(lun->have_ca, initidx)))
11845                 ctl_clear_mask(lun->have_ca, initidx);
11846 #endif
11847
11848         /*
11849          * If the command has this flag set, it handles its own unit
11850          * attention reporting, we shouldn't do anything.  Otherwise we
11851          * check for any pending unit attentions, and send them back to the
11852          * initiator.  We only do this when a command initially comes in,
11853          * not when we pull it off the blocked queue.
11854          *
11855          * According to SAM-3, section 5.3.2, the order that things get
11856          * presented back to the host is basically unit attentions caused
11857          * by some sort of reset event, busy status, reservation conflicts
11858          * or task set full, and finally any other status.
11859          *
11860          * One issue here is that some of the unit attentions we report
11861          * don't fall into the "reset" category (e.g. "reported luns data
11862          * has changed").  So reporting it here, before the reservation
11863          * check, may be technically wrong.  I guess the only thing to do
11864          * would be to check for and report the reset events here, and then
11865          * check for the other unit attention types after we check for a
11866          * reservation conflict.
11867          *
11868          * XXX KDM need to fix this
11869          */
11870         if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
11871                 ctl_ua_type ua_type;
11872
11873                 if (lun->pending_ua[initidx] != CTL_UA_NONE) {
11874                         scsi_sense_data_type sense_format;
11875
11876                         if (lun != NULL)
11877                                 sense_format = (lun->flags &
11878                                     CTL_LUN_SENSE_DESC) ? SSD_TYPE_DESC :
11879                                     SSD_TYPE_FIXED;
11880                         else
11881                                 sense_format = SSD_TYPE_FIXED;
11882
11883                         ua_type = ctl_build_ua(&lun->pending_ua[initidx],
11884                             &ctsio->sense_data, sense_format);
11885                         if (ua_type != CTL_UA_NONE) {
11886                                 ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
11887                                 ctsio->io_hdr.status = CTL_SCSI_ERROR |
11888                                                        CTL_AUTOSENSE;
11889                                 ctsio->sense_len = SSD_FULL_SIZE;
11890                                 mtx_unlock(&lun->lun_lock);
11891                                 ctl_done((union ctl_io *)ctsio);
11892                                 return (retval);
11893                         }
11894                 }
11895         }
11896
11897
11898         if (ctl_scsiio_lun_check(ctl_softc, lun, entry, ctsio) != 0) {
11899                 mtx_unlock(&lun->lun_lock);
11900                 ctl_done((union ctl_io *)ctsio);
11901                 return (retval);
11902         }
11903
11904         /*
11905          * XXX CHD this is where we want to send IO to other side if
11906          * this LUN is secondary on this SC. We will need to make a copy
11907          * of the IO and flag the IO on this side as SENT_2OTHER and the flag
11908          * the copy we send as FROM_OTHER.
11909          * We also need to stuff the address of the original IO so we can
11910          * find it easily. Something similar will need be done on the other
11911          * side so when we are done we can find the copy.
11912          */
11913         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
11914                 union ctl_ha_msg msg_info;
11915                 int isc_retval;
11916
11917                 ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11918
11919                 msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
11920                 msg_info.hdr.original_sc = (union ctl_io *)ctsio;
11921 #if 0
11922                 printf("1. ctsio %p\n", ctsio);
11923 #endif
11924                 msg_info.hdr.serializing_sc = NULL;
11925                 msg_info.hdr.nexus = ctsio->io_hdr.nexus;
11926                 msg_info.scsi.tag_num = ctsio->tag_num;
11927                 msg_info.scsi.tag_type = ctsio->tag_type;
11928                 memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
11929
11930                 ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11931
11932                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11933                     (void *)&msg_info, sizeof(msg_info), 0)) >
11934                     CTL_HA_STATUS_SUCCESS) {
11935                         printf("CTL:precheck, ctl_ha_msg_send returned %d\n",
11936                                isc_retval);
11937                         printf("CTL:opcode is %x\n", ctsio->cdb[0]);
11938                 } else {
11939 #if 0
11940                         printf("CTL:Precheck sent msg, opcode is %x\n",opcode);
11941 #endif
11942                 }
11943
11944                 /*
11945                  * XXX KDM this I/O is off the incoming queue, but hasn't
11946                  * been inserted on any other queue.  We may need to come
11947                  * up with a holding queue while we wait for serialization
11948                  * so that we have an idea of what we're waiting for from
11949                  * the other side.
11950                  */
11951                 mtx_unlock(&lun->lun_lock);
11952                 return (retval);
11953         }
11954
11955         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
11956                               (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
11957                               ctl_ooaq, ooa_links))) {
11958         case CTL_ACTION_BLOCK:
11959                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
11960                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
11961                                   blocked_links);
11962                 mtx_unlock(&lun->lun_lock);
11963                 return (retval);
11964         case CTL_ACTION_PASS:
11965         case CTL_ACTION_SKIP:
11966                 ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11967                 mtx_unlock(&lun->lun_lock);
11968                 ctl_enqueue_rtr((union ctl_io *)ctsio);
11969                 break;
11970         case CTL_ACTION_OVERLAP:
11971                 mtx_unlock(&lun->lun_lock);
11972                 ctl_set_overlapped_cmd(ctsio);
11973                 ctl_done((union ctl_io *)ctsio);
11974                 break;
11975         case CTL_ACTION_OVERLAP_TAG:
11976                 mtx_unlock(&lun->lun_lock);
11977                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
11978                 ctl_done((union ctl_io *)ctsio);
11979                 break;
11980         case CTL_ACTION_ERROR:
11981         default:
11982                 mtx_unlock(&lun->lun_lock);
11983                 ctl_set_internal_failure(ctsio,
11984                                          /*sks_valid*/ 0,
11985                                          /*retry_count*/ 0);
11986                 ctl_done((union ctl_io *)ctsio);
11987                 break;
11988         }
11989         return (retval);
11990 }
11991
11992 const struct ctl_cmd_entry *
11993 ctl_get_cmd_entry(struct ctl_scsiio *ctsio, int *sa)
11994 {
11995         const struct ctl_cmd_entry *entry;
11996         int service_action;
11997
11998         entry = &ctl_cmd_table[ctsio->cdb[0]];
11999         if (sa)
12000                 *sa = ((entry->flags & CTL_CMD_FLAG_SA5) != 0);
12001         if (entry->flags & CTL_CMD_FLAG_SA5) {
12002                 service_action = ctsio->cdb[1] & SERVICE_ACTION_MASK;
12003                 entry = &((const struct ctl_cmd_entry *)
12004                     entry->execute)[service_action];
12005         }
12006         return (entry);
12007 }
12008
12009 const struct ctl_cmd_entry *
12010 ctl_validate_command(struct ctl_scsiio *ctsio)
12011 {
12012         const struct ctl_cmd_entry *entry;
12013         int i, sa;
12014         uint8_t diff;
12015
12016         entry = ctl_get_cmd_entry(ctsio, &sa);
12017         if (entry->execute == NULL) {
12018                 if (sa)
12019                         ctl_set_invalid_field(ctsio,
12020                                               /*sks_valid*/ 1,
12021                                               /*command*/ 1,
12022                                               /*field*/ 1,
12023                                               /*bit_valid*/ 1,
12024                                               /*bit*/ 4);
12025                 else
12026                         ctl_set_invalid_opcode(ctsio);
12027                 ctl_done((union ctl_io *)ctsio);
12028                 return (NULL);
12029         }
12030         KASSERT(entry->length > 0,
12031             ("Not defined length for command 0x%02x/0x%02x",
12032              ctsio->cdb[0], ctsio->cdb[1]));
12033         for (i = 1; i < entry->length; i++) {
12034                 diff = ctsio->cdb[i] & ~entry->usage[i - 1];
12035                 if (diff == 0)
12036                         continue;
12037                 ctl_set_invalid_field(ctsio,
12038                                       /*sks_valid*/ 1,
12039                                       /*command*/ 1,
12040                                       /*field*/ i,
12041                                       /*bit_valid*/ 1,
12042                                       /*bit*/ fls(diff) - 1);
12043                 ctl_done((union ctl_io *)ctsio);
12044                 return (NULL);
12045         }
12046         return (entry);
12047 }
12048
12049 static int
12050 ctl_cmd_applicable(uint8_t lun_type, const struct ctl_cmd_entry *entry)
12051 {
12052
12053         switch (lun_type) {
12054         case T_PROCESSOR:
12055                 if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0) &&
12056                     ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
12057                         return (0);
12058                 break;
12059         case T_DIRECT:
12060                 if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0) &&
12061                     ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
12062                         return (0);
12063                 break;
12064         default:
12065                 return (0);
12066         }
12067         return (1);
12068 }
12069
12070 static int
12071 ctl_scsiio(struct ctl_scsiio *ctsio)
12072 {
12073         int retval;
12074         const struct ctl_cmd_entry *entry;
12075
12076         retval = CTL_RETVAL_COMPLETE;
12077
12078         CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
12079
12080         entry = ctl_get_cmd_entry(ctsio, NULL);
12081
12082         /*
12083          * If this I/O has been aborted, just send it straight to
12084          * ctl_done() without executing it.
12085          */
12086         if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
12087                 ctl_done((union ctl_io *)ctsio);
12088                 goto bailout;
12089         }
12090
12091         /*
12092          * All the checks should have been handled by ctl_scsiio_precheck().
12093          * We should be clear now to just execute the I/O.
12094          */
12095         retval = entry->execute(ctsio);
12096
12097 bailout:
12098         return (retval);
12099 }
12100
12101 /*
12102  * Since we only implement one target right now, a bus reset simply resets
12103  * our single target.
12104  */
12105 static int
12106 ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io)
12107 {
12108         return(ctl_target_reset(ctl_softc, io, CTL_UA_BUS_RESET));
12109 }
12110
12111 static int
12112 ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
12113                  ctl_ua_type ua_type)
12114 {
12115         struct ctl_lun *lun;
12116         int retval;
12117
12118         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12119                 union ctl_ha_msg msg_info;
12120
12121                 io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
12122                 msg_info.hdr.nexus = io->io_hdr.nexus;
12123                 if (ua_type==CTL_UA_TARG_RESET)
12124                         msg_info.task.task_action = CTL_TASK_TARGET_RESET;
12125                 else
12126                         msg_info.task.task_action = CTL_TASK_BUS_RESET;
12127                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
12128                 msg_info.hdr.original_sc = NULL;
12129                 msg_info.hdr.serializing_sc = NULL;
12130                 if (CTL_HA_STATUS_SUCCESS != ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12131                     (void *)&msg_info, sizeof(msg_info), 0)) {
12132                 }
12133         }
12134         retval = 0;
12135
12136         mtx_lock(&ctl_softc->ctl_lock);
12137         STAILQ_FOREACH(lun, &ctl_softc->lun_list, links)
12138                 retval += ctl_lun_reset(lun, io, ua_type);
12139         mtx_unlock(&ctl_softc->ctl_lock);
12140
12141         return (retval);
12142 }
12143
12144 /*
12145  * The LUN should always be set.  The I/O is optional, and is used to
12146  * distinguish between I/Os sent by this initiator, and by other
12147  * initiators.  We set unit attention for initiators other than this one.
12148  * SAM-3 is vague on this point.  It does say that a unit attention should
12149  * be established for other initiators when a LUN is reset (see section
12150  * 5.7.3), but it doesn't specifically say that the unit attention should
12151  * be established for this particular initiator when a LUN is reset.  Here
12152  * is the relevant text, from SAM-3 rev 8:
12153  *
12154  * 5.7.2 When a SCSI initiator port aborts its own tasks
12155  *
12156  * When a SCSI initiator port causes its own task(s) to be aborted, no
12157  * notification that the task(s) have been aborted shall be returned to
12158  * the SCSI initiator port other than the completion response for the
12159  * command or task management function action that caused the task(s) to
12160  * be aborted and notification(s) associated with related effects of the
12161  * action (e.g., a reset unit attention condition).
12162  *
12163  * XXX KDM for now, we're setting unit attention for all initiators.
12164  */
12165 static int
12166 ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
12167 {
12168         union ctl_io *xio;
12169 #if 0
12170         uint32_t initindex;
12171 #endif
12172         int i;
12173
12174         mtx_lock(&lun->lun_lock);
12175         /*
12176          * Run through the OOA queue and abort each I/O.
12177          */
12178 #if 0
12179         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
12180 #endif
12181         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
12182              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
12183                 xio->io_hdr.flags |= CTL_FLAG_ABORT | CTL_FLAG_ABORT_STATUS;
12184         }
12185
12186         /*
12187          * This version sets unit attention for every
12188          */
12189 #if 0
12190         initindex = ctl_get_initindex(&io->io_hdr.nexus);
12191         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
12192                 if (initindex == i)
12193                         continue;
12194                 lun->pending_ua[i] |= ua_type;
12195         }
12196 #endif
12197
12198         /*
12199          * A reset (any kind, really) clears reservations established with
12200          * RESERVE/RELEASE.  It does not clear reservations established
12201          * with PERSISTENT RESERVE OUT, but we don't support that at the
12202          * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
12203          * reservations made with the RESERVE/RELEASE commands, because
12204          * those commands are obsolete in SPC-3.
12205          */
12206         lun->flags &= ~CTL_LUN_RESERVED;
12207
12208         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
12209 #ifdef CTL_WITH_CA
12210                 ctl_clear_mask(lun->have_ca, i);
12211 #endif
12212                 lun->pending_ua[i] |= ua_type;
12213         }
12214         mtx_unlock(&lun->lun_lock);
12215
12216         return (0);
12217 }
12218
12219 static void
12220 ctl_abort_tasks_lun(struct ctl_lun *lun, uint32_t targ_port, uint32_t init_id,
12221     int other_sc)
12222 {
12223         union ctl_io *xio;
12224
12225         mtx_assert(&lun->lun_lock, MA_OWNED);
12226
12227         /*
12228          * Run through the OOA queue and attempt to find the given I/O.
12229          * The target port, initiator ID, tag type and tag number have to
12230          * match the values that we got from the initiator.  If we have an
12231          * untagged command to abort, simply abort the first untagged command
12232          * we come to.  We only allow one untagged command at a time of course.
12233          */
12234         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
12235              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
12236
12237                 if ((targ_port == UINT32_MAX ||
12238                      targ_port == xio->io_hdr.nexus.targ_port) &&
12239                     (init_id == UINT32_MAX ||
12240                      init_id == xio->io_hdr.nexus.initid.id)) {
12241                         if (targ_port != xio->io_hdr.nexus.targ_port ||
12242                             init_id != xio->io_hdr.nexus.initid.id)
12243                                 xio->io_hdr.flags |= CTL_FLAG_ABORT_STATUS;
12244                         xio->io_hdr.flags |= CTL_FLAG_ABORT;
12245                         if (!other_sc && !(lun->flags & CTL_LUN_PRIMARY_SC)) {
12246                                 union ctl_ha_msg msg_info;
12247
12248                                 msg_info.hdr.nexus = xio->io_hdr.nexus;
12249                                 msg_info.task.task_action = CTL_TASK_ABORT_TASK;
12250                                 msg_info.task.tag_num = xio->scsiio.tag_num;
12251                                 msg_info.task.tag_type = xio->scsiio.tag_type;
12252                                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
12253                                 msg_info.hdr.original_sc = NULL;
12254                                 msg_info.hdr.serializing_sc = NULL;
12255                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12256                                     (void *)&msg_info, sizeof(msg_info), 0);
12257                         }
12258                 }
12259         }
12260 }
12261
12262 static int
12263 ctl_abort_task_set(union ctl_io *io)
12264 {
12265         struct ctl_softc *softc = control_softc;
12266         struct ctl_lun *lun;
12267         uint32_t targ_lun;
12268
12269         /*
12270          * Look up the LUN.
12271          */
12272         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12273         mtx_lock(&softc->ctl_lock);
12274         if ((targ_lun < CTL_MAX_LUNS) && (softc->ctl_luns[targ_lun] != NULL))
12275                 lun = softc->ctl_luns[targ_lun];
12276         else {
12277                 mtx_unlock(&softc->ctl_lock);
12278                 return (1);
12279         }
12280
12281         mtx_lock(&lun->lun_lock);
12282         mtx_unlock(&softc->ctl_lock);
12283         if (io->taskio.task_action == CTL_TASK_ABORT_TASK_SET) {
12284                 ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
12285                     io->io_hdr.nexus.initid.id,
12286                     (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
12287         } else { /* CTL_TASK_CLEAR_TASK_SET */
12288                 ctl_abort_tasks_lun(lun, UINT32_MAX, UINT32_MAX,
12289                     (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
12290         }
12291         mtx_unlock(&lun->lun_lock);
12292         return (0);
12293 }
12294
12295 static int
12296 ctl_i_t_nexus_reset(union ctl_io *io)
12297 {
12298         struct ctl_softc *softc = control_softc;
12299         struct ctl_lun *lun;
12300         uint32_t initindex, residx;
12301
12302         initindex = ctl_get_initindex(&io->io_hdr.nexus);
12303         residx = ctl_get_resindex(&io->io_hdr.nexus);
12304         mtx_lock(&softc->ctl_lock);
12305         STAILQ_FOREACH(lun, &softc->lun_list, links) {
12306                 mtx_lock(&lun->lun_lock);
12307                 ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
12308                     io->io_hdr.nexus.initid.id,
12309                     (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
12310 #ifdef CTL_WITH_CA
12311                 ctl_clear_mask(lun->have_ca, initindex);
12312 #endif
12313                 if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == residx))
12314                         lun->flags &= ~CTL_LUN_RESERVED;
12315                 lun->pending_ua[initindex] |= CTL_UA_I_T_NEXUS_LOSS;
12316                 mtx_unlock(&lun->lun_lock);
12317         }
12318         mtx_unlock(&softc->ctl_lock);
12319         return (0);
12320 }
12321
12322 static int
12323 ctl_abort_task(union ctl_io *io)
12324 {
12325         union ctl_io *xio;
12326         struct ctl_lun *lun;
12327         struct ctl_softc *ctl_softc;
12328 #if 0
12329         struct sbuf sb;
12330         char printbuf[128];
12331 #endif
12332         int found;
12333         uint32_t targ_lun;
12334
12335         ctl_softc = control_softc;
12336         found = 0;
12337
12338         /*
12339          * Look up the LUN.
12340          */
12341         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12342         mtx_lock(&ctl_softc->ctl_lock);
12343         if ((targ_lun < CTL_MAX_LUNS)
12344          && (ctl_softc->ctl_luns[targ_lun] != NULL))
12345                 lun = ctl_softc->ctl_luns[targ_lun];
12346         else {
12347                 mtx_unlock(&ctl_softc->ctl_lock);
12348                 return (1);
12349         }
12350
12351 #if 0
12352         printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
12353                lun->lun, io->taskio.tag_num, io->taskio.tag_type);
12354 #endif
12355
12356         mtx_lock(&lun->lun_lock);
12357         mtx_unlock(&ctl_softc->ctl_lock);
12358         /*
12359          * Run through the OOA queue and attempt to find the given I/O.
12360          * The target port, initiator ID, tag type and tag number have to
12361          * match the values that we got from the initiator.  If we have an
12362          * untagged command to abort, simply abort the first untagged command
12363          * we come to.  We only allow one untagged command at a time of course.
12364          */
12365 #if 0
12366         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
12367 #endif
12368         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
12369              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
12370 #if 0
12371                 sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
12372
12373                 sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
12374                             lun->lun, xio->scsiio.tag_num,
12375                             xio->scsiio.tag_type,
12376                             (xio->io_hdr.blocked_links.tqe_prev
12377                             == NULL) ? "" : " BLOCKED",
12378                             (xio->io_hdr.flags &
12379                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
12380                             (xio->io_hdr.flags &
12381                             CTL_FLAG_ABORT) ? " ABORT" : "",
12382                             (xio->io_hdr.flags &
12383                             CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
12384                 ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
12385                 sbuf_finish(&sb);
12386                 printf("%s\n", sbuf_data(&sb));
12387 #endif
12388
12389                 if ((xio->io_hdr.nexus.targ_port == io->io_hdr.nexus.targ_port)
12390                  && (xio->io_hdr.nexus.initid.id ==
12391                      io->io_hdr.nexus.initid.id)) {
12392                         /*
12393                          * If the abort says that the task is untagged, the
12394                          * task in the queue must be untagged.  Otherwise,
12395                          * we just check to see whether the tag numbers
12396                          * match.  This is because the QLogic firmware
12397                          * doesn't pass back the tag type in an abort
12398                          * request.
12399                          */
12400 #if 0
12401                         if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
12402                           && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
12403                          || (xio->scsiio.tag_num == io->taskio.tag_num)) {
12404 #endif
12405                         /*
12406                          * XXX KDM we've got problems with FC, because it
12407                          * doesn't send down a tag type with aborts.  So we
12408                          * can only really go by the tag number...
12409                          * This may cause problems with parallel SCSI.
12410                          * Need to figure that out!!
12411                          */
12412                         if (xio->scsiio.tag_num == io->taskio.tag_num) {
12413                                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
12414                                 found = 1;
12415                                 if ((io->io_hdr.flags &
12416                                      CTL_FLAG_FROM_OTHER_SC) == 0 &&
12417                                     !(lun->flags & CTL_LUN_PRIMARY_SC)) {
12418                                         union ctl_ha_msg msg_info;
12419
12420                                         io->io_hdr.flags |=
12421                                                         CTL_FLAG_SENT_2OTHER_SC;
12422                                         msg_info.hdr.nexus = io->io_hdr.nexus;
12423                                         msg_info.task.task_action =
12424                                                 CTL_TASK_ABORT_TASK;
12425                                         msg_info.task.tag_num =
12426                                                 io->taskio.tag_num;
12427                                         msg_info.task.tag_type =
12428                                                 io->taskio.tag_type;
12429                                         msg_info.hdr.msg_type =
12430                                                 CTL_MSG_MANAGE_TASKS;
12431                                         msg_info.hdr.original_sc = NULL;
12432                                         msg_info.hdr.serializing_sc = NULL;
12433 #if 0
12434                                         printf("Sent Abort to other side\n");
12435 #endif
12436                                         if (CTL_HA_STATUS_SUCCESS !=
12437                                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12438                                                 (void *)&msg_info,
12439                                                 sizeof(msg_info), 0)) {
12440                                         }
12441                                 }
12442 #if 0
12443                                 printf("ctl_abort_task: found I/O to abort\n");
12444 #endif
12445                                 break;
12446                         }
12447                 }
12448         }
12449         mtx_unlock(&lun->lun_lock);
12450
12451         if (found == 0) {
12452                 /*
12453                  * This isn't really an error.  It's entirely possible for
12454                  * the abort and command completion to cross on the wire.
12455                  * This is more of an informative/diagnostic error.
12456                  */
12457 #if 0
12458                 printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
12459                        "%d:%d:%d:%d tag %d type %d\n",
12460                        io->io_hdr.nexus.initid.id,
12461                        io->io_hdr.nexus.targ_port,
12462                        io->io_hdr.nexus.targ_target.id,
12463                        io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
12464                        io->taskio.tag_type);
12465 #endif
12466         }
12467         return (0);
12468 }
12469
12470 static void
12471 ctl_run_task(union ctl_io *io)
12472 {
12473         struct ctl_softc *ctl_softc = control_softc;
12474         int retval = 1;
12475         const char *task_desc;
12476
12477         CTL_DEBUG_PRINT(("ctl_run_task\n"));
12478
12479         KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
12480             ("ctl_run_task: Unextected io_type %d\n",
12481              io->io_hdr.io_type));
12482
12483         task_desc = ctl_scsi_task_string(&io->taskio);
12484         if (task_desc != NULL) {
12485 #ifdef NEEDTOPORT
12486                 csevent_log(CSC_CTL | CSC_SHELF_SW |
12487                             CTL_TASK_REPORT,
12488                             csevent_LogType_Trace,
12489                             csevent_Severity_Information,
12490                             csevent_AlertLevel_Green,
12491                             csevent_FRU_Firmware,
12492                             csevent_FRU_Unknown,
12493                             "CTL: received task: %s",task_desc);
12494 #endif
12495         } else {
12496 #ifdef NEEDTOPORT
12497                 csevent_log(CSC_CTL | CSC_SHELF_SW |
12498                             CTL_TASK_REPORT,
12499                             csevent_LogType_Trace,
12500                             csevent_Severity_Information,
12501                             csevent_AlertLevel_Green,
12502                             csevent_FRU_Firmware,
12503                             csevent_FRU_Unknown,
12504                             "CTL: received unknown task "
12505                             "type: %d (%#x)",
12506                             io->taskio.task_action,
12507                             io->taskio.task_action);
12508 #endif
12509         }
12510         switch (io->taskio.task_action) {
12511         case CTL_TASK_ABORT_TASK:
12512                 retval = ctl_abort_task(io);
12513                 break;
12514         case CTL_TASK_ABORT_TASK_SET:
12515         case CTL_TASK_CLEAR_TASK_SET:
12516                 retval = ctl_abort_task_set(io);
12517                 break;
12518         case CTL_TASK_CLEAR_ACA:
12519                 break;
12520         case CTL_TASK_I_T_NEXUS_RESET:
12521                 retval = ctl_i_t_nexus_reset(io);
12522                 break;
12523         case CTL_TASK_LUN_RESET: {
12524                 struct ctl_lun *lun;
12525                 uint32_t targ_lun;
12526
12527                 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12528                 mtx_lock(&ctl_softc->ctl_lock);
12529                 if ((targ_lun < CTL_MAX_LUNS)
12530                  && (ctl_softc->ctl_luns[targ_lun] != NULL))
12531                         lun = ctl_softc->ctl_luns[targ_lun];
12532                 else {
12533                         mtx_unlock(&ctl_softc->ctl_lock);
12534                         retval = 1;
12535                         break;
12536                 }
12537
12538                 if (!(io->io_hdr.flags &
12539                     CTL_FLAG_FROM_OTHER_SC)) {
12540                         union ctl_ha_msg msg_info;
12541
12542                         io->io_hdr.flags |=
12543                                 CTL_FLAG_SENT_2OTHER_SC;
12544                         msg_info.hdr.msg_type =
12545                                 CTL_MSG_MANAGE_TASKS;
12546                         msg_info.hdr.nexus = io->io_hdr.nexus;
12547                         msg_info.task.task_action =
12548                                 CTL_TASK_LUN_RESET;
12549                         msg_info.hdr.original_sc = NULL;
12550                         msg_info.hdr.serializing_sc = NULL;
12551                         if (CTL_HA_STATUS_SUCCESS !=
12552                             ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12553                             (void *)&msg_info,
12554                             sizeof(msg_info), 0)) {
12555                         }
12556                 }
12557
12558                 retval = ctl_lun_reset(lun, io,
12559                                        CTL_UA_LUN_RESET);
12560                 mtx_unlock(&ctl_softc->ctl_lock);
12561                 break;
12562         }
12563         case CTL_TASK_TARGET_RESET:
12564                 retval = ctl_target_reset(ctl_softc, io, CTL_UA_TARG_RESET);
12565                 break;
12566         case CTL_TASK_BUS_RESET:
12567                 retval = ctl_bus_reset(ctl_softc, io);
12568                 break;
12569         case CTL_TASK_PORT_LOGIN:
12570                 break;
12571         case CTL_TASK_PORT_LOGOUT:
12572                 break;
12573         default:
12574                 printf("ctl_run_task: got unknown task management event %d\n",
12575                        io->taskio.task_action);
12576                 break;
12577         }
12578         if (retval == 0)
12579                 io->io_hdr.status = CTL_SUCCESS;
12580         else
12581                 io->io_hdr.status = CTL_ERROR;
12582         ctl_done(io);
12583 }
12584
12585 /*
12586  * For HA operation.  Handle commands that come in from the other
12587  * controller.
12588  */
12589 static void
12590 ctl_handle_isc(union ctl_io *io)
12591 {
12592         int free_io;
12593         struct ctl_lun *lun;
12594         struct ctl_softc *ctl_softc;
12595         uint32_t targ_lun;
12596
12597         ctl_softc = control_softc;
12598
12599         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12600         lun = ctl_softc->ctl_luns[targ_lun];
12601
12602         switch (io->io_hdr.msg_type) {
12603         case CTL_MSG_SERIALIZE:
12604                 free_io = ctl_serialize_other_sc_cmd(&io->scsiio);
12605                 break;
12606         case CTL_MSG_R2R: {
12607                 const struct ctl_cmd_entry *entry;
12608
12609                 /*
12610                  * This is only used in SER_ONLY mode.
12611                  */
12612                 free_io = 0;
12613                 entry = ctl_get_cmd_entry(&io->scsiio, NULL);
12614                 mtx_lock(&lun->lun_lock);
12615                 if (ctl_scsiio_lun_check(ctl_softc, lun,
12616                     entry, (struct ctl_scsiio *)io) != 0) {
12617                         mtx_unlock(&lun->lun_lock);
12618                         ctl_done(io);
12619                         break;
12620                 }
12621                 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
12622                 mtx_unlock(&lun->lun_lock);
12623                 ctl_enqueue_rtr(io);
12624                 break;
12625         }
12626         case CTL_MSG_FINISH_IO:
12627                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
12628                         free_io = 0;
12629                         ctl_done(io);
12630                 } else {
12631                         free_io = 1;
12632                         mtx_lock(&lun->lun_lock);
12633                         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
12634                                      ooa_links);
12635                         ctl_check_blocked(lun);
12636                         mtx_unlock(&lun->lun_lock);
12637                 }
12638                 break;
12639         case CTL_MSG_PERS_ACTION:
12640                 ctl_hndl_per_res_out_on_other_sc(
12641                         (union ctl_ha_msg *)&io->presio.pr_msg);
12642                 free_io = 1;
12643                 break;
12644         case CTL_MSG_BAD_JUJU:
12645                 free_io = 0;
12646                 ctl_done(io);
12647                 break;
12648         case CTL_MSG_DATAMOVE:
12649                 /* Only used in XFER mode */
12650                 free_io = 0;
12651                 ctl_datamove_remote(io);
12652                 break;
12653         case CTL_MSG_DATAMOVE_DONE:
12654                 /* Only used in XFER mode */
12655                 free_io = 0;
12656                 io->scsiio.be_move_done(io);
12657                 break;
12658         default:
12659                 free_io = 1;
12660                 printf("%s: Invalid message type %d\n",
12661                        __func__, io->io_hdr.msg_type);
12662                 break;
12663         }
12664         if (free_io)
12665                 ctl_free_io(io);
12666
12667 }
12668
12669
12670 /*
12671  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
12672  * there is no match.
12673  */
12674 static ctl_lun_error_pattern
12675 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
12676 {
12677         const struct ctl_cmd_entry *entry;
12678         ctl_lun_error_pattern filtered_pattern, pattern;
12679
12680         pattern = desc->error_pattern;
12681
12682         /*
12683          * XXX KDM we need more data passed into this function to match a
12684          * custom pattern, and we actually need to implement custom pattern
12685          * matching.
12686          */
12687         if (pattern & CTL_LUN_PAT_CMD)
12688                 return (CTL_LUN_PAT_CMD);
12689
12690         if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
12691                 return (CTL_LUN_PAT_ANY);
12692
12693         entry = ctl_get_cmd_entry(ctsio, NULL);
12694
12695         filtered_pattern = entry->pattern & pattern;
12696
12697         /*
12698          * If the user requested specific flags in the pattern (e.g.
12699          * CTL_LUN_PAT_RANGE), make sure the command supports all of those
12700          * flags.
12701          *
12702          * If the user did not specify any flags, it doesn't matter whether
12703          * or not the command supports the flags.
12704          */
12705         if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
12706              (pattern & ~CTL_LUN_PAT_MASK))
12707                 return (CTL_LUN_PAT_NONE);
12708
12709         /*
12710          * If the user asked for a range check, see if the requested LBA
12711          * range overlaps with this command's LBA range.
12712          */
12713         if (filtered_pattern & CTL_LUN_PAT_RANGE) {
12714                 uint64_t lba1;
12715                 uint64_t len1;
12716                 ctl_action action;
12717                 int retval;
12718
12719                 retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
12720                 if (retval != 0)
12721                         return (CTL_LUN_PAT_NONE);
12722
12723                 action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
12724                                               desc->lba_range.len);
12725                 /*
12726                  * A "pass" means that the LBA ranges don't overlap, so
12727                  * this doesn't match the user's range criteria.
12728                  */
12729                 if (action == CTL_ACTION_PASS)
12730                         return (CTL_LUN_PAT_NONE);
12731         }
12732
12733         return (filtered_pattern);
12734 }
12735
12736 static void
12737 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
12738 {
12739         struct ctl_error_desc *desc, *desc2;
12740
12741         mtx_assert(&lun->lun_lock, MA_OWNED);
12742
12743         STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
12744                 ctl_lun_error_pattern pattern;
12745                 /*
12746                  * Check to see whether this particular command matches
12747                  * the pattern in the descriptor.
12748                  */
12749                 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
12750                 if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
12751                         continue;
12752
12753                 switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
12754                 case CTL_LUN_INJ_ABORTED:
12755                         ctl_set_aborted(&io->scsiio);
12756                         break;
12757                 case CTL_LUN_INJ_MEDIUM_ERR:
12758                         ctl_set_medium_error(&io->scsiio);
12759                         break;
12760                 case CTL_LUN_INJ_UA:
12761                         /* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
12762                          * OCCURRED */
12763                         ctl_set_ua(&io->scsiio, 0x29, 0x00);
12764                         break;
12765                 case CTL_LUN_INJ_CUSTOM:
12766                         /*
12767                          * We're assuming the user knows what he is doing.
12768                          * Just copy the sense information without doing
12769                          * checks.
12770                          */
12771                         bcopy(&desc->custom_sense, &io->scsiio.sense_data,
12772                               ctl_min(sizeof(desc->custom_sense),
12773                                       sizeof(io->scsiio.sense_data)));
12774                         io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
12775                         io->scsiio.sense_len = SSD_FULL_SIZE;
12776                         io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
12777                         break;
12778                 case CTL_LUN_INJ_NONE:
12779                 default:
12780                         /*
12781                          * If this is an error injection type we don't know
12782                          * about, clear the continuous flag (if it is set)
12783                          * so it will get deleted below.
12784                          */
12785                         desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
12786                         break;
12787                 }
12788                 /*
12789                  * By default, each error injection action is a one-shot
12790                  */
12791                 if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
12792                         continue;
12793
12794                 STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
12795
12796                 free(desc, M_CTL);
12797         }
12798 }
12799
12800 #ifdef CTL_IO_DELAY
12801 static void
12802 ctl_datamove_timer_wakeup(void *arg)
12803 {
12804         union ctl_io *io;
12805
12806         io = (union ctl_io *)arg;
12807
12808         ctl_datamove(io);
12809 }
12810 #endif /* CTL_IO_DELAY */
12811
12812 void
12813 ctl_datamove(union ctl_io *io)
12814 {
12815         void (*fe_datamove)(union ctl_io *io);
12816
12817         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
12818
12819         CTL_DEBUG_PRINT(("ctl_datamove\n"));
12820
12821 #ifdef CTL_TIME_IO
12822         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12823                 char str[256];
12824                 char path_str[64];
12825                 struct sbuf sb;
12826
12827                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12828                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12829
12830                 sbuf_cat(&sb, path_str);
12831                 switch (io->io_hdr.io_type) {
12832                 case CTL_IO_SCSI:
12833                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12834                         sbuf_printf(&sb, "\n");
12835                         sbuf_cat(&sb, path_str);
12836                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12837                                     io->scsiio.tag_num, io->scsiio.tag_type);
12838                         break;
12839                 case CTL_IO_TASK:
12840                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12841                                     "Tag Type: %d\n", io->taskio.task_action,
12842                                     io->taskio.tag_num, io->taskio.tag_type);
12843                         break;
12844                 default:
12845                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12846                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12847                         break;
12848                 }
12849                 sbuf_cat(&sb, path_str);
12850                 sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
12851                             (intmax_t)time_uptime - io->io_hdr.start_time);
12852                 sbuf_finish(&sb);
12853                 printf("%s", sbuf_data(&sb));
12854         }
12855 #endif /* CTL_TIME_IO */
12856
12857 #ifdef CTL_IO_DELAY
12858         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12859                 struct ctl_lun *lun;
12860
12861                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12862
12863                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12864         } else {
12865                 struct ctl_lun *lun;
12866
12867                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12868                 if ((lun != NULL)
12869                  && (lun->delay_info.datamove_delay > 0)) {
12870                         struct callout *callout;
12871
12872                         callout = (struct callout *)&io->io_hdr.timer_bytes;
12873                         callout_init(callout, /*mpsafe*/ 1);
12874                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12875                         callout_reset(callout,
12876                                       lun->delay_info.datamove_delay * hz,
12877                                       ctl_datamove_timer_wakeup, io);
12878                         if (lun->delay_info.datamove_type ==
12879                             CTL_DELAY_TYPE_ONESHOT)
12880                                 lun->delay_info.datamove_delay = 0;
12881                         return;
12882                 }
12883         }
12884 #endif
12885
12886         /*
12887          * This command has been aborted.  Set the port status, so we fail
12888          * the data move.
12889          */
12890         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12891                 printf("ctl_datamove: tag 0x%04x on (%ju:%d:%ju:%d) aborted\n",
12892                        io->scsiio.tag_num,(uintmax_t)io->io_hdr.nexus.initid.id,
12893                        io->io_hdr.nexus.targ_port,
12894                        (uintmax_t)io->io_hdr.nexus.targ_target.id,
12895                        io->io_hdr.nexus.targ_lun);
12896                 io->io_hdr.port_status = 31337;
12897                 /*
12898                  * Note that the backend, in this case, will get the
12899                  * callback in its context.  In other cases it may get
12900                  * called in the frontend's interrupt thread context.
12901                  */
12902                 io->scsiio.be_move_done(io);
12903                 return;
12904         }
12905
12906         /*
12907          * If we're in XFER mode and this I/O is from the other shelf
12908          * controller, we need to send the DMA to the other side to
12909          * actually transfer the data to/from the host.  In serialize only
12910          * mode the transfer happens below CTL and ctl_datamove() is only
12911          * called on the machine that originally received the I/O.
12912          */
12913         if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
12914          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12915                 union ctl_ha_msg msg;
12916                 uint32_t sg_entries_sent;
12917                 int do_sg_copy;
12918                 int i;
12919
12920                 memset(&msg, 0, sizeof(msg));
12921                 msg.hdr.msg_type = CTL_MSG_DATAMOVE;
12922                 msg.hdr.original_sc = io->io_hdr.original_sc;
12923                 msg.hdr.serializing_sc = io;
12924                 msg.hdr.nexus = io->io_hdr.nexus;
12925                 msg.dt.flags = io->io_hdr.flags;
12926                 /*
12927                  * We convert everything into a S/G list here.  We can't
12928                  * pass by reference, only by value between controllers.
12929                  * So we can't pass a pointer to the S/G list, only as many
12930                  * S/G entries as we can fit in here.  If it's possible for
12931                  * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
12932                  * then we need to break this up into multiple transfers.
12933                  */
12934                 if (io->scsiio.kern_sg_entries == 0) {
12935                         msg.dt.kern_sg_entries = 1;
12936                         /*
12937                          * If this is in cached memory, flush the cache
12938                          * before we send the DMA request to the other
12939                          * controller.  We want to do this in either the
12940                          * read or the write case.  The read case is
12941                          * straightforward.  In the write case, we want to
12942                          * make sure nothing is in the local cache that
12943                          * could overwrite the DMAed data.
12944                          */
12945                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12946                                 /*
12947                                  * XXX KDM use bus_dmamap_sync() here.
12948                                  */
12949                         }
12950
12951                         /*
12952                          * Convert to a physical address if this is a
12953                          * virtual address.
12954                          */
12955                         if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
12956                                 msg.dt.sg_list[0].addr =
12957                                         io->scsiio.kern_data_ptr;
12958                         } else {
12959                                 /*
12960                                  * XXX KDM use busdma here!
12961                                  */
12962 #if 0
12963                                 msg.dt.sg_list[0].addr = (void *)
12964                                         vtophys(io->scsiio.kern_data_ptr);
12965 #endif
12966                         }
12967
12968                         msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
12969                         do_sg_copy = 0;
12970                 } else {
12971                         struct ctl_sg_entry *sgl;
12972
12973                         do_sg_copy = 1;
12974                         msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
12975                         sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
12976                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12977                                 /*
12978                                  * XXX KDM use bus_dmamap_sync() here.
12979                                  */
12980                         }
12981                 }
12982
12983                 msg.dt.kern_data_len = io->scsiio.kern_data_len;
12984                 msg.dt.kern_total_len = io->scsiio.kern_total_len;
12985                 msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
12986                 msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
12987                 msg.dt.sg_sequence = 0;
12988
12989                 /*
12990                  * Loop until we've sent all of the S/G entries.  On the
12991                  * other end, we'll recompose these S/G entries into one
12992                  * contiguous list before passing it to the
12993                  */
12994                 for (sg_entries_sent = 0; sg_entries_sent <
12995                      msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
12996                         msg.dt.cur_sg_entries = ctl_min((sizeof(msg.dt.sg_list)/
12997                                 sizeof(msg.dt.sg_list[0])),
12998                                 msg.dt.kern_sg_entries - sg_entries_sent);
12999
13000                         if (do_sg_copy != 0) {
13001                                 struct ctl_sg_entry *sgl;
13002                                 int j;
13003
13004                                 sgl = (struct ctl_sg_entry *)
13005                                         io->scsiio.kern_data_ptr;
13006                                 /*
13007                                  * If this is in cached memory, flush the cache
13008                                  * before we send the DMA request to the other
13009                                  * controller.  We want to do this in either
13010                                  * the * read or the write case.  The read
13011                                  * case is straightforward.  In the write
13012                                  * case, we want to make sure nothing is
13013                                  * in the local cache that could overwrite
13014                                  * the DMAed data.
13015                                  */
13016
13017                                 for (i = sg_entries_sent, j = 0;
13018                                      i < msg.dt.cur_sg_entries; i++, j++) {
13019                                         if ((io->io_hdr.flags &
13020                                              CTL_FLAG_NO_DATASYNC) == 0) {
13021                                                 /*
13022                                                  * XXX KDM use bus_dmamap_sync()
13023                                                  */
13024                                         }
13025                                         if ((io->io_hdr.flags &
13026                                              CTL_FLAG_BUS_ADDR) == 0) {
13027                                                 /*
13028                                                  * XXX KDM use busdma.
13029                                                  */
13030 #if 0
13031                                                 msg.dt.sg_list[j].addr =(void *)
13032                                                        vtophys(sgl[i].addr);
13033 #endif
13034                                         } else {
13035                                                 msg.dt.sg_list[j].addr =
13036                                                         sgl[i].addr;
13037                                         }
13038                                         msg.dt.sg_list[j].len = sgl[i].len;
13039                                 }
13040                         }
13041
13042                         sg_entries_sent += msg.dt.cur_sg_entries;
13043                         if (sg_entries_sent >= msg.dt.kern_sg_entries)
13044                                 msg.dt.sg_last = 1;
13045                         else
13046                                 msg.dt.sg_last = 0;
13047
13048                         /*
13049                          * XXX KDM drop and reacquire the lock here?
13050                          */
13051                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13052                             sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
13053                                 /*
13054                                  * XXX do something here.
13055                                  */
13056                         }
13057
13058                         msg.dt.sent_sg_entries = sg_entries_sent;
13059                 }
13060                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
13061                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER)
13062                         ctl_failover_io(io, /*have_lock*/ 0);
13063
13064         } else {
13065
13066                 /*
13067                  * Lookup the fe_datamove() function for this particular
13068                  * front end.
13069                  */
13070                 fe_datamove =
13071                     control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
13072
13073                 fe_datamove(io);
13074         }
13075 }
13076
13077 static void
13078 ctl_send_datamove_done(union ctl_io *io, int have_lock)
13079 {
13080         union ctl_ha_msg msg;
13081         int isc_status;
13082
13083         memset(&msg, 0, sizeof(msg));
13084
13085         msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
13086         msg.hdr.original_sc = io;
13087         msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
13088         msg.hdr.nexus = io->io_hdr.nexus;
13089         msg.hdr.status = io->io_hdr.status;
13090         msg.scsi.tag_num = io->scsiio.tag_num;
13091         msg.scsi.tag_type = io->scsiio.tag_type;
13092         msg.scsi.scsi_status = io->scsiio.scsi_status;
13093         memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
13094                sizeof(io->scsiio.sense_data));
13095         msg.scsi.sense_len = io->scsiio.sense_len;
13096         msg.scsi.sense_residual = io->scsiio.sense_residual;
13097         msg.scsi.fetd_status = io->io_hdr.port_status;
13098         msg.scsi.residual = io->scsiio.residual;
13099         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
13100
13101         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
13102                 ctl_failover_io(io, /*have_lock*/ have_lock);
13103                 return;
13104         }
13105
13106         isc_status = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0);
13107         if (isc_status > CTL_HA_STATUS_SUCCESS) {
13108                 /* XXX do something if this fails */
13109         }
13110
13111 }
13112
13113 /*
13114  * The DMA to the remote side is done, now we need to tell the other side
13115  * we're done so it can continue with its data movement.
13116  */
13117 static void
13118 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
13119 {
13120         union ctl_io *io;
13121
13122         io = rq->context;
13123
13124         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
13125                 printf("%s: ISC DMA write failed with error %d", __func__,
13126                        rq->ret);
13127                 ctl_set_internal_failure(&io->scsiio,
13128                                          /*sks_valid*/ 1,
13129                                          /*retry_count*/ rq->ret);
13130         }
13131
13132         ctl_dt_req_free(rq);
13133
13134         /*
13135          * In this case, we had to malloc the memory locally.  Free it.
13136          */
13137         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
13138                 int i;
13139                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13140                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
13141         }
13142         /*
13143          * The data is in local and remote memory, so now we need to send
13144          * status (good or back) back to the other side.
13145          */
13146         ctl_send_datamove_done(io, /*have_lock*/ 0);
13147 }
13148
13149 /*
13150  * We've moved the data from the host/controller into local memory.  Now we
13151  * need to push it over to the remote controller's memory.
13152  */
13153 static int
13154 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
13155 {
13156         int retval;
13157
13158         retval = 0;
13159
13160         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
13161                                           ctl_datamove_remote_write_cb);
13162
13163         return (retval);
13164 }
13165
13166 static void
13167 ctl_datamove_remote_write(union ctl_io *io)
13168 {
13169         int retval;
13170         void (*fe_datamove)(union ctl_io *io);
13171
13172         /*
13173          * - Get the data from the host/HBA into local memory.
13174          * - DMA memory from the local controller to the remote controller.
13175          * - Send status back to the remote controller.
13176          */
13177
13178         retval = ctl_datamove_remote_sgl_setup(io);
13179         if (retval != 0)
13180                 return;
13181
13182         /* Switch the pointer over so the FETD knows what to do */
13183         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
13184
13185         /*
13186          * Use a custom move done callback, since we need to send completion
13187          * back to the other controller, not to the backend on this side.
13188          */
13189         io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
13190
13191         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
13192
13193         fe_datamove(io);
13194
13195         return;
13196
13197 }
13198
13199 static int
13200 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
13201 {
13202 #if 0
13203         char str[256];
13204         char path_str[64];
13205         struct sbuf sb;
13206 #endif
13207
13208         /*
13209          * In this case, we had to malloc the memory locally.  Free it.
13210          */
13211         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
13212                 int i;
13213                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13214                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
13215         }
13216
13217 #if 0
13218         scsi_path_string(io, path_str, sizeof(path_str));
13219         sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
13220         sbuf_cat(&sb, path_str);
13221         scsi_command_string(&io->scsiio, NULL, &sb);
13222         sbuf_printf(&sb, "\n");
13223         sbuf_cat(&sb, path_str);
13224         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
13225                     io->scsiio.tag_num, io->scsiio.tag_type);
13226         sbuf_cat(&sb, path_str);
13227         sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
13228                     io->io_hdr.flags, io->io_hdr.status);
13229         sbuf_finish(&sb);
13230         printk("%s", sbuf_data(&sb));
13231 #endif
13232
13233
13234         /*
13235          * The read is done, now we need to send status (good or bad) back
13236          * to the other side.
13237          */
13238         ctl_send_datamove_done(io, /*have_lock*/ 0);
13239
13240         return (0);
13241 }
13242
13243 static void
13244 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
13245 {
13246         union ctl_io *io;
13247         void (*fe_datamove)(union ctl_io *io);
13248
13249         io = rq->context;
13250
13251         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
13252                 printf("%s: ISC DMA read failed with error %d", __func__,
13253                        rq->ret);
13254                 ctl_set_internal_failure(&io->scsiio,
13255                                          /*sks_valid*/ 1,
13256                                          /*retry_count*/ rq->ret);
13257         }
13258
13259         ctl_dt_req_free(rq);
13260
13261         /* Switch the pointer over so the FETD knows what to do */
13262         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
13263
13264         /*
13265          * Use a custom move done callback, since we need to send completion
13266          * back to the other controller, not to the backend on this side.
13267          */
13268         io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
13269
13270         /* XXX KDM add checks like the ones in ctl_datamove? */
13271
13272         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
13273
13274         fe_datamove(io);
13275 }
13276
13277 static int
13278 ctl_datamove_remote_sgl_setup(union ctl_io *io)
13279 {
13280         struct ctl_sg_entry *local_sglist, *remote_sglist;
13281         struct ctl_sg_entry *local_dma_sglist, *remote_dma_sglist;
13282         struct ctl_softc *softc;
13283         int retval;
13284         int i;
13285
13286         retval = 0;
13287         softc = control_softc;
13288
13289         local_sglist = io->io_hdr.local_sglist;
13290         local_dma_sglist = io->io_hdr.local_dma_sglist;
13291         remote_sglist = io->io_hdr.remote_sglist;
13292         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
13293
13294         if (io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) {
13295                 for (i = 0; i < io->scsiio.kern_sg_entries; i++) {
13296                         local_sglist[i].len = remote_sglist[i].len;
13297
13298                         /*
13299                          * XXX Detect the situation where the RS-level I/O
13300                          * redirector on the other side has already read the
13301                          * data off of the AOR RS on this side, and
13302                          * transferred it to remote (mirror) memory on the
13303                          * other side.  Since we already have the data in
13304                          * memory here, we just need to use it.
13305                          *
13306                          * XXX KDM this can probably be removed once we
13307                          * get the cache device code in and take the
13308                          * current AOR implementation out.
13309                          */
13310 #ifdef NEEDTOPORT
13311                         if ((remote_sglist[i].addr >=
13312                              (void *)vtophys(softc->mirr->addr))
13313                          && (remote_sglist[i].addr <
13314                              ((void *)vtophys(softc->mirr->addr) +
13315                              CacheMirrorOffset))) {
13316                                 local_sglist[i].addr = remote_sglist[i].addr -
13317                                         CacheMirrorOffset;
13318                                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13319                                      CTL_FLAG_DATA_IN)
13320                                         io->io_hdr.flags |= CTL_FLAG_REDIR_DONE;
13321                         } else {
13322                                 local_sglist[i].addr = remote_sglist[i].addr +
13323                                         CacheMirrorOffset;
13324                         }
13325 #endif
13326 #if 0
13327                         printf("%s: local %p, remote %p, len %d\n",
13328                                __func__, local_sglist[i].addr,
13329                                remote_sglist[i].addr, local_sglist[i].len);
13330 #endif
13331                 }
13332         } else {
13333                 uint32_t len_to_go;
13334
13335                 /*
13336                  * In this case, we don't have automatically allocated
13337                  * memory for this I/O on this controller.  This typically
13338                  * happens with internal CTL I/O -- e.g. inquiry, mode
13339                  * sense, etc.  Anything coming from RAIDCore will have
13340                  * a mirror area available.
13341                  */
13342                 len_to_go = io->scsiio.kern_data_len;
13343
13344                 /*
13345                  * Clear the no datasync flag, we have to use malloced
13346                  * buffers.
13347                  */
13348                 io->io_hdr.flags &= ~CTL_FLAG_NO_DATASYNC;
13349
13350                 /*
13351                  * The difficult thing here is that the size of the various
13352                  * S/G segments may be different than the size from the
13353                  * remote controller.  That'll make it harder when DMAing
13354                  * the data back to the other side.
13355                  */
13356                 for (i = 0; (i < sizeof(io->io_hdr.remote_sglist) /
13357                      sizeof(io->io_hdr.remote_sglist[0])) &&
13358                      (len_to_go > 0); i++) {
13359                         local_sglist[i].len = ctl_min(len_to_go, 131072);
13360                         CTL_SIZE_8B(local_dma_sglist[i].len,
13361                                     local_sglist[i].len);
13362                         local_sglist[i].addr =
13363                                 malloc(local_dma_sglist[i].len, M_CTL,M_WAITOK);
13364
13365                         local_dma_sglist[i].addr = local_sglist[i].addr;
13366
13367                         if (local_sglist[i].addr == NULL) {
13368                                 int j;
13369
13370                                 printf("malloc failed for %zd bytes!",
13371                                        local_dma_sglist[i].len);
13372                                 for (j = 0; j < i; j++) {
13373                                         free(local_sglist[j].addr, M_CTL);
13374                                 }
13375                                 ctl_set_internal_failure(&io->scsiio,
13376                                                          /*sks_valid*/ 1,
13377                                                          /*retry_count*/ 4857);
13378                                 retval = 1;
13379                                 goto bailout_error;
13380
13381                         }
13382                         /* XXX KDM do we need a sync here? */
13383
13384                         len_to_go -= local_sglist[i].len;
13385                 }
13386                 /*
13387                  * Reset the number of S/G entries accordingly.  The
13388                  * original number of S/G entries is available in
13389                  * rem_sg_entries.
13390                  */
13391                 io->scsiio.kern_sg_entries = i;
13392
13393 #if 0
13394                 printf("%s: kern_sg_entries = %d\n", __func__,
13395                        io->scsiio.kern_sg_entries);
13396                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13397                         printf("%s: sg[%d] = %p, %d (DMA: %d)\n", __func__, i,
13398                                local_sglist[i].addr, local_sglist[i].len,
13399                                local_dma_sglist[i].len);
13400 #endif
13401         }
13402
13403
13404         return (retval);
13405
13406 bailout_error:
13407
13408         ctl_send_datamove_done(io, /*have_lock*/ 0);
13409
13410         return (retval);
13411 }
13412
13413 static int
13414 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
13415                          ctl_ha_dt_cb callback)
13416 {
13417         struct ctl_ha_dt_req *rq;
13418         struct ctl_sg_entry *remote_sglist, *local_sglist;
13419         struct ctl_sg_entry *remote_dma_sglist, *local_dma_sglist;
13420         uint32_t local_used, remote_used, total_used;
13421         int retval;
13422         int i, j;
13423
13424         retval = 0;
13425
13426         rq = ctl_dt_req_alloc();
13427
13428         /*
13429          * If we failed to allocate the request, and if the DMA didn't fail
13430          * anyway, set busy status.  This is just a resource allocation
13431          * failure.
13432          */
13433         if ((rq == NULL)
13434          && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
13435                 ctl_set_busy(&io->scsiio);
13436
13437         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
13438
13439                 if (rq != NULL)
13440                         ctl_dt_req_free(rq);
13441
13442                 /*
13443                  * The data move failed.  We need to return status back
13444                  * to the other controller.  No point in trying to DMA
13445                  * data to the remote controller.
13446                  */
13447
13448                 ctl_send_datamove_done(io, /*have_lock*/ 0);
13449
13450                 retval = 1;
13451
13452                 goto bailout;
13453         }
13454
13455         local_sglist = io->io_hdr.local_sglist;
13456         local_dma_sglist = io->io_hdr.local_dma_sglist;
13457         remote_sglist = io->io_hdr.remote_sglist;
13458         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
13459         local_used = 0;
13460         remote_used = 0;
13461         total_used = 0;
13462
13463         if (io->io_hdr.flags & CTL_FLAG_REDIR_DONE) {
13464                 rq->ret = CTL_HA_STATUS_SUCCESS;
13465                 rq->context = io;
13466                 callback(rq);
13467                 goto bailout;
13468         }
13469
13470         /*
13471          * Pull/push the data over the wire from/to the other controller.
13472          * This takes into account the possibility that the local and
13473          * remote sglists may not be identical in terms of the size of
13474          * the elements and the number of elements.
13475          *
13476          * One fundamental assumption here is that the length allocated for
13477          * both the local and remote sglists is identical.  Otherwise, we've
13478          * essentially got a coding error of some sort.
13479          */
13480         for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
13481                 int isc_ret;
13482                 uint32_t cur_len, dma_length;
13483                 uint8_t *tmp_ptr;
13484
13485                 rq->id = CTL_HA_DATA_CTL;
13486                 rq->command = command;
13487                 rq->context = io;
13488
13489                 /*
13490                  * Both pointers should be aligned.  But it is possible
13491                  * that the allocation length is not.  They should both
13492                  * also have enough slack left over at the end, though,
13493                  * to round up to the next 8 byte boundary.
13494                  */
13495                 cur_len = ctl_min(local_sglist[i].len - local_used,
13496                                   remote_sglist[j].len - remote_used);
13497
13498                 /*
13499                  * In this case, we have a size issue and need to decrease
13500                  * the size, except in the case where we actually have less
13501                  * than 8 bytes left.  In that case, we need to increase
13502                  * the DMA length to get the last bit.
13503                  */
13504                 if ((cur_len & 0x7) != 0) {
13505                         if (cur_len > 0x7) {
13506                                 cur_len = cur_len - (cur_len & 0x7);
13507                                 dma_length = cur_len;
13508                         } else {
13509                                 CTL_SIZE_8B(dma_length, cur_len);
13510                         }
13511
13512                 } else
13513                         dma_length = cur_len;
13514
13515                 /*
13516                  * If we had to allocate memory for this I/O, instead of using
13517                  * the non-cached mirror memory, we'll need to flush the cache
13518                  * before trying to DMA to the other controller.
13519                  *
13520                  * We could end up doing this multiple times for the same
13521                  * segment if we have a larger local segment than remote
13522                  * segment.  That shouldn't be an issue.
13523                  */
13524                 if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
13525                         /*
13526                          * XXX KDM use bus_dmamap_sync() here.
13527                          */
13528                 }
13529
13530                 rq->size = dma_length;
13531
13532                 tmp_ptr = (uint8_t *)local_sglist[i].addr;
13533                 tmp_ptr += local_used;
13534
13535                 /* Use physical addresses when talking to ISC hardware */
13536                 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
13537                         /* XXX KDM use busdma */
13538 #if 0
13539                         rq->local = vtophys(tmp_ptr);
13540 #endif
13541                 } else
13542                         rq->local = tmp_ptr;
13543
13544                 tmp_ptr = (uint8_t *)remote_sglist[j].addr;
13545                 tmp_ptr += remote_used;
13546                 rq->remote = tmp_ptr;
13547
13548                 rq->callback = NULL;
13549
13550                 local_used += cur_len;
13551                 if (local_used >= local_sglist[i].len) {
13552                         i++;
13553                         local_used = 0;
13554                 }
13555
13556                 remote_used += cur_len;
13557                 if (remote_used >= remote_sglist[j].len) {
13558                         j++;
13559                         remote_used = 0;
13560                 }
13561                 total_used += cur_len;
13562
13563                 if (total_used >= io->scsiio.kern_data_len)
13564                         rq->callback = callback;
13565
13566                 if ((rq->size & 0x7) != 0) {
13567                         printf("%s: warning: size %d is not on 8b boundary\n",
13568                                __func__, rq->size);
13569                 }
13570                 if (((uintptr_t)rq->local & 0x7) != 0) {
13571                         printf("%s: warning: local %p not on 8b boundary\n",
13572                                __func__, rq->local);
13573                 }
13574                 if (((uintptr_t)rq->remote & 0x7) != 0) {
13575                         printf("%s: warning: remote %p not on 8b boundary\n",
13576                                __func__, rq->local);
13577                 }
13578 #if 0
13579                 printf("%s: %s: local %#x remote %#x size %d\n", __func__,
13580                        (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
13581                        rq->local, rq->remote, rq->size);
13582 #endif
13583
13584                 isc_ret = ctl_dt_single(rq);
13585                 if (isc_ret == CTL_HA_STATUS_WAIT)
13586                         continue;
13587
13588                 if (isc_ret == CTL_HA_STATUS_DISCONNECT) {
13589                         rq->ret = CTL_HA_STATUS_SUCCESS;
13590                 } else {
13591                         rq->ret = isc_ret;
13592                 }
13593                 callback(rq);
13594                 goto bailout;
13595         }
13596
13597 bailout:
13598         return (retval);
13599
13600 }
13601
13602 static void
13603 ctl_datamove_remote_read(union ctl_io *io)
13604 {
13605         int retval;
13606         int i;
13607
13608         /*
13609          * This will send an error to the other controller in the case of a
13610          * failure.
13611          */
13612         retval = ctl_datamove_remote_sgl_setup(io);
13613         if (retval != 0)
13614                 return;
13615
13616         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
13617                                           ctl_datamove_remote_read_cb);
13618         if ((retval != 0)
13619          && ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0)) {
13620                 /*
13621                  * Make sure we free memory if there was an error..  The
13622                  * ctl_datamove_remote_xfer() function will send the
13623                  * datamove done message, or call the callback with an
13624                  * error if there is a problem.
13625                  */
13626                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13627                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
13628         }
13629
13630         return;
13631 }
13632
13633 /*
13634  * Process a datamove request from the other controller.  This is used for
13635  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
13636  * first.  Once that is complete, the data gets DMAed into the remote
13637  * controller's memory.  For reads, we DMA from the remote controller's
13638  * memory into our memory first, and then move it out to the FETD.
13639  */
13640 static void
13641 ctl_datamove_remote(union ctl_io *io)
13642 {
13643         struct ctl_softc *softc;
13644
13645         softc = control_softc;
13646
13647         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
13648
13649         /*
13650          * Note that we look for an aborted I/O here, but don't do some of
13651          * the other checks that ctl_datamove() normally does.
13652          * We don't need to run the datamove delay code, since that should
13653          * have been done if need be on the other controller.
13654          */
13655         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
13656                 printf("%s: tag 0x%04x on (%d:%d:%d:%d) aborted\n", __func__,
13657                        io->scsiio.tag_num, io->io_hdr.nexus.initid.id,
13658                        io->io_hdr.nexus.targ_port,
13659                        io->io_hdr.nexus.targ_target.id,
13660                        io->io_hdr.nexus.targ_lun);
13661                 io->io_hdr.port_status = 31338;
13662                 ctl_send_datamove_done(io, /*have_lock*/ 0);
13663                 return;
13664         }
13665
13666         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT) {
13667                 ctl_datamove_remote_write(io);
13668         } else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN){
13669                 ctl_datamove_remote_read(io);
13670         } else {
13671                 union ctl_ha_msg msg;
13672                 struct scsi_sense_data *sense;
13673                 uint8_t sks[3];
13674                 int retry_count;
13675
13676                 memset(&msg, 0, sizeof(msg));
13677
13678                 msg.hdr.msg_type = CTL_MSG_BAD_JUJU;
13679                 msg.hdr.status = CTL_SCSI_ERROR;
13680                 msg.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
13681
13682                 retry_count = 4243;
13683
13684                 sense = &msg.scsi.sense_data;
13685                 sks[0] = SSD_SCS_VALID;
13686                 sks[1] = (retry_count >> 8) & 0xff;
13687                 sks[2] = retry_count & 0xff;
13688
13689                 /* "Internal target failure" */
13690                 scsi_set_sense_data(sense,
13691                                     /*sense_format*/ SSD_TYPE_NONE,
13692                                     /*current_error*/ 1,
13693                                     /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
13694                                     /*asc*/ 0x44,
13695                                     /*ascq*/ 0x00,
13696                                     /*type*/ SSD_ELEM_SKS,
13697                                     /*size*/ sizeof(sks),
13698                                     /*data*/ sks,
13699                                     SSD_ELEM_NONE);
13700
13701                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
13702                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
13703                         ctl_failover_io(io, /*have_lock*/ 1);
13704                         return;
13705                 }
13706
13707                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0) >
13708                     CTL_HA_STATUS_SUCCESS) {
13709                         /* XXX KDM what to do if this fails? */
13710                 }
13711                 return;
13712         }
13713
13714 }
13715
13716 static int
13717 ctl_process_done(union ctl_io *io)
13718 {
13719         struct ctl_lun *lun;
13720         struct ctl_softc *ctl_softc;
13721         void (*fe_done)(union ctl_io *io);
13722         uint32_t targ_port = ctl_port_idx(io->io_hdr.nexus.targ_port);
13723
13724         CTL_DEBUG_PRINT(("ctl_process_done\n"));
13725
13726         fe_done =
13727             control_softc->ctl_ports[targ_port]->fe_done;
13728
13729 #ifdef CTL_TIME_IO
13730         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
13731                 char str[256];
13732                 char path_str[64];
13733                 struct sbuf sb;
13734
13735                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
13736                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
13737
13738                 sbuf_cat(&sb, path_str);
13739                 switch (io->io_hdr.io_type) {
13740                 case CTL_IO_SCSI:
13741                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
13742                         sbuf_printf(&sb, "\n");
13743                         sbuf_cat(&sb, path_str);
13744                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
13745                                     io->scsiio.tag_num, io->scsiio.tag_type);
13746                         break;
13747                 case CTL_IO_TASK:
13748                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
13749                                     "Tag Type: %d\n", io->taskio.task_action,
13750                                     io->taskio.tag_num, io->taskio.tag_type);
13751                         break;
13752                 default:
13753                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
13754                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
13755                         break;
13756                 }
13757                 sbuf_cat(&sb, path_str);
13758                 sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
13759                             (intmax_t)time_uptime - io->io_hdr.start_time);
13760                 sbuf_finish(&sb);
13761                 printf("%s", sbuf_data(&sb));
13762         }
13763 #endif /* CTL_TIME_IO */
13764
13765         switch (io->io_hdr.io_type) {
13766         case CTL_IO_SCSI:
13767                 break;
13768         case CTL_IO_TASK:
13769                 if (bootverbose || verbose > 0)
13770                         ctl_io_error_print(io, NULL);
13771                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
13772                         ctl_free_io(io);
13773                 else
13774                         fe_done(io);
13775                 return (CTL_RETVAL_COMPLETE);
13776                 break;
13777         default:
13778                 printf("ctl_process_done: invalid io type %d\n",
13779                        io->io_hdr.io_type);
13780                 panic("ctl_process_done: invalid io type %d\n",
13781                       io->io_hdr.io_type);
13782                 break; /* NOTREACHED */
13783         }
13784
13785         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13786         if (lun == NULL) {
13787                 CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
13788                                  io->io_hdr.nexus.targ_mapped_lun));
13789                 fe_done(io);
13790                 goto bailout;
13791         }
13792         ctl_softc = lun->ctl_softc;
13793
13794         mtx_lock(&lun->lun_lock);
13795
13796         /*
13797          * Check to see if we have any errors to inject here.  We only
13798          * inject errors for commands that don't already have errors set.
13799          */
13800         if ((STAILQ_FIRST(&lun->error_list) != NULL)
13801          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))
13802                 ctl_inject_error(lun, io);
13803
13804         /*
13805          * XXX KDM how do we treat commands that aren't completed
13806          * successfully?
13807          *
13808          * XXX KDM should we also track I/O latency?
13809          */
13810         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
13811             io->io_hdr.io_type == CTL_IO_SCSI) {
13812 #ifdef CTL_TIME_IO
13813                 struct bintime cur_bt;
13814 #endif
13815                 int type;
13816
13817                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13818                     CTL_FLAG_DATA_IN)
13819                         type = CTL_STATS_READ;
13820                 else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13821                     CTL_FLAG_DATA_OUT)
13822                         type = CTL_STATS_WRITE;
13823                 else
13824                         type = CTL_STATS_NO_IO;
13825
13826                 lun->stats.ports[targ_port].bytes[type] +=
13827                     io->scsiio.kern_total_len;
13828                 lun->stats.ports[targ_port].operations[type]++;
13829 #ifdef CTL_TIME_IO
13830                 bintime_add(&lun->stats.ports[targ_port].dma_time[type],
13831                    &io->io_hdr.dma_bt);
13832                 lun->stats.ports[targ_port].num_dmas[type] +=
13833                     io->io_hdr.num_dmas;
13834                 getbintime(&cur_bt);
13835                 bintime_sub(&cur_bt, &io->io_hdr.start_bt);
13836                 bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
13837 #endif
13838         }
13839
13840         /*
13841          * Remove this from the OOA queue.
13842          */
13843         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
13844
13845         /*
13846          * Run through the blocked queue on this LUN and see if anything
13847          * has become unblocked, now that this transaction is done.
13848          */
13849         ctl_check_blocked(lun);
13850
13851         /*
13852          * If the LUN has been invalidated, free it if there is nothing
13853          * left on its OOA queue.
13854          */
13855         if ((lun->flags & CTL_LUN_INVALID)
13856          && TAILQ_EMPTY(&lun->ooa_queue)) {
13857                 mtx_unlock(&lun->lun_lock);
13858                 mtx_lock(&ctl_softc->ctl_lock);
13859                 ctl_free_lun(lun);
13860                 mtx_unlock(&ctl_softc->ctl_lock);
13861         } else
13862                 mtx_unlock(&lun->lun_lock);
13863
13864         /*
13865          * If this command has been aborted, make sure we set the status
13866          * properly.  The FETD is responsible for freeing the I/O and doing
13867          * whatever it needs to do to clean up its state.
13868          */
13869         if (io->io_hdr.flags & CTL_FLAG_ABORT)
13870                 ctl_set_task_aborted(&io->scsiio);
13871
13872         /*
13873          * We print out status for every task management command.  For SCSI
13874          * commands, we filter out any unit attention errors; they happen
13875          * on every boot, and would clutter up the log.  Note:  task
13876          * management commands aren't printed here, they are printed above,
13877          * since they should never even make it down here.
13878          */
13879         switch (io->io_hdr.io_type) {
13880         case CTL_IO_SCSI: {
13881                 int error_code, sense_key, asc, ascq;
13882
13883                 sense_key = 0;
13884
13885                 if (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR)
13886                  && (io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)) {
13887                         /*
13888                          * Since this is just for printing, no need to
13889                          * show errors here.
13890                          */
13891                         scsi_extract_sense_len(&io->scsiio.sense_data,
13892                                                io->scsiio.sense_len,
13893                                                &error_code,
13894                                                &sense_key,
13895                                                &asc,
13896                                                &ascq,
13897                                                /*show_errors*/ 0);
13898                 }
13899
13900                 if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
13901                  && (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SCSI_ERROR)
13902                   || (io->scsiio.scsi_status != SCSI_STATUS_CHECK_COND)
13903                   || (sense_key != SSD_KEY_UNIT_ATTENTION))) {
13904
13905                         if ((time_uptime - ctl_softc->last_print_jiffies) <= 0){
13906                                 ctl_softc->skipped_prints++;
13907                         } else {
13908                                 uint32_t skipped_prints;
13909
13910                                 skipped_prints = ctl_softc->skipped_prints;
13911
13912                                 ctl_softc->skipped_prints = 0;
13913                                 ctl_softc->last_print_jiffies = time_uptime;
13914
13915                                 if (skipped_prints > 0) {
13916 #ifdef NEEDTOPORT
13917                                         csevent_log(CSC_CTL | CSC_SHELF_SW |
13918                                             CTL_ERROR_REPORT,
13919                                             csevent_LogType_Trace,
13920                                             csevent_Severity_Information,
13921                                             csevent_AlertLevel_Green,
13922                                             csevent_FRU_Firmware,
13923                                             csevent_FRU_Unknown,
13924                                             "High CTL error volume, %d prints "
13925                                             "skipped", skipped_prints);
13926 #endif
13927                                 }
13928                                 if (bootverbose || verbose > 0)
13929                                         ctl_io_error_print(io, NULL);
13930                         }
13931                 }
13932                 break;
13933         }
13934         case CTL_IO_TASK:
13935                 if (bootverbose || verbose > 0)
13936                         ctl_io_error_print(io, NULL);
13937                 break;
13938         default:
13939                 break;
13940         }
13941
13942         /*
13943          * Tell the FETD or the other shelf controller we're done with this
13944          * command.  Note that only SCSI commands get to this point.  Task
13945          * management commands are completed above.
13946          *
13947          * We only send status to the other controller if we're in XFER
13948          * mode.  In SER_ONLY mode, the I/O is done on the controller that
13949          * received the I/O (from CTL's perspective), and so the status is
13950          * generated there.
13951          *
13952          * XXX KDM if we hold the lock here, we could cause a deadlock
13953          * if the frontend comes back in in this context to queue
13954          * something.
13955          */
13956         if ((ctl_softc->ha_mode == CTL_HA_MODE_XFER)
13957          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
13958                 union ctl_ha_msg msg;
13959
13960                 memset(&msg, 0, sizeof(msg));
13961                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
13962                 msg.hdr.original_sc = io->io_hdr.original_sc;
13963                 msg.hdr.nexus = io->io_hdr.nexus;
13964                 msg.hdr.status = io->io_hdr.status;
13965                 msg.scsi.scsi_status = io->scsiio.scsi_status;
13966                 msg.scsi.tag_num = io->scsiio.tag_num;
13967                 msg.scsi.tag_type = io->scsiio.tag_type;
13968                 msg.scsi.sense_len = io->scsiio.sense_len;
13969                 msg.scsi.sense_residual = io->scsiio.sense_residual;
13970                 msg.scsi.residual = io->scsiio.residual;
13971                 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
13972                        sizeof(io->scsiio.sense_data));
13973                 /*
13974                  * We copy this whether or not this is an I/O-related
13975                  * command.  Otherwise, we'd have to go and check to see
13976                  * whether it's a read/write command, and it really isn't
13977                  * worth it.
13978                  */
13979                 memcpy(&msg.scsi.lbalen,
13980                        &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
13981                        sizeof(msg.scsi.lbalen));
13982
13983                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13984                                 sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
13985                         /* XXX do something here */
13986                 }
13987
13988                 ctl_free_io(io);
13989         } else
13990                 fe_done(io);
13991
13992 bailout:
13993
13994         return (CTL_RETVAL_COMPLETE);
13995 }
13996
13997 #ifdef CTL_WITH_CA
13998 /*
13999  * Front end should call this if it doesn't do autosense.  When the request
14000  * sense comes back in from the initiator, we'll dequeue this and send it.
14001  */
14002 int
14003 ctl_queue_sense(union ctl_io *io)
14004 {
14005         struct ctl_lun *lun;
14006         struct ctl_softc *ctl_softc;
14007         uint32_t initidx, targ_lun;
14008
14009         ctl_softc = control_softc;
14010
14011         CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
14012
14013         /*
14014          * LUN lookup will likely move to the ctl_work_thread() once we
14015          * have our new queueing infrastructure (that doesn't put things on
14016          * a per-LUN queue initially).  That is so that we can handle
14017          * things like an INQUIRY to a LUN that we don't have enabled.  We
14018          * can't deal with that right now.
14019          */
14020         mtx_lock(&ctl_softc->ctl_lock);
14021
14022         /*
14023          * If we don't have a LUN for this, just toss the sense
14024          * information.
14025          */
14026         targ_lun = io->io_hdr.nexus.targ_lun;
14027         targ_lun = ctl_map_lun(io->io_hdr.nexus.targ_port, targ_lun);
14028         if ((targ_lun < CTL_MAX_LUNS)
14029          && (ctl_softc->ctl_luns[targ_lun] != NULL))
14030                 lun = ctl_softc->ctl_luns[targ_lun];
14031         else
14032                 goto bailout;
14033
14034         initidx = ctl_get_initindex(&io->io_hdr.nexus);
14035
14036         mtx_lock(&lun->lun_lock);
14037         /*
14038          * Already have CA set for this LUN...toss the sense information.
14039          */
14040         if (ctl_is_set(lun->have_ca, initidx)) {
14041                 mtx_unlock(&lun->lun_lock);
14042                 goto bailout;
14043         }
14044
14045         memcpy(&lun->pending_sense[initidx], &io->scsiio.sense_data,
14046                ctl_min(sizeof(lun->pending_sense[initidx]),
14047                sizeof(io->scsiio.sense_data)));
14048         ctl_set_mask(lun->have_ca, initidx);
14049         mtx_unlock(&lun->lun_lock);
14050
14051 bailout:
14052         mtx_unlock(&ctl_softc->ctl_lock);
14053
14054         ctl_free_io(io);
14055
14056         return (CTL_RETVAL_COMPLETE);
14057 }
14058 #endif
14059
14060 /*
14061  * Primary command inlet from frontend ports.  All SCSI and task I/O
14062  * requests must go through this function.
14063  */
14064 int
14065 ctl_queue(union ctl_io *io)
14066 {
14067         struct ctl_softc *ctl_softc;
14068
14069         CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
14070
14071         ctl_softc = control_softc;
14072
14073 #ifdef CTL_TIME_IO
14074         io->io_hdr.start_time = time_uptime;
14075         getbintime(&io->io_hdr.start_bt);
14076 #endif /* CTL_TIME_IO */
14077
14078         /* Map FE-specific LUN ID into global one. */
14079         io->io_hdr.nexus.targ_mapped_lun =
14080             ctl_map_lun(io->io_hdr.nexus.targ_port, io->io_hdr.nexus.targ_lun);
14081
14082         switch (io->io_hdr.io_type) {
14083         case CTL_IO_SCSI:
14084         case CTL_IO_TASK:
14085                 ctl_enqueue_incoming(io);
14086                 break;
14087         default:
14088                 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
14089                 return (EINVAL);
14090         }
14091
14092         return (CTL_RETVAL_COMPLETE);
14093 }
14094
14095 #ifdef CTL_IO_DELAY
14096 static void
14097 ctl_done_timer_wakeup(void *arg)
14098 {
14099         union ctl_io *io;
14100
14101         io = (union ctl_io *)arg;
14102         ctl_done(io);
14103 }
14104 #endif /* CTL_IO_DELAY */
14105
14106 void
14107 ctl_done(union ctl_io *io)
14108 {
14109         struct ctl_softc *ctl_softc;
14110
14111         ctl_softc = control_softc;
14112
14113         /*
14114          * Enable this to catch duplicate completion issues.
14115          */
14116 #if 0
14117         if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
14118                 printf("%s: type %d msg %d cdb %x iptl: "
14119                        "%d:%d:%d:%d tag 0x%04x "
14120                        "flag %#x status %x\n",
14121                         __func__,
14122                         io->io_hdr.io_type,
14123                         io->io_hdr.msg_type,
14124                         io->scsiio.cdb[0],
14125                         io->io_hdr.nexus.initid.id,
14126                         io->io_hdr.nexus.targ_port,
14127                         io->io_hdr.nexus.targ_target.id,
14128                         io->io_hdr.nexus.targ_lun,
14129                         (io->io_hdr.io_type ==
14130                         CTL_IO_TASK) ?
14131                         io->taskio.tag_num :
14132                         io->scsiio.tag_num,
14133                         io->io_hdr.flags,
14134                         io->io_hdr.status);
14135         } else
14136                 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
14137 #endif
14138
14139         /*
14140          * This is an internal copy of an I/O, and should not go through
14141          * the normal done processing logic.
14142          */
14143         if (io->io_hdr.flags & CTL_FLAG_INT_COPY)
14144                 return;
14145
14146         /*
14147          * We need to send a msg to the serializing shelf to finish the IO
14148          * as well.  We don't send a finish message to the other shelf if
14149          * this is a task management command.  Task management commands
14150          * aren't serialized in the OOA queue, but rather just executed on
14151          * both shelf controllers for commands that originated on that
14152          * controller.
14153          */
14154         if ((io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)
14155          && (io->io_hdr.io_type != CTL_IO_TASK)) {
14156                 union ctl_ha_msg msg_io;
14157
14158                 msg_io.hdr.msg_type = CTL_MSG_FINISH_IO;
14159                 msg_io.hdr.serializing_sc = io->io_hdr.serializing_sc;
14160                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_io,
14161                     sizeof(msg_io), 0 ) != CTL_HA_STATUS_SUCCESS) {
14162                 }
14163                 /* continue on to finish IO */
14164         }
14165 #ifdef CTL_IO_DELAY
14166         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
14167                 struct ctl_lun *lun;
14168
14169                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
14170
14171                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
14172         } else {
14173                 struct ctl_lun *lun;
14174
14175                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
14176
14177                 if ((lun != NULL)
14178                  && (lun->delay_info.done_delay > 0)) {
14179                         struct callout *callout;
14180
14181                         callout = (struct callout *)&io->io_hdr.timer_bytes;
14182                         callout_init(callout, /*mpsafe*/ 1);
14183                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
14184                         callout_reset(callout,
14185                                       lun->delay_info.done_delay * hz,
14186                                       ctl_done_timer_wakeup, io);
14187                         if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
14188                                 lun->delay_info.done_delay = 0;
14189                         return;
14190                 }
14191         }
14192 #endif /* CTL_IO_DELAY */
14193
14194         ctl_enqueue_done(io);
14195 }
14196
14197 int
14198 ctl_isc(struct ctl_scsiio *ctsio)
14199 {
14200         struct ctl_lun *lun;
14201         int retval;
14202
14203         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
14204
14205         CTL_DEBUG_PRINT(("ctl_isc: command: %02x\n", ctsio->cdb[0]));
14206
14207         CTL_DEBUG_PRINT(("ctl_isc: calling data_submit()\n"));
14208
14209         retval = lun->backend->data_submit((union ctl_io *)ctsio);
14210
14211         return (retval);
14212 }
14213
14214
14215 static void
14216 ctl_work_thread(void *arg)
14217 {
14218         struct ctl_thread *thr = (struct ctl_thread *)arg;
14219         struct ctl_softc *softc = thr->ctl_softc;
14220         union ctl_io *io;
14221         int retval;
14222
14223         CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
14224
14225         for (;;) {
14226                 retval = 0;
14227
14228                 /*
14229                  * We handle the queues in this order:
14230                  * - ISC
14231                  * - done queue (to free up resources, unblock other commands)
14232                  * - RtR queue
14233                  * - incoming queue
14234                  *
14235                  * If those queues are empty, we break out of the loop and
14236                  * go to sleep.
14237                  */
14238                 mtx_lock(&thr->queue_lock);
14239                 io = (union ctl_io *)STAILQ_FIRST(&thr->isc_queue);
14240                 if (io != NULL) {
14241                         STAILQ_REMOVE_HEAD(&thr->isc_queue, links);
14242                         mtx_unlock(&thr->queue_lock);
14243                         ctl_handle_isc(io);
14244                         continue;
14245                 }
14246                 io = (union ctl_io *)STAILQ_FIRST(&thr->done_queue);
14247                 if (io != NULL) {
14248                         STAILQ_REMOVE_HEAD(&thr->done_queue, links);
14249                         /* clear any blocked commands, call fe_done */
14250                         mtx_unlock(&thr->queue_lock);
14251                         retval = ctl_process_done(io);
14252                         continue;
14253                 }
14254                 io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
14255                 if (io != NULL) {
14256                         STAILQ_REMOVE_HEAD(&thr->incoming_queue, links);
14257                         mtx_unlock(&thr->queue_lock);
14258                         if (io->io_hdr.io_type == CTL_IO_TASK)
14259                                 ctl_run_task(io);
14260                         else
14261                                 ctl_scsiio_precheck(softc, &io->scsiio);
14262                         continue;
14263                 }
14264                 if (!ctl_pause_rtr) {
14265                         io = (union ctl_io *)STAILQ_FIRST(&thr->rtr_queue);
14266                         if (io != NULL) {
14267                                 STAILQ_REMOVE_HEAD(&thr->rtr_queue, links);
14268                                 mtx_unlock(&thr->queue_lock);
14269                                 retval = ctl_scsiio(&io->scsiio);
14270                                 if (retval != CTL_RETVAL_COMPLETE)
14271                                         CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
14272                                 continue;
14273                         }
14274                 }
14275
14276                 /* Sleep until we have something to do. */
14277                 mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0);
14278         }
14279 }
14280
14281 static void
14282 ctl_lun_thread(void *arg)
14283 {
14284         struct ctl_softc *softc = (struct ctl_softc *)arg;
14285         struct ctl_be_lun *be_lun;
14286         int retval;
14287
14288         CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
14289
14290         for (;;) {
14291                 retval = 0;
14292                 mtx_lock(&softc->ctl_lock);
14293                 be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
14294                 if (be_lun != NULL) {
14295                         STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
14296                         mtx_unlock(&softc->ctl_lock);
14297                         ctl_create_lun(be_lun);
14298                         continue;
14299                 }
14300
14301                 /* Sleep until we have something to do. */
14302                 mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock,
14303                     PDROP | PRIBIO, "-", 0);
14304         }
14305 }
14306
14307 static void
14308 ctl_enqueue_incoming(union ctl_io *io)
14309 {
14310         struct ctl_softc *softc = control_softc;
14311         struct ctl_thread *thr;
14312         u_int idx;
14313
14314         idx = (io->io_hdr.nexus.targ_port * 127 +
14315                io->io_hdr.nexus.initid.id) % worker_threads;
14316         thr = &softc->threads[idx];
14317         mtx_lock(&thr->queue_lock);
14318         STAILQ_INSERT_TAIL(&thr->incoming_queue, &io->io_hdr, links);
14319         mtx_unlock(&thr->queue_lock);
14320         wakeup(thr);
14321 }
14322
14323 static void
14324 ctl_enqueue_rtr(union ctl_io *io)
14325 {
14326         struct ctl_softc *softc = control_softc;
14327         struct ctl_thread *thr;
14328
14329         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
14330         mtx_lock(&thr->queue_lock);
14331         STAILQ_INSERT_TAIL(&thr->rtr_queue, &io->io_hdr, links);
14332         mtx_unlock(&thr->queue_lock);
14333         wakeup(thr);
14334 }
14335
14336 static void
14337 ctl_enqueue_done(union ctl_io *io)
14338 {
14339         struct ctl_softc *softc = control_softc;
14340         struct ctl_thread *thr;
14341
14342         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
14343         mtx_lock(&thr->queue_lock);
14344         STAILQ_INSERT_TAIL(&thr->done_queue, &io->io_hdr, links);
14345         mtx_unlock(&thr->queue_lock);
14346         wakeup(thr);
14347 }
14348
14349 static void
14350 ctl_enqueue_isc(union ctl_io *io)
14351 {
14352         struct ctl_softc *softc = control_softc;
14353         struct ctl_thread *thr;
14354
14355         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
14356         mtx_lock(&thr->queue_lock);
14357         STAILQ_INSERT_TAIL(&thr->isc_queue, &io->io_hdr, links);
14358         mtx_unlock(&thr->queue_lock);
14359         wakeup(thr);
14360 }
14361
14362 /* Initialization and failover */
14363
14364 void
14365 ctl_init_isc_msg(void)
14366 {
14367         printf("CTL: Still calling this thing\n");
14368 }
14369
14370 /*
14371  * Init component
14372  *      Initializes component into configuration defined by bootMode
14373  *      (see hasc-sv.c)
14374  *      returns hasc_Status:
14375  *              OK
14376  *              ERROR - fatal error
14377  */
14378 static ctl_ha_comp_status
14379 ctl_isc_init(struct ctl_ha_component *c)
14380 {
14381         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
14382
14383         c->status = ret;
14384         return ret;
14385 }
14386
14387 /* Start component
14388  *      Starts component in state requested. If component starts successfully,
14389  *      it must set its own state to the requestrd state
14390  *      When requested state is HASC_STATE_HA, the component may refine it
14391  *      by adding _SLAVE or _MASTER flags.
14392  *      Currently allowed state transitions are:
14393  *      UNKNOWN->HA             - initial startup
14394  *      UNKNOWN->SINGLE - initial startup when no parter detected
14395  *      HA->SINGLE              - failover
14396  * returns ctl_ha_comp_status:
14397  *              OK      - component successfully started in requested state
14398  *              FAILED  - could not start the requested state, failover may
14399  *                        be possible
14400  *              ERROR   - fatal error detected, no future startup possible
14401  */
14402 static ctl_ha_comp_status
14403 ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
14404 {
14405         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
14406
14407         printf("%s: go\n", __func__);
14408
14409         // UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
14410         if (c->state == CTL_HA_STATE_UNKNOWN ) {
14411                 ctl_is_single = 0;
14412                 if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
14413                     != CTL_HA_STATUS_SUCCESS) {
14414                         printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
14415                         ret = CTL_HA_COMP_STATUS_ERROR;
14416                 }
14417         } else if (CTL_HA_STATE_IS_HA(c->state)
14418                 && CTL_HA_STATE_IS_SINGLE(state)){
14419                 // HA->SINGLE transition
14420                 ctl_failover();
14421                 ctl_is_single = 1;
14422         } else {
14423                 printf("ctl_isc_start:Invalid state transition %X->%X\n",
14424                        c->state, state);
14425                 ret = CTL_HA_COMP_STATUS_ERROR;
14426         }
14427         if (CTL_HA_STATE_IS_SINGLE(state))
14428                 ctl_is_single = 1;
14429
14430         c->state = state;
14431         c->status = ret;
14432         return ret;
14433 }
14434
14435 /*
14436  * Quiesce component
14437  * The component must clear any error conditions (set status to OK) and
14438  * prepare itself to another Start call
14439  * returns ctl_ha_comp_status:
14440  *      OK
14441  *      ERROR
14442  */
14443 static ctl_ha_comp_status
14444 ctl_isc_quiesce(struct ctl_ha_component *c)
14445 {
14446         int ret = CTL_HA_COMP_STATUS_OK;
14447
14448         ctl_pause_rtr = 1;
14449         c->status = ret;
14450         return ret;
14451 }
14452
14453 struct ctl_ha_component ctl_ha_component_ctlisc =
14454 {
14455         .name = "CTL ISC",
14456         .state = CTL_HA_STATE_UNKNOWN,
14457         .init = ctl_isc_init,
14458         .start = ctl_isc_start,
14459         .quiesce = ctl_isc_quiesce
14460 };
14461
14462 /*
14463  *  vim: ts=8
14464  */