sys/cam/ctl/ctl.c

   1 /*-
   2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
   3  * Copyright (c) 2012 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Edward Tomasz Napierala
   7  * under sponsorship from the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  16  *    substantially similar to the "NO WARRANTY" disclaimer below
  17  *    ("Disclaimer") and any redistribution must be conditioned upon
  18  *    including a substantially similar Disclaimer requirement for further
  19  *    binary redistribution.
  20  *
  21  * NO WARRANTY
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  30  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  31  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGES.
  33  *
  34  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl.c#8 $
  35  */
  36 /*
  37  * CAM Target Layer, a SCSI device emulation subsystem.
  38  *
  39  * Author: Ken Merry <ken@FreeBSD.org>
  40  */
  41
  42 #define _CTL_C
  43
  44 #include <sys/cdefs.h>
  45 __FBSDID("$FreeBSD$");
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/types.h>
  51 #include <sys/kthread.h>
  52 #include <sys/bio.h>
  53 #include <sys/fcntl.h>
  54 #include <sys/lock.h>
  55 #include <sys/module.h>
  56 #include <sys/mutex.h>
  57 #include <sys/condvar.h>
  58 #include <sys/malloc.h>
  59 #include <sys/conf.h>
  60 #include <sys/ioccom.h>
  61 #include <sys/queue.h>
  62 #include <sys/sbuf.h>
  63 #include <sys/smp.h>
  64 #include <sys/endian.h>
  65 #include <sys/sysctl.h>
  66
  67 #include <cam/cam.h>
  68 #include <cam/scsi/scsi_all.h>
  69 #include <cam/scsi/scsi_da.h>
  70 #include <cam/ctl/ctl_io.h>
  71 #include <cam/ctl/ctl.h>
  72 #include <cam/ctl/ctl_frontend.h>
  73 #include <cam/ctl/ctl_frontend_internal.h>
  74 #include <cam/ctl/ctl_util.h>
  75 #include <cam/ctl/ctl_backend.h>
  76 #include <cam/ctl/ctl_ioctl.h>
  77 #include <cam/ctl/ctl_ha.h>
  78 #include <cam/ctl/ctl_private.h>
  79 #include <cam/ctl/ctl_debug.h>
  80 #include <cam/ctl/ctl_scsi_all.h>
  81 #include <cam/ctl/ctl_error.h>
  82
  83 struct ctl_softc *control_softc = NULL;
  84
  85 /*
  86  * The default is to run with CTL_DONE_THREAD turned on.  Completed
  87  * transactions are queued for processing by the CTL work thread.  When
  88  * CTL_DONE_THREAD is not defined, completed transactions are processed in
  89  * the caller's context.
  90  */
  91 #define CTL_DONE_THREAD
  92
  93 /*
  94  * Use the serial number and device ID provided by the backend, rather than
  95  * making up our own.
  96  */
  97 #define CTL_USE_BACKEND_SN
  98
  99 /*
 100  * Size and alignment macros needed for Copan-specific HA hardware.  These
 101  * can go away when the HA code is re-written, and uses busdma for any
 102  * hardware.
 103  */
 104 #define CTL_ALIGN_8B(target, source, type)                              \
 105         if (((uint32_t)source & 0x7) != 0)                              \
 106                 target = (type)(source + (0x8 - ((uint32_t)source & 0x7)));\
 107         else                                                            \
 108                 target = (type)source;
 109
 110 #define CTL_SIZE_8B(target, size)                                       \
 111         if ((size & 0x7) != 0)                                          \
 112                 target = size + (0x8 - (size & 0x7));                   \
 113         else                                                            \
 114                 target = size;
 115
 116 #define CTL_ALIGN_8B_MARGIN     16
 117
 118 /*
 119  * Template mode pages.
 120  */
 121
 122 /*
 123  * Note that these are default values only.  The actual values will be
 124  * filled in when the user does a mode sense.
 125  */
 126 static struct copan_power_subpage power_page_default = {
 127         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 128         /*subpage*/ PWR_SUBPAGE_CODE,
 129         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 130                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 131         /*page_version*/ PWR_VERSION,
 132         /* total_luns */ 26,
 133         /* max_active_luns*/ PWR_DFLT_MAX_LUNS,
 134         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 135                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 136                       0, 0, 0, 0, 0, 0}
 137 };
 138
 139 static struct copan_power_subpage power_page_changeable = {
 140         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 141         /*subpage*/ PWR_SUBPAGE_CODE,
 142         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 143                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 144         /*page_version*/ 0,
 145         /* total_luns */ 0,
 146         /* max_active_luns*/ 0,
 147         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 148                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 149                       0, 0, 0, 0, 0, 0}
 150 };
 151
 152 static struct copan_aps_subpage aps_page_default = {
 153         APS_PAGE_CODE | SMPH_SPF, //page_code
 154         APS_SUBPAGE_CODE, //subpage
 155         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 156          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 157         APS_VERSION, //page_version
 158         0, //lock_active
 159         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 160         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 161         0, 0, 0, 0, 0} //reserved
 162 };
 163
 164 static struct copan_aps_subpage aps_page_changeable = {
 165         APS_PAGE_CODE | SMPH_SPF, //page_code
 166         APS_SUBPAGE_CODE, //subpage
 167         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 168          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 169         0, //page_version
 170         0, //lock_active
 171         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 172         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 173         0, 0, 0, 0, 0} //reserved
 174 };
 175
 176 static struct copan_debugconf_subpage debugconf_page_default = {
 177         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 178         DBGCNF_SUBPAGE_CODE,            /* subpage */
 179         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 180          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 181         DBGCNF_VERSION,                 /* page_version */
 182         {CTL_TIME_IO_DEFAULT_SECS>>8,
 183          CTL_TIME_IO_DEFAULT_SECS>>0},  /* ctl_time_io_secs */
 184 };
 185
 186 static struct copan_debugconf_subpage debugconf_page_changeable = {
 187         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 188         DBGCNF_SUBPAGE_CODE,            /* subpage */
 189         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 190          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 191         0,                              /* page_version */
 192         {0xff,0xff},                    /* ctl_time_io_secs */
 193 };
 194
 195 static struct scsi_format_page format_page_default = {
 196         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 197         /*page_length*/sizeof(struct scsi_format_page) - 2,
 198         /*tracks_per_zone*/ {0, 0},
 199         /*alt_sectors_per_zone*/ {0, 0},
 200         /*alt_tracks_per_zone*/ {0, 0},
 201         /*alt_tracks_per_lun*/ {0, 0},
 202         /*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
 203                                 CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
 204         /*bytes_per_sector*/ {0, 0},
 205         /*interleave*/ {0, 0},
 206         /*track_skew*/ {0, 0},
 207         /*cylinder_skew*/ {0, 0},
 208         /*flags*/ SFP_HSEC,
 209         /*reserved*/ {0, 0, 0}
 210 };
 211
 212 static struct scsi_format_page format_page_changeable = {
 213         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 214         /*page_length*/sizeof(struct scsi_format_page) - 2,
 215         /*tracks_per_zone*/ {0, 0},
 216         /*alt_sectors_per_zone*/ {0, 0},
 217         /*alt_tracks_per_zone*/ {0, 0},
 218         /*alt_tracks_per_lun*/ {0, 0},
 219         /*sectors_per_track*/ {0, 0},
 220         /*bytes_per_sector*/ {0, 0},
 221         /*interleave*/ {0, 0},
 222         /*track_skew*/ {0, 0},
 223         /*cylinder_skew*/ {0, 0},
 224         /*flags*/ 0,
 225         /*reserved*/ {0, 0, 0}
 226 };
 227
 228 static struct scsi_rigid_disk_page rigid_disk_page_default = {
 229         /*page_code*/SMS_RIGID_DISK_PAGE,
 230         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 231         /*cylinders*/ {0, 0, 0},
 232         /*heads*/ CTL_DEFAULT_HEADS,
 233         /*start_write_precomp*/ {0, 0, 0},
 234         /*start_reduced_current*/ {0, 0, 0},
 235         /*step_rate*/ {0, 0},
 236         /*landing_zone_cylinder*/ {0, 0, 0},
 237         /*rpl*/ SRDP_RPL_DISABLED,
 238         /*rotational_offset*/ 0,
 239         /*reserved1*/ 0,
 240         /*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
 241                            CTL_DEFAULT_ROTATION_RATE & 0xff},
 242         /*reserved2*/ {0, 0}
 243 };
 244
 245 static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
 246         /*page_code*/SMS_RIGID_DISK_PAGE,
 247         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 248         /*cylinders*/ {0, 0, 0},
 249         /*heads*/ 0,
 250         /*start_write_precomp*/ {0, 0, 0},
 251         /*start_reduced_current*/ {0, 0, 0},
 252         /*step_rate*/ {0, 0},
 253         /*landing_zone_cylinder*/ {0, 0, 0},
 254         /*rpl*/ 0,
 255         /*rotational_offset*/ 0,
 256         /*reserved1*/ 0,
 257         /*rotation_rate*/ {0, 0},
 258         /*reserved2*/ {0, 0}
 259 };
 260
 261 static struct scsi_caching_page caching_page_default = {
 262         /*page_code*/SMS_CACHING_PAGE,
 263         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 264         /*flags1*/ SCP_DISC | SCP_WCE,
 265         /*ret_priority*/ 0,
 266         /*disable_pf_transfer_len*/ {0xff, 0xff},
 267         /*min_prefetch*/ {0, 0},
 268         /*max_prefetch*/ {0xff, 0xff},
 269         /*max_pf_ceiling*/ {0xff, 0xff},
 270         /*flags2*/ 0,
 271         /*cache_segments*/ 0,
 272         /*cache_seg_size*/ {0, 0},
 273         /*reserved*/ 0,
 274         /*non_cache_seg_size*/ {0, 0, 0}
 275 };
 276
 277 static struct scsi_caching_page caching_page_changeable = {
 278         /*page_code*/SMS_CACHING_PAGE,
 279         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 280         /*flags1*/ 0,
 281         /*ret_priority*/ 0,
 282         /*disable_pf_transfer_len*/ {0, 0},
 283         /*min_prefetch*/ {0, 0},
 284         /*max_prefetch*/ {0, 0},
 285         /*max_pf_ceiling*/ {0, 0},
 286         /*flags2*/ 0,
 287         /*cache_segments*/ 0,
 288         /*cache_seg_size*/ {0, 0},
 289         /*reserved*/ 0,
 290         /*non_cache_seg_size*/ {0, 0, 0}
 291 };
 292
 293 static struct scsi_control_page control_page_default = {
 294         /*page_code*/SMS_CONTROL_MODE_PAGE,
 295         /*page_length*/sizeof(struct scsi_control_page) - 2,
 296         /*rlec*/0,
 297         /*queue_flags*/0,
 298         /*eca_and_aen*/0,
 299         /*reserved*/0,
 300         /*aen_holdoff_period*/{0, 0}
 301 };
 302
 303 static struct scsi_control_page control_page_changeable = {
 304         /*page_code*/SMS_CONTROL_MODE_PAGE,
 305         /*page_length*/sizeof(struct scsi_control_page) - 2,
 306         /*rlec*/SCP_DSENSE,
 307         /*queue_flags*/0,
 308         /*eca_and_aen*/0,
 309         /*reserved*/0,
 310         /*aen_holdoff_period*/{0, 0}
 311 };
 312
 313
 314 /*
 315  * XXX KDM move these into the softc.
 316  */
 317 static int rcv_sync_msg;
 318 static int persis_offset;
 319 static uint8_t ctl_pause_rtr;
 320 static int     ctl_is_single = 1;
 321 static int     index_to_aps_page;
 322
 323 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 324 static int worker_threads = 1;
 325 TUNABLE_INT("kern.cam.ctl.worker_threads", &worker_threads);
 326 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
 327     &worker_threads, 1, "Number of worker threads");
 328 static int verbose = 0;
 329 TUNABLE_INT("kern.cam.ctl.verbose", &verbose);
 330 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, verbose, CTLFLAG_RWTUN,
 331     &verbose, 0, "Show SCSI errors returned to initiator");
 332
 333 /*
 334  * Serial number (0x80), device id (0x83), supported pages (0x00),
 335  * Block limits (0xB0) and Logical Block Provisioning (0xB2)
 336  */
 337 #define SCSI_EVPD_NUM_SUPPORTED_PAGES   5
 338
 339 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 340                                   int param);
 341 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 342 static int ctl_init(void);
 343 void ctl_shutdown(void);
 344 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 345 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 346 static void ctl_ioctl_online(void *arg);
 347 static void ctl_ioctl_offline(void *arg);
 348 static int ctl_ioctl_targ_enable(void *arg, struct ctl_id targ_id);
 349 static int ctl_ioctl_targ_disable(void *arg, struct ctl_id targ_id);
 350 static int ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id);
 351 static int ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id);
 352 static int ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio);
 353 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio, int have_lock);
 354 static int ctl_ioctl_submit_wait(union ctl_io *io);
 355 static void ctl_ioctl_datamove(union ctl_io *io);
 356 static void ctl_ioctl_done(union ctl_io *io);
 357 static void ctl_ioctl_hard_startstop_callback(void *arg,
 358                                               struct cfi_metatask *metatask);
 359 static void ctl_ioctl_bbrread_callback(void *arg,struct cfi_metatask *metatask);
 360 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 361                               struct ctl_ooa *ooa_hdr,
 362                               struct ctl_ooa_entry *kern_entries);
 363 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 364                      struct thread *td);
 365 uint32_t ctl_get_resindex(struct ctl_nexus *nexus);
 366 uint32_t ctl_port_idx(int port_num);
 367 #ifdef unused
 368 static union ctl_io *ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port,
 369                                    uint32_t targ_target, uint32_t targ_lun,
 370                                    int can_wait);
 371 static void ctl_kfree_io(union ctl_io *io);
 372 #endif /* unused */
 373 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
 374                          struct ctl_be_lun *be_lun, struct ctl_id target_id);
 375 static int ctl_free_lun(struct ctl_lun *lun);
 376 static void ctl_create_lun(struct ctl_be_lun *be_lun);
 377 /**
 378 static void ctl_failover_change_pages(struct ctl_softc *softc,
 379                                       struct ctl_scsiio *ctsio, int master);
 380 **/
 381
 382 static int ctl_do_mode_select(union ctl_io *io);
 383 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
 384                            uint64_t res_key, uint64_t sa_res_key,
 385                            uint8_t type, uint32_t residx,
 386                            struct ctl_scsiio *ctsio,
 387                            struct scsi_per_res_out *cdb,
 388                            struct scsi_per_res_out_parms* param);
 389 static void ctl_pro_preempt_other(struct ctl_lun *lun,
 390                                   union ctl_ha_msg *msg);
 391 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
 392 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
 393 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
 394 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
 395 static int ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio,
 396                                          int alloc_len);
 397 static int ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len);
 398 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
 399 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
 400 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len);
 401 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2);
 402 static ctl_action ctl_check_for_blockage(union ctl_io *pending_io,
 403                                          union ctl_io *ooa_io);
 404 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
 405                                 union ctl_io *starting_io);
 406 static int ctl_check_blocked(struct ctl_lun *lun);
 407 static int ctl_scsiio_lun_check(struct ctl_softc *ctl_softc,
 408                                 struct ctl_lun *lun,
 409                                 struct ctl_cmd_entry *entry,
 410                                 struct ctl_scsiio *ctsio);
 411 //static int ctl_check_rtr(union ctl_io *pending_io, struct ctl_softc *softc);
 412 static void ctl_failover(void);
 413 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 414                                struct ctl_scsiio *ctsio);
 415 static int ctl_scsiio(struct ctl_scsiio *ctsio);
 416
 417 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 418 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 419                             ctl_ua_type ua_type);
 420 static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 421                          ctl_ua_type ua_type);
 422 static int ctl_abort_task(union ctl_io *io);
 423 static void ctl_run_task_queue(struct ctl_softc *ctl_softc);
 424 #ifdef CTL_IO_DELAY
 425 static void ctl_datamove_timer_wakeup(void *arg);
 426 static void ctl_done_timer_wakeup(void *arg);
 427 #endif /* CTL_IO_DELAY */
 428
 429 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 430 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
 431 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
 432 static void ctl_datamove_remote_write(union ctl_io *io);
 433 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
 434 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 435 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 436 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 437                                     ctl_ha_dt_cb callback);
 438 static void ctl_datamove_remote_read(union ctl_io *io);
 439 static void ctl_datamove_remote(union ctl_io *io);
 440 static int ctl_process_done(union ctl_io *io, int have_lock);
 441 static void ctl_work_thread(void *arg);
 442
 443 /*
 444  * Load the serialization table.  This isn't very pretty, but is probably
 445  * the easiest way to do it.
 446  */
 447 #include "ctl_ser_table.c"
 448
 449 /*
 450  * We only need to define open, close and ioctl routines for this driver.
 451  */
 452 static struct cdevsw ctl_cdevsw = {
 453         .d_version =    D_VERSION,
 454         .d_flags =      0,
 455         .d_open =       ctl_open,
 456         .d_close =      ctl_close,
 457         .d_ioctl =      ctl_ioctl,
 458         .d_name =       "ctl",
 459 };
 460
 461
 462 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 463
 464 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 465
 466 static moduledata_t ctl_moduledata = {
 467         "ctl",
 468         ctl_module_event_handler,
 469         NULL
 470 };
 471
 472 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
 473 MODULE_VERSION(ctl, 1);
 474
 475 static void
 476 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 477                             union ctl_ha_msg *msg_info)
 478 {
 479         struct ctl_scsiio *ctsio;
 480
 481         if (msg_info->hdr.original_sc == NULL) {
 482                 printf("%s: original_sc == NULL!\n", __func__);
 483                 /* XXX KDM now what? */
 484                 return;
 485         }
 486
 487         ctsio = &msg_info->hdr.original_sc->scsiio;
 488         ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 489         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 490         ctsio->io_hdr.status = msg_info->hdr.status;
 491         ctsio->scsi_status = msg_info->scsi.scsi_status;
 492         ctsio->sense_len = msg_info->scsi.sense_len;
 493         ctsio->sense_residual = msg_info->scsi.sense_residual;
 494         ctsio->residual = msg_info->scsi.residual;
 495         memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 496                sizeof(ctsio->sense_data));
 497         memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
 498                &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));
 499         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue, &ctsio->io_hdr, links);
 500         ctl_wakeup_thread();
 501 }
 502
 503 static void
 504 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
 505                                 union ctl_ha_msg *msg_info)
 506 {
 507         struct ctl_scsiio *ctsio;
 508
 509         if (msg_info->hdr.serializing_sc == NULL) {
 510                 printf("%s: serializing_sc == NULL!\n", __func__);
 511                 /* XXX KDM now what? */
 512                 return;
 513         }
 514
 515         ctsio = &msg_info->hdr.serializing_sc->scsiio;
 516 #if 0
 517         /*
 518          * Attempt to catch the situation where an I/O has
 519          * been freed, and we're using it again.
 520          */
 521         if (ctsio->io_hdr.io_type == 0xff) {
 522                 union ctl_io *tmp_io;
 523                 tmp_io = (union ctl_io *)ctsio;
 524                 printf("%s: %p use after free!\n", __func__,
 525                        ctsio);
 526                 printf("%s: type %d msg %d cdb %x iptl: "
 527                        "%d:%d:%d:%d tag 0x%04x "
 528                        "flag %#x status %x\n",
 529                         __func__,
 530                         tmp_io->io_hdr.io_type,
 531                         tmp_io->io_hdr.msg_type,
 532                         tmp_io->scsiio.cdb[0],
 533                         tmp_io->io_hdr.nexus.initid.id,
 534                         tmp_io->io_hdr.nexus.targ_port,
 535                         tmp_io->io_hdr.nexus.targ_target.id,
 536                         tmp_io->io_hdr.nexus.targ_lun,
 537                         (tmp_io->io_hdr.io_type ==
 538                         CTL_IO_TASK) ?
 539                         tmp_io->taskio.tag_num :
 540                         tmp_io->scsiio.tag_num,
 541                         tmp_io->io_hdr.flags,
 542                         tmp_io->io_hdr.status);
 543         }
 544 #endif
 545         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 546         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue, &ctsio->io_hdr, links);
 547         ctl_wakeup_thread();
 548 }
 549
 550 /*
 551  * ISC (Inter Shelf Communication) event handler.  Events from the HA
 552  * subsystem come in here.
 553  */
 554 static void
 555 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 556 {
 557         struct ctl_softc *ctl_softc;
 558         union ctl_io *io;
 559         struct ctl_prio *presio;
 560         ctl_ha_status isc_status;
 561
 562         ctl_softc = control_softc;
 563         io = NULL;
 564
 565
 566 #if 0
 567         printf("CTL: Isc Msg event %d\n", event);
 568 #endif
 569         if (event == CTL_HA_EVT_MSG_RECV) {
 570                 union ctl_ha_msg msg_info;
 571
 572                 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
 573                                              sizeof(msg_info), /*wait*/ 0);
 574 #if 0
 575                 printf("CTL: msg_type %d\n", msg_info.msg_type);
 576 #endif
 577                 if (isc_status != 0) {
 578                         printf("Error receiving message, status = %d\n",
 579                                isc_status);
 580                         return;
 581                 }
 582                 mtx_lock(&ctl_softc->ctl_lock);
 583
 584                 switch (msg_info.hdr.msg_type) {
 585                 case CTL_MSG_SERIALIZE:
 586 #if 0
 587                         printf("Serialize\n");
 588 #endif
 589                         io = ctl_alloc_io((void *)ctl_softc->othersc_pool);
 590                         if (io == NULL) {
 591                                 printf("ctl_isc_event_handler: can't allocate "
 592                                        "ctl_io!\n");
 593                                 /* Bad Juju */
 594                                 /* Need to set busy and send msg back */
 595                                 mtx_unlock(&ctl_softc->ctl_lock);
 596                                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
 597                                 msg_info.hdr.status = CTL_SCSI_ERROR;
 598                                 msg_info.scsi.scsi_status = SCSI_STATUS_BUSY;
 599                                 msg_info.scsi.sense_len = 0;
 600                                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
 601                                     sizeof(msg_info), 0) > CTL_HA_STATUS_SUCCESS){
 602                                 }
 603                                 goto bailout;
 604                         }
 605                         ctl_zero_io(io);
 606                         // populate ctsio from msg_info
 607                         io->io_hdr.io_type = CTL_IO_SCSI;
 608                         io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
 609                         io->io_hdr.original_sc = msg_info.hdr.original_sc;
 610 #if 0
 611                         printf("pOrig %x\n", (int)msg_info.original_sc);
 612 #endif
 613                         io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
 614                                             CTL_FLAG_IO_ACTIVE;
 615                         /*
 616                          * If we're in serialization-only mode, we don't
 617                          * want to go through full done processing.  Thus
 618                          * the COPY flag.
 619                          *
 620                          * XXX KDM add another flag that is more specific.
 621                          */
 622                         if (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)
 623                                 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
 624                         io->io_hdr.nexus = msg_info.hdr.nexus;
 625 #if 0
 626                         printf("targ %d, port %d, iid %d, lun %d\n",
 627                                io->io_hdr.nexus.targ_target.id,
 628                                io->io_hdr.nexus.targ_port,
 629                                io->io_hdr.nexus.initid.id,
 630                                io->io_hdr.nexus.targ_lun);
 631 #endif
 632                         io->scsiio.tag_num = msg_info.scsi.tag_num;
 633                         io->scsiio.tag_type = msg_info.scsi.tag_type;
 634                         memcpy(io->scsiio.cdb, msg_info.scsi.cdb,
 635                                CTL_MAX_CDBLEN);
 636                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
 637                                 struct ctl_cmd_entry *entry;
 638                                 uint8_t opcode;
 639
 640                                 opcode = io->scsiio.cdb[0];
 641                                 entry = &ctl_cmd_table[opcode];
 642                                 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
 643                                 io->io_hdr.flags |=
 644                                         entry->flags & CTL_FLAG_DATA_MASK;
 645                         }
 646                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 647                                            &io->io_hdr, links);
 648                         ctl_wakeup_thread();
 649                         break;
 650
 651                 /* Performed on the Originating SC, XFER mode only */
 652                 case CTL_MSG_DATAMOVE: {
 653                         struct ctl_sg_entry *sgl;
 654                         int i, j;
 655
 656                         io = msg_info.hdr.original_sc;
 657                         if (io == NULL) {
 658                                 printf("%s: original_sc == NULL!\n", __func__);
 659                                 /* XXX KDM do something here */
 660                                 break;
 661                         }
 662                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
 663                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 664                         /*
 665                          * Keep track of this, we need to send it back over
 666                          * when the datamove is complete.
 667                          */
 668                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 669
 670                         if (msg_info.dt.sg_sequence == 0) {
 671                                 /*
 672                                  * XXX KDM we use the preallocated S/G list
 673                                  * here, but we'll need to change this to
 674                                  * dynamic allocation if we need larger S/G
 675                                  * lists.
 676                                  */
 677                                 if (msg_info.dt.kern_sg_entries >
 678                                     sizeof(io->io_hdr.remote_sglist) /
 679                                     sizeof(io->io_hdr.remote_sglist[0])) {
 680                                         printf("%s: number of S/G entries "
 681                                             "needed %u > allocated num %zd\n",
 682                                             __func__,
 683                                             msg_info.dt.kern_sg_entries,
 684                                             sizeof(io->io_hdr.remote_sglist)/
 685                                             sizeof(io->io_hdr.remote_sglist[0]));
 686
 687                                         /*
 688                                          * XXX KDM send a message back to
 689                                          * the other side to shut down the
 690                                          * DMA.  The error will come back
 691                                          * through via the normal channel.
 692                                          */
 693                                         break;
 694                                 }
 695                                 sgl = io->io_hdr.remote_sglist;
 696                                 memset(sgl, 0,
 697                                        sizeof(io->io_hdr.remote_sglist));
 698
 699                                 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
 700
 701                                 io->scsiio.kern_sg_entries =
 702                                         msg_info.dt.kern_sg_entries;
 703                                 io->scsiio.rem_sg_entries =
 704                                         msg_info.dt.kern_sg_entries;
 705                                 io->scsiio.kern_data_len =
 706                                         msg_info.dt.kern_data_len;
 707                                 io->scsiio.kern_total_len =
 708                                         msg_info.dt.kern_total_len;
 709                                 io->scsiio.kern_data_resid =
 710                                         msg_info.dt.kern_data_resid;
 711                                 io->scsiio.kern_rel_offset =
 712                                         msg_info.dt.kern_rel_offset;
 713                                 /*
 714                                  * Clear out per-DMA flags.
 715                                  */
 716                                 io->io_hdr.flags &= ~CTL_FLAG_RDMA_MASK;
 717                                 /*
 718                                  * Add per-DMA flags that are set for this
 719                                  * particular DMA request.
 720                                  */
 721                                 io->io_hdr.flags |= msg_info.dt.flags &
 722                                                     CTL_FLAG_RDMA_MASK;
 723                         } else
 724                                 sgl = (struct ctl_sg_entry *)
 725                                         io->scsiio.kern_data_ptr;
 726
 727                         for (i = msg_info.dt.sent_sg_entries, j = 0;
 728                              i < (msg_info.dt.sent_sg_entries +
 729                              msg_info.dt.cur_sg_entries); i++, j++) {
 730                                 sgl[i].addr = msg_info.dt.sg_list[j].addr;
 731                                 sgl[i].len = msg_info.dt.sg_list[j].len;
 732
 733 #if 0
 734                                 printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
 735                                        __func__,
 736                                        msg_info.dt.sg_list[j].addr,
 737                                        msg_info.dt.sg_list[j].len,
 738                                        sgl[i].addr, sgl[i].len, j, i);
 739 #endif
 740                         }
 741 #if 0
 742                         memcpy(&sgl[msg_info.dt.sent_sg_entries],
 743                                msg_info.dt.sg_list,
 744                                sizeof(*sgl) * msg_info.dt.cur_sg_entries);
 745 #endif
 746
 747                         /*
 748                          * If this is the last piece of the I/O, we've got
 749                          * the full S/G list.  Queue processing in the thread.
 750                          * Otherwise wait for the next piece.
 751                          */
 752                         if (msg_info.dt.sg_last != 0) {
 753                                 STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 754                                                    &io->io_hdr, links);
 755                                 ctl_wakeup_thread();
 756                         }
 757                         break;
 758                 }
 759                 /* Performed on the Serializing (primary) SC, XFER mode only */
 760                 case CTL_MSG_DATAMOVE_DONE: {
 761                         if (msg_info.hdr.serializing_sc == NULL) {
 762                                 printf("%s: serializing_sc == NULL!\n",
 763                                        __func__);
 764                                 /* XXX KDM now what? */
 765                                 break;
 766                         }
 767                         /*
 768                          * We grab the sense information here in case
 769                          * there was a failure, so we can return status
 770                          * back to the initiator.
 771                          */
 772                         io = msg_info.hdr.serializing_sc;
 773                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
 774                         io->io_hdr.status = msg_info.hdr.status;
 775                         io->scsiio.scsi_status = msg_info.scsi.scsi_status;
 776                         io->scsiio.sense_len = msg_info.scsi.sense_len;
 777                         io->scsiio.sense_residual =msg_info.scsi.sense_residual;
 778                         io->io_hdr.port_status = msg_info.scsi.fetd_status;
 779                         io->scsiio.residual = msg_info.scsi.residual;
 780                         memcpy(&io->scsiio.sense_data,&msg_info.scsi.sense_data,
 781                                sizeof(io->scsiio.sense_data));
 782
 783                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 784                                            &io->io_hdr, links);
 785                         ctl_wakeup_thread();
 786                         break;
 787                 }
 788
 789                 /* Preformed on Originating SC, SER_ONLY mode */
 790                 case CTL_MSG_R2R:
 791                         io = msg_info.hdr.original_sc;
 792                         if (io == NULL) {
 793                                 printf("%s: Major Bummer\n", __func__);
 794                                 mtx_unlock(&ctl_softc->ctl_lock);
 795                                 return;
 796                         } else {
 797 #if 0
 798                                 printf("pOrig %x\n",(int) ctsio);
 799 #endif
 800                         }
 801                         io->io_hdr.msg_type = CTL_MSG_R2R;
 802                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 803                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 804                                            &io->io_hdr, links);
 805                         ctl_wakeup_thread();
 806                         break;
 807
 808                 /*
 809                  * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
 810                  * mode.
 811                  * Performed on the Originating (i.e. secondary) SC in XFER
 812                  * mode
 813                  */
 814                 case CTL_MSG_FINISH_IO:
 815                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER)
 816                                 ctl_isc_handler_finish_xfer(ctl_softc,
 817                                                             &msg_info);
 818                         else
 819                                 ctl_isc_handler_finish_ser_only(ctl_softc,
 820                                                                 &msg_info);
 821                         break;
 822
 823                 /* Preformed on Originating SC */
 824                 case CTL_MSG_BAD_JUJU:
 825                         io = msg_info.hdr.original_sc;
 826                         if (io == NULL) {
 827                                 printf("%s: Bad JUJU!, original_sc is NULL!\n",
 828                                        __func__);
 829                                 break;
 830                         }
 831                         ctl_copy_sense_data(&msg_info, io);
 832                         /*
 833                          * IO should have already been cleaned up on other
 834                          * SC so clear this flag so we won't send a message
 835                          * back to finish the IO there.
 836                          */
 837                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
 838                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 839
 840                         /* io = msg_info.hdr.serializing_sc; */
 841                         io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
 842                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 843                                            &io->io_hdr, links);
 844                         ctl_wakeup_thread();
 845                         break;
 846
 847                 /* Handle resets sent from the other side */
 848                 case CTL_MSG_MANAGE_TASKS: {
 849                         struct ctl_taskio *taskio;
 850                         taskio = (struct ctl_taskio *)ctl_alloc_io(
 851                                 (void *)ctl_softc->othersc_pool);
 852                         if (taskio == NULL) {
 853                                 printf("ctl_isc_event_handler: can't allocate "
 854                                        "ctl_io!\n");
 855                                 /* Bad Juju */
 856                                 /* should I just call the proper reset func
 857                                    here??? */
 858                                 mtx_unlock(&ctl_softc->ctl_lock);
 859                                 goto bailout;
 860                         }
 861                         ctl_zero_io((union ctl_io *)taskio);
 862                         taskio->io_hdr.io_type = CTL_IO_TASK;
 863                         taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
 864                         taskio->io_hdr.nexus = msg_info.hdr.nexus;
 865                         taskio->task_action = msg_info.task.task_action;
 866                         taskio->tag_num = msg_info.task.tag_num;
 867                         taskio->tag_type = msg_info.task.tag_type;
 868 #ifdef CTL_TIME_IO
 869                         taskio->io_hdr.start_time = time_uptime;
 870                         getbintime(&taskio->io_hdr.start_bt);
 871 #if 0
 872                         cs_prof_gettime(&taskio->io_hdr.start_ticks);
 873 #endif
 874 #endif /* CTL_TIME_IO */
 875                         STAILQ_INSERT_TAIL(&ctl_softc->task_queue,
 876                                            &taskio->io_hdr, links);
 877                         ctl_softc->flags |= CTL_FLAG_TASK_PENDING;
 878                         ctl_wakeup_thread();
 879                         break;
 880                 }
 881                 /* Persistent Reserve action which needs attention */
 882                 case CTL_MSG_PERS_ACTION:
 883                         presio = (struct ctl_prio *)ctl_alloc_io(
 884                                 (void *)ctl_softc->othersc_pool);
 885                         if (presio == NULL) {
 886                                 printf("ctl_isc_event_handler: can't allocate "
 887                                        "ctl_io!\n");
 888                                 /* Bad Juju */
 889                                 /* Need to set busy and send msg back */
 890                                 mtx_unlock(&ctl_softc->ctl_lock);
 891                                 goto bailout;
 892                         }
 893                         ctl_zero_io((union ctl_io *)presio);
 894                         presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
 895                         presio->pr_msg = msg_info.pr;
 896                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 897                                            &presio->io_hdr, links);
 898                         ctl_wakeup_thread();
 899                         break;
 900                 case CTL_MSG_SYNC_FE:
 901                         rcv_sync_msg = 1;
 902                         break;
 903                 case CTL_MSG_APS_LOCK: {
 904                         // It's quicker to execute this then to
 905                         // queue it.
 906                         struct ctl_lun *lun;
 907                         struct ctl_page_index *page_index;
 908                         struct copan_aps_subpage *current_sp;
 909                         uint32_t targ_lun;
 910
 911                         targ_lun = msg_info.hdr.nexus.targ_lun;
 912                         if (msg_info.hdr.nexus.lun_map_fn != NULL)
 913                                 targ_lun = msg_info.hdr.nexus.lun_map_fn(msg_info.hdr.nexus.lun_map_arg, targ_lun);
 914
 915                         lun = ctl_softc->ctl_luns[targ_lun];
 916                         page_index = &lun->mode_pages.index[index_to_aps_page];
 917                         current_sp = (struct copan_aps_subpage *)
 918                                      (page_index->page_data +
 919                                      (page_index->page_len * CTL_PAGE_CURRENT));
 920
 921                         current_sp->lock_active = msg_info.aps.lock_flag;
 922                         break;
 923                 }
 924                 default:
 925                         printf("How did I get here?\n");
 926                 }
 927                 mtx_unlock(&ctl_softc->ctl_lock);
 928         } else if (event == CTL_HA_EVT_MSG_SENT) {
 929                 if (param != CTL_HA_STATUS_SUCCESS) {
 930                         printf("Bad status from ctl_ha_msg_send status %d\n",
 931                                param);
 932                 }
 933                 return;
 934         } else if (event == CTL_HA_EVT_DISCONNECT) {
 935                 printf("CTL: Got a disconnect from Isc\n");
 936                 return;
 937         } else {
 938                 printf("ctl_isc_event_handler: Unknown event %d\n", event);
 939                 return;
 940         }
 941
 942 bailout:
 943         return;
 944 }
 945
 946 static void
 947 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
 948 {
 949         struct scsi_sense_data *sense;
 950
 951         sense = &dest->scsiio.sense_data;
 952         bcopy(&src->scsi.sense_data, sense, sizeof(*sense));
 953         dest->scsiio.scsi_status = src->scsi.scsi_status;
 954         dest->scsiio.sense_len = src->scsi.sense_len;
 955         dest->io_hdr.status = src->hdr.status;
 956 }
 957
 958 static int
 959 ctl_init(void)
 960 {
 961         struct ctl_softc *softc;
 962         struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
 963         struct ctl_frontend *fe;
 964         struct ctl_lun *lun;
 965         uint8_t sc_id =0;
 966         int i, error, retval;
 967         //int isc_retval;
 968
 969         retval = 0;
 970         ctl_pause_rtr = 0;
 971         rcv_sync_msg = 0;
 972
 973         control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
 974                                M_WAITOK | M_ZERO);
 975         softc = control_softc;
 976
 977         softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
 978                               "cam/ctl");
 979
 980         softc->dev->si_drv1 = softc;
 981
 982         /*
 983          * By default, return a "bad LUN" peripheral qualifier for unknown
 984          * LUNs.  The user can override this default using the tunable or
 985          * sysctl.  See the comment in ctl_inquiry_std() for more details.
 986          */
 987         softc->inquiry_pq_no_lun = 1;
 988         TUNABLE_INT_FETCH("kern.cam.ctl.inquiry_pq_no_lun",
 989                           &softc->inquiry_pq_no_lun);
 990         sysctl_ctx_init(&softc->sysctl_ctx);
 991         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
 992                 SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
 993                 CTLFLAG_RD, 0, "CAM Target Layer");
 994
 995         if (softc->sysctl_tree == NULL) {
 996                 printf("%s: unable to allocate sysctl tree\n", __func__);
 997                 destroy_dev(softc->dev);
 998                 free(control_softc, M_DEVBUF);
 999                 control_softc = NULL;
1000                 return (ENOMEM);
1001         }
1002
1003         SYSCTL_ADD_INT(&softc->sysctl_ctx,
1004                        SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1005                        "inquiry_pq_no_lun", CTLFLAG_RW,
1006                        &softc->inquiry_pq_no_lun, 0,
1007                        "Report no lun possible for invalid LUNs");
1008
1009         mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
1010         mtx_init(&softc->pool_lock, "CTL pool mutex", NULL, MTX_DEF);
1011         softc->open_count = 0;
1012
1013         /*
1014          * Default to actually sending a SYNCHRONIZE CACHE command down to
1015          * the drive.
1016          */
1017         softc->flags = CTL_FLAG_REAL_SYNC;
1018
1019         /*
1020          * In Copan's HA scheme, the "master" and "slave" roles are
1021          * figured out through the slot the controller is in.  Although it
1022          * is an active/active system, someone has to be in charge.
1023          */
1024 #ifdef NEEDTOPORT
1025         scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
1026 #endif
1027
1028         if (sc_id == 0) {
1029                 softc->flags |= CTL_FLAG_MASTER_SHELF;
1030                 persis_offset = 0;
1031         } else
1032                 persis_offset = CTL_MAX_INITIATORS;
1033
1034         /*
1035          * XXX KDM need to figure out where we want to get our target ID
1036          * and WWID.  Is it different on each port?
1037          */
1038         softc->target.id = 0;
1039         softc->target.wwid[0] = 0x12345678;
1040         softc->target.wwid[1] = 0x87654321;
1041         STAILQ_INIT(&softc->lun_list);
1042         STAILQ_INIT(&softc->pending_lun_queue);
1043         STAILQ_INIT(&softc->task_queue);
1044         STAILQ_INIT(&softc->incoming_queue);
1045         STAILQ_INIT(&softc->rtr_queue);
1046         STAILQ_INIT(&softc->done_queue);
1047         STAILQ_INIT(&softc->isc_queue);
1048         STAILQ_INIT(&softc->fe_list);
1049         STAILQ_INIT(&softc->be_list);
1050         STAILQ_INIT(&softc->io_pools);
1051
1052         lun = &softc->lun;
1053
1054         /*
1055          * We don't bother calling these with ctl_lock held here, because,
1056          * in theory, no one else can try to do anything while we're in our
1057          * module init routine.
1058          */
1059         if (ctl_pool_create(softc, CTL_POOL_INTERNAL, CTL_POOL_ENTRIES_INTERNAL,
1060                             &internal_pool)!= 0){
1061                 printf("ctl: can't allocate %d entry internal pool, "
1062                        "exiting\n", CTL_POOL_ENTRIES_INTERNAL);
1063                 return (ENOMEM);
1064         }
1065
1066         if (ctl_pool_create(softc, CTL_POOL_EMERGENCY,
1067                             CTL_POOL_ENTRIES_EMERGENCY, &emergency_pool) != 0) {
1068                 printf("ctl: can't allocate %d entry emergency pool, "
1069                        "exiting\n", CTL_POOL_ENTRIES_EMERGENCY);
1070                 ctl_pool_free(internal_pool);
1071                 return (ENOMEM);
1072         }
1073
1074         if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC,
1075                             &other_pool) != 0)
1076         {
1077                 printf("ctl: can't allocate %d entry other SC pool, "
1078                        "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1079                 ctl_pool_free(internal_pool);
1080                 ctl_pool_free(emergency_pool);
1081                 return (ENOMEM);
1082         }
1083
1084         softc->internal_pool = internal_pool;
1085         softc->emergency_pool = emergency_pool;
1086         softc->othersc_pool = other_pool;
1087
1088         /*
1089          * We used to allocate a processor LUN here.  The new scheme is to
1090          * just let the user allocate LUNs as he sees fit.
1091          */
1092 #if 0
1093         mtx_lock(&softc->ctl_lock);
1094         ctl_alloc_lun(softc, lun, /*be_lun*/NULL, /*target*/softc->target);
1095         mtx_unlock(&softc->ctl_lock);
1096 #endif
1097
1098         if (worker_threads > MAXCPU || worker_threads == 0) {
1099                 printf("invalid kern.cam.ctl.worker_threads value; "
1100                     "setting to 1");
1101                 worker_threads = 1;
1102         } else if (worker_threads < 0) {
1103                 if (mp_ncpus > 2) {
1104                         /*
1105                          * Using more than two worker threads actually hurts
1106                          * performance due to lock contention.
1107                          */
1108                         worker_threads = 2;
1109                 } else {
1110                         worker_threads = 1;
1111                 }
1112         }
1113
1114         for (i = 0; i < worker_threads; i++) {
1115                 error = kproc_kthread_add(ctl_work_thread, softc,
1116                     &softc->work_thread, NULL, 0, 0, "ctl", "work%d", i);
1117                 if (error != 0) {
1118                         printf("error creating CTL work thread!\n");
1119                         mtx_lock(&softc->ctl_lock);
1120                         ctl_free_lun(lun);
1121                         mtx_unlock(&softc->ctl_lock);
1122                         ctl_pool_free(internal_pool);
1123                         ctl_pool_free(emergency_pool);
1124                         ctl_pool_free(other_pool);
1125                         return (error);
1126                 }
1127         }
1128         if (bootverbose)
1129                 printf("ctl: CAM Target Layer loaded\n");
1130
1131         /*
1132          * Initialize the initiator and portname mappings
1133          */
1134         memset(softc->wwpn_iid, 0, sizeof(softc->wwpn_iid));
1135
1136         /*
1137          * Initialize the ioctl front end.
1138          */
1139         fe = &softc->ioctl_info.fe;
1140         sprintf(softc->ioctl_info.port_name, "CTL ioctl");
1141         fe->port_type = CTL_PORT_IOCTL;
1142         fe->num_requested_ctl_io = 100;
1143         fe->port_name = softc->ioctl_info.port_name;
1144         fe->port_online = ctl_ioctl_online;
1145         fe->port_offline = ctl_ioctl_offline;
1146         fe->onoff_arg = &softc->ioctl_info;
1147         fe->targ_enable = ctl_ioctl_targ_enable;
1148         fe->targ_disable = ctl_ioctl_targ_disable;
1149         fe->lun_enable = ctl_ioctl_lun_enable;
1150         fe->lun_disable = ctl_ioctl_lun_disable;
1151         fe->targ_lun_arg = &softc->ioctl_info;
1152         fe->fe_datamove = ctl_ioctl_datamove;
1153         fe->fe_done = ctl_ioctl_done;
1154         fe->max_targets = 15;
1155         fe->max_target_id = 15;
1156
1157         if (ctl_frontend_register(&softc->ioctl_info.fe,
1158                           (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
1159                 printf("ctl: ioctl front end registration failed, will "
1160                        "continue anyway\n");
1161         }
1162
1163 #ifdef CTL_IO_DELAY
1164         if (sizeof(struct callout) > CTL_TIMER_BYTES) {
1165                 printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
1166                        sizeof(struct callout), CTL_TIMER_BYTES);
1167                 return (EINVAL);
1168         }
1169 #endif /* CTL_IO_DELAY */
1170
1171         return (0);
1172 }
1173
1174 void
1175 ctl_shutdown(void)
1176 {
1177         struct ctl_softc *softc;
1178         struct ctl_lun *lun, *next_lun;
1179         struct ctl_io_pool *pool;
1180
1181         softc = (struct ctl_softc *)control_softc;
1182
1183         if (ctl_frontend_deregister(&softc->ioctl_info.fe) != 0)
1184                 printf("ctl: ioctl front end deregistration failed\n");
1185
1186         mtx_lock(&softc->ctl_lock);
1187
1188         /*
1189          * Free up each LUN.
1190          */
1191         for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1192                 next_lun = STAILQ_NEXT(lun, links);
1193                 ctl_free_lun(lun);
1194         }
1195
1196         mtx_unlock(&softc->ctl_lock);
1197
1198         /*
1199          * This will rip the rug out from under any FETDs or anyone else
1200          * that has a pool allocated.  Since we increment our module
1201          * refcount any time someone outside the main CTL module allocates
1202          * a pool, we shouldn't have any problems here.  The user won't be
1203          * able to unload the CTL module until client modules have
1204          * successfully unloaded.
1205          */
1206         while ((pool = STAILQ_FIRST(&softc->io_pools)) != NULL)
1207                 ctl_pool_free(pool);
1208
1209 #if 0
1210         ctl_shutdown_thread(softc->work_thread);
1211 #endif
1212
1213         mtx_destroy(&softc->pool_lock);
1214         mtx_destroy(&softc->ctl_lock);
1215
1216         destroy_dev(softc->dev);
1217
1218         sysctl_ctx_free(&softc->sysctl_ctx);
1219
1220         free(control_softc, M_DEVBUF);
1221         control_softc = NULL;
1222
1223         if (bootverbose)
1224                 printf("ctl: CAM Target Layer unloaded\n");
1225 }
1226
1227 static int
1228 ctl_module_event_handler(module_t mod, int what, void *arg)
1229 {
1230
1231         switch (what) {
1232         case MOD_LOAD:
1233                 return (ctl_init());
1234         case MOD_UNLOAD:
1235                 return (EBUSY);
1236         default:
1237                 return (EOPNOTSUPP);
1238         }
1239 }
1240
1241 /*
1242  * XXX KDM should we do some access checks here?  Bump a reference count to
1243  * prevent a CTL module from being unloaded while someone has it open?
1244  */
1245 static int
1246 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1247 {
1248         return (0);
1249 }
1250
1251 static int
1252 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1253 {
1254         return (0);
1255 }
1256
1257 int
1258 ctl_port_enable(ctl_port_type port_type)
1259 {
1260         struct ctl_softc *softc;
1261         struct ctl_frontend *fe;
1262
1263         if (ctl_is_single == 0) {
1264                 union ctl_ha_msg msg_info;
1265                 int isc_retval;
1266
1267 #if 0
1268                 printf("%s: HA mode, synchronizing frontend enable\n",
1269                         __func__);
1270 #endif
1271                 msg_info.hdr.msg_type = CTL_MSG_SYNC_FE;
1272                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1273                         sizeof(msg_info), 1 )) > CTL_HA_STATUS_SUCCESS) {
1274                         printf("Sync msg send error retval %d\n", isc_retval);
1275                 }
1276                 if (!rcv_sync_msg) {
1277                         isc_retval=ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
1278                                 sizeof(msg_info), 1);
1279                 }
1280 #if 0
1281                 printf("CTL:Frontend Enable\n");
1282         } else {
1283                 printf("%s: single mode, skipping frontend synchronization\n",
1284                         __func__);
1285 #endif
1286         }
1287
1288         softc = control_softc;
1289
1290         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1291                 if (port_type & fe->port_type)
1292                 {
1293 #if 0
1294                         printf("port %d\n", fe->targ_port);
1295 #endif
1296                         ctl_frontend_online(fe);
1297                 }
1298         }
1299
1300         return (0);
1301 }
1302
1303 int
1304 ctl_port_disable(ctl_port_type port_type)
1305 {
1306         struct ctl_softc *softc;
1307         struct ctl_frontend *fe;
1308
1309         softc = control_softc;
1310
1311         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1312                 if (port_type & fe->port_type)
1313                         ctl_frontend_offline(fe);
1314         }
1315
1316         return (0);
1317 }
1318
1319 /*
1320  * Returns 0 for success, 1 for failure.
1321  * Currently the only failure mode is if there aren't enough entries
1322  * allocated.  So, in case of a failure, look at num_entries_dropped,
1323  * reallocate and try again.
1324  */
1325 int
1326 ctl_port_list(struct ctl_port_entry *entries, int num_entries_alloced,
1327               int *num_entries_filled, int *num_entries_dropped,
1328               ctl_port_type port_type, int no_virtual)
1329 {
1330         struct ctl_softc *softc;
1331         struct ctl_frontend *fe;
1332         int entries_dropped, entries_filled;
1333         int retval;
1334         int i;
1335
1336         softc = control_softc;
1337
1338         retval = 0;
1339         entries_filled = 0;
1340         entries_dropped = 0;
1341
1342         i = 0;
1343         mtx_lock(&softc->ctl_lock);
1344         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1345                 struct ctl_port_entry *entry;
1346
1347                 if ((fe->port_type & port_type) == 0)
1348                         continue;
1349
1350                 if ((no_virtual != 0)
1351                  && (fe->virtual_port != 0))
1352                         continue;
1353
1354                 if (entries_filled >= num_entries_alloced) {
1355                         entries_dropped++;
1356                         continue;
1357                 }
1358                 entry = &entries[i];
1359
1360                 entry->port_type = fe->port_type;
1361                 strlcpy(entry->port_name, fe->port_name,
1362                         sizeof(entry->port_name));
1363                 entry->physical_port = fe->physical_port;
1364                 entry->virtual_port = fe->virtual_port;
1365                 entry->wwnn = fe->wwnn;
1366                 entry->wwpn = fe->wwpn;
1367
1368                 i++;
1369                 entries_filled++;
1370         }
1371
1372         mtx_unlock(&softc->ctl_lock);
1373
1374         if (entries_dropped > 0)
1375                 retval = 1;
1376
1377         *num_entries_dropped = entries_dropped;
1378         *num_entries_filled = entries_filled;
1379
1380         return (retval);
1381 }
1382
1383 static void
1384 ctl_ioctl_online(void *arg)
1385 {
1386         struct ctl_ioctl_info *ioctl_info;
1387
1388         ioctl_info = (struct ctl_ioctl_info *)arg;
1389
1390         ioctl_info->flags |= CTL_IOCTL_FLAG_ENABLED;
1391 }
1392
1393 static void
1394 ctl_ioctl_offline(void *arg)
1395 {
1396         struct ctl_ioctl_info *ioctl_info;
1397
1398         ioctl_info = (struct ctl_ioctl_info *)arg;
1399
1400         ioctl_info->flags &= ~CTL_IOCTL_FLAG_ENABLED;
1401 }
1402
1403 /*
1404  * Remove an initiator by port number and initiator ID.
1405  * Returns 0 for success, 1 for failure.
1406  */
1407 int
1408 ctl_remove_initiator(int32_t targ_port, uint32_t iid)
1409 {
1410         struct ctl_softc *softc;
1411
1412         softc = control_softc;
1413
1414         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1415
1416         if ((targ_port < 0)
1417          || (targ_port > CTL_MAX_PORTS)) {
1418                 printf("%s: invalid port number %d\n", __func__, targ_port);
1419                 return (1);
1420         }
1421         if (iid > CTL_MAX_INIT_PER_PORT) {
1422                 printf("%s: initiator ID %u > maximun %u!\n",
1423                        __func__, iid, CTL_MAX_INIT_PER_PORT);
1424                 return (1);
1425         }
1426
1427         mtx_lock(&softc->ctl_lock);
1428
1429         softc->wwpn_iid[targ_port][iid].in_use = 0;
1430
1431         mtx_unlock(&softc->ctl_lock);
1432
1433         return (0);
1434 }
1435
1436 /*
1437  * Add an initiator to the initiator map.
1438  * Returns 0 for success, 1 for failure.
1439  */
1440 int
1441 ctl_add_initiator(uint64_t wwpn, int32_t targ_port, uint32_t iid)
1442 {
1443         struct ctl_softc *softc;
1444         int retval;
1445
1446         softc = control_softc;
1447
1448         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1449
1450         retval = 0;
1451
1452         if ((targ_port < 0)
1453          || (targ_port > CTL_MAX_PORTS)) {
1454                 printf("%s: invalid port number %d\n", __func__, targ_port);
1455                 return (1);
1456         }
1457         if (iid > CTL_MAX_INIT_PER_PORT) {
1458                 printf("%s: WWPN %#jx initiator ID %u > maximun %u!\n",
1459                        __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1460                 return (1);
1461         }
1462
1463         mtx_lock(&softc->ctl_lock);
1464
1465         if (softc->wwpn_iid[targ_port][iid].in_use != 0) {
1466                 /*
1467                  * We don't treat this as an error.
1468                  */
1469                 if (softc->wwpn_iid[targ_port][iid].wwpn == wwpn) {
1470                         printf("%s: port %d iid %u WWPN %#jx arrived again?\n",
1471                                __func__, targ_port, iid, (uintmax_t)wwpn);
1472                         goto bailout;
1473                 }
1474
1475                 /*
1476                  * This is an error, but what do we do about it?  The
1477                  * driver is telling us we have a new WWPN for this
1478                  * initiator ID, so we pretty much need to use it.
1479                  */
1480                 printf("%s: port %d iid %u WWPN %#jx arrived, WWPN %#jx is "
1481                        "still at that address\n", __func__, targ_port, iid,
1482                        (uintmax_t)wwpn,
1483                        (uintmax_t)softc->wwpn_iid[targ_port][iid].wwpn);
1484
1485                 /*
1486                  * XXX KDM clear have_ca and ua_pending on each LUN for
1487                  * this initiator.
1488                  */
1489         }
1490         softc->wwpn_iid[targ_port][iid].in_use = 1;
1491         softc->wwpn_iid[targ_port][iid].iid = iid;
1492         softc->wwpn_iid[targ_port][iid].wwpn = wwpn;
1493         softc->wwpn_iid[targ_port][iid].port = targ_port;
1494
1495 bailout:
1496
1497         mtx_unlock(&softc->ctl_lock);
1498
1499         return (retval);
1500 }
1501
1502 /*
1503  * XXX KDM should we pretend to do something in the target/lun
1504  * enable/disable functions?
1505  */
1506 static int
1507 ctl_ioctl_targ_enable(void *arg, struct ctl_id targ_id)
1508 {
1509         return (0);
1510 }
1511
1512 static int
1513 ctl_ioctl_targ_disable(void *arg, struct ctl_id targ_id)
1514 {
1515         return (0);
1516 }
1517
1518 static int
1519 ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
1520 {
1521         return (0);
1522 }
1523
1524 static int
1525 ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
1526 {
1527         return (0);
1528 }
1529
1530 /*
1531  * Data movement routine for the CTL ioctl frontend port.
1532  */
1533 static int
1534 ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
1535 {
1536         struct ctl_sg_entry *ext_sglist, *kern_sglist;
1537         struct ctl_sg_entry ext_entry, kern_entry;
1538         int ext_sglen, ext_sg_entries, kern_sg_entries;
1539         int ext_sg_start, ext_offset;
1540         int len_to_copy, len_copied;
1541         int kern_watermark, ext_watermark;
1542         int ext_sglist_malloced;
1543         int i, j;
1544
1545         ext_sglist_malloced = 0;
1546         ext_sg_start = 0;
1547         ext_offset = 0;
1548
1549         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove\n"));
1550
1551         /*
1552          * If this flag is set, fake the data transfer.
1553          */
1554         if (ctsio->io_hdr.flags & CTL_FLAG_NO_DATAMOVE) {
1555                 ctsio->ext_data_filled = ctsio->ext_data_len;
1556                 goto bailout;
1557         }
1558
1559         /*
1560          * To simplify things here, if we have a single buffer, stick it in
1561          * a S/G entry and just make it a single entry S/G list.
1562          */
1563         if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
1564                 int len_seen;
1565
1566                 ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
1567
1568                 ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
1569                                                            M_WAITOK);
1570                 ext_sglist_malloced = 1;
1571                 if (copyin(ctsio->ext_data_ptr, ext_sglist,
1572                                    ext_sglen) != 0) {
1573                         ctl_set_internal_failure(ctsio,
1574                                                  /*sks_valid*/ 0,
1575                                                  /*retry_count*/ 0);
1576                         goto bailout;
1577                 }
1578                 ext_sg_entries = ctsio->ext_sg_entries;
1579                 len_seen = 0;
1580                 for (i = 0; i < ext_sg_entries; i++) {
1581                         if ((len_seen + ext_sglist[i].len) >=
1582                              ctsio->ext_data_filled) {
1583                                 ext_sg_start = i;
1584                                 ext_offset = ctsio->ext_data_filled - len_seen;
1585                                 break;
1586                         }
1587                         len_seen += ext_sglist[i].len;
1588                 }
1589         } else {
1590                 ext_sglist = &ext_entry;
1591                 ext_sglist->addr = ctsio->ext_data_ptr;
1592                 ext_sglist->len = ctsio->ext_data_len;
1593                 ext_sg_entries = 1;
1594                 ext_sg_start = 0;
1595                 ext_offset = ctsio->ext_data_filled;
1596         }
1597
1598         if (ctsio->kern_sg_entries > 0) {
1599                 kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
1600                 kern_sg_entries = ctsio->kern_sg_entries;
1601         } else {
1602                 kern_sglist = &kern_entry;
1603                 kern_sglist->addr = ctsio->kern_data_ptr;
1604                 kern_sglist->len = ctsio->kern_data_len;
1605                 kern_sg_entries = 1;
1606         }
1607
1608
1609         kern_watermark = 0;
1610         ext_watermark = ext_offset;
1611         len_copied = 0;
1612         for (i = ext_sg_start, j = 0;
1613              i < ext_sg_entries && j < kern_sg_entries;) {
1614                 uint8_t *ext_ptr, *kern_ptr;
1615
1616                 len_to_copy = ctl_min(ext_sglist[i].len - ext_watermark,
1617                                       kern_sglist[j].len - kern_watermark);
1618
1619                 ext_ptr = (uint8_t *)ext_sglist[i].addr;
1620                 ext_ptr = ext_ptr + ext_watermark;
1621                 if (ctsio->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
1622                         /*
1623                          * XXX KDM fix this!
1624                          */
1625                         panic("need to implement bus address support");
1626 #if 0
1627                         kern_ptr = bus_to_virt(kern_sglist[j].addr);
1628 #endif
1629                 } else
1630                         kern_ptr = (uint8_t *)kern_sglist[j].addr;
1631                 kern_ptr = kern_ptr + kern_watermark;
1632
1633                 kern_watermark += len_to_copy;
1634                 ext_watermark += len_to_copy;
1635
1636                 if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
1637                      CTL_FLAG_DATA_IN) {
1638                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1639                                          "bytes to user\n", len_to_copy));
1640                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1641                                          "to %p\n", kern_ptr, ext_ptr));
1642                         if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
1643                                 ctl_set_internal_failure(ctsio,
1644                                                          /*sks_valid*/ 0,
1645                                                          /*retry_count*/ 0);
1646                                 goto bailout;
1647                         }
1648                 } else {
1649                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1650                                          "bytes from user\n", len_to_copy));
1651                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1652                                          "to %p\n", ext_ptr, kern_ptr));
1653                         if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
1654                                 ctl_set_internal_failure(ctsio,
1655                                                          /*sks_valid*/ 0,
1656                                                          /*retry_count*/0);
1657                                 goto bailout;
1658                         }
1659                 }
1660
1661                 len_copied += len_to_copy;
1662
1663                 if (ext_sglist[i].len == ext_watermark) {
1664                         i++;
1665                         ext_watermark = 0;
1666                 }
1667
1668                 if (kern_sglist[j].len == kern_watermark) {
1669                         j++;
1670                         kern_watermark = 0;
1671                 }
1672         }
1673
1674         ctsio->ext_data_filled += len_copied;
1675
1676         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_sg_entries: %d, "
1677                          "kern_sg_entries: %d\n", ext_sg_entries,
1678                          kern_sg_entries));
1679         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_data_len = %d, "
1680                          "kern_data_len = %d\n", ctsio->ext_data_len,
1681                          ctsio->kern_data_len));
1682
1683
1684         /* XXX KDM set residual?? */
1685 bailout:
1686
1687         if (ext_sglist_malloced != 0)
1688                 free(ext_sglist, M_CTL);
1689
1690         return (CTL_RETVAL_COMPLETE);
1691 }
1692
1693 /*
1694  * Serialize a command that went down the "wrong" side, and so was sent to
1695  * this controller for execution.  The logic is a little different than the
1696  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1697  * sent back to the other side, but in the success case, we execute the
1698  * command on this side (XFER mode) or tell the other side to execute it
1699  * (SER_ONLY mode).
1700  */
1701 static int
1702 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio, int have_lock)
1703 {
1704         struct ctl_softc *ctl_softc;
1705         union ctl_ha_msg msg_info;
1706         struct ctl_lun *lun;
1707         int retval = 0;
1708         uint32_t targ_lun;
1709
1710         ctl_softc = control_softc;
1711         if (have_lock == 0)
1712                 mtx_lock(&ctl_softc->ctl_lock);
1713
1714         targ_lun = ctsio->io_hdr.nexus.targ_lun;
1715         if (ctsio->io_hdr.nexus.lun_map_fn != NULL)
1716                 targ_lun = ctsio->io_hdr.nexus.lun_map_fn(ctsio->io_hdr.nexus.lun_map_arg, targ_lun);
1717         lun = ctl_softc->ctl_luns[targ_lun];
1718         if (lun==NULL)
1719         {
1720                 /*
1721                  * Why isn't LUN defined? The other side wouldn't
1722                  * send a cmd if the LUN is undefined.
1723                  */
1724                 printf("%s: Bad JUJU!, LUN is NULL!\n", __func__);
1725
1726                 /* "Logical unit not supported" */
1727                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1728                                    lun,
1729                                    /*sense_format*/SSD_TYPE_NONE,
1730                                    /*current_error*/ 1,
1731                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1732                                    /*asc*/ 0x25,
1733                                    /*ascq*/ 0x00,
1734                                    SSD_ELEM_NONE);
1735
1736                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1737                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1738                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1739                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1740                 msg_info.hdr.serializing_sc = NULL;
1741                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1742                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1743                                 sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1744                 }
1745                 if (have_lock == 0)
1746                         mtx_unlock(&ctl_softc->ctl_lock);
1747                 return(1);
1748
1749         }
1750
1751         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1752
1753         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1754                 (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1755                  ooa_links))) {
1756         case CTL_ACTION_BLOCK:
1757                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
1758                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
1759                                   blocked_links);
1760                 break;
1761         case CTL_ACTION_PASS:
1762         case CTL_ACTION_SKIP:
1763                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
1764                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
1765                         STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
1766                                            &ctsio->io_hdr, links);
1767                 } else {
1768
1769                         /* send msg back to other side */
1770                         msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1771                         msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
1772                         msg_info.hdr.msg_type = CTL_MSG_R2R;
1773 #if 0
1774                         printf("2. pOrig %x\n", (int)msg_info.hdr.original_sc);
1775 #endif
1776                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1777                             sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1778                         }
1779                 }
1780                 break;
1781         case CTL_ACTION_OVERLAP:
1782                 /* OVERLAPPED COMMANDS ATTEMPTED */
1783                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1784                                    lun,
1785                                    /*sense_format*/SSD_TYPE_NONE,
1786                                    /*current_error*/ 1,
1787                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1788                                    /*asc*/ 0x4E,
1789                                    /*ascq*/ 0x00,
1790                                    SSD_ELEM_NONE);
1791
1792                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1793                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1794                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1795                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1796                 msg_info.hdr.serializing_sc = NULL;
1797                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1798 #if 0
1799                 printf("BAD JUJU:Major Bummer Overlap\n");
1800 #endif
1801                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1802                 retval = 1;
1803                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1804                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1805                 }
1806                 break;
1807         case CTL_ACTION_OVERLAP_TAG:
1808                 /* TAGGED OVERLAPPED COMMANDS (NN = QUEUE TAG) */
1809                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1810                                    lun,
1811                                    /*sense_format*/SSD_TYPE_NONE,
1812                                    /*current_error*/ 1,
1813                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1814                                    /*asc*/ 0x4D,
1815                                    /*ascq*/ ctsio->tag_num & 0xff,
1816                                    SSD_ELEM_NONE);
1817
1818                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1819                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1820                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1821                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1822                 msg_info.hdr.serializing_sc = NULL;
1823                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1824 #if 0
1825                 printf("BAD JUJU:Major Bummer Overlap Tag\n");
1826 #endif
1827                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1828                 retval = 1;
1829                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1830                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1831                 }
1832                 break;
1833         case CTL_ACTION_ERROR:
1834         default:
1835                 /* "Internal target failure" */
1836                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1837                                    lun,
1838                                    /*sense_format*/SSD_TYPE_NONE,
1839                                    /*current_error*/ 1,
1840                                    /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
1841                                    /*asc*/ 0x44,
1842                                    /*ascq*/ 0x00,
1843                                    SSD_ELEM_NONE);
1844
1845                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1846                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1847                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1848                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1849                 msg_info.hdr.serializing_sc = NULL;
1850                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1851 #if 0
1852                 printf("BAD JUJU:Major Bummer HW Error\n");
1853 #endif
1854                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1855                 retval = 1;
1856                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1857                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1858                 }
1859                 break;
1860         }
1861         if (have_lock == 0)
1862                 mtx_unlock(&ctl_softc->ctl_lock);
1863         return (retval);
1864 }
1865
1866 static int
1867 ctl_ioctl_submit_wait(union ctl_io *io)
1868 {
1869         struct ctl_fe_ioctl_params params;
1870         ctl_fe_ioctl_state last_state;
1871         int done, retval;
1872
1873         retval = 0;
1874
1875         bzero(&params, sizeof(params));
1876
1877         mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
1878         cv_init(&params.sem, "ctlioccv");
1879         params.state = CTL_IOCTL_INPROG;
1880         last_state = params.state;
1881
1882         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = &params;
1883
1884         CTL_DEBUG_PRINT(("ctl_ioctl_submit_wait\n"));
1885
1886         /* This shouldn't happen */
1887         if ((retval = ctl_queue(io)) != CTL_RETVAL_COMPLETE)
1888                 return (retval);
1889
1890         done = 0;
1891
1892         do {
1893                 mtx_lock(&params.ioctl_mtx);
1894                 /*
1895                  * Check the state here, and don't sleep if the state has
1896                  * already changed (i.e. wakeup has already occured, but we
1897                  * weren't waiting yet).
1898                  */
1899                 if (params.state == last_state) {
1900                         /* XXX KDM cv_wait_sig instead? */
1901                         cv_wait(&params.sem, &params.ioctl_mtx);
1902                 }
1903                 last_state = params.state;
1904
1905                 switch (params.state) {
1906                 case CTL_IOCTL_INPROG:
1907                         /* Why did we wake up? */
1908                         /* XXX KDM error here? */
1909                         mtx_unlock(&params.ioctl_mtx);
1910                         break;
1911                 case CTL_IOCTL_DATAMOVE:
1912                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DATAMOVE\n"));
1913
1914                         /*
1915                          * change last_state back to INPROG to avoid
1916                          * deadlock on subsequent data moves.
1917                          */
1918                         params.state = last_state = CTL_IOCTL_INPROG;
1919
1920                         mtx_unlock(&params.ioctl_mtx);
1921                         ctl_ioctl_do_datamove(&io->scsiio);
1922                         /*
1923                          * Note that in some cases, most notably writes,
1924                          * this will queue the I/O and call us back later.
1925                          * In other cases, generally reads, this routine
1926                          * will immediately call back and wake us up,
1927                          * probably using our own context.
1928                          */
1929                         io->scsiio.be_move_done(io);
1930                         break;
1931                 case CTL_IOCTL_DONE:
1932                         mtx_unlock(&params.ioctl_mtx);
1933                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DONE\n"));
1934                         done = 1;
1935                         break;
1936                 default:
1937                         mtx_unlock(&params.ioctl_mtx);
1938                         /* XXX KDM error here? */
1939                         break;
1940                 }
1941         } while (done == 0);
1942
1943         mtx_destroy(&params.ioctl_mtx);
1944         cv_destroy(&params.sem);
1945
1946         return (CTL_RETVAL_COMPLETE);
1947 }
1948
1949 static void
1950 ctl_ioctl_datamove(union ctl_io *io)
1951 {
1952         struct ctl_fe_ioctl_params *params;
1953
1954         params = (struct ctl_fe_ioctl_params *)
1955                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1956
1957         mtx_lock(&params->ioctl_mtx);
1958         params->state = CTL_IOCTL_DATAMOVE;
1959         cv_broadcast(&params->sem);
1960         mtx_unlock(&params->ioctl_mtx);
1961 }
1962
1963 static void
1964 ctl_ioctl_done(union ctl_io *io)
1965 {
1966         struct ctl_fe_ioctl_params *params;
1967
1968         params = (struct ctl_fe_ioctl_params *)
1969                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1970
1971         mtx_lock(&params->ioctl_mtx);
1972         params->state = CTL_IOCTL_DONE;
1973         cv_broadcast(&params->sem);
1974         mtx_unlock(&params->ioctl_mtx);
1975 }
1976
1977 static void
1978 ctl_ioctl_hard_startstop_callback(void *arg, struct cfi_metatask *metatask)
1979 {
1980         struct ctl_fe_ioctl_startstop_info *sd_info;
1981
1982         sd_info = (struct ctl_fe_ioctl_startstop_info *)arg;
1983
1984         sd_info->hs_info.status = metatask->status;
1985         sd_info->hs_info.total_luns = metatask->taskinfo.startstop.total_luns;
1986         sd_info->hs_info.luns_complete =
1987                 metatask->taskinfo.startstop.luns_complete;
1988         sd_info->hs_info.luns_failed = metatask->taskinfo.startstop.luns_failed;
1989
1990         cv_broadcast(&sd_info->sem);
1991 }
1992
1993 static void
1994 ctl_ioctl_bbrread_callback(void *arg, struct cfi_metatask *metatask)
1995 {
1996         struct ctl_fe_ioctl_bbrread_info *fe_bbr_info;
1997
1998         fe_bbr_info = (struct ctl_fe_ioctl_bbrread_info *)arg;
1999
2000         mtx_lock(fe_bbr_info->lock);
2001         fe_bbr_info->bbr_info->status = metatask->status;
2002         fe_bbr_info->bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2003         fe_bbr_info->wakeup_done = 1;
2004         mtx_unlock(fe_bbr_info->lock);
2005
2006         cv_broadcast(&fe_bbr_info->sem);
2007 }
2008
2009 /*
2010  * Returns 0 for success, errno for failure.
2011  */
2012 static int
2013 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
2014                    struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
2015 {
2016         union ctl_io *io;
2017         int retval;
2018
2019         retval = 0;
2020
2021         mtx_assert(&control_softc->ctl_lock, MA_OWNED);
2022
2023         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
2024              (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2025              ooa_links)) {
2026                 struct ctl_ooa_entry *entry;
2027
2028                 /*
2029                  * If we've got more than we can fit, just count the
2030                  * remaining entries.
2031                  */
2032                 if (*cur_fill_num >= ooa_hdr->alloc_num)
2033                         continue;
2034
2035                 entry = &kern_entries[*cur_fill_num];
2036
2037                 entry->tag_num = io->scsiio.tag_num;
2038                 entry->lun_num = lun->lun;
2039 #ifdef CTL_TIME_IO
2040                 entry->start_bt = io->io_hdr.start_bt;
2041 #endif
2042                 bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
2043                 entry->cdb_len = io->scsiio.cdb_len;
2044                 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
2045                         entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
2046
2047                 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
2048                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
2049
2050                 if (io->io_hdr.flags & CTL_FLAG_ABORT)
2051                         entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
2052
2053                 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
2054                         entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
2055
2056                 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
2057                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
2058         }
2059
2060         return (retval);
2061 }
2062
2063 static void *
2064 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2065                  size_t error_str_len)
2066 {
2067         void *kptr;
2068
2069         kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2070
2071         if (copyin(user_addr, kptr, len) != 0) {
2072                 snprintf(error_str, error_str_len, "Error copying %d bytes "
2073                          "from user address %p to kernel address %p", len,
2074                          user_addr, kptr);
2075                 free(kptr, M_CTL);
2076                 return (NULL);
2077         }
2078
2079         return (kptr);
2080 }
2081
2082 static void
2083 ctl_free_args(int num_be_args, struct ctl_be_arg *be_args)
2084 {
2085         int i;
2086
2087         if (be_args == NULL)
2088                 return;
2089
2090         for (i = 0; i < num_be_args; i++) {
2091                 free(be_args[i].kname, M_CTL);
2092                 free(be_args[i].kvalue, M_CTL);
2093         }
2094
2095         free(be_args, M_CTL);
2096 }
2097
2098 static struct ctl_be_arg *
2099 ctl_copyin_args(int num_be_args, struct ctl_be_arg *be_args,
2100                 char *error_str, size_t error_str_len)
2101 {
2102         struct ctl_be_arg *args;
2103         int i;
2104
2105         args = ctl_copyin_alloc(be_args, num_be_args * sizeof(*be_args),
2106                                 error_str, error_str_len);
2107
2108         if (args == NULL)
2109                 goto bailout;
2110
2111         for (i = 0; i < num_be_args; i++) {
2112                 args[i].kname = NULL;
2113                 args[i].kvalue = NULL;
2114         }
2115
2116         for (i = 0; i < num_be_args; i++) {
2117                 uint8_t *tmpptr;
2118
2119                 args[i].kname = ctl_copyin_alloc(args[i].name,
2120                         args[i].namelen, error_str, error_str_len);
2121                 if (args[i].kname == NULL)
2122                         goto bailout;
2123
2124                 if (args[i].kname[args[i].namelen - 1] != '\0') {
2125                         snprintf(error_str, error_str_len, "Argument %d "
2126                                  "name is not NUL-terminated", i);
2127                         goto bailout;
2128                 }
2129
2130                 args[i].kvalue = NULL;
2131
2132                 tmpptr = ctl_copyin_alloc(args[i].value,
2133                         args[i].vallen, error_str, error_str_len);
2134                 if (tmpptr == NULL)
2135                         goto bailout;
2136
2137                 args[i].kvalue = tmpptr;
2138
2139                 if ((args[i].flags & CTL_BEARG_ASCII)
2140                  && (tmpptr[args[i].vallen - 1] != '\0')) {
2141                         snprintf(error_str, error_str_len, "Argument %d "
2142                                  "value is not NUL-terminated", i);
2143                         goto bailout;
2144                 }
2145         }
2146
2147         return (args);
2148 bailout:
2149
2150         ctl_free_args(num_be_args, args);
2151
2152         return (NULL);
2153 }
2154
2155 /*
2156  * Escape characters that are illegal or not recommended in XML.
2157  */
2158 int
2159 ctl_sbuf_printf_esc(struct sbuf *sb, char *str)
2160 {
2161         int retval;
2162
2163         retval = 0;
2164
2165         for (; *str; str++) {
2166                 switch (*str) {
2167                 case '&':
2168                         retval = sbuf_printf(sb, "&amp;");
2169                         break;
2170                 case '>':
2171                         retval = sbuf_printf(sb, "&gt;");
2172                         break;
2173                 case '<':
2174                         retval = sbuf_printf(sb, "&lt;");
2175                         break;
2176                 default:
2177                         retval = sbuf_putc(sb, *str);
2178                         break;
2179                 }
2180
2181                 if (retval != 0)
2182                         break;
2183
2184         }
2185
2186         return (retval);
2187 }
2188
2189 static int
2190 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2191           struct thread *td)
2192 {
2193         struct ctl_softc *softc;
2194         int retval;
2195
2196         softc = control_softc;
2197
2198         retval = 0;
2199
2200         switch (cmd) {
2201         case CTL_IO: {
2202                 union ctl_io *io;
2203                 void *pool_tmp;
2204
2205                 /*
2206                  * If we haven't been "enabled", don't allow any SCSI I/O
2207                  * to this FETD.
2208                  */
2209                 if ((softc->ioctl_info.flags & CTL_IOCTL_FLAG_ENABLED) == 0) {
2210                         retval = -EPERM;
2211                         break;
2212                 }
2213
2214                 io = ctl_alloc_io(softc->ioctl_info.fe.ctl_pool_ref);
2215                 if (io == NULL) {
2216                         printf("ctl_ioctl: can't allocate ctl_io!\n");
2217                         retval = -ENOSPC;
2218                         break;
2219                 }
2220
2221                 /*
2222                  * Need to save the pool reference so it doesn't get
2223                  * spammed by the user's ctl_io.
2224                  */
2225                 pool_tmp = io->io_hdr.pool;
2226
2227                 memcpy(io, (void *)addr, sizeof(*io));
2228
2229                 io->io_hdr.pool = pool_tmp;
2230                 /*
2231                  * No status yet, so make sure the status is set properly.
2232                  */
2233                 io->io_hdr.status = CTL_STATUS_NONE;
2234
2235                 /*
2236                  * The user sets the initiator ID, target and LUN IDs.
2237                  */
2238                 io->io_hdr.nexus.targ_port = softc->ioctl_info.fe.targ_port;
2239                 io->io_hdr.flags |= CTL_FLAG_USER_REQ;
2240                 if ((io->io_hdr.io_type == CTL_IO_SCSI)
2241                  && (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
2242                         io->scsiio.tag_num = softc->ioctl_info.cur_tag_num++;
2243
2244                 retval = ctl_ioctl_submit_wait(io);
2245
2246                 if (retval != 0) {
2247                         ctl_free_io(io);
2248                         break;
2249                 }
2250
2251                 memcpy((void *)addr, io, sizeof(*io));
2252
2253                 /* return this to our pool */
2254                 ctl_free_io(io);
2255
2256                 break;
2257         }
2258         case CTL_ENABLE_PORT:
2259         case CTL_DISABLE_PORT:
2260         case CTL_SET_PORT_WWNS: {
2261                 struct ctl_frontend *fe;
2262                 struct ctl_port_entry *entry;
2263
2264                 entry = (struct ctl_port_entry *)addr;
2265
2266                 mtx_lock(&softc->ctl_lock);
2267                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2268                         int action, done;
2269
2270                         action = 0;
2271                         done = 0;
2272
2273                         if ((entry->port_type == CTL_PORT_NONE)
2274                          && (entry->targ_port == fe->targ_port)) {
2275                                 /*
2276                                  * If the user only wants to enable or
2277                                  * disable or set WWNs on a specific port,
2278                                  * do the operation and we're done.
2279                                  */
2280                                 action = 1;
2281                                 done = 1;
2282                         } else if (entry->port_type & fe->port_type) {
2283                                 /*
2284                                  * Compare the user's type mask with the
2285                                  * particular frontend type to see if we
2286                                  * have a match.
2287                                  */
2288                                 action = 1;
2289                                 done = 0;
2290
2291                                 /*
2292                                  * Make sure the user isn't trying to set
2293                                  * WWNs on multiple ports at the same time.
2294                                  */
2295                                 if (cmd == CTL_SET_PORT_WWNS) {
2296                                         printf("%s: Can't set WWNs on "
2297                                                "multiple ports\n", __func__);
2298                                         retval = EINVAL;
2299                                         break;
2300                                 }
2301                         }
2302                         if (action != 0) {
2303                                 /*
2304                                  * XXX KDM we have to drop the lock here,
2305                                  * because the online/offline operations
2306                                  * can potentially block.  We need to
2307                                  * reference count the frontends so they
2308                                  * can't go away,
2309                                  */
2310                                 mtx_unlock(&softc->ctl_lock);
2311
2312                                 if (cmd == CTL_ENABLE_PORT) {
2313                                         struct ctl_lun *lun;
2314
2315                                         STAILQ_FOREACH(lun, &softc->lun_list,
2316                                                        links) {
2317                                                 fe->lun_enable(fe->targ_lun_arg,
2318                                                     lun->target,
2319                                                     lun->lun);
2320                                         }
2321
2322                                         ctl_frontend_online(fe);
2323                                 } else if (cmd == CTL_DISABLE_PORT) {
2324                                         struct ctl_lun *lun;
2325
2326                                         ctl_frontend_offline(fe);
2327
2328                                         STAILQ_FOREACH(lun, &softc->lun_list,
2329                                                        links) {
2330                                                 fe->lun_disable(
2331                                                     fe->targ_lun_arg,
2332                                                     lun->target,
2333                                                     lun->lun);
2334                                         }
2335                                 }
2336
2337                                 mtx_lock(&softc->ctl_lock);
2338
2339                                 if (cmd == CTL_SET_PORT_WWNS)
2340                                         ctl_frontend_set_wwns(fe,
2341                                             (entry->flags & CTL_PORT_WWNN_VALID) ?
2342                                             1 : 0, entry->wwnn,
2343                                             (entry->flags & CTL_PORT_WWPN_VALID) ?
2344                                             1 : 0, entry->wwpn);
2345                         }
2346                         if (done != 0)
2347                                 break;
2348                 }
2349                 mtx_unlock(&softc->ctl_lock);
2350                 break;
2351         }
2352         case CTL_GET_PORT_LIST: {
2353                 struct ctl_frontend *fe;
2354                 struct ctl_port_list *list;
2355                 int i;
2356
2357                 list = (struct ctl_port_list *)addr;
2358
2359                 if (list->alloc_len != (list->alloc_num *
2360                     sizeof(struct ctl_port_entry))) {
2361                         printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2362                                "alloc_num %u * sizeof(struct ctl_port_entry) "
2363                                "%zu\n", __func__, list->alloc_len,
2364                                list->alloc_num, sizeof(struct ctl_port_entry));
2365                         retval = EINVAL;
2366                         break;
2367                 }
2368                 list->fill_len = 0;
2369                 list->fill_num = 0;
2370                 list->dropped_num = 0;
2371                 i = 0;
2372                 mtx_lock(&softc->ctl_lock);
2373                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2374                         struct ctl_port_entry entry, *list_entry;
2375
2376                         if (list->fill_num >= list->alloc_num) {
2377                                 list->dropped_num++;
2378                                 continue;
2379                         }
2380
2381                         entry.port_type = fe->port_type;
2382                         strlcpy(entry.port_name, fe->port_name,
2383                                 sizeof(entry.port_name));
2384                         entry.targ_port = fe->targ_port;
2385                         entry.physical_port = fe->physical_port;
2386                         entry.virtual_port = fe->virtual_port;
2387                         entry.wwnn = fe->wwnn;
2388                         entry.wwpn = fe->wwpn;
2389                         if (fe->status & CTL_PORT_STATUS_ONLINE)
2390                                 entry.online = 1;
2391                         else
2392                                 entry.online = 0;
2393
2394                         list_entry = &list->entries[i];
2395
2396                         retval = copyout(&entry, list_entry, sizeof(entry));
2397                         if (retval != 0) {
2398                                 printf("%s: CTL_GET_PORT_LIST: copyout "
2399                                        "returned %d\n", __func__, retval);
2400                                 break;
2401                         }
2402                         i++;
2403                         list->fill_num++;
2404                         list->fill_len += sizeof(entry);
2405                 }
2406                 mtx_unlock(&softc->ctl_lock);
2407
2408                 /*
2409                  * If this is non-zero, we had a copyout fault, so there's
2410                  * probably no point in attempting to set the status inside
2411                  * the structure.
2412                  */
2413                 if (retval != 0)
2414                         break;
2415
2416                 if (list->dropped_num > 0)
2417                         list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2418                 else
2419                         list->status = CTL_PORT_LIST_OK;
2420                 break;
2421         }
2422         case CTL_DUMP_OOA: {
2423                 struct ctl_lun *lun;
2424                 union ctl_io *io;
2425                 char printbuf[128];
2426                 struct sbuf sb;
2427
2428                 mtx_lock(&softc->ctl_lock);
2429                 printf("Dumping OOA queues:\n");
2430                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2431                         for (io = (union ctl_io *)TAILQ_FIRST(
2432                              &lun->ooa_queue); io != NULL;
2433                              io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2434                              ooa_links)) {
2435                                 sbuf_new(&sb, printbuf, sizeof(printbuf),
2436                                          SBUF_FIXEDLEN);
2437                                 sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2438                                             (intmax_t)lun->lun,
2439                                             io->scsiio.tag_num,
2440                                             (io->io_hdr.flags &
2441                                             CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2442                                             (io->io_hdr.flags &
2443                                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2444                                             (io->io_hdr.flags &
2445                                             CTL_FLAG_ABORT) ? " ABORT" : "",
2446                                             (io->io_hdr.flags &
2447                                         CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2448                                 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2449                                 sbuf_finish(&sb);
2450                                 printf("%s\n", sbuf_data(&sb));
2451                         }
2452                 }
2453                 printf("OOA queues dump done\n");
2454                 mtx_unlock(&softc->ctl_lock);
2455                 break;
2456         }
2457         case CTL_GET_OOA: {
2458                 struct ctl_lun *lun;
2459                 struct ctl_ooa *ooa_hdr;
2460                 struct ctl_ooa_entry *entries;
2461                 uint32_t cur_fill_num;
2462
2463                 ooa_hdr = (struct ctl_ooa *)addr;
2464
2465                 if ((ooa_hdr->alloc_len == 0)
2466                  || (ooa_hdr->alloc_num == 0)) {
2467                         printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2468                                "must be non-zero\n", __func__,
2469                                ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2470                         retval = EINVAL;
2471                         break;
2472                 }
2473
2474                 if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2475                     sizeof(struct ctl_ooa_entry))) {
2476                         printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2477                                "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2478                                __func__, ooa_hdr->alloc_len,
2479                                ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2480                         retval = EINVAL;
2481                         break;
2482                 }
2483
2484                 entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2485                 if (entries == NULL) {
2486                         printf("%s: could not allocate %d bytes for OOA "
2487                                "dump\n", __func__, ooa_hdr->alloc_len);
2488                         retval = ENOMEM;
2489                         break;
2490                 }
2491
2492                 mtx_lock(&softc->ctl_lock);
2493                 if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2494                  && ((ooa_hdr->lun_num > CTL_MAX_LUNS)
2495                   || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2496                         mtx_unlock(&softc->ctl_lock);
2497                         free(entries, M_CTL);
2498                         printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2499                                __func__, (uintmax_t)ooa_hdr->lun_num);
2500                         retval = EINVAL;
2501                         break;
2502                 }
2503
2504                 cur_fill_num = 0;
2505
2506                 if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2507                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
2508                                 retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2509                                         ooa_hdr, entries);
2510                                 if (retval != 0)
2511                                         break;
2512                         }
2513                         if (retval != 0) {
2514                                 mtx_unlock(&softc->ctl_lock);
2515                                 free(entries, M_CTL);
2516                                 break;
2517                         }
2518                 } else {
2519                         lun = softc->ctl_luns[ooa_hdr->lun_num];
2520
2521                         retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
2522                                                     entries);
2523                 }
2524                 mtx_unlock(&softc->ctl_lock);
2525
2526                 ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2527                 ooa_hdr->fill_len = ooa_hdr->fill_num *
2528                         sizeof(struct ctl_ooa_entry);
2529                 retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2530                 if (retval != 0) {
2531                         printf("%s: error copying out %d bytes for OOA dump\n",
2532                                __func__, ooa_hdr->fill_len);
2533                 }
2534
2535                 getbintime(&ooa_hdr->cur_bt);
2536
2537                 if (cur_fill_num > ooa_hdr->alloc_num) {
2538                         ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2539                         ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2540                 } else {
2541                         ooa_hdr->dropped_num = 0;
2542                         ooa_hdr->status = CTL_OOA_OK;
2543                 }
2544
2545                 free(entries, M_CTL);
2546                 break;
2547         }
2548         case CTL_CHECK_OOA: {
2549                 union ctl_io *io;
2550                 struct ctl_lun *lun;
2551                 struct ctl_ooa_info *ooa_info;
2552
2553
2554                 ooa_info = (struct ctl_ooa_info *)addr;
2555
2556                 if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2557                         ooa_info->status = CTL_OOA_INVALID_LUN;
2558                         break;
2559                 }
2560                 mtx_lock(&softc->ctl_lock);
2561                 lun = softc->ctl_luns[ooa_info->lun_id];
2562                 if (lun == NULL) {
2563                         mtx_unlock(&softc->ctl_lock);
2564                         ooa_info->status = CTL_OOA_INVALID_LUN;
2565                         break;
2566                 }
2567
2568                 ooa_info->num_entries = 0;
2569                 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2570                      io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2571                      &io->io_hdr, ooa_links)) {
2572                         ooa_info->num_entries++;
2573                 }
2574
2575                 mtx_unlock(&softc->ctl_lock);
2576                 ooa_info->status = CTL_OOA_SUCCESS;
2577
2578                 break;
2579         }
2580         case CTL_HARD_START:
2581         case CTL_HARD_STOP: {
2582                 struct ctl_fe_ioctl_startstop_info ss_info;
2583                 struct cfi_metatask *metatask;
2584                 struct mtx hs_mtx;
2585
2586                 mtx_init(&hs_mtx, "HS Mutex", NULL, MTX_DEF);
2587
2588                 cv_init(&ss_info.sem, "hard start/stop cv" );
2589
2590                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2591                 if (metatask == NULL) {
2592                         retval = ENOMEM;
2593                         mtx_destroy(&hs_mtx);
2594                         break;
2595                 }
2596
2597                 if (cmd == CTL_HARD_START)
2598                         metatask->tasktype = CFI_TASK_STARTUP;
2599                 else
2600                         metatask->tasktype = CFI_TASK_SHUTDOWN;
2601
2602                 metatask->callback = ctl_ioctl_hard_startstop_callback;
2603                 metatask->callback_arg = &ss_info;
2604
2605                 cfi_action(metatask);
2606
2607                 /* Wait for the callback */
2608                 mtx_lock(&hs_mtx);
2609                 cv_wait_sig(&ss_info.sem, &hs_mtx);
2610                 mtx_unlock(&hs_mtx);
2611
2612                 /*
2613                  * All information has been copied from the metatask by the
2614                  * time cv_broadcast() is called, so we free the metatask here.
2615                  */
2616                 cfi_free_metatask(metatask);
2617
2618                 memcpy((void *)addr, &ss_info.hs_info, sizeof(ss_info.hs_info));
2619
2620                 mtx_destroy(&hs_mtx);
2621                 break;
2622         }
2623         case CTL_BBRREAD: {
2624                 struct ctl_bbrread_info *bbr_info;
2625                 struct ctl_fe_ioctl_bbrread_info fe_bbr_info;
2626                 struct mtx bbr_mtx;
2627                 struct cfi_metatask *metatask;
2628
2629                 bbr_info = (struct ctl_bbrread_info *)addr;
2630
2631                 bzero(&fe_bbr_info, sizeof(fe_bbr_info));
2632
2633                 bzero(&bbr_mtx, sizeof(bbr_mtx));
2634                 mtx_init(&bbr_mtx, "BBR Mutex", NULL, MTX_DEF);
2635
2636                 fe_bbr_info.bbr_info = bbr_info;
2637                 fe_bbr_info.lock = &bbr_mtx;
2638
2639                 cv_init(&fe_bbr_info.sem, "BBR read cv");
2640                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2641
2642                 if (metatask == NULL) {
2643                         mtx_destroy(&bbr_mtx);
2644                         cv_destroy(&fe_bbr_info.sem);
2645                         retval = ENOMEM;
2646                         break;
2647                 }
2648                 metatask->tasktype = CFI_TASK_BBRREAD;
2649                 metatask->callback = ctl_ioctl_bbrread_callback;
2650                 metatask->callback_arg = &fe_bbr_info;
2651                 metatask->taskinfo.bbrread.lun_num = bbr_info->lun_num;
2652                 metatask->taskinfo.bbrread.lba = bbr_info->lba;
2653                 metatask->taskinfo.bbrread.len = bbr_info->len;
2654
2655                 cfi_action(metatask);
2656
2657                 mtx_lock(&bbr_mtx);
2658                 while (fe_bbr_info.wakeup_done == 0)
2659                         cv_wait_sig(&fe_bbr_info.sem, &bbr_mtx);
2660                 mtx_unlock(&bbr_mtx);
2661
2662                 bbr_info->status = metatask->status;
2663                 bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2664                 bbr_info->scsi_status = metatask->taskinfo.bbrread.scsi_status;
2665                 memcpy(&bbr_info->sense_data,
2666                        &metatask->taskinfo.bbrread.sense_data,
2667                        ctl_min(sizeof(bbr_info->sense_data),
2668                                sizeof(metatask->taskinfo.bbrread.sense_data)));
2669
2670                 cfi_free_metatask(metatask);
2671
2672                 mtx_destroy(&bbr_mtx);
2673                 cv_destroy(&fe_bbr_info.sem);
2674
2675                 break;
2676         }
2677         case CTL_DELAY_IO: {
2678                 struct ctl_io_delay_info *delay_info;
2679 #ifdef CTL_IO_DELAY
2680                 struct ctl_lun *lun;
2681 #endif /* CTL_IO_DELAY */
2682
2683                 delay_info = (struct ctl_io_delay_info *)addr;
2684
2685 #ifdef CTL_IO_DELAY
2686                 mtx_lock(&softc->ctl_lock);
2687
2688                 if ((delay_info->lun_id > CTL_MAX_LUNS)
2689                  || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2690                         delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2691                 } else {
2692                         lun = softc->ctl_luns[delay_info->lun_id];
2693
2694                         delay_info->status = CTL_DELAY_STATUS_OK;
2695
2696                         switch (delay_info->delay_type) {
2697                         case CTL_DELAY_TYPE_CONT:
2698                                 break;
2699                         case CTL_DELAY_TYPE_ONESHOT:
2700                                 break;
2701                         default:
2702                                 delay_info->status =
2703                                         CTL_DELAY_STATUS_INVALID_TYPE;
2704                                 break;
2705                         }
2706
2707                         switch (delay_info->delay_loc) {
2708                         case CTL_DELAY_LOC_DATAMOVE:
2709                                 lun->delay_info.datamove_type =
2710                                         delay_info->delay_type;
2711                                 lun->delay_info.datamove_delay =
2712                                         delay_info->delay_secs;
2713                                 break;
2714                         case CTL_DELAY_LOC_DONE:
2715                                 lun->delay_info.done_type =
2716                                         delay_info->delay_type;
2717                                 lun->delay_info.done_delay =
2718                                         delay_info->delay_secs;
2719                                 break;
2720                         default:
2721                                 delay_info->status =
2722                                         CTL_DELAY_STATUS_INVALID_LOC;
2723                                 break;
2724                         }
2725                 }
2726
2727                 mtx_unlock(&softc->ctl_lock);
2728 #else
2729                 delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2730 #endif /* CTL_IO_DELAY */
2731                 break;
2732         }
2733         case CTL_REALSYNC_SET: {
2734                 int *syncstate;
2735
2736                 syncstate = (int *)addr;
2737
2738                 mtx_lock(&softc->ctl_lock);
2739                 switch (*syncstate) {
2740                 case 0:
2741                         softc->flags &= ~CTL_FLAG_REAL_SYNC;
2742                         break;
2743                 case 1:
2744                         softc->flags |= CTL_FLAG_REAL_SYNC;
2745                         break;
2746                 default:
2747                         retval = -EINVAL;
2748                         break;
2749                 }
2750                 mtx_unlock(&softc->ctl_lock);
2751                 break;
2752         }
2753         case CTL_REALSYNC_GET: {
2754                 int *syncstate;
2755
2756                 syncstate = (int*)addr;
2757
2758                 mtx_lock(&softc->ctl_lock);
2759                 if (softc->flags & CTL_FLAG_REAL_SYNC)
2760                         *syncstate = 1;
2761                 else
2762                         *syncstate = 0;
2763                 mtx_unlock(&softc->ctl_lock);
2764
2765                 break;
2766         }
2767         case CTL_SETSYNC:
2768         case CTL_GETSYNC: {
2769                 struct ctl_sync_info *sync_info;
2770                 struct ctl_lun *lun;
2771
2772                 sync_info = (struct ctl_sync_info *)addr;
2773
2774                 mtx_lock(&softc->ctl_lock);
2775                 lun = softc->ctl_luns[sync_info->lun_id];
2776                 if (lun == NULL) {
2777                         mtx_unlock(&softc->ctl_lock);
2778                         sync_info->status = CTL_GS_SYNC_NO_LUN;
2779                 }
2780                 /*
2781                  * Get or set the sync interval.  We're not bounds checking
2782                  * in the set case, hopefully the user won't do something
2783                  * silly.
2784                  */
2785                 if (cmd == CTL_GETSYNC)
2786                         sync_info->sync_interval = lun->sync_interval;
2787                 else
2788                         lun->sync_interval = sync_info->sync_interval;
2789
2790                 mtx_unlock(&softc->ctl_lock);
2791
2792                 sync_info->status = CTL_GS_SYNC_OK;
2793
2794                 break;
2795         }
2796         case CTL_GETSTATS: {
2797                 struct ctl_stats *stats;
2798                 struct ctl_lun *lun;
2799                 int i;
2800
2801                 stats = (struct ctl_stats *)addr;
2802
2803                 if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2804                      stats->alloc_len) {
2805                         stats->status = CTL_SS_NEED_MORE_SPACE;
2806                         stats->num_luns = softc->num_luns;
2807                         break;
2808                 }
2809                 /*
2810                  * XXX KDM no locking here.  If the LUN list changes,
2811                  * things can blow up.
2812                  */
2813                 for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2814                      i++, lun = STAILQ_NEXT(lun, links)) {
2815                         retval = copyout(&lun->stats, &stats->lun_stats[i],
2816                                          sizeof(lun->stats));
2817                         if (retval != 0)
2818                                 break;
2819                 }
2820                 stats->num_luns = softc->num_luns;
2821                 stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2822                                  softc->num_luns;
2823                 stats->status = CTL_SS_OK;
2824 #ifdef CTL_TIME_IO
2825                 stats->flags = CTL_STATS_FLAG_TIME_VALID;
2826 #else
2827                 stats->flags = CTL_STATS_FLAG_NONE;
2828 #endif
2829                 getnanouptime(&stats->timestamp);
2830                 break;
2831         }
2832         case CTL_ERROR_INJECT: {
2833                 struct ctl_error_desc *err_desc, *new_err_desc;
2834                 struct ctl_lun *lun;
2835
2836                 err_desc = (struct ctl_error_desc *)addr;
2837
2838                 new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2839                                       M_WAITOK | M_ZERO);
2840                 bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2841
2842                 mtx_lock(&softc->ctl_lock);
2843                 lun = softc->ctl_luns[err_desc->lun_id];
2844                 if (lun == NULL) {
2845                         mtx_unlock(&softc->ctl_lock);
2846                         printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2847                                __func__, (uintmax_t)err_desc->lun_id);
2848                         retval = EINVAL;
2849                         break;
2850                 }
2851
2852                 /*
2853                  * We could do some checking here to verify the validity
2854                  * of the request, but given the complexity of error
2855                  * injection requests, the checking logic would be fairly
2856                  * complex.
2857                  *
2858                  * For now, if the request is invalid, it just won't get
2859                  * executed and might get deleted.
2860                  */
2861                 STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2862
2863                 /*
2864                  * XXX KDM check to make sure the serial number is unique,
2865                  * in case we somehow manage to wrap.  That shouldn't
2866                  * happen for a very long time, but it's the right thing to
2867                  * do.
2868                  */
2869                 new_err_desc->serial = lun->error_serial;
2870                 err_desc->serial = lun->error_serial;
2871                 lun->error_serial++;
2872
2873                 mtx_unlock(&softc->ctl_lock);
2874                 break;
2875         }
2876         case CTL_ERROR_INJECT_DELETE: {
2877                 struct ctl_error_desc *delete_desc, *desc, *desc2;
2878                 struct ctl_lun *lun;
2879                 int delete_done;
2880
2881                 delete_desc = (struct ctl_error_desc *)addr;
2882                 delete_done = 0;
2883
2884                 mtx_lock(&softc->ctl_lock);
2885                 lun = softc->ctl_luns[delete_desc->lun_id];
2886                 if (lun == NULL) {
2887                         mtx_unlock(&softc->ctl_lock);
2888                         printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
2889                                __func__, (uintmax_t)delete_desc->lun_id);
2890                         retval = EINVAL;
2891                         break;
2892                 }
2893                 STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
2894                         if (desc->serial != delete_desc->serial)
2895                                 continue;
2896
2897                         STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
2898                                       links);
2899                         free(desc, M_CTL);
2900                         delete_done = 1;
2901                 }
2902                 mtx_unlock(&softc->ctl_lock);
2903                 if (delete_done == 0) {
2904                         printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
2905                                "error serial %ju on LUN %u\n", __func__,
2906                                delete_desc->serial, delete_desc->lun_id);
2907                         retval = EINVAL;
2908                         break;
2909                 }
2910                 break;
2911         }
2912         case CTL_DUMP_STRUCTS: {
2913                 int i, j, k;
2914                 struct ctl_frontend *fe;
2915
2916                 printf("CTL IID to WWPN map start:\n");
2917                 for (i = 0; i < CTL_MAX_PORTS; i++) {
2918                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
2919                                 if (softc->wwpn_iid[i][j].in_use == 0)
2920                                         continue;
2921
2922                                 printf("port %d iid %u WWPN %#jx\n",
2923                                        softc->wwpn_iid[i][j].port,
2924                                        softc->wwpn_iid[i][j].iid,
2925                                        (uintmax_t)softc->wwpn_iid[i][j].wwpn);
2926                         }
2927                 }
2928                 printf("CTL IID to WWPN map end\n");
2929                 printf("CTL Persistent Reservation information start:\n");
2930                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2931                         struct ctl_lun *lun;
2932
2933                         lun = softc->ctl_luns[i];
2934
2935                         if ((lun == NULL)
2936                          || ((lun->flags & CTL_LUN_DISABLED) != 0))
2937                                 continue;
2938
2939                         for (j = 0; j < (CTL_MAX_PORTS * 2); j++) {
2940                                 for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
2941                                         if (lun->per_res[j+k].registered == 0)
2942                                                 continue;
2943                                         printf("LUN %d port %d iid %d key "
2944                                                "%#jx\n", i, j, k,
2945                                                (uintmax_t)scsi_8btou64(
2946                                                lun->per_res[j+k].res_key.key));
2947                                 }
2948                         }
2949                 }
2950                 printf("CTL Persistent Reservation information end\n");
2951                 printf("CTL Frontends:\n");
2952                 /*
2953                  * XXX KDM calling this without a lock.  We'd likely want
2954                  * to drop the lock before calling the frontend's dump
2955                  * routine anyway.
2956                  */
2957                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2958                         printf("Frontend %s Type %u pport %d vport %d WWNN "
2959                                "%#jx WWPN %#jx\n", fe->port_name, fe->port_type,
2960                                fe->physical_port, fe->virtual_port,
2961                                (uintmax_t)fe->wwnn, (uintmax_t)fe->wwpn);
2962
2963                         /*
2964                          * Frontends are not required to support the dump
2965                          * routine.
2966                          */
2967                         if (fe->fe_dump == NULL)
2968                                 continue;
2969
2970                         fe->fe_dump();
2971                 }
2972                 printf("CTL Frontend information end\n");
2973                 break;
2974         }
2975         case CTL_LUN_REQ: {
2976                 struct ctl_lun_req *lun_req;
2977                 struct ctl_backend_driver *backend;
2978
2979                 lun_req = (struct ctl_lun_req *)addr;
2980
2981                 backend = ctl_backend_find(lun_req->backend);
2982                 if (backend == NULL) {
2983                         lun_req->status = CTL_LUN_ERROR;
2984                         snprintf(lun_req->error_str,
2985                                  sizeof(lun_req->error_str),
2986                                  "Backend \"%s\" not found.",
2987                                  lun_req->backend);
2988                         break;
2989                 }
2990                 if (lun_req->num_be_args > 0) {
2991                         lun_req->kern_be_args = ctl_copyin_args(
2992                                 lun_req->num_be_args,
2993                                 lun_req->be_args,
2994                                 lun_req->error_str,
2995                                 sizeof(lun_req->error_str));
2996                         if (lun_req->kern_be_args == NULL) {
2997                                 lun_req->status = CTL_LUN_ERROR;
2998                                 break;
2999                         }
3000                 }
3001
3002                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3003
3004                 if (lun_req->num_be_args > 0) {
3005                         ctl_free_args(lun_req->num_be_args,
3006                                       lun_req->kern_be_args);
3007                 }
3008                 break;
3009         }
3010         case CTL_LUN_LIST: {
3011                 struct sbuf *sb;
3012                 struct ctl_lun *lun;
3013                 struct ctl_lun_list *list;
3014                 struct ctl_be_lun_option *opt;
3015
3016                 list = (struct ctl_lun_list *)addr;
3017
3018                 /*
3019                  * Allocate a fixed length sbuf here, based on the length
3020                  * of the user's buffer.  We could allocate an auto-extending
3021                  * buffer, and then tell the user how much larger our
3022                  * amount of data is than his buffer, but that presents
3023                  * some problems:
3024                  *
3025                  * 1.  The sbuf(9) routines use a blocking malloc, and so
3026                  *     we can't hold a lock while calling them with an
3027                  *     auto-extending buffer.
3028                  *
3029                  * 2.  There is not currently a LUN reference counting
3030                  *     mechanism, outside of outstanding transactions on
3031                  *     the LUN's OOA queue.  So a LUN could go away on us
3032                  *     while we're getting the LUN number, backend-specific
3033                  *     information, etc.  Thus, given the way things
3034                  *     currently work, we need to hold the CTL lock while
3035                  *     grabbing LUN information.
3036                  *
3037                  * So, from the user's standpoint, the best thing to do is
3038                  * allocate what he thinks is a reasonable buffer length,
3039                  * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
3040                  * double the buffer length and try again.  (And repeat
3041                  * that until he succeeds.)
3042                  */
3043                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3044                 if (sb == NULL) {
3045                         list->status = CTL_LUN_LIST_ERROR;
3046                         snprintf(list->error_str, sizeof(list->error_str),
3047                                  "Unable to allocate %d bytes for LUN list",
3048                                  list->alloc_len);
3049                         break;
3050                 }
3051
3052                 sbuf_printf(sb, "<ctllunlist>\n");
3053
3054                 mtx_lock(&softc->ctl_lock);
3055
3056                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
3057                         retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3058                                              (uintmax_t)lun->lun);
3059
3060                         /*
3061                          * Bail out as soon as we see that we've overfilled
3062                          * the buffer.
3063                          */
3064                         if (retval != 0)
3065                                 break;
3066
3067                         retval = sbuf_printf(sb, "<backend_type>%s"
3068                                              "</backend_type>\n",
3069                                              (lun->backend == NULL) ?  "none" :
3070                                              lun->backend->name);
3071
3072                         if (retval != 0)
3073                                 break;
3074
3075                         retval = sbuf_printf(sb, "<lun_type>%d</lun_type>\n",
3076                                              lun->be_lun->lun_type);
3077
3078                         if (retval != 0)
3079                                 break;
3080
3081                         if (lun->backend == NULL) {
3082                                 retval = sbuf_printf(sb, "</lun>\n");
3083                                 if (retval != 0)
3084                                         break;
3085                                 continue;
3086                         }
3087
3088                         retval = sbuf_printf(sb, "<size>%ju</size>\n",
3089                                              (lun->be_lun->maxlba > 0) ?
3090                                              lun->be_lun->maxlba + 1 : 0);
3091
3092                         if (retval != 0)
3093                                 break;
3094
3095                         retval = sbuf_printf(sb, "<blocksize>%u</blocksize>\n",
3096                                              lun->be_lun->blocksize);
3097
3098                         if (retval != 0)
3099                                 break;
3100
3101                         retval = sbuf_printf(sb, "<serial_number>");
3102
3103                         if (retval != 0)
3104                                 break;
3105
3106                         retval = ctl_sbuf_printf_esc(sb,
3107                                                      lun->be_lun->serial_num);
3108
3109                         if (retval != 0)
3110                                 break;
3111
3112                         retval = sbuf_printf(sb, "</serial_number>\n");
3113
3114                         if (retval != 0)
3115                                 break;
3116
3117                         retval = sbuf_printf(sb, "<device_id>");
3118
3119                         if (retval != 0)
3120                                 break;
3121
3122                         retval = ctl_sbuf_printf_esc(sb,lun->be_lun->device_id);
3123
3124                         if (retval != 0)
3125                                 break;
3126
3127                         retval = sbuf_printf(sb, "</device_id>\n");
3128
3129                         if (retval != 0)
3130                                 break;
3131
3132                         if (lun->backend->lun_info != NULL) {
3133                                 retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3134                                 if (retval != 0)
3135                                         break;
3136                         }
3137                         STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3138                                 retval = sbuf_printf(sb, "<%s>%s</%s>", opt->name, opt->value, opt->name);
3139                                 if (retval != 0)
3140                                         break;
3141                         }
3142
3143                         retval = sbuf_printf(sb, "</lun>\n");
3144
3145                         if (retval != 0)
3146                                 break;
3147                 }
3148                 mtx_unlock(&softc->ctl_lock);
3149
3150                 if ((retval != 0)
3151                  || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3152                         retval = 0;
3153                         sbuf_delete(sb);
3154                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3155                         snprintf(list->error_str, sizeof(list->error_str),
3156                                  "Out of space, %d bytes is too small",
3157                                  list->alloc_len);
3158                         break;
3159                 }
3160
3161                 sbuf_finish(sb);
3162
3163                 retval = copyout(sbuf_data(sb), list->lun_xml,
3164                                  sbuf_len(sb) + 1);
3165
3166                 list->fill_len = sbuf_len(sb) + 1;
3167                 list->status = CTL_LUN_LIST_OK;
3168                 sbuf_delete(sb);
3169                 break;
3170         }
3171         case CTL_ISCSI: {
3172                 struct ctl_iscsi *ci;
3173                 struct ctl_frontend *fe;
3174
3175                 ci = (struct ctl_iscsi *)addr;
3176
3177                 mtx_lock(&softc->ctl_lock);
3178                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
3179                         if (strcmp(fe->port_name, "iscsi") == 0)
3180                                 break;
3181                 }
3182                 mtx_unlock(&softc->ctl_lock);
3183
3184                 if (fe == NULL) {
3185                         ci->status = CTL_ISCSI_ERROR;
3186                         snprintf(ci->error_str, sizeof(ci->error_str), "Backend \"iscsi\" not found.");
3187                         break;
3188                 }
3189
3190                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3191                 break;
3192         }
3193         default: {
3194                 /* XXX KDM should we fix this? */
3195 #if 0
3196                 struct ctl_backend_driver *backend;
3197                 unsigned int type;
3198                 int found;
3199
3200                 found = 0;
3201
3202                 /*
3203                  * We encode the backend type as the ioctl type for backend
3204                  * ioctls.  So parse it out here, and then search for a
3205                  * backend of this type.
3206                  */
3207                 type = _IOC_TYPE(cmd);
3208
3209                 STAILQ_FOREACH(backend, &softc->be_list, links) {
3210                         if (backend->type == type) {
3211                                 found = 1;
3212                                 break;
3213                         }
3214                 }
3215                 if (found == 0) {
3216                         printf("ctl: unknown ioctl command %#lx or backend "
3217                                "%d\n", cmd, type);
3218                         retval = -EINVAL;
3219                         break;
3220                 }
3221                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3222 #endif
3223                 retval = ENOTTY;
3224                 break;
3225         }
3226         }
3227         return (retval);
3228 }
3229
3230 uint32_t
3231 ctl_get_initindex(struct ctl_nexus *nexus)
3232 {
3233         if (nexus->targ_port < CTL_MAX_PORTS)
3234                 return (nexus->initid.id +
3235                         (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3236         else
3237                 return (nexus->initid.id +
3238                        ((nexus->targ_port - CTL_MAX_PORTS) *
3239                         CTL_MAX_INIT_PER_PORT));
3240 }
3241
3242 uint32_t
3243 ctl_get_resindex(struct ctl_nexus *nexus)
3244 {
3245         return (nexus->initid.id + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3246 }
3247
3248 uint32_t
3249 ctl_port_idx(int port_num)
3250 {
3251         if (port_num < CTL_MAX_PORTS)
3252                 return(port_num);
3253         else
3254                 return(port_num - CTL_MAX_PORTS);
3255 }
3256
3257 /*
3258  * Note:  This only works for bitmask sizes that are at least 32 bits, and
3259  * that are a power of 2.
3260  */
3261 int
3262 ctl_ffz(uint32_t *mask, uint32_t size)
3263 {
3264         uint32_t num_chunks, num_pieces;
3265         int i, j;
3266
3267         num_chunks = (size >> 5);
3268         if (num_chunks == 0)
3269                 num_chunks++;
3270         num_pieces = ctl_min((sizeof(uint32_t) * 8), size);
3271
3272         for (i = 0; i < num_chunks; i++) {
3273                 for (j = 0; j < num_pieces; j++) {
3274                         if ((mask[i] & (1 << j)) == 0)
3275                                 return ((i << 5) + j);
3276                 }
3277         }
3278
3279         return (-1);
3280 }
3281
3282 int
3283 ctl_set_mask(uint32_t *mask, uint32_t bit)
3284 {
3285         uint32_t chunk, piece;
3286
3287         chunk = bit >> 5;
3288         piece = bit % (sizeof(uint32_t) * 8);
3289
3290         if ((mask[chunk] & (1 << piece)) != 0)
3291                 return (-1);
3292         else
3293                 mask[chunk] |= (1 << piece);
3294
3295         return (0);
3296 }
3297
3298 int
3299 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3300 {
3301         uint32_t chunk, piece;
3302
3303         chunk = bit >> 5;
3304         piece = bit % (sizeof(uint32_t) * 8);
3305
3306         if ((mask[chunk] & (1 << piece)) == 0)
3307                 return (-1);
3308         else
3309                 mask[chunk] &= ~(1 << piece);
3310
3311         return (0);
3312 }
3313
3314 int
3315 ctl_is_set(uint32_t *mask, uint32_t bit)
3316 {
3317         uint32_t chunk, piece;
3318
3319         chunk = bit >> 5;
3320         piece = bit % (sizeof(uint32_t) * 8);
3321
3322         if ((mask[chunk] & (1 << piece)) == 0)
3323                 return (0);
3324         else
3325                 return (1);
3326 }
3327
3328 #ifdef unused
3329 /*
3330  * The bus, target and lun are optional, they can be filled in later.
3331  * can_wait is used to determine whether we can wait on the malloc or not.
3332  */
3333 union ctl_io*
3334 ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port, uint32_t targ_target,
3335               uint32_t targ_lun, int can_wait)
3336 {
3337         union ctl_io *io;
3338
3339         if (can_wait)
3340                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_WAITOK);
3341         else
3342                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3343
3344         if (io != NULL) {
3345                 io->io_hdr.io_type = io_type;
3346                 io->io_hdr.targ_port = targ_port;
3347                 /*
3348                  * XXX KDM this needs to change/go away.  We need to move
3349                  * to a preallocated pool of ctl_scsiio structures.
3350                  */
3351                 io->io_hdr.nexus.targ_target.id = targ_target;
3352                 io->io_hdr.nexus.targ_lun = targ_lun;
3353         }
3354
3355         return (io);
3356 }
3357
3358 void
3359 ctl_kfree_io(union ctl_io *io)
3360 {
3361         free(io, M_CTL);
3362 }
3363 #endif /* unused */
3364
3365 /*
3366  * ctl_softc, pool_type, total_ctl_io are passed in.
3367  * npool is passed out.
3368  */
3369 int
3370 ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type,
3371                 uint32_t total_ctl_io, struct ctl_io_pool **npool)
3372 {
3373         uint32_t i;
3374         union ctl_io *cur_io, *next_io;
3375         struct ctl_io_pool *pool;
3376         int retval;
3377
3378         retval = 0;
3379
3380         pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3381                                             M_NOWAIT | M_ZERO);
3382         if (pool == NULL) {
3383                 retval = -ENOMEM;
3384                 goto bailout;
3385         }
3386
3387         pool->type = pool_type;
3388         pool->ctl_softc = ctl_softc;
3389
3390         mtx_lock(&ctl_softc->pool_lock);
3391         pool->id = ctl_softc->cur_pool_id++;
3392         mtx_unlock(&ctl_softc->pool_lock);
3393
3394         pool->flags = CTL_POOL_FLAG_NONE;
3395         pool->refcount = 1;             /* Reference for validity. */
3396         STAILQ_INIT(&pool->free_queue);
3397
3398         /*
3399          * XXX KDM other options here:
3400          * - allocate a page at a time
3401          * - allocate one big chunk of memory.
3402          * Page allocation might work well, but would take a little more
3403          * tracking.
3404          */
3405         for (i = 0; i < total_ctl_io; i++) {
3406                 cur_io = (union ctl_io *)malloc(sizeof(*cur_io), M_CTL,
3407                                                 M_NOWAIT);
3408                 if (cur_io == NULL) {
3409                         retval = ENOMEM;
3410                         break;
3411                 }
3412                 cur_io->io_hdr.pool = pool;
3413                 STAILQ_INSERT_TAIL(&pool->free_queue, &cur_io->io_hdr, links);
3414                 pool->total_ctl_io++;
3415                 pool->free_ctl_io++;
3416         }
3417
3418         if (retval != 0) {
3419                 for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3420                      cur_io != NULL; cur_io = next_io) {
3421                         next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr,
3422                                                               links);
3423                         STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr,
3424                                       ctl_io_hdr, links);
3425                         free(cur_io, M_CTL);
3426                 }
3427
3428                 free(pool, M_CTL);
3429                 goto bailout;
3430         }
3431         mtx_lock(&ctl_softc->pool_lock);
3432         ctl_softc->num_pools++;
3433         STAILQ_INSERT_TAIL(&ctl_softc->io_pools, pool, links);
3434         /*
3435          * Increment our usage count if this is an external consumer, so we
3436          * can't get unloaded until the external consumer (most likely a
3437          * FETD) unloads and frees his pool.
3438          *
3439          * XXX KDM will this increment the caller's module use count, or
3440          * mine?
3441          */
3442 #if 0
3443         if ((pool_type != CTL_POOL_EMERGENCY)
3444          && (pool_type != CTL_POOL_INTERNAL)
3445          && (pool_type != CTL_POOL_IOCTL)
3446          && (pool_type != CTL_POOL_4OTHERSC))
3447                 MOD_INC_USE_COUNT;
3448 #endif
3449
3450         mtx_unlock(&ctl_softc->pool_lock);
3451
3452         *npool = pool;
3453
3454 bailout:
3455
3456         return (retval);
3457 }
3458
3459 static int
3460 ctl_pool_acquire(struct ctl_io_pool *pool)
3461 {
3462
3463         mtx_assert(&pool->ctl_softc->pool_lock, MA_OWNED);
3464
3465         if (pool->flags & CTL_POOL_FLAG_INVALID)
3466                 return (-EINVAL);
3467
3468         pool->refcount++;
3469
3470         return (0);
3471 }
3472
3473 static void
3474 ctl_pool_release(struct ctl_io_pool *pool)
3475 {
3476         struct ctl_softc *ctl_softc = pool->ctl_softc;
3477         union ctl_io *io;
3478
3479         mtx_assert(&ctl_softc->pool_lock, MA_OWNED);
3480
3481         if (--pool->refcount != 0)
3482                 return;
3483
3484         while ((io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue)) != NULL) {
3485                 STAILQ_REMOVE(&pool->free_queue, &io->io_hdr, ctl_io_hdr,
3486                               links);
3487                 free(io, M_CTL);
3488         }
3489
3490         STAILQ_REMOVE(&ctl_softc->io_pools, pool, ctl_io_pool, links);
3491         ctl_softc->num_pools--;
3492
3493         /*
3494          * XXX KDM will this decrement the caller's usage count or mine?
3495          */
3496 #if 0
3497         if ((pool->type != CTL_POOL_EMERGENCY)
3498          && (pool->type != CTL_POOL_INTERNAL)
3499          && (pool->type != CTL_POOL_IOCTL))
3500                 MOD_DEC_USE_COUNT;
3501 #endif
3502
3503         free(pool, M_CTL);
3504 }
3505
3506 void
3507 ctl_pool_free(struct ctl_io_pool *pool)
3508 {
3509         struct ctl_softc *ctl_softc;
3510
3511         if (pool == NULL)
3512                 return;
3513
3514         ctl_softc = pool->ctl_softc;
3515         mtx_lock(&ctl_softc->pool_lock);
3516         pool->flags |= CTL_POOL_FLAG_INVALID;
3517         ctl_pool_release(pool);
3518         mtx_unlock(&ctl_softc->pool_lock);
3519 }
3520
3521 /*
3522  * This routine does not block (except for spinlocks of course).
3523  * It tries to allocate a ctl_io union from the caller's pool as quickly as
3524  * possible.
3525  */
3526 union ctl_io *
3527 ctl_alloc_io(void *pool_ref)
3528 {
3529         union ctl_io *io;
3530         struct ctl_softc *ctl_softc;
3531         struct ctl_io_pool *pool, *npool;
3532         struct ctl_io_pool *emergency_pool;
3533
3534         pool = (struct ctl_io_pool *)pool_ref;
3535
3536         if (pool == NULL) {
3537                 printf("%s: pool is NULL\n", __func__);
3538                 return (NULL);
3539         }
3540
3541         emergency_pool = NULL;
3542
3543         ctl_softc = pool->ctl_softc;
3544
3545         mtx_lock(&ctl_softc->pool_lock);
3546         /*
3547          * First, try to get the io structure from the user's pool.
3548          */
3549         if (ctl_pool_acquire(pool) == 0) {
3550                 io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3551                 if (io != NULL) {
3552                         STAILQ_REMOVE_HEAD(&pool->free_queue, links);
3553                         pool->total_allocated++;
3554                         pool->free_ctl_io--;
3555                         mtx_unlock(&ctl_softc->pool_lock);
3556                         return (io);
3557                 } else
3558                         ctl_pool_release(pool);
3559         }
3560         /*
3561          * If he doesn't have any io structures left, search for an
3562          * emergency pool and grab one from there.
3563          */
3564         STAILQ_FOREACH(npool, &ctl_softc->io_pools, links) {
3565                 if (npool->type != CTL_POOL_EMERGENCY)
3566                         continue;
3567
3568                 if (ctl_pool_acquire(npool) != 0)
3569                         continue;
3570
3571                 emergency_pool = npool;
3572
3573                 io = (union ctl_io *)STAILQ_FIRST(&npool->free_queue);
3574                 if (io != NULL) {
3575                         STAILQ_REMOVE_HEAD(&npool->free_queue, links);
3576                         npool->total_allocated++;
3577                         npool->free_ctl_io--;
3578                         mtx_unlock(&ctl_softc->pool_lock);
3579                         return (io);
3580                 } else
3581                         ctl_pool_release(npool);
3582         }
3583
3584         /* Drop the spinlock before we malloc */
3585         mtx_unlock(&ctl_softc->pool_lock);
3586
3587         /*
3588          * The emergency pool (if it exists) didn't have one, so try an
3589          * atomic (i.e. nonblocking) malloc and see if we get lucky.
3590          */
3591         io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3592         if (io != NULL) {
3593                 /*
3594                  * If the emergency pool exists but is empty, add this
3595                  * ctl_io to its list when it gets freed.
3596                  */
3597                 if (emergency_pool != NULL) {
3598                         mtx_lock(&ctl_softc->pool_lock);
3599                         if (ctl_pool_acquire(emergency_pool) == 0) {
3600                                 io->io_hdr.pool = emergency_pool;
3601                                 emergency_pool->total_ctl_io++;
3602                                 /*
3603                                  * Need to bump this, otherwise
3604                                  * total_allocated and total_freed won't
3605                                  * match when we no longer have anything
3606                                  * outstanding.
3607                                  */
3608                                 emergency_pool->total_allocated++;
3609                         }
3610                         mtx_unlock(&ctl_softc->pool_lock);
3611                 } else
3612                         io->io_hdr.pool = NULL;
3613         }
3614
3615         return (io);
3616 }
3617
3618 void
3619 ctl_free_io(union ctl_io *io)
3620 {
3621         if (io == NULL)
3622                 return;
3623
3624         /*
3625          * If this ctl_io has a pool, return it to that pool.
3626          */
3627         if (io->io_hdr.pool != NULL) {
3628                 struct ctl_io_pool *pool;
3629 #if 0
3630                 struct ctl_softc *ctl_softc;
3631                 union ctl_io *tmp_io;
3632                 unsigned long xflags;
3633                 int i;
3634
3635                 ctl_softc = control_softc;
3636 #endif
3637
3638                 pool = (struct ctl_io_pool *)io->io_hdr.pool;
3639
3640                 mtx_lock(&pool->ctl_softc->pool_lock);
3641 #if 0
3642                 save_flags(xflags);
3643
3644                 for (i = 0, tmp_io = (union ctl_io *)STAILQ_FIRST(
3645                      &ctl_softc->task_queue); tmp_io != NULL; i++,
3646                      tmp_io = (union ctl_io *)STAILQ_NEXT(&tmp_io->io_hdr,
3647                      links)) {
3648                         if (tmp_io == io) {
3649                                 printf("%s: %p is still on the task queue!\n",
3650                                        __func__, tmp_io);
3651                                 printf("%s: (%d): type %d "
3652                                        "msg %d cdb %x iptl: "
3653                                        "%d:%d:%d:%d tag 0x%04x "
3654                                        "flg %#lx\n",
3655                                         __func__, i,
3656                                         tmp_io->io_hdr.io_type,
3657                                         tmp_io->io_hdr.msg_type,
3658                                         tmp_io->scsiio.cdb[0],
3659                                         tmp_io->io_hdr.nexus.initid.id,
3660                                         tmp_io->io_hdr.nexus.targ_port,
3661                                         tmp_io->io_hdr.nexus.targ_target.id,
3662                                         tmp_io->io_hdr.nexus.targ_lun,
3663                                         (tmp_io->io_hdr.io_type ==
3664                                         CTL_IO_TASK) ?
3665                                         tmp_io->taskio.tag_num :
3666                                         tmp_io->scsiio.tag_num,
3667                                         xflags);
3668                                 panic("I/O still on the task queue!");
3669                         }
3670                 }
3671 #endif
3672                 io->io_hdr.io_type = 0xff;
3673                 STAILQ_INSERT_TAIL(&pool->free_queue, &io->io_hdr, links);
3674                 pool->total_freed++;
3675                 pool->free_ctl_io++;
3676                 ctl_pool_release(pool);
3677                 mtx_unlock(&pool->ctl_softc->pool_lock);
3678         } else {
3679                 /*
3680                  * Otherwise, just free it.  We probably malloced it and
3681                  * the emergency pool wasn't available.
3682                  */
3683                 free(io, M_CTL);
3684         }
3685
3686 }
3687
3688 void
3689 ctl_zero_io(union ctl_io *io)
3690 {
3691         void *pool_ref;
3692
3693         if (io == NULL)
3694                 return;
3695
3696         /*
3697          * May need to preserve linked list pointers at some point too.
3698          */
3699         pool_ref = io->io_hdr.pool;
3700
3701         memset(io, 0, sizeof(*io));
3702
3703         io->io_hdr.pool = pool_ref;
3704 }
3705
3706 /*
3707  * This routine is currently used for internal copies of ctl_ios that need
3708  * to persist for some reason after we've already returned status to the
3709  * FETD.  (Thus the flag set.)
3710  *
3711  * XXX XXX
3712  * Note that this makes a blind copy of all fields in the ctl_io, except
3713  * for the pool reference.  This includes any memory that has been
3714  * allocated!  That memory will no longer be valid after done has been
3715  * called, so this would be VERY DANGEROUS for command that actually does
3716  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3717  * start and stop commands, which don't transfer any data, so this is not a
3718  * problem.  If it is used for anything else, the caller would also need to
3719  * allocate data buffer space and this routine would need to be modified to
3720  * copy the data buffer(s) as well.
3721  */
3722 void
3723 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3724 {
3725         void *pool_ref;
3726
3727         if ((src == NULL)
3728          || (dest == NULL))
3729                 return;
3730
3731         /*
3732          * May need to preserve linked list pointers at some point too.
3733          */
3734         pool_ref = dest->io_hdr.pool;
3735
3736         memcpy(dest, src, ctl_min(sizeof(*src), sizeof(*dest)));
3737
3738         dest->io_hdr.pool = pool_ref;
3739         /*
3740          * We need to know that this is an internal copy, and doesn't need
3741          * to get passed back to the FETD that allocated it.
3742          */
3743         dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3744 }
3745
3746 #ifdef NEEDTOPORT
3747 static void
3748 ctl_update_power_subpage(struct copan_power_subpage *page)
3749 {
3750         int num_luns, num_partitions, config_type;
3751         struct ctl_softc *softc;
3752         cs_BOOL_t aor_present, shelf_50pct_power;
3753         cs_raidset_personality_t rs_type;
3754         int max_active_luns;
3755
3756         softc = control_softc;
3757
3758         /* subtract out the processor LUN */
3759         num_luns = softc->num_luns - 1;
3760         /*
3761          * Default to 7 LUNs active, which was the only number we allowed
3762          * in the past.
3763          */
3764         max_active_luns = 7;
3765
3766         num_partitions = config_GetRsPartitionInfo();
3767         config_type = config_GetConfigType();
3768         shelf_50pct_power = config_GetShelfPowerMode();
3769         aor_present = config_IsAorRsPresent();
3770
3771         rs_type = ddb_GetRsRaidType(1);
3772         if ((rs_type != CS_RAIDSET_PERSONALITY_RAID5)
3773          && (rs_type != CS_RAIDSET_PERSONALITY_RAID1)) {
3774                 EPRINT(0, "Unsupported RS type %d!", rs_type);
3775         }
3776
3777
3778         page->total_luns = num_luns;
3779
3780         switch (config_type) {
3781         case 40:
3782                 /*
3783                  * In a 40 drive configuration, it doesn't matter what DC
3784                  * cards we have, whether we have AOR enabled or not,
3785                  * partitioning or not, or what type of RAIDset we have.
3786                  * In that scenario, we can power up every LUN we present
3787                  * to the user.
3788                  */
3789                 max_active_luns = num_luns;
3790
3791                 break;
3792         case 64:
3793                 if (shelf_50pct_power == CS_FALSE) {
3794                         /* 25% power */
3795                         if (aor_present == CS_TRUE) {
3796                                 if (rs_type ==
3797                                      CS_RAIDSET_PERSONALITY_RAID5) {
3798                                         max_active_luns = 7;
3799                                 } else if (rs_type ==
3800                                          CS_RAIDSET_PERSONALITY_RAID1){
3801                                         max_active_luns = 14;
3802                                 } else {
3803                                         /* XXX KDM now what?? */
3804                                 }
3805                         } else {
3806                                 if (rs_type ==
3807                                      CS_RAIDSET_PERSONALITY_RAID5) {
3808                                         max_active_luns = 8;
3809                                 } else if (rs_type ==
3810                                          CS_RAIDSET_PERSONALITY_RAID1){
3811                                         max_active_luns = 16;
3812                                 } else {
3813                                         /* XXX KDM now what?? */
3814                                 }
3815                         }
3816                 } else {
3817                         /* 50% power */
3818                         /*
3819                          * With 50% power in a 64 drive configuration, we
3820                          * can power all LUNs we present.
3821                          */
3822                         max_active_luns = num_luns;
3823                 }
3824                 break;
3825         case 112:
3826                 if (shelf_50pct_power == CS_FALSE) {
3827                         /* 25% power */
3828                         if (aor_present == CS_TRUE) {
3829                                 if (rs_type ==
3830                                      CS_RAIDSET_PERSONALITY_RAID5) {
3831                                         max_active_luns = 7;
3832                                 } else if (rs_type ==
3833                                          CS_RAIDSET_PERSONALITY_RAID1){
3834                                         max_active_luns = 14;
3835                                 } else {
3836                                         /* XXX KDM now what?? */
3837                                 }
3838                         } else {
3839                                 if (rs_type ==
3840                                      CS_RAIDSET_PERSONALITY_RAID5) {
3841                                         max_active_luns = 8;
3842                                 } else if (rs_type ==
3843                                          CS_RAIDSET_PERSONALITY_RAID1){
3844                                         max_active_luns = 16;
3845                                 } else {
3846                                         /* XXX KDM now what?? */
3847                                 }
3848                         }
3849                 } else {
3850                         /* 50% power */
3851                         if (aor_present == CS_TRUE) {
3852                                 if (rs_type ==
3853                                      CS_RAIDSET_PERSONALITY_RAID5) {
3854                                         max_active_luns = 14;
3855                                 } else if (rs_type ==
3856                                          CS_RAIDSET_PERSONALITY_RAID1){
3857                                         /*
3858                                          * We're assuming here that disk
3859                                          * caching is enabled, and so we're
3860                                          * able to power up half of each
3861                                          * LUN, and cache all writes.
3862                                          */
3863                                         max_active_luns = num_luns;
3864                                 } else {
3865                                         /* XXX KDM now what?? */
3866                                 }
3867                         } else {
3868                                 if (rs_type ==
3869                                      CS_RAIDSET_PERSONALITY_RAID5) {
3870                                         max_active_luns = 15;
3871                                 } else if (rs_type ==
3872                                          CS_RAIDSET_PERSONALITY_RAID1){
3873                                         max_active_luns = 30;
3874                                 } else {
3875                                         /* XXX KDM now what?? */
3876                                 }
3877                         }
3878                 }
3879                 break;
3880         default:
3881                 /*
3882                  * In this case, we have an unknown configuration, so we
3883                  * just use the default from above.
3884                  */
3885                 break;
3886         }
3887
3888         page->max_active_luns = max_active_luns;
3889 #if 0
3890         printk("%s: total_luns = %d, max_active_luns = %d\n", __func__,
3891                page->total_luns, page->max_active_luns);
3892 #endif
3893 }
3894 #endif /* NEEDTOPORT */
3895
3896 /*
3897  * This routine could be used in the future to load default and/or saved
3898  * mode page parameters for a particuar lun.
3899  */
3900 static int
3901 ctl_init_page_index(struct ctl_lun *lun)
3902 {
3903         int i;
3904         struct ctl_page_index *page_index;
3905         struct ctl_softc *softc;
3906
3907         memcpy(&lun->mode_pages.index, page_index_template,
3908                sizeof(page_index_template));
3909
3910         softc = lun->ctl_softc;
3911
3912         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
3913
3914                 page_index = &lun->mode_pages.index[i];
3915                 /*
3916                  * If this is a disk-only mode page, there's no point in
3917                  * setting it up.  For some pages, we have to have some
3918                  * basic information about the disk in order to calculate the
3919                  * mode page data.
3920                  */
3921                 if ((lun->be_lun->lun_type != T_DIRECT)
3922                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
3923                         continue;
3924
3925                 switch (page_index->page_code & SMPH_PC_MASK) {
3926                 case SMS_FORMAT_DEVICE_PAGE: {
3927                         struct scsi_format_page *format_page;
3928
3929                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3930                                 panic("subpage is incorrect!");
3931
3932                         /*
3933                          * Sectors per track are set above.  Bytes per
3934                          * sector need to be set here on a per-LUN basis.
3935                          */
3936                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
3937                                &format_page_default,
3938                                sizeof(format_page_default));
3939                         memcpy(&lun->mode_pages.format_page[
3940                                CTL_PAGE_CHANGEABLE], &format_page_changeable,
3941                                sizeof(format_page_changeable));
3942                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
3943                                &format_page_default,
3944                                sizeof(format_page_default));
3945                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
3946                                &format_page_default,
3947                                sizeof(format_page_default));
3948
3949                         format_page = &lun->mode_pages.format_page[
3950                                 CTL_PAGE_CURRENT];
3951                         scsi_ulto2b(lun->be_lun->blocksize,
3952                                     format_page->bytes_per_sector);
3953
3954                         format_page = &lun->mode_pages.format_page[
3955                                 CTL_PAGE_DEFAULT];
3956                         scsi_ulto2b(lun->be_lun->blocksize,
3957                                     format_page->bytes_per_sector);
3958
3959                         format_page = &lun->mode_pages.format_page[
3960                                 CTL_PAGE_SAVED];
3961                         scsi_ulto2b(lun->be_lun->blocksize,
3962                                     format_page->bytes_per_sector);
3963
3964                         page_index->page_data =
3965                                 (uint8_t *)lun->mode_pages.format_page;
3966                         break;
3967                 }
3968                 case SMS_RIGID_DISK_PAGE: {
3969                         struct scsi_rigid_disk_page *rigid_disk_page;
3970                         uint32_t sectors_per_cylinder;
3971                         uint64_t cylinders;
3972 #ifndef __XSCALE__
3973                         int shift;
3974 #endif /* !__XSCALE__ */
3975
3976                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3977                                 panic("invalid subpage value %d",
3978                                       page_index->subpage);
3979
3980                         /*
3981                          * Rotation rate and sectors per track are set
3982                          * above.  We calculate the cylinders here based on
3983                          * capacity.  Due to the number of heads and
3984                          * sectors per track we're using, smaller arrays
3985                          * may turn out to have 0 cylinders.  Linux and
3986                          * FreeBSD don't pay attention to these mode pages
3987                          * to figure out capacity, but Solaris does.  It
3988                          * seems to deal with 0 cylinders just fine, and
3989                          * works out a fake geometry based on the capacity.
3990                          */
3991                         memcpy(&lun->mode_pages.rigid_disk_page[
3992                                CTL_PAGE_CURRENT], &rigid_disk_page_default,
3993                                sizeof(rigid_disk_page_default));
3994                         memcpy(&lun->mode_pages.rigid_disk_page[
3995                                CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
3996                                sizeof(rigid_disk_page_changeable));
3997                         memcpy(&lun->mode_pages.rigid_disk_page[
3998                                CTL_PAGE_DEFAULT], &rigid_disk_page_default,
3999                                sizeof(rigid_disk_page_default));
4000                         memcpy(&lun->mode_pages.rigid_disk_page[
4001                                CTL_PAGE_SAVED], &rigid_disk_page_default,
4002                                sizeof(rigid_disk_page_default));
4003
4004                         sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
4005                                 CTL_DEFAULT_HEADS;
4006
4007                         /*
4008                          * The divide method here will be more accurate,
4009                          * probably, but results in floating point being
4010                          * used in the kernel on i386 (__udivdi3()).  On the
4011                          * XScale, though, __udivdi3() is implemented in
4012                          * software.
4013                          *
4014                          * The shift method for cylinder calculation is
4015                          * accurate if sectors_per_cylinder is a power of
4016                          * 2.  Otherwise it might be slightly off -- you
4017                          * might have a bit of a truncation problem.
4018                          */
4019 #ifdef  __XSCALE__
4020                         cylinders = (lun->be_lun->maxlba + 1) /
4021                                 sectors_per_cylinder;
4022 #else
4023                         for (shift = 31; shift > 0; shift--) {
4024                                 if (sectors_per_cylinder & (1 << shift))
4025                                         break;
4026                         }
4027                         cylinders = (lun->be_lun->maxlba + 1) >> shift;
4028 #endif
4029
4030                         /*
4031                          * We've basically got 3 bytes, or 24 bits for the
4032                          * cylinder size in the mode page.  If we're over,
4033                          * just round down to 2^24.
4034                          */
4035                         if (cylinders > 0xffffff)
4036                                 cylinders = 0xffffff;
4037
4038                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4039                                 CTL_PAGE_CURRENT];
4040                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4041
4042                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4043                                 CTL_PAGE_DEFAULT];
4044                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4045
4046                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4047                                 CTL_PAGE_SAVED];
4048                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4049
4050                         page_index->page_data =
4051                                 (uint8_t *)lun->mode_pages.rigid_disk_page;
4052                         break;
4053                 }
4054                 case SMS_CACHING_PAGE: {
4055
4056                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4057                                 panic("invalid subpage value %d",
4058                                       page_index->subpage);
4059                         /*
4060                          * Defaults should be okay here, no calculations
4061                          * needed.
4062                          */
4063                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
4064                                &caching_page_default,
4065                                sizeof(caching_page_default));
4066                         memcpy(&lun->mode_pages.caching_page[
4067                                CTL_PAGE_CHANGEABLE], &caching_page_changeable,
4068                                sizeof(caching_page_changeable));
4069                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
4070                                &caching_page_default,
4071                                sizeof(caching_page_default));
4072                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4073                                &caching_page_default,
4074                                sizeof(caching_page_default));
4075                         page_index->page_data =
4076                                 (uint8_t *)lun->mode_pages.caching_page;
4077                         break;
4078                 }
4079                 case SMS_CONTROL_MODE_PAGE: {
4080
4081                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4082                                 panic("invalid subpage value %d",
4083                                       page_index->subpage);
4084
4085                         /*
4086                          * Defaults should be okay here, no calculations
4087                          * needed.
4088                          */
4089                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
4090                                &control_page_default,
4091                                sizeof(control_page_default));
4092                         memcpy(&lun->mode_pages.control_page[
4093                                CTL_PAGE_CHANGEABLE], &control_page_changeable,
4094                                sizeof(control_page_changeable));
4095                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
4096                                &control_page_default,
4097                                sizeof(control_page_default));
4098                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
4099                                &control_page_default,
4100                                sizeof(control_page_default));
4101                         page_index->page_data =
4102                                 (uint8_t *)lun->mode_pages.control_page;
4103                         break;
4104
4105                 }
4106                 case SMS_VENDOR_SPECIFIC_PAGE:{
4107                         switch (page_index->subpage) {
4108                         case PWR_SUBPAGE_CODE: {
4109                                 struct copan_power_subpage *current_page,
4110                                                            *saved_page;
4111
4112                                 memcpy(&lun->mode_pages.power_subpage[
4113                                        CTL_PAGE_CURRENT],
4114                                        &power_page_default,
4115                                        sizeof(power_page_default));
4116                                 memcpy(&lun->mode_pages.power_subpage[
4117                                        CTL_PAGE_CHANGEABLE],
4118                                        &power_page_changeable,
4119                                        sizeof(power_page_changeable));
4120                                 memcpy(&lun->mode_pages.power_subpage[
4121                                        CTL_PAGE_DEFAULT],
4122                                        &power_page_default,
4123                                        sizeof(power_page_default));
4124                                 memcpy(&lun->mode_pages.power_subpage[
4125                                        CTL_PAGE_SAVED],
4126                                        &power_page_default,
4127                                        sizeof(power_page_default));
4128                                 page_index->page_data =
4129                                     (uint8_t *)lun->mode_pages.power_subpage;
4130
4131                                 current_page = (struct copan_power_subpage *)
4132                                         (page_index->page_data +
4133                                          (page_index->page_len *
4134                                           CTL_PAGE_CURRENT));
4135                                 saved_page = (struct copan_power_subpage *)
4136                                         (page_index->page_data +
4137                                          (page_index->page_len *
4138                                           CTL_PAGE_SAVED));
4139                                 break;
4140                         }
4141                         case APS_SUBPAGE_CODE: {
4142                                 struct copan_aps_subpage *current_page,
4143                                                          *saved_page;
4144
4145                                 // This gets set multiple times but
4146                                 // it should always be the same. It's
4147                                 // only done during init so who cares.
4148                                 index_to_aps_page = i;
4149
4150                                 memcpy(&lun->mode_pages.aps_subpage[
4151                                        CTL_PAGE_CURRENT],
4152                                        &aps_page_default,
4153                                        sizeof(aps_page_default));
4154                                 memcpy(&lun->mode_pages.aps_subpage[
4155                                        CTL_PAGE_CHANGEABLE],
4156                                        &aps_page_changeable,
4157                                        sizeof(aps_page_changeable));
4158                                 memcpy(&lun->mode_pages.aps_subpage[
4159                                        CTL_PAGE_DEFAULT],
4160                                        &aps_page_default,
4161                                        sizeof(aps_page_default));
4162                                 memcpy(&lun->mode_pages.aps_subpage[
4163                                        CTL_PAGE_SAVED],
4164                                        &aps_page_default,
4165                                        sizeof(aps_page_default));
4166                                 page_index->page_data =
4167                                         (uint8_t *)lun->mode_pages.aps_subpage;
4168
4169                                 current_page = (struct copan_aps_subpage *)
4170                                         (page_index->page_data +
4171                                          (page_index->page_len *
4172                                           CTL_PAGE_CURRENT));
4173                                 saved_page = (struct copan_aps_subpage *)
4174                                         (page_index->page_data +
4175                                          (page_index->page_len *
4176                                           CTL_PAGE_SAVED));
4177                                 break;
4178                         }
4179                         case DBGCNF_SUBPAGE_CODE: {
4180                                 struct copan_debugconf_subpage *current_page,
4181                                                                *saved_page;
4182
4183                                 memcpy(&lun->mode_pages.debugconf_subpage[
4184                                        CTL_PAGE_CURRENT],
4185                                        &debugconf_page_default,
4186                                        sizeof(debugconf_page_default));
4187                                 memcpy(&lun->mode_pages.debugconf_subpage[
4188                                        CTL_PAGE_CHANGEABLE],
4189                                        &debugconf_page_changeable,
4190                                        sizeof(debugconf_page_changeable));
4191                                 memcpy(&lun->mode_pages.debugconf_subpage[
4192                                        CTL_PAGE_DEFAULT],
4193                                        &debugconf_page_default,
4194                                        sizeof(debugconf_page_default));
4195                                 memcpy(&lun->mode_pages.debugconf_subpage[
4196                                        CTL_PAGE_SAVED],
4197                                        &debugconf_page_default,
4198                                        sizeof(debugconf_page_default));
4199                                 page_index->page_data =
4200                                         (uint8_t *)lun->mode_pages.debugconf_subpage;
4201
4202                                 current_page = (struct copan_debugconf_subpage *)
4203                                         (page_index->page_data +
4204                                          (page_index->page_len *
4205                                           CTL_PAGE_CURRENT));
4206                                 saved_page = (struct copan_debugconf_subpage *)
4207                                         (page_index->page_data +
4208                                          (page_index->page_len *
4209                                           CTL_PAGE_SAVED));
4210                                 break;
4211                         }
4212                         default:
4213                                 panic("invalid subpage value %d",
4214                                       page_index->subpage);
4215                                 break;
4216                         }
4217                         break;
4218                 }
4219                 default:
4220                         panic("invalid page value %d",
4221                               page_index->page_code & SMPH_PC_MASK);
4222                         break;
4223         }
4224         }
4225
4226         return (CTL_RETVAL_COMPLETE);
4227 }
4228
4229 /*
4230  * LUN allocation.
4231  *
4232  * Requirements:
4233  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4234  *   wants us to allocate the LUN and he can block.
4235  * - ctl_softc is always set
4236  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4237  *
4238  * Returns 0 for success, non-zero (errno) for failure.
4239  */
4240 static int
4241 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4242               struct ctl_be_lun *const be_lun, struct ctl_id target_id)
4243 {
4244         struct ctl_lun *nlun, *lun;
4245         struct ctl_frontend *fe;
4246         int lun_number, i, lun_malloced;
4247
4248         if (be_lun == NULL)
4249                 return (EINVAL);
4250
4251         /*
4252          * We currently only support Direct Access or Processor LUN types.
4253          */
4254         switch (be_lun->lun_type) {
4255         case T_DIRECT:
4256                 break;
4257         case T_PROCESSOR:
4258                 break;
4259         case T_SEQUENTIAL:
4260         case T_CHANGER:
4261         default:
4262                 be_lun->lun_config_status(be_lun->be_lun,
4263                                           CTL_LUN_CONFIG_FAILURE);
4264                 break;
4265         }
4266         if (ctl_lun == NULL) {
4267                 lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4268                 lun_malloced = 1;
4269         } else {
4270                 lun_malloced = 0;
4271                 lun = ctl_lun;
4272         }
4273
4274         memset(lun, 0, sizeof(*lun));
4275         if (lun_malloced)
4276                 lun->flags = CTL_LUN_MALLOCED;
4277
4278         mtx_lock(&ctl_softc->ctl_lock);
4279         /*
4280          * See if the caller requested a particular LUN number.  If so, see
4281          * if it is available.  Otherwise, allocate the first available LUN.
4282          */
4283         if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4284                 if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4285                  || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4286                         mtx_unlock(&ctl_softc->ctl_lock);
4287                         if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4288                                 printf("ctl: requested LUN ID %d is higher "
4289                                        "than CTL_MAX_LUNS - 1 (%d)\n",
4290                                        be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4291                         } else {
4292                                 /*
4293                                  * XXX KDM return an error, or just assign
4294                                  * another LUN ID in this case??
4295                                  */
4296                                 printf("ctl: requested LUN ID %d is already "
4297                                        "in use\n", be_lun->req_lun_id);
4298                         }
4299                         if (lun->flags & CTL_LUN_MALLOCED)
4300                                 free(lun, M_CTL);
4301                         be_lun->lun_config_status(be_lun->be_lun,
4302                                                   CTL_LUN_CONFIG_FAILURE);
4303                         return (ENOSPC);
4304                 }
4305                 lun_number = be_lun->req_lun_id;
4306         } else {
4307                 lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, CTL_MAX_LUNS);
4308                 if (lun_number == -1) {
4309                         mtx_unlock(&ctl_softc->ctl_lock);
4310                         printf("ctl: can't allocate LUN on target %ju, out of "
4311                                "LUNs\n", (uintmax_t)target_id.id);
4312                         if (lun->flags & CTL_LUN_MALLOCED)
4313                                 free(lun, M_CTL);
4314                         be_lun->lun_config_status(be_lun->be_lun,
4315                                                   CTL_LUN_CONFIG_FAILURE);
4316                         return (ENOSPC);
4317                 }
4318         }
4319         ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4320
4321         lun->target = target_id;
4322         lun->lun = lun_number;
4323         lun->be_lun = be_lun;
4324         /*
4325          * The processor LUN is always enabled.  Disk LUNs come on line
4326          * disabled, and must be enabled by the backend.
4327          */
4328         lun->flags |= CTL_LUN_DISABLED;
4329         lun->backend = be_lun->be;
4330         be_lun->ctl_lun = lun;
4331         be_lun->lun_id = lun_number;
4332         atomic_add_int(&be_lun->be->num_luns, 1);
4333         if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4334                 lun->flags |= CTL_LUN_STOPPED;
4335
4336         if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4337                 lun->flags |= CTL_LUN_INOPERABLE;
4338
4339         if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4340                 lun->flags |= CTL_LUN_PRIMARY_SC;
4341
4342         lun->ctl_softc = ctl_softc;
4343         TAILQ_INIT(&lun->ooa_queue);
4344         TAILQ_INIT(&lun->blocked_queue);
4345         STAILQ_INIT(&lun->error_list);
4346
4347         /*
4348          * Initialize the mode page index.
4349          */
4350         ctl_init_page_index(lun);
4351
4352         /*
4353          * Set the poweron UA for all initiators on this LUN only.
4354          */
4355         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4356                 lun->pending_sense[i].ua_pending = CTL_UA_POWERON;
4357
4358         /*
4359          * Now, before we insert this lun on the lun list, set the lun
4360          * inventory changed UA for all other luns.
4361          */
4362         STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4363                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4364                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4365                 }
4366         }
4367
4368         STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4369
4370         ctl_softc->ctl_luns[lun_number] = lun;
4371
4372         ctl_softc->num_luns++;
4373
4374         /* Setup statistics gathering */
4375         lun->stats.device_type = be_lun->lun_type;
4376         lun->stats.lun_number = lun_number;
4377         if (lun->stats.device_type == T_DIRECT)
4378                 lun->stats.blocksize = be_lun->blocksize;
4379         else
4380                 lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4381         for (i = 0;i < CTL_MAX_PORTS;i++)
4382                 lun->stats.ports[i].targ_port = i;
4383
4384         mtx_unlock(&ctl_softc->ctl_lock);
4385
4386         lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4387
4388         /*
4389          * Run through each registered FETD and bring it online if it isn't
4390          * already.  Enable the target ID if it hasn't been enabled, and
4391          * enable this particular LUN.
4392          */
4393         STAILQ_FOREACH(fe, &ctl_softc->fe_list, links) {
4394                 int retval;
4395
4396                 /*
4397                  * XXX KDM this only works for ONE TARGET ID.  We'll need
4398                  * to do things differently if we go to a multiple target
4399                  * ID scheme.
4400                  */
4401                 if ((fe->status & CTL_PORT_STATUS_TARG_ONLINE) == 0) {
4402
4403                         retval = fe->targ_enable(fe->targ_lun_arg, target_id);
4404                         if (retval != 0) {
4405                                 printf("ctl_alloc_lun: FETD %s port %d "
4406                                        "returned error %d for targ_enable on "
4407                                        "target %ju\n", fe->port_name,
4408                                        fe->targ_port, retval,
4409                                        (uintmax_t)target_id.id);
4410                         } else
4411                                 fe->status |= CTL_PORT_STATUS_TARG_ONLINE;
4412                 }
4413
4414                 retval = fe->lun_enable(fe->targ_lun_arg, target_id,lun_number);
4415                 if (retval != 0) {
4416                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4417                                "%d for lun_enable on target %ju lun %d\n",
4418                                fe->port_name, fe->targ_port, retval,
4419                                (uintmax_t)target_id.id, lun_number);
4420                 } else
4421                         fe->status |= CTL_PORT_STATUS_LUN_ONLINE;
4422         }
4423         return (0);
4424 }
4425
4426 /*
4427  * Delete a LUN.
4428  * Assumptions:
4429  * - LUN has already been marked invalid and any pending I/O has been taken
4430  *   care of.
4431  */
4432 static int
4433 ctl_free_lun(struct ctl_lun *lun)
4434 {
4435         struct ctl_softc *softc;
4436 #if 0
4437         struct ctl_frontend *fe;
4438 #endif
4439         struct ctl_lun *nlun;
4440         union ctl_io *io, *next_io;
4441         int i;
4442
4443         softc = lun->ctl_softc;
4444
4445         mtx_assert(&softc->ctl_lock, MA_OWNED);
4446
4447         STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4448
4449         ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4450
4451         softc->ctl_luns[lun->lun] = NULL;
4452
4453         if (TAILQ_FIRST(&lun->ooa_queue) != NULL) {
4454                 printf("ctl_free_lun: aieee!! freeing a LUN with "
4455                        "outstanding I/O!!\n");
4456         }
4457
4458         /*
4459          * If we have anything pending on the RtR queue, remove it.
4460          */
4461         for (io = (union ctl_io *)STAILQ_FIRST(&softc->rtr_queue); io != NULL;
4462              io = next_io) {
4463                 uint32_t targ_lun;
4464
4465                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
4466                 targ_lun = io->io_hdr.nexus.targ_lun;
4467                 if (io->io_hdr.nexus.lun_map_fn != NULL)
4468                         targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
4469                 if ((io->io_hdr.nexus.targ_target.id == lun->target.id)
4470                  && (targ_lun == lun->lun))
4471                         STAILQ_REMOVE(&softc->rtr_queue, &io->io_hdr,
4472                                       ctl_io_hdr, links);
4473         }
4474
4475         /*
4476          * Then remove everything from the blocked queue.
4477          */
4478         for (io = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue); io != NULL;
4479              io = next_io) {
4480                 next_io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,blocked_links);
4481                 TAILQ_REMOVE(&lun->blocked_queue, &io->io_hdr, blocked_links);
4482                 io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
4483         }
4484
4485         /*
4486          * Now clear out the OOA queue, and free all the I/O.
4487          * XXX KDM should we notify the FETD here?  We probably need to
4488          * quiesce the LUN before deleting it.
4489          */
4490         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); io != NULL;
4491              io = next_io) {
4492                 next_io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr, ooa_links);
4493                 TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
4494                 ctl_free_io(io);
4495         }
4496
4497         softc->num_luns--;
4498
4499         /*
4500          * XXX KDM this scheme only works for a single target/multiple LUN
4501          * setup.  It needs to be revamped for a multiple target scheme.
4502          *
4503          * XXX KDM this results in fe->lun_disable() getting called twice,
4504          * once when ctl_disable_lun() is called, and a second time here.
4505          * We really need to re-think the LUN disable semantics.  There
4506          * should probably be several steps/levels to LUN removal:
4507          *  - disable
4508          *  - invalidate
4509          *  - free
4510          *
4511          * Right now we only have a disable method when communicating to
4512          * the front end ports, at least for individual LUNs.
4513          */
4514 #if 0
4515         STAILQ_FOREACH(fe, &softc->fe_list, links) {
4516                 int retval;
4517
4518                 retval = fe->lun_disable(fe->targ_lun_arg, lun->target,
4519                                          lun->lun);
4520                 if (retval != 0) {
4521                         printf("ctl_free_lun: FETD %s port %d returned error "
4522                                "%d for lun_disable on target %ju lun %jd\n",
4523                                fe->port_name, fe->targ_port, retval,
4524                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4525                 }
4526
4527                 if (STAILQ_FIRST(&softc->lun_list) == NULL) {
4528                         fe->status &= ~CTL_PORT_STATUS_LUN_ONLINE;
4529
4530                         retval = fe->targ_disable(fe->targ_lun_arg,lun->target);
4531                         if (retval != 0) {
4532                                 printf("ctl_free_lun: FETD %s port %d "
4533                                        "returned error %d for targ_disable on "
4534                                        "target %ju\n", fe->port_name,
4535                                        fe->targ_port, retval,
4536                                        (uintmax_t)lun->target.id);
4537                         } else
4538                                 fe->status &= ~CTL_PORT_STATUS_TARG_ONLINE;
4539
4540                         if ((fe->status & CTL_PORT_STATUS_TARG_ONLINE) != 0)
4541                                 continue;
4542
4543 #if 0
4544                         fe->port_offline(fe->onoff_arg);
4545                         fe->status &= ~CTL_PORT_STATUS_ONLINE;
4546 #endif
4547                 }
4548         }
4549 #endif
4550
4551         /*
4552          * Tell the backend to free resources, if this LUN has a backend.
4553          */
4554         atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4555         lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4556
4557         if (lun->flags & CTL_LUN_MALLOCED)
4558                 free(lun, M_CTL);
4559
4560         STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4561                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4562                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4563                 }
4564         }
4565
4566         return (0);
4567 }
4568
4569 static void
4570 ctl_create_lun(struct ctl_be_lun *be_lun)
4571 {
4572         struct ctl_softc *ctl_softc;
4573
4574         ctl_softc = control_softc;
4575
4576         /*
4577          * ctl_alloc_lun() should handle all potential failure cases.
4578          */
4579         ctl_alloc_lun(ctl_softc, NULL, be_lun, ctl_softc->target);
4580 }
4581
4582 int
4583 ctl_add_lun(struct ctl_be_lun *be_lun)
4584 {
4585         struct ctl_softc *ctl_softc;
4586
4587         ctl_softc = control_softc;
4588
4589         mtx_lock(&ctl_softc->ctl_lock);
4590         STAILQ_INSERT_TAIL(&ctl_softc->pending_lun_queue, be_lun, links);
4591         mtx_unlock(&ctl_softc->ctl_lock);
4592
4593         ctl_wakeup_thread();
4594
4595         return (0);
4596 }
4597
4598 int
4599 ctl_enable_lun(struct ctl_be_lun *be_lun)
4600 {
4601         struct ctl_softc *ctl_softc;
4602         struct ctl_frontend *fe, *nfe;
4603         struct ctl_lun *lun;
4604         int retval;
4605
4606         ctl_softc = control_softc;
4607
4608         lun = (struct ctl_lun *)be_lun->ctl_lun;
4609
4610         mtx_lock(&ctl_softc->ctl_lock);
4611         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4612                 /*
4613                  * eh?  Why did we get called if the LUN is already
4614                  * enabled?
4615                  */
4616                 mtx_unlock(&ctl_softc->ctl_lock);
4617                 return (0);
4618         }
4619         lun->flags &= ~CTL_LUN_DISABLED;
4620
4621         for (fe = STAILQ_FIRST(&ctl_softc->fe_list); fe != NULL; fe = nfe) {
4622                 nfe = STAILQ_NEXT(fe, links);
4623
4624                 /*
4625                  * Drop the lock while we call the FETD's enable routine.
4626                  * This can lead to a callback into CTL (at least in the
4627                  * case of the internal initiator frontend.
4628                  */
4629                 mtx_unlock(&ctl_softc->ctl_lock);
4630                 retval = fe->lun_enable(fe->targ_lun_arg, lun->target,lun->lun);
4631                 mtx_lock(&ctl_softc->ctl_lock);
4632                 if (retval != 0) {
4633                         printf("%s: FETD %s port %d returned error "
4634                                "%d for lun_enable on target %ju lun %jd\n",
4635                                __func__, fe->port_name, fe->targ_port, retval,
4636                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4637                 }
4638 #if 0
4639                  else {
4640             /* NOTE:  TODO:  why does lun enable affect port status? */
4641                         fe->status |= CTL_PORT_STATUS_LUN_ONLINE;
4642                 }
4643 #endif
4644         }
4645
4646         mtx_unlock(&ctl_softc->ctl_lock);
4647
4648         return (0);
4649 }
4650
4651 int
4652 ctl_disable_lun(struct ctl_be_lun *be_lun)
4653 {
4654         struct ctl_softc *ctl_softc;
4655         struct ctl_frontend *fe;
4656         struct ctl_lun *lun;
4657         int retval;
4658
4659         ctl_softc = control_softc;
4660
4661         lun = (struct ctl_lun *)be_lun->ctl_lun;
4662
4663         mtx_lock(&ctl_softc->ctl_lock);
4664
4665         if (lun->flags & CTL_LUN_DISABLED) {
4666                 mtx_unlock(&ctl_softc->ctl_lock);
4667                 return (0);
4668         }
4669         lun->flags |= CTL_LUN_DISABLED;
4670
4671         STAILQ_FOREACH(fe, &ctl_softc->fe_list, links) {
4672                 mtx_unlock(&ctl_softc->ctl_lock);
4673                 /*
4674                  * Drop the lock before we call the frontend's disable
4675                  * routine, to avoid lock order reversals.
4676                  *
4677                  * XXX KDM what happens if the frontend list changes while
4678                  * we're traversing it?  It's unlikely, but should be handled.
4679                  */
4680                 retval = fe->lun_disable(fe->targ_lun_arg, lun->target,
4681                                          lun->lun);
4682                 mtx_lock(&ctl_softc->ctl_lock);
4683                 if (retval != 0) {
4684                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4685                                "%d for lun_disable on target %ju lun %jd\n",
4686                                fe->port_name, fe->targ_port, retval,
4687                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4688                 }
4689         }
4690
4691         mtx_unlock(&ctl_softc->ctl_lock);
4692
4693         return (0);
4694 }
4695
4696 int
4697 ctl_start_lun(struct ctl_be_lun *be_lun)
4698 {
4699         struct ctl_softc *ctl_softc;
4700         struct ctl_lun *lun;
4701
4702         ctl_softc = control_softc;
4703
4704         lun = (struct ctl_lun *)be_lun->ctl_lun;
4705
4706         mtx_lock(&ctl_softc->ctl_lock);
4707         lun->flags &= ~CTL_LUN_STOPPED;
4708         mtx_unlock(&ctl_softc->ctl_lock);
4709
4710         return (0);
4711 }
4712
4713 int
4714 ctl_stop_lun(struct ctl_be_lun *be_lun)
4715 {
4716         struct ctl_softc *ctl_softc;
4717         struct ctl_lun *lun;
4718
4719         ctl_softc = control_softc;
4720
4721         lun = (struct ctl_lun *)be_lun->ctl_lun;
4722
4723         mtx_lock(&ctl_softc->ctl_lock);
4724         lun->flags |= CTL_LUN_STOPPED;
4725         mtx_unlock(&ctl_softc->ctl_lock);
4726
4727         return (0);
4728 }
4729
4730 int
4731 ctl_lun_offline(struct ctl_be_lun *be_lun)
4732 {
4733         struct ctl_softc *ctl_softc;
4734         struct ctl_lun *lun;
4735
4736         ctl_softc = control_softc;
4737
4738         lun = (struct ctl_lun *)be_lun->ctl_lun;
4739
4740         mtx_lock(&ctl_softc->ctl_lock);
4741         lun->flags |= CTL_LUN_OFFLINE;
4742         mtx_unlock(&ctl_softc->ctl_lock);
4743
4744         return (0);
4745 }
4746
4747 int
4748 ctl_lun_online(struct ctl_be_lun *be_lun)
4749 {
4750         struct ctl_softc *ctl_softc;
4751         struct ctl_lun *lun;
4752
4753         ctl_softc = control_softc;
4754
4755         lun = (struct ctl_lun *)be_lun->ctl_lun;
4756
4757         mtx_lock(&ctl_softc->ctl_lock);
4758         lun->flags &= ~CTL_LUN_OFFLINE;
4759         mtx_unlock(&ctl_softc->ctl_lock);
4760
4761         return (0);
4762 }
4763
4764 int
4765 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4766 {
4767         struct ctl_softc *ctl_softc;
4768         struct ctl_lun *lun;
4769
4770         ctl_softc = control_softc;
4771
4772         lun = (struct ctl_lun *)be_lun->ctl_lun;
4773
4774         mtx_lock(&ctl_softc->ctl_lock);
4775
4776         /*
4777          * The LUN needs to be disabled before it can be marked invalid.
4778          */
4779         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4780                 mtx_unlock(&ctl_softc->ctl_lock);
4781                 return (-1);
4782         }
4783         /*
4784          * Mark the LUN invalid.
4785          */
4786         lun->flags |= CTL_LUN_INVALID;
4787
4788         /*
4789          * If there is nothing in the OOA queue, go ahead and free the LUN.
4790          * If we have something in the OOA queue, we'll free it when the
4791          * last I/O completes.
4792          */
4793         if (TAILQ_FIRST(&lun->ooa_queue) == NULL)
4794                 ctl_free_lun(lun);
4795         mtx_unlock(&ctl_softc->ctl_lock);
4796
4797         return (0);
4798 }
4799
4800 int
4801 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
4802 {
4803         struct ctl_softc *ctl_softc;
4804         struct ctl_lun *lun;
4805
4806         ctl_softc = control_softc;
4807         lun = (struct ctl_lun *)be_lun->ctl_lun;
4808
4809         mtx_lock(&ctl_softc->ctl_lock);
4810         lun->flags |= CTL_LUN_INOPERABLE;
4811         mtx_unlock(&ctl_softc->ctl_lock);
4812
4813         return (0);
4814 }
4815
4816 int
4817 ctl_lun_operable(struct ctl_be_lun *be_lun)
4818 {
4819         struct ctl_softc *ctl_softc;
4820         struct ctl_lun *lun;
4821
4822         ctl_softc = control_softc;
4823         lun = (struct ctl_lun *)be_lun->ctl_lun;
4824
4825         mtx_lock(&ctl_softc->ctl_lock);
4826         lun->flags &= ~CTL_LUN_INOPERABLE;
4827         mtx_unlock(&ctl_softc->ctl_lock);
4828
4829         return (0);
4830 }
4831
4832 int
4833 ctl_lun_power_lock(struct ctl_be_lun *be_lun, struct ctl_nexus *nexus,
4834                    int lock)
4835 {
4836         struct ctl_softc *softc;
4837         struct ctl_lun *lun;
4838         struct copan_aps_subpage *current_sp;
4839         struct ctl_page_index *page_index;
4840         int i;
4841
4842         softc = control_softc;
4843
4844         mtx_lock(&softc->ctl_lock);
4845
4846         lun = (struct ctl_lun *)be_lun->ctl_lun;
4847
4848         page_index = NULL;
4849         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4850                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
4851                      APS_PAGE_CODE)
4852                         continue;
4853
4854                 if (lun->mode_pages.index[i].subpage != APS_SUBPAGE_CODE)
4855                         continue;
4856                 page_index = &lun->mode_pages.index[i];
4857         }
4858
4859         if (page_index == NULL) {
4860                 mtx_unlock(&softc->ctl_lock);
4861                 printf("%s: APS subpage not found for lun %ju!\n", __func__,
4862                        (uintmax_t)lun->lun);
4863                 return (1);
4864         }
4865 #if 0
4866         if ((softc->aps_locked_lun != 0)
4867          && (softc->aps_locked_lun != lun->lun)) {
4868                 printf("%s: attempt to lock LUN %llu when %llu is already "
4869                        "locked\n");
4870                 mtx_unlock(&softc->ctl_lock);
4871                 return (1);
4872         }
4873 #endif
4874
4875         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
4876                 (page_index->page_len * CTL_PAGE_CURRENT));
4877
4878         if (lock != 0) {
4879                 current_sp->lock_active = APS_LOCK_ACTIVE;
4880                 softc->aps_locked_lun = lun->lun;
4881         } else {
4882                 current_sp->lock_active = 0;
4883                 softc->aps_locked_lun = 0;
4884         }
4885
4886
4887         /*
4888          * If we're in HA mode, try to send the lock message to the other
4889          * side.
4890          */
4891         if (ctl_is_single == 0) {
4892                 int isc_retval;
4893                 union ctl_ha_msg lock_msg;
4894
4895                 lock_msg.hdr.nexus = *nexus;
4896                 lock_msg.hdr.msg_type = CTL_MSG_APS_LOCK;
4897                 if (lock != 0)
4898                         lock_msg.aps.lock_flag = 1;
4899                 else
4900                         lock_msg.aps.lock_flag = 0;
4901                 isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &lock_msg,
4902                                          sizeof(lock_msg), 0);
4903                 if (isc_retval > CTL_HA_STATUS_SUCCESS) {
4904                         printf("%s: APS (lock=%d) error returned from "
4905                                "ctl_ha_msg_send: %d\n", __func__, lock, isc_retval);
4906                         mtx_unlock(&softc->ctl_lock);
4907                         return (1);
4908                 }
4909         }
4910
4911         mtx_unlock(&softc->ctl_lock);
4912
4913         return (0);
4914 }
4915
4916 void
4917 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
4918 {
4919         struct ctl_lun *lun;
4920         struct ctl_softc *softc;
4921         int i;
4922
4923         softc = control_softc;
4924
4925         mtx_lock(&softc->ctl_lock);
4926
4927         lun = (struct ctl_lun *)be_lun->ctl_lun;
4928
4929         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4930                 lun->pending_sense[i].ua_pending |= CTL_UA_CAPACITY_CHANGED;
4931
4932         mtx_unlock(&softc->ctl_lock);
4933 }
4934
4935 /*
4936  * Backend "memory move is complete" callback for requests that never
4937  * make it down to say RAIDCore's configuration code.
4938  */
4939 int
4940 ctl_config_move_done(union ctl_io *io)
4941 {
4942         int retval;
4943
4944         retval = CTL_RETVAL_COMPLETE;
4945
4946
4947         CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
4948         /*
4949          * XXX KDM this shouldn't happen, but what if it does?
4950          */
4951         if (io->io_hdr.io_type != CTL_IO_SCSI)
4952                 panic("I/O type isn't CTL_IO_SCSI!");
4953
4954         if ((io->io_hdr.port_status == 0)
4955          && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
4956          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
4957                 io->io_hdr.status = CTL_SUCCESS;
4958         else if ((io->io_hdr.port_status != 0)
4959               && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
4960               && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)){
4961                 /*
4962                  * For hardware error sense keys, the sense key
4963                  * specific value is defined to be a retry count,
4964                  * but we use it to pass back an internal FETD
4965                  * error code.  XXX KDM  Hopefully the FETD is only
4966                  * using 16 bits for an error code, since that's
4967                  * all the space we have in the sks field.
4968                  */
4969                 ctl_set_internal_failure(&io->scsiio,
4970                                          /*sks_valid*/ 1,
4971                                          /*retry_count*/
4972                                          io->io_hdr.port_status);
4973                 free(io->scsiio.kern_data_ptr, M_CTL);
4974                 ctl_done(io);
4975                 goto bailout;
4976         }
4977
4978         if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
4979          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
4980          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
4981                 /*
4982                  * XXX KDM just assuming a single pointer here, and not a
4983                  * S/G list.  If we start using S/G lists for config data,
4984                  * we'll need to know how to clean them up here as well.
4985                  */
4986                 free(io->scsiio.kern_data_ptr, M_CTL);
4987                 /* Hopefully the user has already set the status... */
4988                 ctl_done(io);
4989         } else {
4990                 /*
4991                  * XXX KDM now we need to continue data movement.  Some
4992                  * options:
4993                  * - call ctl_scsiio() again?  We don't do this for data
4994                  *   writes, because for those at least we know ahead of
4995                  *   time where the write will go and how long it is.  For
4996                  *   config writes, though, that information is largely
4997                  *   contained within the write itself, thus we need to
4998                  *   parse out the data again.
4999                  *
5000                  * - Call some other function once the data is in?
5001                  */
5002
5003                 /*
5004                  * XXX KDM call ctl_scsiio() again for now, and check flag
5005                  * bits to see whether we're allocated or not.
5006                  */
5007                 retval = ctl_scsiio(&io->scsiio);
5008         }
5009 bailout:
5010         return (retval);
5011 }
5012
5013 /*
5014  * This gets called by a backend driver when it is done with a
5015  * configuration write.
5016  */
5017 void
5018 ctl_config_write_done(union ctl_io *io)
5019 {
5020         /*
5021          * If the IO_CONT flag is set, we need to call the supplied
5022          * function to continue processing the I/O, instead of completing
5023          * the I/O just yet.
5024          *
5025          * If there is an error, though, we don't want to keep processing.
5026          * Instead, just send status back to the initiator.
5027          */
5028         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT)
5029          && (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)
5030           || ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))) {
5031                 io->scsiio.io_cont(io);
5032                 return;
5033         }
5034         /*
5035          * Since a configuration write can be done for commands that actually
5036          * have data allocated, like write buffer, and commands that have
5037          * no data, like start/stop unit, we need to check here.
5038          */
5039         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
5040                 free(io->scsiio.kern_data_ptr, M_CTL);
5041         ctl_done(io);
5042 }
5043
5044 /*
5045  * SCSI release command.
5046  */
5047 int
5048 ctl_scsi_release(struct ctl_scsiio *ctsio)
5049 {
5050         int length, longid, thirdparty_id, resv_id;
5051         struct ctl_softc *ctl_softc;
5052         struct ctl_lun *lun;
5053
5054         length = 0;
5055         resv_id = 0;
5056
5057         CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
5058
5059         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5060         ctl_softc = control_softc;
5061
5062         switch (ctsio->cdb[0]) {
5063         case RELEASE: {
5064                 struct scsi_release *cdb;
5065
5066                 cdb = (struct scsi_release *)ctsio->cdb;
5067                 if ((cdb->byte2 & 0x1f) != 0) {
5068                         ctl_set_invalid_field(ctsio,
5069                                               /*sks_valid*/ 1,
5070                                               /*command*/ 1,
5071                                               /*field*/ 1,
5072                                               /*bit_valid*/ 0,
5073                                               /*bit*/ 0);
5074                         ctl_done((union ctl_io *)ctsio);
5075                         return (CTL_RETVAL_COMPLETE);
5076                 }
5077                 break;
5078         }
5079         case RELEASE_10: {
5080                 struct scsi_release_10 *cdb;
5081
5082                 cdb = (struct scsi_release_10 *)ctsio->cdb;
5083
5084                 if ((cdb->byte2 & SR10_EXTENT) != 0) {
5085                         ctl_set_invalid_field(ctsio,
5086                                               /*sks_valid*/ 1,
5087                                               /*command*/ 1,
5088                                               /*field*/ 1,
5089                                               /*bit_valid*/ 1,
5090                                               /*bit*/ 0);
5091                         ctl_done((union ctl_io *)ctsio);
5092                         return (CTL_RETVAL_COMPLETE);
5093
5094                 }
5095
5096                 if ((cdb->byte2 & SR10_3RDPTY) != 0) {
5097                         ctl_set_invalid_field(ctsio,
5098                                               /*sks_valid*/ 1,
5099                                               /*command*/ 1,
5100                                               /*field*/ 1,
5101                                               /*bit_valid*/ 1,
5102                                               /*bit*/ 4);
5103                         ctl_done((union ctl_io *)ctsio);
5104                         return (CTL_RETVAL_COMPLETE);
5105                 }
5106
5107                 if (cdb->byte2 & SR10_LONGID)
5108                         longid = 1;
5109                 else
5110                         thirdparty_id = cdb->thirdparty_id;
5111
5112                 resv_id = cdb->resv_id;
5113                 length = scsi_2btoul(cdb->length);
5114                 break;
5115         }
5116         }
5117
5118
5119         /*
5120          * XXX KDM right now, we only support LUN reservation.  We don't
5121          * support 3rd party reservations, or extent reservations, which
5122          * might actually need the parameter list.  If we've gotten this
5123          * far, we've got a LUN reservation.  Anything else got kicked out
5124          * above.  So, according to SPC, ignore the length.
5125          */
5126         length = 0;
5127
5128         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5129          && (length > 0)) {
5130                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5131                 ctsio->kern_data_len = length;
5132                 ctsio->kern_total_len = length;
5133                 ctsio->kern_data_resid = 0;
5134                 ctsio->kern_rel_offset = 0;
5135                 ctsio->kern_sg_entries = 0;
5136                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5137                 ctsio->be_move_done = ctl_config_move_done;
5138                 ctl_datamove((union ctl_io *)ctsio);
5139
5140                 return (CTL_RETVAL_COMPLETE);
5141         }
5142
5143         if (length > 0)
5144                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5145
5146         mtx_lock(&ctl_softc->ctl_lock);
5147
5148         /*
5149          * According to SPC, it is not an error for an intiator to attempt
5150          * to release a reservation on a LUN that isn't reserved, or that
5151          * is reserved by another initiator.  The reservation can only be
5152          * released, though, by the initiator who made it or by one of
5153          * several reset type events.
5154          */
5155         if (lun->flags & CTL_LUN_RESERVED) {
5156                 if ((ctsio->io_hdr.nexus.initid.id == lun->rsv_nexus.initid.id)
5157                  && (ctsio->io_hdr.nexus.targ_port == lun->rsv_nexus.targ_port)
5158                  && (ctsio->io_hdr.nexus.targ_target.id ==
5159                      lun->rsv_nexus.targ_target.id)) {
5160                         lun->flags &= ~CTL_LUN_RESERVED;
5161                 }
5162         }
5163
5164         ctsio->scsi_status = SCSI_STATUS_OK;
5165         ctsio->io_hdr.status = CTL_SUCCESS;
5166
5167         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5168                 free(ctsio->kern_data_ptr, M_CTL);
5169                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5170         }
5171
5172         mtx_unlock(&ctl_softc->ctl_lock);
5173
5174         ctl_done((union ctl_io *)ctsio);
5175         return (CTL_RETVAL_COMPLETE);
5176 }
5177
5178 int
5179 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5180 {
5181         int extent, thirdparty, longid;
5182         int resv_id, length;
5183         uint64_t thirdparty_id;
5184         struct ctl_softc *ctl_softc;
5185         struct ctl_lun *lun;
5186
5187         extent = 0;
5188         thirdparty = 0;
5189         longid = 0;
5190         resv_id = 0;
5191         length = 0;
5192         thirdparty_id = 0;
5193
5194         CTL_DEBUG_PRINT(("ctl_reserve\n"));
5195
5196         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5197         ctl_softc = control_softc;
5198
5199         switch (ctsio->cdb[0]) {
5200         case RESERVE: {
5201                 struct scsi_reserve *cdb;
5202
5203                 cdb = (struct scsi_reserve *)ctsio->cdb;
5204                 if ((cdb->byte2 & 0x1f) != 0) {
5205                         ctl_set_invalid_field(ctsio,
5206                                               /*sks_valid*/ 1,
5207                                               /*command*/ 1,
5208                                               /*field*/ 1,
5209                                               /*bit_valid*/ 0,
5210                                               /*bit*/ 0);
5211                         ctl_done((union ctl_io *)ctsio);
5212                         return (CTL_RETVAL_COMPLETE);
5213                 }
5214                 resv_id = cdb->resv_id;
5215                 length = scsi_2btoul(cdb->length);
5216                 break;
5217         }
5218         case RESERVE_10: {
5219                 struct scsi_reserve_10 *cdb;
5220
5221                 cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5222
5223                 if ((cdb->byte2 & SR10_EXTENT) != 0) {
5224                         ctl_set_invalid_field(ctsio,
5225                                               /*sks_valid*/ 1,
5226                                               /*command*/ 1,
5227                                               /*field*/ 1,
5228                                               /*bit_valid*/ 1,
5229                                               /*bit*/ 0);
5230                         ctl_done((union ctl_io *)ctsio);
5231                         return (CTL_RETVAL_COMPLETE);
5232                 }
5233                 if ((cdb->byte2 & SR10_3RDPTY) != 0) {
5234                         ctl_set_invalid_field(ctsio,
5235                                               /*sks_valid*/ 1,
5236                                               /*command*/ 1,
5237                                               /*field*/ 1,
5238                                               /*bit_valid*/ 1,
5239                                               /*bit*/ 4);
5240                         ctl_done((union ctl_io *)ctsio);
5241                         return (CTL_RETVAL_COMPLETE);
5242                 }
5243                 if (cdb->byte2 & SR10_LONGID)
5244                         longid = 1;
5245                 else
5246                         thirdparty_id = cdb->thirdparty_id;
5247
5248                 resv_id = cdb->resv_id;
5249                 length = scsi_2btoul(cdb->length);
5250                 break;
5251         }
5252         }
5253
5254         /*
5255          * XXX KDM right now, we only support LUN reservation.  We don't
5256          * support 3rd party reservations, or extent reservations, which
5257          * might actually need the parameter list.  If we've gotten this
5258          * far, we've got a LUN reservation.  Anything else got kicked out
5259          * above.  So, according to SPC, ignore the length.
5260          */
5261         length = 0;
5262
5263         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5264          && (length > 0)) {
5265                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5266                 ctsio->kern_data_len = length;
5267                 ctsio->kern_total_len = length;
5268                 ctsio->kern_data_resid = 0;
5269                 ctsio->kern_rel_offset = 0;
5270                 ctsio->kern_sg_entries = 0;
5271                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5272                 ctsio->be_move_done = ctl_config_move_done;
5273                 ctl_datamove((union ctl_io *)ctsio);
5274
5275                 return (CTL_RETVAL_COMPLETE);
5276         }
5277
5278         if (length > 0)
5279                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5280
5281         mtx_lock(&ctl_softc->ctl_lock);
5282         if (lun->flags & CTL_LUN_RESERVED) {
5283                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
5284                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
5285                  || (ctsio->io_hdr.nexus.targ_target.id !=
5286                      lun->rsv_nexus.targ_target.id)) {
5287                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
5288                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
5289                         goto bailout;
5290                 }
5291         }
5292
5293         lun->flags |= CTL_LUN_RESERVED;
5294         lun->rsv_nexus = ctsio->io_hdr.nexus;
5295
5296         ctsio->scsi_status = SCSI_STATUS_OK;
5297         ctsio->io_hdr.status = CTL_SUCCESS;
5298
5299 bailout:
5300         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5301                 free(ctsio->kern_data_ptr, M_CTL);
5302                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5303         }
5304
5305         mtx_unlock(&ctl_softc->ctl_lock);
5306
5307         ctl_done((union ctl_io *)ctsio);
5308         return (CTL_RETVAL_COMPLETE);
5309 }
5310
5311 int
5312 ctl_start_stop(struct ctl_scsiio *ctsio)
5313 {
5314         struct scsi_start_stop_unit *cdb;
5315         struct ctl_lun *lun;
5316         struct ctl_softc *ctl_softc;
5317         int retval;
5318
5319         CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5320
5321         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5322         ctl_softc = control_softc;
5323         retval = 0;
5324
5325         cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5326
5327         /*
5328          * XXX KDM
5329          * We don't support the immediate bit on a stop unit.  In order to
5330          * do that, we would need to code up a way to know that a stop is
5331          * pending, and hold off any new commands until it completes, one
5332          * way or another.  Then we could accept or reject those commands
5333          * depending on its status.  We would almost need to do the reverse
5334          * of what we do below for an immediate start -- return the copy of
5335          * the ctl_io to the FETD with status to send to the host (and to
5336          * free the copy!) and then free the original I/O once the stop
5337          * actually completes.  That way, the OOA queue mechanism can work
5338          * to block commands that shouldn't proceed.  Another alternative
5339          * would be to put the copy in the queue in place of the original,
5340          * and return the original back to the caller.  That could be
5341          * slightly safer..
5342          */
5343         if ((cdb->byte2 & SSS_IMMED)
5344          && ((cdb->how & SSS_START) == 0)) {
5345                 ctl_set_invalid_field(ctsio,
5346                                       /*sks_valid*/ 1,
5347                                       /*command*/ 1,
5348                                       /*field*/ 1,
5349                                       /*bit_valid*/ 1,
5350                                       /*bit*/ 0);
5351                 ctl_done((union ctl_io *)ctsio);
5352                 return (CTL_RETVAL_COMPLETE);
5353         }
5354
5355         /*
5356          * We don't support the power conditions field.  We need to check
5357          * this prior to checking the load/eject and start/stop bits.
5358          */
5359         if ((cdb->how & SSS_PC_MASK) != SSS_PC_START_VALID) {
5360                 ctl_set_invalid_field(ctsio,
5361                                       /*sks_valid*/ 1,
5362                                       /*command*/ 1,
5363                                       /*field*/ 4,
5364                                       /*bit_valid*/ 1,
5365                                       /*bit*/ 4);
5366                 ctl_done((union ctl_io *)ctsio);
5367                 return (CTL_RETVAL_COMPLETE);
5368         }
5369
5370         /*
5371          * Media isn't removable, so we can't load or eject it.
5372          */
5373         if ((cdb->how & SSS_LOEJ) != 0) {
5374                 ctl_set_invalid_field(ctsio,
5375                                       /*sks_valid*/ 1,
5376                                       /*command*/ 1,
5377                                       /*field*/ 4,
5378                                       /*bit_valid*/ 1,
5379                                       /*bit*/ 1);
5380                 ctl_done((union ctl_io *)ctsio);
5381                 return (CTL_RETVAL_COMPLETE);
5382         }
5383
5384         if ((lun->flags & CTL_LUN_PR_RESERVED)
5385          && ((cdb->how & SSS_START)==0)) {
5386                 uint32_t residx;
5387
5388                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5389                 if (!lun->per_res[residx].registered
5390                  || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5391
5392                         ctl_set_reservation_conflict(ctsio);
5393                         ctl_done((union ctl_io *)ctsio);
5394                         return (CTL_RETVAL_COMPLETE);
5395                 }
5396         }
5397
5398         /*
5399          * If there is no backend on this device, we can't start or stop
5400          * it.  In theory we shouldn't get any start/stop commands in the
5401          * first place at this level if the LUN doesn't have a backend.
5402          * That should get stopped by the command decode code.
5403          */
5404         if (lun->backend == NULL) {
5405                 ctl_set_invalid_opcode(ctsio);
5406                 ctl_done((union ctl_io *)ctsio);
5407                 return (CTL_RETVAL_COMPLETE);
5408         }
5409
5410         /*
5411          * XXX KDM Copan-specific offline behavior.
5412          * Figure out a reasonable way to port this?
5413          */
5414 #ifdef NEEDTOPORT
5415         mtx_lock(&ctl_softc->ctl_lock);
5416
5417         if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5418          && (lun->flags & CTL_LUN_OFFLINE)) {
5419                 /*
5420                  * If the LUN is offline, and the on/offline bit isn't set,
5421                  * reject the start or stop.  Otherwise, let it through.
5422                  */
5423                 mtx_unlock(&ctl_softc->ctl_lock);
5424                 ctl_set_lun_not_ready(ctsio);
5425                 ctl_done((union ctl_io *)ctsio);
5426         } else {
5427                 mtx_unlock(&ctl_softc->ctl_lock);
5428 #endif /* NEEDTOPORT */
5429                 /*
5430                  * This could be a start or a stop when we're online,
5431                  * or a stop/offline or start/online.  A start or stop when
5432                  * we're offline is covered in the case above.
5433                  */
5434                 /*
5435                  * In the non-immediate case, we send the request to
5436                  * the backend and return status to the user when
5437                  * it is done.
5438                  *
5439                  * In the immediate case, we allocate a new ctl_io
5440                  * to hold a copy of the request, and send that to
5441                  * the backend.  We then set good status on the
5442                  * user's request and return it immediately.
5443                  */
5444                 if (cdb->byte2 & SSS_IMMED) {
5445                         union ctl_io *new_io;
5446
5447                         new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5448                         if (new_io == NULL) {
5449                                 ctl_set_busy(ctsio);
5450                                 ctl_done((union ctl_io *)ctsio);
5451                         } else {
5452                                 ctl_copy_io((union ctl_io *)ctsio,
5453                                             new_io);
5454                                 retval = lun->backend->config_write(new_io);
5455                                 ctl_set_success(ctsio);
5456                                 ctl_done((union ctl_io *)ctsio);
5457                         }
5458                 } else {
5459                         retval = lun->backend->config_write(
5460                                 (union ctl_io *)ctsio);
5461                 }
5462 #ifdef NEEDTOPORT
5463         }
5464 #endif
5465         return (retval);
5466 }
5467
5468 /*
5469  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5470  * we don't really do anything with the LBA and length fields if the user
5471  * passes them in.  Instead we'll just flush out the cache for the entire
5472  * LUN.
5473  */
5474 int
5475 ctl_sync_cache(struct ctl_scsiio *ctsio)
5476 {
5477         struct ctl_lun *lun;
5478         struct ctl_softc *ctl_softc;
5479         uint64_t starting_lba;
5480         uint32_t block_count;
5481         int reladr, immed;
5482         int retval;
5483
5484         CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5485
5486         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5487         ctl_softc = control_softc;
5488         retval = 0;
5489         reladr = 0;
5490         immed = 0;
5491
5492         switch (ctsio->cdb[0]) {
5493         case SYNCHRONIZE_CACHE: {
5494                 struct scsi_sync_cache *cdb;
5495                 cdb = (struct scsi_sync_cache *)ctsio->cdb;
5496
5497                 if (cdb->byte2 & SSC_RELADR)
5498                         reladr = 1;
5499
5500                 if (cdb->byte2 & SSC_IMMED)
5501                         immed = 1;
5502
5503                 starting_lba = scsi_4btoul(cdb->begin_lba);
5504                 block_count = scsi_2btoul(cdb->lb_count);
5505                 break;
5506         }
5507         case SYNCHRONIZE_CACHE_16: {
5508                 struct scsi_sync_cache_16 *cdb;
5509                 cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5510
5511                 if (cdb->byte2 & SSC_RELADR)
5512                         reladr = 1;
5513
5514                 if (cdb->byte2 & SSC_IMMED)
5515                         immed = 1;
5516
5517                 starting_lba = scsi_8btou64(cdb->begin_lba);
5518                 block_count = scsi_4btoul(cdb->lb_count);
5519                 break;
5520         }
5521         default:
5522                 ctl_set_invalid_opcode(ctsio);
5523                 ctl_done((union ctl_io *)ctsio);
5524                 goto bailout;
5525                 break; /* NOTREACHED */
5526         }
5527
5528         if (immed) {
5529                 /*
5530                  * We don't support the immediate bit.  Since it's in the
5531                  * same place for the 10 and 16 byte SYNCHRONIZE CACHE
5532                  * commands, we can just return the same error in either
5533                  * case.
5534                  */
5535                 ctl_set_invalid_field(ctsio,
5536                                       /*sks_valid*/ 1,
5537                                       /*command*/ 1,
5538                                       /*field*/ 1,
5539                                       /*bit_valid*/ 1,
5540                                       /*bit*/ 1);
5541                 ctl_done((union ctl_io *)ctsio);
5542                 goto bailout;
5543         }
5544
5545         if (reladr) {
5546                 /*
5547                  * We don't support the reladr bit either.  It can only be
5548                  * used with linked commands, and we don't support linked
5549                  * commands.  Since the bit is in the same place for the
5550                  * 10 and 16 byte SYNCHRONIZE CACHE * commands, we can
5551                  * just return the same error in either case.
5552                  */
5553                 ctl_set_invalid_field(ctsio,
5554                                       /*sks_valid*/ 1,
5555                                       /*command*/ 1,
5556                                       /*field*/ 1,
5557                                       /*bit_valid*/ 1,
5558                                       /*bit*/ 0);
5559                 ctl_done((union ctl_io *)ctsio);
5560                 goto bailout;
5561         }
5562
5563         /*
5564          * We check the LBA and length, but don't do anything with them.
5565          * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5566          * get flushed.  This check will just help satisfy anyone who wants
5567          * to see an error for an out of range LBA.
5568          */
5569         if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5570                 ctl_set_lba_out_of_range(ctsio);
5571                 ctl_done((union ctl_io *)ctsio);
5572                 goto bailout;
5573         }
5574
5575         /*
5576          * If this LUN has no backend, we can't flush the cache anyway.
5577          */
5578         if (lun->backend == NULL) {
5579                 ctl_set_invalid_opcode(ctsio);
5580                 ctl_done((union ctl_io *)ctsio);
5581                 goto bailout;
5582         }
5583
5584         /*
5585          * Check to see whether we're configured to send the SYNCHRONIZE
5586          * CACHE command directly to the back end.
5587          */
5588         mtx_lock(&ctl_softc->ctl_lock);
5589         if ((ctl_softc->flags & CTL_FLAG_REAL_SYNC)
5590          && (++(lun->sync_count) >= lun->sync_interval)) {
5591                 lun->sync_count = 0;
5592                 mtx_unlock(&ctl_softc->ctl_lock);
5593                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5594         } else {
5595                 mtx_unlock(&ctl_softc->ctl_lock);
5596                 ctl_set_success(ctsio);
5597                 ctl_done((union ctl_io *)ctsio);
5598         }
5599
5600 bailout:
5601
5602         return (retval);
5603 }
5604
5605 int
5606 ctl_format(struct ctl_scsiio *ctsio)
5607 {
5608         struct scsi_format *cdb;
5609         struct ctl_lun *lun;
5610         struct ctl_softc *ctl_softc;
5611         int length, defect_list_len;
5612
5613         CTL_DEBUG_PRINT(("ctl_format\n"));
5614
5615         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5616         ctl_softc = control_softc;
5617
5618         cdb = (struct scsi_format *)ctsio->cdb;
5619
5620         length = 0;
5621         if (cdb->byte2 & SF_FMTDATA) {
5622                 if (cdb->byte2 & SF_LONGLIST)
5623                         length = sizeof(struct scsi_format_header_long);
5624                 else
5625                         length = sizeof(struct scsi_format_header_short);
5626         }
5627
5628         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5629          && (length > 0)) {
5630                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5631                 ctsio->kern_data_len = length;
5632                 ctsio->kern_total_len = length;
5633                 ctsio->kern_data_resid = 0;
5634                 ctsio->kern_rel_offset = 0;
5635                 ctsio->kern_sg_entries = 0;
5636                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5637                 ctsio->be_move_done = ctl_config_move_done;
5638                 ctl_datamove((union ctl_io *)ctsio);
5639
5640                 return (CTL_RETVAL_COMPLETE);
5641         }
5642
5643         defect_list_len = 0;
5644
5645         if (cdb->byte2 & SF_FMTDATA) {
5646                 if (cdb->byte2 & SF_LONGLIST) {
5647                         struct scsi_format_header_long *header;
5648
5649                         header = (struct scsi_format_header_long *)
5650                                 ctsio->kern_data_ptr;
5651
5652                         defect_list_len = scsi_4btoul(header->defect_list_len);
5653                         if (defect_list_len != 0) {
5654                                 ctl_set_invalid_field(ctsio,
5655                                                       /*sks_valid*/ 1,
5656                                                       /*command*/ 0,
5657                                                       /*field*/ 2,
5658                                                       /*bit_valid*/ 0,
5659                                                       /*bit*/ 0);
5660                                 goto bailout;
5661                         }
5662                 } else {
5663                         struct scsi_format_header_short *header;
5664
5665                         header = (struct scsi_format_header_short *)
5666                                 ctsio->kern_data_ptr;
5667
5668                         defect_list_len = scsi_2btoul(header->defect_list_len);
5669                         if (defect_list_len != 0) {
5670                                 ctl_set_invalid_field(ctsio,
5671                                                       /*sks_valid*/ 1,
5672                                                       /*command*/ 0,
5673                                                       /*field*/ 2,
5674                                                       /*bit_valid*/ 0,
5675                                                       /*bit*/ 0);
5676                                 goto bailout;
5677                         }
5678                 }
5679         }
5680
5681         /*
5682          * The format command will clear out the "Medium format corrupted"
5683          * status if set by the configuration code.  That status is really
5684          * just a way to notify the host that we have lost the media, and
5685          * get them to issue a command that will basically make them think
5686          * they're blowing away the media.
5687          */
5688         mtx_lock(&ctl_softc->ctl_lock);
5689         lun->flags &= ~CTL_LUN_INOPERABLE;
5690         mtx_unlock(&ctl_softc->ctl_lock);
5691
5692         ctsio->scsi_status = SCSI_STATUS_OK;
5693         ctsio->io_hdr.status = CTL_SUCCESS;
5694 bailout:
5695
5696         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5697                 free(ctsio->kern_data_ptr, M_CTL);
5698                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5699         }
5700
5701         ctl_done((union ctl_io *)ctsio);
5702         return (CTL_RETVAL_COMPLETE);
5703 }
5704
5705 int
5706 ctl_write_buffer(struct ctl_scsiio *ctsio)
5707 {
5708         struct scsi_write_buffer *cdb;
5709         struct copan_page_header *header;
5710         struct ctl_lun *lun;
5711         struct ctl_softc *ctl_softc;
5712         int buffer_offset, len;
5713         int retval;
5714
5715         header = NULL;
5716
5717         retval = CTL_RETVAL_COMPLETE;
5718
5719         CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5720
5721         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5722         ctl_softc = control_softc;
5723         cdb = (struct scsi_write_buffer *)ctsio->cdb;
5724
5725         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5726                 ctl_set_invalid_field(ctsio,
5727                                       /*sks_valid*/ 1,
5728                                       /*command*/ 1,
5729                                       /*field*/ 1,
5730                                       /*bit_valid*/ 1,
5731                                       /*bit*/ 4);
5732                 ctl_done((union ctl_io *)ctsio);
5733                 return (CTL_RETVAL_COMPLETE);
5734         }
5735         if (cdb->buffer_id != 0) {
5736                 ctl_set_invalid_field(ctsio,
5737                                       /*sks_valid*/ 1,
5738                                       /*command*/ 1,
5739                                       /*field*/ 2,
5740                                       /*bit_valid*/ 0,
5741                                       /*bit*/ 0);
5742                 ctl_done((union ctl_io *)ctsio);
5743                 return (CTL_RETVAL_COMPLETE);
5744         }
5745
5746         len = scsi_3btoul(cdb->length);
5747         buffer_offset = scsi_3btoul(cdb->offset);
5748
5749         if (len > sizeof(lun->write_buffer)) {
5750                 ctl_set_invalid_field(ctsio,
5751                                       /*sks_valid*/ 1,
5752                                       /*command*/ 1,
5753                                       /*field*/ 6,
5754                                       /*bit_valid*/ 0,
5755                                       /*bit*/ 0);
5756                 ctl_done((union ctl_io *)ctsio);
5757                 return (CTL_RETVAL_COMPLETE);
5758         }
5759
5760         if (buffer_offset != 0) {
5761                 ctl_set_invalid_field(ctsio,
5762                                       /*sks_valid*/ 1,
5763                                       /*command*/ 1,
5764                                       /*field*/ 3,
5765                                       /*bit_valid*/ 0,
5766                                       /*bit*/ 0);
5767                 ctl_done((union ctl_io *)ctsio);
5768                 return (CTL_RETVAL_COMPLETE);
5769         }
5770
5771         /*
5772          * If we've got a kernel request that hasn't been malloced yet,
5773          * malloc it and tell the caller the data buffer is here.
5774          */
5775         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5776                 ctsio->kern_data_ptr = lun->write_buffer;
5777                 ctsio->kern_data_len = len;
5778                 ctsio->kern_total_len = len;
5779                 ctsio->kern_data_resid = 0;
5780                 ctsio->kern_rel_offset = 0;
5781                 ctsio->kern_sg_entries = 0;
5782                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5783                 ctsio->be_move_done = ctl_config_move_done;
5784                 ctl_datamove((union ctl_io *)ctsio);
5785
5786                 return (CTL_RETVAL_COMPLETE);
5787         }
5788
5789         ctl_done((union ctl_io *)ctsio);
5790
5791         return (CTL_RETVAL_COMPLETE);
5792 }
5793
5794 int
5795 ctl_write_same(struct ctl_scsiio *ctsio)
5796 {
5797         struct ctl_lun *lun;
5798         struct ctl_lba_len_flags lbalen;
5799         uint64_t lba;
5800         uint32_t num_blocks;
5801         int len, retval;
5802         uint8_t byte2;
5803
5804         retval = CTL_RETVAL_COMPLETE;
5805
5806         CTL_DEBUG_PRINT(("ctl_write_same\n"));
5807
5808         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5809
5810         switch (ctsio->cdb[0]) {
5811         case WRITE_SAME_10: {
5812                 struct scsi_write_same_10 *cdb;
5813
5814                 cdb = (struct scsi_write_same_10 *)ctsio->cdb;
5815
5816                 lba = scsi_4btoul(cdb->addr);
5817                 num_blocks = scsi_2btoul(cdb->length);
5818                 byte2 = cdb->byte2;
5819                 break;
5820         }
5821         case WRITE_SAME_16: {
5822                 struct scsi_write_same_16 *cdb;
5823
5824                 cdb = (struct scsi_write_same_16 *)ctsio->cdb;
5825
5826                 lba = scsi_8btou64(cdb->addr);
5827                 num_blocks = scsi_4btoul(cdb->length);
5828                 byte2 = cdb->byte2;
5829                 break;
5830         }
5831         default:
5832                 /*
5833                  * We got a command we don't support.  This shouldn't
5834                  * happen, commands should be filtered out above us.
5835                  */
5836                 ctl_set_invalid_opcode(ctsio);
5837                 ctl_done((union ctl_io *)ctsio);
5838
5839                 return (CTL_RETVAL_COMPLETE);
5840                 break; /* NOTREACHED */
5841         }
5842
5843         /*
5844          * The first check is to make sure we're in bounds, the second
5845          * check is to catch wrap-around problems.  If the lba + num blocks
5846          * is less than the lba, then we've wrapped around and the block
5847          * range is invalid anyway.
5848          */
5849         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5850          || ((lba + num_blocks) < lba)) {
5851                 ctl_set_lba_out_of_range(ctsio);
5852                 ctl_done((union ctl_io *)ctsio);
5853                 return (CTL_RETVAL_COMPLETE);
5854         }
5855
5856         /* Zero number of blocks means "to the last logical block" */
5857         if (num_blocks == 0) {
5858                 if ((lun->be_lun->maxlba + 1) - lba > UINT32_MAX) {
5859                         ctl_set_invalid_field(ctsio,
5860                                               /*sks_valid*/ 0,
5861                                               /*command*/ 1,
5862                                               /*field*/ 0,
5863                                               /*bit_valid*/ 0,
5864                                               /*bit*/ 0);
5865                         ctl_done((union ctl_io *)ctsio);
5866                         return (CTL_RETVAL_COMPLETE);
5867                 }
5868                 num_blocks = (lun->be_lun->maxlba + 1) - lba;
5869         }
5870
5871         len = lun->be_lun->blocksize;
5872
5873         /*
5874          * If we've got a kernel request that hasn't been malloced yet,
5875          * malloc it and tell the caller the data buffer is here.
5876          */
5877         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5878                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5879                 ctsio->kern_data_len = len;
5880                 ctsio->kern_total_len = len;
5881                 ctsio->kern_data_resid = 0;
5882                 ctsio->kern_rel_offset = 0;
5883                 ctsio->kern_sg_entries = 0;
5884                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5885                 ctsio->be_move_done = ctl_config_move_done;
5886                 ctl_datamove((union ctl_io *)ctsio);
5887
5888                 return (CTL_RETVAL_COMPLETE);
5889         }
5890
5891         lbalen.lba = lba;
5892         lbalen.len = num_blocks;
5893         lbalen.flags = byte2;
5894         memcpy(ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes, &lbalen,
5895                sizeof(lbalen));
5896         retval = lun->backend->config_write((union ctl_io *)ctsio);
5897
5898         return (retval);
5899 }
5900
5901 int
5902 ctl_unmap(struct ctl_scsiio *ctsio)
5903 {
5904         struct ctl_lun *lun;
5905         struct scsi_unmap *cdb;
5906         struct ctl_ptr_len_flags ptrlen;
5907         struct scsi_unmap_header *hdr;
5908         struct scsi_unmap_desc *buf, *end;
5909         uint64_t lba;
5910         uint32_t num_blocks;
5911         int len, retval;
5912         uint8_t byte2;
5913
5914         retval = CTL_RETVAL_COMPLETE;
5915
5916         CTL_DEBUG_PRINT(("ctl_unmap\n"));
5917
5918         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5919         cdb = (struct scsi_unmap *)ctsio->cdb;
5920
5921         len = scsi_2btoul(cdb->length);
5922         byte2 = cdb->byte2;
5923
5924         /*
5925          * If we've got a kernel request that hasn't been malloced yet,
5926          * malloc it and tell the caller the data buffer is here.
5927          */
5928         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5929                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5930                 ctsio->kern_data_len = len;
5931                 ctsio->kern_total_len = len;
5932                 ctsio->kern_data_resid = 0;
5933                 ctsio->kern_rel_offset = 0;
5934                 ctsio->kern_sg_entries = 0;
5935                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5936                 ctsio->be_move_done = ctl_config_move_done;
5937                 ctl_datamove((union ctl_io *)ctsio);
5938
5939                 return (CTL_RETVAL_COMPLETE);
5940         }
5941
5942         len = ctsio->kern_total_len - ctsio->kern_data_resid;
5943         hdr = (struct scsi_unmap_header *)ctsio->kern_data_ptr;
5944         if (len < sizeof (*hdr) ||
5945             len < (scsi_2btoul(hdr->length) + sizeof(hdr->length)) ||
5946             len < (scsi_2btoul(hdr->desc_length) + sizeof (*hdr)) ||
5947             scsi_2btoul(hdr->desc_length) % sizeof(*buf) != 0) {
5948                 ctl_set_invalid_field(ctsio,
5949                                       /*sks_valid*/ 0,
5950                                       /*command*/ 0,
5951                                       /*field*/ 0,
5952                                       /*bit_valid*/ 0,
5953                                       /*bit*/ 0);
5954                 ctl_done((union ctl_io *)ctsio);
5955                 return (CTL_RETVAL_COMPLETE);
5956         }
5957         len = scsi_2btoul(hdr->desc_length);
5958         buf = (struct scsi_unmap_desc *)(hdr + 1);
5959         end = buf + len / sizeof(*buf);
5960
5961         ptrlen.ptr = (void *)buf;
5962         ptrlen.len = len;
5963         ptrlen.flags = byte2;
5964         memcpy(ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes, &ptrlen,
5965                sizeof(ptrlen));
5966
5967         for (; buf < end; buf++) {
5968                 lba = scsi_8btou64(buf->lba);
5969                 num_blocks = scsi_4btoul(buf->length);
5970                 if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5971                  || ((lba + num_blocks) < lba)) {
5972                         ctl_set_lba_out_of_range(ctsio);
5973                         ctl_done((union ctl_io *)ctsio);
5974                         return (CTL_RETVAL_COMPLETE);
5975                 }
5976         }
5977
5978         retval = lun->backend->config_write((union ctl_io *)ctsio);
5979
5980         return (retval);
5981 }
5982
5983 /*
5984  * Note that this function currently doesn't actually do anything inside
5985  * CTL to enforce things if the DQue bit is turned on.
5986  *
5987  * Also note that this function can't be used in the default case, because
5988  * the DQue bit isn't set in the changeable mask for the control mode page
5989  * anyway.  This is just here as an example for how to implement a page
5990  * handler, and a placeholder in case we want to allow the user to turn
5991  * tagged queueing on and off.
5992  *
5993  * The D_SENSE bit handling is functional, however, and will turn
5994  * descriptor sense on and off for a given LUN.
5995  */
5996 int
5997 ctl_control_page_handler(struct ctl_scsiio *ctsio,
5998                          struct ctl_page_index *page_index, uint8_t *page_ptr)
5999 {
6000         struct scsi_control_page *current_cp, *saved_cp, *user_cp;
6001         struct ctl_lun *lun;
6002         struct ctl_softc *softc;
6003         int set_ua;
6004         uint32_t initidx;
6005
6006         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6007         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
6008         set_ua = 0;
6009
6010         user_cp = (struct scsi_control_page *)page_ptr;
6011         current_cp = (struct scsi_control_page *)
6012                 (page_index->page_data + (page_index->page_len *
6013                 CTL_PAGE_CURRENT));
6014         saved_cp = (struct scsi_control_page *)
6015                 (page_index->page_data + (page_index->page_len *
6016                 CTL_PAGE_SAVED));
6017
6018         softc = control_softc;
6019
6020         mtx_lock(&softc->ctl_lock);
6021         if (((current_cp->rlec & SCP_DSENSE) == 0)
6022          && ((user_cp->rlec & SCP_DSENSE) != 0)) {
6023                 /*
6024                  * Descriptor sense is currently turned off and the user
6025                  * wants to turn it on.
6026                  */
6027                 current_cp->rlec |= SCP_DSENSE;
6028                 saved_cp->rlec |= SCP_DSENSE;
6029                 lun->flags |= CTL_LUN_SENSE_DESC;
6030                 set_ua = 1;
6031         } else if (((current_cp->rlec & SCP_DSENSE) != 0)
6032                 && ((user_cp->rlec & SCP_DSENSE) == 0)) {
6033                 /*
6034                  * Descriptor sense is currently turned on, and the user
6035                  * wants to turn it off.
6036                  */
6037                 current_cp->rlec &= ~SCP_DSENSE;
6038                 saved_cp->rlec &= ~SCP_DSENSE;
6039                 lun->flags &= ~CTL_LUN_SENSE_DESC;
6040                 set_ua = 1;
6041         }
6042         if (current_cp->queue_flags & SCP_QUEUE_DQUE) {
6043                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
6044 #ifdef NEEDTOPORT
6045                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6046                                     CTL_UNTAG_TO_UNTAG,
6047                                     csevent_LogType_Trace,
6048                                     csevent_Severity_Information,
6049                                     csevent_AlertLevel_Green,
6050                                     csevent_FRU_Firmware,
6051                                     csevent_FRU_Unknown,
6052                                     "Received untagged to untagged transition");
6053 #endif /* NEEDTOPORT */
6054                 } else {
6055 #ifdef NEEDTOPORT
6056                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6057                                     CTL_UNTAG_TO_TAG,
6058                                     csevent_LogType_ConfigChange,
6059                                     csevent_Severity_Information,
6060                                     csevent_AlertLevel_Green,
6061                                     csevent_FRU_Firmware,
6062                                     csevent_FRU_Unknown,
6063                                     "Received untagged to tagged "
6064                                     "queueing transition");
6065 #endif /* NEEDTOPORT */
6066
6067                         current_cp->queue_flags &= ~SCP_QUEUE_DQUE;
6068                         saved_cp->queue_flags &= ~SCP_QUEUE_DQUE;
6069                         set_ua = 1;
6070                 }
6071         } else {
6072                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
6073 #ifdef NEEDTOPORT
6074                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6075                                     CTL_TAG_TO_UNTAG,
6076                                     csevent_LogType_ConfigChange,
6077                                     csevent_Severity_Warning,
6078                                     csevent_AlertLevel_Yellow,
6079                                     csevent_FRU_Firmware,
6080                                     csevent_FRU_Unknown,
6081                                     "Received tagged queueing to untagged "
6082                                     "transition");
6083 #endif /* NEEDTOPORT */
6084
6085                         current_cp->queue_flags |= SCP_QUEUE_DQUE;
6086                         saved_cp->queue_flags |= SCP_QUEUE_DQUE;
6087                         set_ua = 1;
6088                 } else {
6089 #ifdef NEEDTOPORT
6090                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6091                                     CTL_TAG_TO_TAG,
6092                                     csevent_LogType_Trace,
6093                                     csevent_Severity_Information,
6094                                     csevent_AlertLevel_Green,
6095                                     csevent_FRU_Firmware,
6096                                     csevent_FRU_Unknown,
6097                                     "Received tagged queueing to tagged "
6098                                     "queueing transition");
6099 #endif /* NEEDTOPORT */
6100                 }
6101         }
6102         if (set_ua != 0) {
6103                 int i;
6104                 /*
6105                  * Let other initiators know that the mode
6106                  * parameters for this LUN have changed.
6107                  */
6108                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
6109                         if (i == initidx)
6110                                 continue;
6111
6112                         lun->pending_sense[i].ua_pending |=
6113                                 CTL_UA_MODE_CHANGE;
6114                 }
6115         }
6116         mtx_unlock(&softc->ctl_lock);
6117
6118         return (0);
6119 }
6120
6121 int
6122 ctl_power_sp_handler(struct ctl_scsiio *ctsio,
6123                      struct ctl_page_index *page_index, uint8_t *page_ptr)
6124 {
6125         return (0);
6126 }
6127
6128 int
6129 ctl_power_sp_sense_handler(struct ctl_scsiio *ctsio,
6130                            struct ctl_page_index *page_index, int pc)
6131 {
6132         struct copan_power_subpage *page;
6133
6134         page = (struct copan_power_subpage *)page_index->page_data +
6135                 (page_index->page_len * pc);
6136
6137         switch (pc) {
6138         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6139                 /*
6140                  * We don't update the changable bits for this page.
6141                  */
6142                 break;
6143         case SMS_PAGE_CTRL_CURRENT >> 6:
6144         case SMS_PAGE_CTRL_DEFAULT >> 6:
6145         case SMS_PAGE_CTRL_SAVED >> 6:
6146 #ifdef NEEDTOPORT
6147                 ctl_update_power_subpage(page);
6148 #endif
6149                 break;
6150         default:
6151 #ifdef NEEDTOPORT
6152                 EPRINT(0, "Invalid PC %d!!", pc);
6153 #endif
6154                 break;
6155         }
6156         return (0);
6157 }
6158
6159
6160 int
6161 ctl_aps_sp_handler(struct ctl_scsiio *ctsio,
6162                    struct ctl_page_index *page_index, uint8_t *page_ptr)
6163 {
6164         struct copan_aps_subpage *user_sp;
6165         struct copan_aps_subpage *current_sp;
6166         union ctl_modepage_info *modepage_info;
6167         struct ctl_softc *softc;
6168         struct ctl_lun *lun;
6169         int retval;
6170
6171         retval = CTL_RETVAL_COMPLETE;
6172         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
6173                      (page_index->page_len * CTL_PAGE_CURRENT));
6174         softc = control_softc;
6175         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6176
6177         user_sp = (struct copan_aps_subpage *)page_ptr;
6178
6179         modepage_info = (union ctl_modepage_info *)
6180                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6181
6182         modepage_info->header.page_code = page_index->page_code & SMPH_PC_MASK;
6183         modepage_info->header.subpage = page_index->subpage;
6184         modepage_info->aps.lock_active = user_sp->lock_active;
6185
6186         mtx_lock(&softc->ctl_lock);
6187
6188         /*
6189          * If there is a request to lock the LUN and another LUN is locked
6190          * this is an error. If the requested LUN is already locked ignore
6191          * the request. If no LUN is locked attempt to lock it.
6192          * if there is a request to unlock the LUN and the LUN is currently
6193          * locked attempt to unlock it. Otherwise ignore the request. i.e.
6194          * if another LUN is locked or no LUN is locked.
6195          */
6196         if (user_sp->lock_active & APS_LOCK_ACTIVE) {
6197                 if (softc->aps_locked_lun == lun->lun) {
6198                         /*
6199                          * This LUN is already locked, so we're done.
6200                          */
6201                         retval = CTL_RETVAL_COMPLETE;
6202                 } else if (softc->aps_locked_lun == 0) {
6203                         /*
6204                          * No one has the lock, pass the request to the
6205                          * backend.
6206                          */
6207                         retval = lun->backend->config_write(
6208                                 (union ctl_io *)ctsio);
6209                 } else {
6210                         /*
6211                          * Someone else has the lock, throw out the request.
6212                          */
6213                         ctl_set_already_locked(ctsio);
6214                         free(ctsio->kern_data_ptr, M_CTL);
6215                         ctl_done((union ctl_io *)ctsio);
6216
6217                         /*
6218                          * Set the return value so that ctl_do_mode_select()
6219                          * won't try to complete the command.  We already
6220                          * completed it here.
6221                          */
6222                         retval = CTL_RETVAL_ERROR;
6223                 }
6224         } else if (softc->aps_locked_lun == lun->lun) {
6225                 /*
6226                  * This LUN is locked, so pass the unlock request to the
6227                  * backend.
6228                  */
6229                 retval = lun->backend->config_write((union ctl_io *)ctsio);
6230         }
6231         mtx_unlock(&softc->ctl_lock);
6232
6233         return (retval);
6234 }
6235
6236 int
6237 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6238                                 struct ctl_page_index *page_index,
6239                                 uint8_t *page_ptr)
6240 {
6241         uint8_t *c;
6242         int i;
6243
6244         c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6245         ctl_time_io_secs =
6246                 (c[0] << 8) |
6247                 (c[1] << 0) |
6248                 0;
6249         CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6250         printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6251         printf("page data:");
6252         for (i=0; i<8; i++)
6253                 printf(" %.2x",page_ptr[i]);
6254         printf("\n");
6255         return (0);
6256 }
6257
6258 int
6259 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6260                                struct ctl_page_index *page_index,
6261                                int pc)
6262 {
6263         struct copan_debugconf_subpage *page;
6264
6265         page = (struct copan_debugconf_subpage *)page_index->page_data +
6266                 (page_index->page_len * pc);
6267
6268         switch (pc) {
6269         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6270         case SMS_PAGE_CTRL_DEFAULT >> 6:
6271         case SMS_PAGE_CTRL_SAVED >> 6:
6272                 /*
6273                  * We don't update the changable or default bits for this page.
6274                  */
6275                 break;
6276         case SMS_PAGE_CTRL_CURRENT >> 6:
6277                 page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6278                 page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6279                 break;
6280         default:
6281 #ifdef NEEDTOPORT
6282                 EPRINT(0, "Invalid PC %d!!", pc);
6283 #endif /* NEEDTOPORT */
6284                 break;
6285         }
6286         return (0);
6287 }
6288
6289
6290 static int
6291 ctl_do_mode_select(union ctl_io *io)
6292 {
6293         struct scsi_mode_page_header *page_header;
6294         struct ctl_page_index *page_index;
6295         struct ctl_scsiio *ctsio;
6296         int control_dev, page_len;
6297         int page_len_offset, page_len_size;
6298         union ctl_modepage_info *modepage_info;
6299         struct ctl_lun *lun;
6300         int *len_left, *len_used;
6301         int retval, i;
6302
6303         ctsio = &io->scsiio;
6304         page_index = NULL;
6305         page_len = 0;
6306         retval = CTL_RETVAL_COMPLETE;
6307
6308         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6309
6310         if (lun->be_lun->lun_type != T_DIRECT)
6311                 control_dev = 1;
6312         else
6313                 control_dev = 0;
6314
6315         modepage_info = (union ctl_modepage_info *)
6316                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6317         len_left = &modepage_info->header.len_left;
6318         len_used = &modepage_info->header.len_used;
6319
6320 do_next_page:
6321
6322         page_header = (struct scsi_mode_page_header *)
6323                 (ctsio->kern_data_ptr + *len_used);
6324
6325         if (*len_left == 0) {
6326                 free(ctsio->kern_data_ptr, M_CTL);
6327                 ctl_set_success(ctsio);
6328                 ctl_done((union ctl_io *)ctsio);
6329                 return (CTL_RETVAL_COMPLETE);
6330         } else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6331
6332                 free(ctsio->kern_data_ptr, M_CTL);
6333                 ctl_set_param_len_error(ctsio);
6334                 ctl_done((union ctl_io *)ctsio);
6335                 return (CTL_RETVAL_COMPLETE);
6336
6337         } else if ((page_header->page_code & SMPH_SPF)
6338                 && (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6339
6340                 free(ctsio->kern_data_ptr, M_CTL);
6341                 ctl_set_param_len_error(ctsio);
6342                 ctl_done((union ctl_io *)ctsio);
6343                 return (CTL_RETVAL_COMPLETE);
6344         }
6345
6346
6347         /*
6348          * XXX KDM should we do something with the block descriptor?
6349          */
6350         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6351
6352                 if ((control_dev != 0)
6353                  && (lun->mode_pages.index[i].page_flags &
6354                      CTL_PAGE_FLAG_DISK_ONLY))
6355                         continue;
6356
6357                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6358                     (page_header->page_code & SMPH_PC_MASK))
6359                         continue;
6360
6361                 /*
6362                  * If neither page has a subpage code, then we've got a
6363                  * match.
6364                  */
6365                 if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6366                  && ((page_header->page_code & SMPH_SPF) == 0)) {
6367                         page_index = &lun->mode_pages.index[i];
6368                         page_len = page_header->page_length;
6369                         break;
6370                 }
6371
6372                 /*
6373                  * If both pages have subpages, then the subpage numbers
6374                  * have to match.
6375                  */
6376                 if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6377                   && (page_header->page_code & SMPH_SPF)) {
6378                         struct scsi_mode_page_header_sp *sph;
6379
6380                         sph = (struct scsi_mode_page_header_sp *)page_header;
6381
6382                         if (lun->mode_pages.index[i].subpage ==
6383                             sph->subpage) {
6384                                 page_index = &lun->mode_pages.index[i];
6385                                 page_len = scsi_2btoul(sph->page_length);
6386                                 break;
6387                         }
6388                 }
6389         }
6390
6391         /*
6392          * If we couldn't find the page, or if we don't have a mode select
6393          * handler for it, send back an error to the user.
6394          */
6395         if ((page_index == NULL)
6396          || (page_index->select_handler == NULL)) {
6397                 ctl_set_invalid_field(ctsio,
6398                                       /*sks_valid*/ 1,
6399                                       /*command*/ 0,
6400                                       /*field*/ *len_used,
6401                                       /*bit_valid*/ 0,
6402                                       /*bit*/ 0);
6403                 free(ctsio->kern_data_ptr, M_CTL);
6404                 ctl_done((union ctl_io *)ctsio);
6405                 return (CTL_RETVAL_COMPLETE);
6406         }
6407
6408         if (page_index->page_code & SMPH_SPF) {
6409                 page_len_offset = 2;
6410                 page_len_size = 2;
6411         } else {
6412                 page_len_size = 1;
6413                 page_len_offset = 1;
6414         }
6415
6416         /*
6417          * If the length the initiator gives us isn't the one we specify in
6418          * the mode page header, or if they didn't specify enough data in
6419          * the CDB to avoid truncating this page, kick out the request.
6420          */
6421         if ((page_len != (page_index->page_len - page_len_offset -
6422                           page_len_size))
6423          || (*len_left < page_index->page_len)) {
6424
6425
6426                 ctl_set_invalid_field(ctsio,
6427                                       /*sks_valid*/ 1,
6428                                       /*command*/ 0,
6429                                       /*field*/ *len_used + page_len_offset,
6430                                       /*bit_valid*/ 0,
6431                                       /*bit*/ 0);
6432                 free(ctsio->kern_data_ptr, M_CTL);
6433                 ctl_done((union ctl_io *)ctsio);
6434                 return (CTL_RETVAL_COMPLETE);
6435         }
6436
6437         /*
6438          * Run through the mode page, checking to make sure that the bits
6439          * the user changed are actually legal for him to change.
6440          */
6441         for (i = 0; i < page_index->page_len; i++) {
6442                 uint8_t *user_byte, *change_mask, *current_byte;
6443                 int bad_bit;
6444                 int j;
6445
6446                 user_byte = (uint8_t *)page_header + i;
6447                 change_mask = page_index->page_data +
6448                               (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6449                 current_byte = page_index->page_data +
6450                                (page_index->page_len * CTL_PAGE_CURRENT) + i;
6451
6452                 /*
6453                  * Check to see whether the user set any bits in this byte
6454                  * that he is not allowed to set.
6455                  */
6456                 if ((*user_byte & ~(*change_mask)) ==
6457                     (*current_byte & ~(*change_mask)))
6458                         continue;
6459
6460                 /*
6461                  * Go through bit by bit to determine which one is illegal.
6462                  */
6463                 bad_bit = 0;
6464                 for (j = 7; j >= 0; j--) {
6465                         if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6466                             (((1 << i) & ~(*change_mask)) & *current_byte)) {
6467                                 bad_bit = i;
6468                                 break;
6469                         }
6470                 }
6471                 ctl_set_invalid_field(ctsio,
6472                                       /*sks_valid*/ 1,
6473                                       /*command*/ 0,
6474                                       /*field*/ *len_used + i,
6475                                       /*bit_valid*/ 1,
6476                                       /*bit*/ bad_bit);
6477                 free(ctsio->kern_data_ptr, M_CTL);
6478                 ctl_done((union ctl_io *)ctsio);
6479                 return (CTL_RETVAL_COMPLETE);
6480         }
6481
6482         /*
6483          * Decrement these before we call the page handler, since we may
6484          * end up getting called back one way or another before the handler
6485          * returns to this context.
6486          */
6487         *len_left -= page_index->page_len;
6488         *len_used += page_index->page_len;
6489
6490         retval = page_index->select_handler(ctsio, page_index,
6491                                             (uint8_t *)page_header);
6492
6493         /*
6494          * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6495          * wait until this queued command completes to finish processing
6496          * the mode page.  If it returns anything other than
6497          * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6498          * already set the sense information, freed the data pointer, and
6499          * completed the io for us.
6500          */
6501         if (retval != CTL_RETVAL_COMPLETE)
6502                 goto bailout_no_done;
6503
6504         /*
6505          * If the initiator sent us more than one page, parse the next one.
6506          */
6507         if (*len_left > 0)
6508                 goto do_next_page;
6509
6510         ctl_set_success(ctsio);
6511         free(ctsio->kern_data_ptr, M_CTL);
6512         ctl_done((union ctl_io *)ctsio);
6513
6514 bailout_no_done:
6515
6516         return (CTL_RETVAL_COMPLETE);
6517
6518 }
6519
6520 int
6521 ctl_mode_select(struct ctl_scsiio *ctsio)
6522 {
6523         int param_len, pf, sp;
6524         int header_size, bd_len;
6525         int len_left, len_used;
6526         struct ctl_page_index *page_index;
6527         struct ctl_lun *lun;
6528         int control_dev, page_len;
6529         union ctl_modepage_info *modepage_info;
6530         int retval;
6531
6532         pf = 0;
6533         sp = 0;
6534         page_len = 0;
6535         len_used = 0;
6536         len_left = 0;
6537         retval = 0;
6538         bd_len = 0;
6539         page_index = NULL;
6540
6541         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6542
6543         if (lun->be_lun->lun_type != T_DIRECT)
6544                 control_dev = 1;
6545         else
6546                 control_dev = 0;
6547
6548         switch (ctsio->cdb[0]) {
6549         case MODE_SELECT_6: {
6550                 struct scsi_mode_select_6 *cdb;
6551
6552                 cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6553
6554                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6555                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6556
6557                 param_len = cdb->length;
6558                 header_size = sizeof(struct scsi_mode_header_6);
6559                 break;
6560         }
6561         case MODE_SELECT_10: {
6562                 struct scsi_mode_select_10 *cdb;
6563
6564                 cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6565
6566                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6567                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6568
6569                 param_len = scsi_2btoul(cdb->length);
6570                 header_size = sizeof(struct scsi_mode_header_10);
6571                 break;
6572         }
6573         default:
6574                 ctl_set_invalid_opcode(ctsio);
6575                 ctl_done((union ctl_io *)ctsio);
6576                 return (CTL_RETVAL_COMPLETE);
6577                 break; /* NOTREACHED */
6578         }
6579
6580         /*
6581          * From SPC-3:
6582          * "A parameter list length of zero indicates that the Data-Out Buffer
6583          * shall be empty. This condition shall not be considered as an error."
6584          */
6585         if (param_len == 0) {
6586                 ctl_set_success(ctsio);
6587                 ctl_done((union ctl_io *)ctsio);
6588                 return (CTL_RETVAL_COMPLETE);
6589         }
6590
6591         /*
6592          * Since we'll hit this the first time through, prior to
6593          * allocation, we don't need to free a data buffer here.
6594          */
6595         if (param_len < header_size) {
6596                 ctl_set_param_len_error(ctsio);
6597                 ctl_done((union ctl_io *)ctsio);
6598                 return (CTL_RETVAL_COMPLETE);
6599         }
6600
6601         /*
6602          * Allocate the data buffer and grab the user's data.  In theory,
6603          * we shouldn't have to sanity check the parameter list length here
6604          * because the maximum size is 64K.  We should be able to malloc
6605          * that much without too many problems.
6606          */
6607         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6608                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6609                 ctsio->kern_data_len = param_len;
6610                 ctsio->kern_total_len = param_len;
6611                 ctsio->kern_data_resid = 0;
6612                 ctsio->kern_rel_offset = 0;
6613                 ctsio->kern_sg_entries = 0;
6614                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6615                 ctsio->be_move_done = ctl_config_move_done;
6616                 ctl_datamove((union ctl_io *)ctsio);
6617
6618                 return (CTL_RETVAL_COMPLETE);
6619         }
6620
6621         switch (ctsio->cdb[0]) {
6622         case MODE_SELECT_6: {
6623                 struct scsi_mode_header_6 *mh6;
6624
6625                 mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6626                 bd_len = mh6->blk_desc_len;
6627                 break;
6628         }
6629         case MODE_SELECT_10: {
6630                 struct scsi_mode_header_10 *mh10;
6631
6632                 mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6633                 bd_len = scsi_2btoul(mh10->blk_desc_len);
6634                 break;
6635         }
6636         default:
6637                 panic("Invalid CDB type %#x", ctsio->cdb[0]);
6638                 break;
6639         }
6640
6641         if (param_len < (header_size + bd_len)) {
6642                 free(ctsio->kern_data_ptr, M_CTL);
6643                 ctl_set_param_len_error(ctsio);
6644                 ctl_done((union ctl_io *)ctsio);
6645                 return (CTL_RETVAL_COMPLETE);
6646         }
6647
6648         /*
6649          * Set the IO_CONT flag, so that if this I/O gets passed to
6650          * ctl_config_write_done(), it'll get passed back to
6651          * ctl_do_mode_select() for further processing, or completion if
6652          * we're all done.
6653          */
6654         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6655         ctsio->io_cont = ctl_do_mode_select;
6656
6657         modepage_info = (union ctl_modepage_info *)
6658                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6659
6660         memset(modepage_info, 0, sizeof(*modepage_info));
6661
6662         len_left = param_len - header_size - bd_len;
6663         len_used = header_size + bd_len;
6664
6665         modepage_info->header.len_left = len_left;
6666         modepage_info->header.len_used = len_used;
6667
6668         return (ctl_do_mode_select((union ctl_io *)ctsio));
6669 }
6670
6671 int
6672 ctl_mode_sense(struct ctl_scsiio *ctsio)
6673 {
6674         struct ctl_lun *lun;
6675         int pc, page_code, dbd, llba, subpage;
6676         int alloc_len, page_len, header_len, total_len;
6677         struct scsi_mode_block_descr *block_desc;
6678         struct ctl_page_index *page_index;
6679         int control_dev;
6680
6681         dbd = 0;
6682         llba = 0;
6683         block_desc = NULL;
6684         page_index = NULL;
6685
6686         CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6687
6688         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6689
6690         if (lun->be_lun->lun_type != T_DIRECT)
6691                 control_dev = 1;
6692         else
6693                 control_dev = 0;
6694
6695         switch (ctsio->cdb[0]) {
6696         case MODE_SENSE_6: {
6697                 struct scsi_mode_sense_6 *cdb;
6698
6699                 cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6700
6701                 header_len = sizeof(struct scsi_mode_hdr_6);
6702                 if (cdb->byte2 & SMS_DBD)
6703                         dbd = 1;
6704                 else
6705                         header_len += sizeof(struct scsi_mode_block_descr);
6706
6707                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6708                 page_code = cdb->page & SMS_PAGE_CODE;
6709                 subpage = cdb->subpage;
6710                 alloc_len = cdb->length;
6711                 break;
6712         }
6713         case MODE_SENSE_10: {
6714                 struct scsi_mode_sense_10 *cdb;
6715
6716                 cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6717
6718                 header_len = sizeof(struct scsi_mode_hdr_10);
6719
6720                 if (cdb->byte2 & SMS_DBD)
6721                         dbd = 1;
6722                 else
6723                         header_len += sizeof(struct scsi_mode_block_descr);
6724                 if (cdb->byte2 & SMS10_LLBAA)
6725                         llba = 1;
6726                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6727                 page_code = cdb->page & SMS_PAGE_CODE;
6728                 subpage = cdb->subpage;
6729                 alloc_len = scsi_2btoul(cdb->length);
6730                 break;
6731         }
6732         default:
6733                 ctl_set_invalid_opcode(ctsio);
6734                 ctl_done((union ctl_io *)ctsio);
6735                 return (CTL_RETVAL_COMPLETE);
6736                 break; /* NOTREACHED */
6737         }
6738
6739         /*
6740          * We have to make a first pass through to calculate the size of
6741          * the pages that match the user's query.  Then we allocate enough
6742          * memory to hold it, and actually copy the data into the buffer.
6743          */
6744         switch (page_code) {
6745         case SMS_ALL_PAGES_PAGE: {
6746                 int i;
6747
6748                 page_len = 0;
6749
6750                 /*
6751                  * At the moment, values other than 0 and 0xff here are
6752                  * reserved according to SPC-3.
6753                  */
6754                 if ((subpage != SMS_SUBPAGE_PAGE_0)
6755                  && (subpage != SMS_SUBPAGE_ALL)) {
6756                         ctl_set_invalid_field(ctsio,
6757                                               /*sks_valid*/ 1,
6758                                               /*command*/ 1,
6759                                               /*field*/ 3,
6760                                               /*bit_valid*/ 0,
6761                                               /*bit*/ 0);
6762                         ctl_done((union ctl_io *)ctsio);
6763                         return (CTL_RETVAL_COMPLETE);
6764                 }
6765
6766                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6767                         if ((control_dev != 0)
6768                          && (lun->mode_pages.index[i].page_flags &
6769                              CTL_PAGE_FLAG_DISK_ONLY))
6770                                 continue;
6771
6772                         /*
6773                          * We don't use this subpage if the user didn't
6774                          * request all subpages.
6775                          */
6776                         if ((lun->mode_pages.index[i].subpage != 0)
6777                          && (subpage == SMS_SUBPAGE_PAGE_0))
6778                                 continue;
6779
6780 #if 0
6781                         printf("found page %#x len %d\n",
6782                                lun->mode_pages.index[i].page_code &
6783                                SMPH_PC_MASK,
6784                                lun->mode_pages.index[i].page_len);
6785 #endif
6786                         page_len += lun->mode_pages.index[i].page_len;
6787                 }
6788                 break;
6789         }
6790         default: {
6791                 int i;
6792
6793                 page_len = 0;
6794
6795                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6796                         /* Look for the right page code */
6797                         if ((lun->mode_pages.index[i].page_code &
6798                              SMPH_PC_MASK) != page_code)
6799                                 continue;
6800
6801                         /* Look for the right subpage or the subpage wildcard*/
6802                         if ((lun->mode_pages.index[i].subpage != subpage)
6803                          && (subpage != SMS_SUBPAGE_ALL))
6804                                 continue;
6805
6806                         /* Make sure the page is supported for this dev type */
6807                         if ((control_dev != 0)
6808                          && (lun->mode_pages.index[i].page_flags &
6809                              CTL_PAGE_FLAG_DISK_ONLY))
6810                                 continue;
6811
6812 #if 0
6813                         printf("found page %#x len %d\n",
6814                                lun->mode_pages.index[i].page_code &
6815                                SMPH_PC_MASK,
6816                                lun->mode_pages.index[i].page_len);
6817 #endif
6818
6819                         page_len += lun->mode_pages.index[i].page_len;
6820                 }
6821
6822                 if (page_len == 0) {
6823                         ctl_set_invalid_field(ctsio,
6824                                               /*sks_valid*/ 1,
6825                                               /*command*/ 1,
6826                                               /*field*/ 2,
6827                                               /*bit_valid*/ 1,
6828                                               /*bit*/ 5);
6829                         ctl_done((union ctl_io *)ctsio);
6830                         return (CTL_RETVAL_COMPLETE);
6831                 }
6832                 break;
6833         }
6834         }
6835
6836         total_len = header_len + page_len;
6837 #if 0
6838         printf("header_len = %d, page_len = %d, total_len = %d\n",
6839                header_len, page_len, total_len);
6840 #endif
6841
6842         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6843         ctsio->kern_sg_entries = 0;
6844         ctsio->kern_data_resid = 0;
6845         ctsio->kern_rel_offset = 0;
6846         if (total_len < alloc_len) {
6847                 ctsio->residual = alloc_len - total_len;
6848                 ctsio->kern_data_len = total_len;
6849                 ctsio->kern_total_len = total_len;
6850         } else {
6851                 ctsio->residual = 0;
6852                 ctsio->kern_data_len = alloc_len;
6853                 ctsio->kern_total_len = alloc_len;
6854         }
6855
6856         switch (ctsio->cdb[0]) {
6857         case MODE_SENSE_6: {
6858                 struct scsi_mode_hdr_6 *header;
6859
6860                 header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6861
6862                 header->datalen = ctl_min(total_len - 1, 254);
6863
6864                 if (dbd)
6865                         header->block_descr_len = 0;
6866                 else
6867                         header->block_descr_len =
6868                                 sizeof(struct scsi_mode_block_descr);
6869                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6870                 break;
6871         }
6872         case MODE_SENSE_10: {
6873                 struct scsi_mode_hdr_10 *header;
6874                 int datalen;
6875
6876                 header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6877
6878                 datalen = ctl_min(total_len - 2, 65533);
6879                 scsi_ulto2b(datalen, header->datalen);
6880                 if (dbd)
6881                         scsi_ulto2b(0, header->block_descr_len);
6882                 else
6883                         scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
6884                                     header->block_descr_len);
6885                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6886                 break;
6887         }
6888         default:
6889                 panic("invalid CDB type %#x", ctsio->cdb[0]);
6890                 break; /* NOTREACHED */
6891         }
6892
6893         /*
6894          * If we've got a disk, use its blocksize in the block
6895          * descriptor.  Otherwise, just set it to 0.
6896          */
6897         if (dbd == 0) {
6898                 if (control_dev != 0)
6899                         scsi_ulto3b(lun->be_lun->blocksize,
6900                                     block_desc->block_len);
6901                 else
6902                         scsi_ulto3b(0, block_desc->block_len);
6903         }
6904
6905         switch (page_code) {
6906         case SMS_ALL_PAGES_PAGE: {
6907                 int i, data_used;
6908
6909                 data_used = header_len;
6910                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6911                         struct ctl_page_index *page_index;
6912
6913                         page_index = &lun->mode_pages.index[i];
6914
6915                         if ((control_dev != 0)
6916                          && (page_index->page_flags &
6917                             CTL_PAGE_FLAG_DISK_ONLY))
6918                                 continue;
6919
6920                         /*
6921                          * We don't use this subpage if the user didn't
6922                          * request all subpages.  We already checked (above)
6923                          * to make sure the user only specified a subpage
6924                          * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
6925                          */
6926                         if ((page_index->subpage != 0)
6927                          && (subpage == SMS_SUBPAGE_PAGE_0))
6928                                 continue;
6929
6930                         /*
6931                          * Call the handler, if it exists, to update the
6932                          * page to the latest values.
6933                          */
6934                         if (page_index->sense_handler != NULL)
6935                                 page_index->sense_handler(ctsio, page_index,pc);
6936
6937                         memcpy(ctsio->kern_data_ptr + data_used,
6938                                page_index->page_data +
6939                                (page_index->page_len * pc),
6940                                page_index->page_len);
6941                         data_used += page_index->page_len;
6942                 }
6943                 break;
6944         }
6945         default: {
6946                 int i, data_used;
6947
6948                 data_used = header_len;
6949
6950                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6951                         struct ctl_page_index *page_index;
6952
6953                         page_index = &lun->mode_pages.index[i];
6954
6955                         /* Look for the right page code */
6956                         if ((page_index->page_code & SMPH_PC_MASK) != page_code)
6957                                 continue;
6958
6959                         /* Look for the right subpage or the subpage wildcard*/
6960                         if ((page_index->subpage != subpage)
6961                          && (subpage != SMS_SUBPAGE_ALL))
6962                                 continue;
6963
6964                         /* Make sure the page is supported for this dev type */
6965                         if ((control_dev != 0)
6966                          && (page_index->page_flags &
6967                              CTL_PAGE_FLAG_DISK_ONLY))
6968                                 continue;
6969
6970                         /*
6971                          * Call the handler, if it exists, to update the
6972                          * page to the latest values.
6973                          */
6974                         if (page_index->sense_handler != NULL)
6975                                 page_index->sense_handler(ctsio, page_index,pc);
6976
6977                         memcpy(ctsio->kern_data_ptr + data_used,
6978                                page_index->page_data +
6979                                (page_index->page_len * pc),
6980                                page_index->page_len);
6981                         data_used += page_index->page_len;
6982                 }
6983                 break;
6984         }
6985         }
6986
6987         ctsio->scsi_status = SCSI_STATUS_OK;
6988
6989         ctsio->be_move_done = ctl_config_move_done;
6990         ctl_datamove((union ctl_io *)ctsio);
6991
6992         return (CTL_RETVAL_COMPLETE);
6993 }
6994
6995 int
6996 ctl_read_capacity(struct ctl_scsiio *ctsio)
6997 {
6998         struct scsi_read_capacity *cdb;
6999         struct scsi_read_capacity_data *data;
7000         struct ctl_lun *lun;
7001         uint32_t lba;
7002
7003         CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
7004
7005         cdb = (struct scsi_read_capacity *)ctsio->cdb;
7006
7007         lba = scsi_4btoul(cdb->addr);
7008         if (((cdb->pmi & SRC_PMI) == 0)
7009          && (lba != 0)) {
7010                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7011                                       /*sks_valid*/ 1,
7012                                       /*command*/ 1,
7013                                       /*field*/ 2,
7014                                       /*bit_valid*/ 0,
7015                                       /*bit*/ 0);
7016                 ctl_done((union ctl_io *)ctsio);
7017                 return (CTL_RETVAL_COMPLETE);
7018         }
7019
7020         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7021
7022         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7023         data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
7024         ctsio->residual = 0;
7025         ctsio->kern_data_len = sizeof(*data);
7026         ctsio->kern_total_len = sizeof(*data);
7027         ctsio->kern_data_resid = 0;
7028         ctsio->kern_rel_offset = 0;
7029         ctsio->kern_sg_entries = 0;
7030
7031         /*
7032          * If the maximum LBA is greater than 0xfffffffe, the user must
7033          * issue a SERVICE ACTION IN (16) command, with the read capacity
7034          * serivce action set.
7035          */
7036         if (lun->be_lun->maxlba > 0xfffffffe)
7037                 scsi_ulto4b(0xffffffff, data->addr);
7038         else
7039                 scsi_ulto4b(lun->be_lun->maxlba, data->addr);
7040
7041         /*
7042          * XXX KDM this may not be 512 bytes...
7043          */
7044         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7045
7046         ctsio->scsi_status = SCSI_STATUS_OK;
7047
7048         ctsio->be_move_done = ctl_config_move_done;
7049         ctl_datamove((union ctl_io *)ctsio);
7050
7051         return (CTL_RETVAL_COMPLETE);
7052 }
7053
7054 static int
7055 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
7056 {
7057         struct scsi_read_capacity_16 *cdb;
7058         struct scsi_read_capacity_data_long *data;
7059         struct ctl_lun *lun;
7060         uint64_t lba;
7061         uint32_t alloc_len;
7062
7063         CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
7064
7065         cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
7066
7067         alloc_len = scsi_4btoul(cdb->alloc_len);
7068         lba = scsi_8btou64(cdb->addr);
7069
7070         if ((cdb->reladr & SRC16_PMI)
7071          && (lba != 0)) {
7072                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7073                                       /*sks_valid*/ 1,
7074                                       /*command*/ 1,
7075                                       /*field*/ 2,
7076                                       /*bit_valid*/ 0,
7077                                       /*bit*/ 0);
7078                 ctl_done((union ctl_io *)ctsio);
7079                 return (CTL_RETVAL_COMPLETE);
7080         }
7081
7082         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7083
7084         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7085         data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
7086
7087         if (sizeof(*data) < alloc_len) {
7088                 ctsio->residual = alloc_len - sizeof(*data);
7089                 ctsio->kern_data_len = sizeof(*data);
7090                 ctsio->kern_total_len = sizeof(*data);
7091         } else {
7092                 ctsio->residual = 0;
7093                 ctsio->kern_data_len = alloc_len;
7094                 ctsio->kern_total_len = alloc_len;
7095         }
7096         ctsio->kern_data_resid = 0;
7097         ctsio->kern_rel_offset = 0;
7098         ctsio->kern_sg_entries = 0;
7099
7100         scsi_u64to8b(lun->be_lun->maxlba, data->addr);
7101         /* XXX KDM this may not be 512 bytes... */
7102         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7103         data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
7104         scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
7105         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
7106                 data->lalba_lbp[0] |= SRC16_LBPME;
7107
7108         ctsio->scsi_status = SCSI_STATUS_OK;
7109
7110         ctsio->be_move_done = ctl_config_move_done;
7111         ctl_datamove((union ctl_io *)ctsio);
7112
7113         return (CTL_RETVAL_COMPLETE);
7114 }
7115
7116 int
7117 ctl_service_action_in(struct ctl_scsiio *ctsio)
7118 {
7119         struct scsi_service_action_in *cdb;
7120         int retval;
7121
7122         CTL_DEBUG_PRINT(("ctl_service_action_in\n"));
7123
7124         cdb = (struct scsi_service_action_in *)ctsio->cdb;
7125
7126         retval = CTL_RETVAL_COMPLETE;
7127
7128         switch (cdb->service_action) {
7129         case SRC16_SERVICE_ACTION:
7130                 retval = ctl_read_capacity_16(ctsio);
7131                 break;
7132         default:
7133                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7134                                       /*sks_valid*/ 1,
7135                                       /*command*/ 1,
7136                                       /*field*/ 1,
7137                                       /*bit_valid*/ 1,
7138                                       /*bit*/ 4);
7139                 ctl_done((union ctl_io *)ctsio);
7140                 break;
7141         }
7142
7143         return (retval);
7144 }
7145
7146 int
7147 ctl_maintenance_in(struct ctl_scsiio *ctsio)
7148 {
7149         struct scsi_maintenance_in *cdb;
7150         int retval;
7151         int alloc_len, total_len = 0;
7152         int num_target_port_groups, single;
7153         struct ctl_lun *lun;
7154         struct ctl_softc *softc;
7155         struct scsi_target_group_data *rtg_ptr;
7156         struct scsi_target_port_group_descriptor *tpg_desc_ptr1, *tpg_desc_ptr2;
7157         struct scsi_target_port_descriptor  *tp_desc_ptr1_1, *tp_desc_ptr1_2,
7158                                             *tp_desc_ptr2_1, *tp_desc_ptr2_2;
7159
7160         CTL_DEBUG_PRINT(("ctl_maintenance_in\n"));
7161
7162         cdb = (struct scsi_maintenance_in *)ctsio->cdb;
7163         softc = control_softc;
7164         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7165
7166         retval = CTL_RETVAL_COMPLETE;
7167
7168         if ((cdb->byte2 & SERVICE_ACTION_MASK) != SA_RPRT_TRGT_GRP) {
7169                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7170                                       /*sks_valid*/ 1,
7171                                       /*command*/ 1,
7172                                       /*field*/ 1,
7173                                       /*bit_valid*/ 1,
7174                                       /*bit*/ 4);
7175                 ctl_done((union ctl_io *)ctsio);
7176                 return(retval);
7177         }
7178
7179         mtx_lock(&softc->ctl_lock);
7180         single = ctl_is_single;
7181         mtx_unlock(&softc->ctl_lock);
7182
7183         if (single)
7184                 num_target_port_groups = NUM_TARGET_PORT_GROUPS - 1;
7185         else
7186                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
7187
7188         total_len = sizeof(struct scsi_target_group_data) +
7189                 sizeof(struct scsi_target_port_group_descriptor) *
7190                 num_target_port_groups +
7191                 sizeof(struct scsi_target_port_descriptor) *
7192                 NUM_PORTS_PER_GRP * num_target_port_groups;
7193
7194         alloc_len = scsi_4btoul(cdb->length);
7195
7196         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7197
7198         ctsio->kern_sg_entries = 0;
7199
7200         if (total_len < alloc_len) {
7201                 ctsio->residual = alloc_len - total_len;
7202                 ctsio->kern_data_len = total_len;
7203                 ctsio->kern_total_len = total_len;
7204         } else {
7205                 ctsio->residual = 0;
7206                 ctsio->kern_data_len = alloc_len;
7207                 ctsio->kern_total_len = alloc_len;
7208         }
7209         ctsio->kern_data_resid = 0;
7210         ctsio->kern_rel_offset = 0;
7211
7212         rtg_ptr = (struct scsi_target_group_data *)ctsio->kern_data_ptr;
7213
7214         tpg_desc_ptr1 = &rtg_ptr->groups[0];
7215         tp_desc_ptr1_1 = &tpg_desc_ptr1->descriptors[0];
7216         tp_desc_ptr1_2 = (struct scsi_target_port_descriptor *)
7217                 &tp_desc_ptr1_1->desc_list[0];
7218
7219         if (single == 0) {
7220                 tpg_desc_ptr2 = (struct scsi_target_port_group_descriptor *)
7221                         &tp_desc_ptr1_2->desc_list[0];
7222                 tp_desc_ptr2_1 = &tpg_desc_ptr2->descriptors[0];
7223                 tp_desc_ptr2_2 = (struct scsi_target_port_descriptor *)
7224                         &tp_desc_ptr2_1->desc_list[0];
7225         } else {
7226                 tpg_desc_ptr2 = NULL;
7227                 tp_desc_ptr2_1 = NULL;
7228                 tp_desc_ptr2_2 = NULL;
7229         }
7230
7231         scsi_ulto4b(total_len - 4, rtg_ptr->length);
7232         if (single == 0) {
7233                 if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7234                         if (lun->flags & CTL_LUN_PRIMARY_SC) {
7235                                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7236                                 tpg_desc_ptr2->pref_state =
7237                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7238                         } else {
7239                                 tpg_desc_ptr1->pref_state =
7240                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7241                                 tpg_desc_ptr2->pref_state = TPG_PRIMARY;
7242                         }
7243                 } else {
7244                         if (lun->flags & CTL_LUN_PRIMARY_SC) {
7245                                 tpg_desc_ptr1->pref_state =
7246                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7247                                 tpg_desc_ptr2->pref_state = TPG_PRIMARY;
7248                         } else {
7249                                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7250                                 tpg_desc_ptr2->pref_state =
7251                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7252                         }
7253                 }
7254         } else {
7255                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7256         }
7257         tpg_desc_ptr1->support = 0;
7258         tpg_desc_ptr1->target_port_group[1] = 1;
7259         tpg_desc_ptr1->status = TPG_IMPLICIT;
7260         tpg_desc_ptr1->target_port_count= NUM_PORTS_PER_GRP;
7261
7262         if (single == 0) {
7263                 tpg_desc_ptr2->support = 0;
7264                 tpg_desc_ptr2->target_port_group[1] = 2;
7265                 tpg_desc_ptr2->status = TPG_IMPLICIT;
7266                 tpg_desc_ptr2->target_port_count = NUM_PORTS_PER_GRP;
7267
7268                 tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7269                 tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7270
7271                 tp_desc_ptr2_1->relative_target_port_identifier[1] = 9;
7272                 tp_desc_ptr2_2->relative_target_port_identifier[1] = 10;
7273         } else {
7274                 if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7275                         tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7276                         tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7277                 } else {
7278                         tp_desc_ptr1_1->relative_target_port_identifier[1] = 9;
7279                         tp_desc_ptr1_2->relative_target_port_identifier[1] = 10;
7280                 }
7281         }
7282
7283         ctsio->be_move_done = ctl_config_move_done;
7284
7285         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7286                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7287                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7288                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7289                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7290
7291         ctl_datamove((union ctl_io *)ctsio);
7292         return(retval);
7293 }
7294
7295 int
7296 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7297 {
7298         struct scsi_per_res_in *cdb;
7299         int alloc_len, total_len = 0;
7300         /* struct scsi_per_res_in_rsrv in_data; */
7301         struct ctl_lun *lun;
7302         struct ctl_softc *softc;
7303
7304         CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7305
7306         softc = control_softc;
7307
7308         cdb = (struct scsi_per_res_in *)ctsio->cdb;
7309
7310         alloc_len = scsi_2btoul(cdb->length);
7311
7312         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7313
7314 retry:
7315         mtx_lock(&softc->ctl_lock);
7316         switch (cdb->action) {
7317         case SPRI_RK: /* read keys */
7318                 total_len = sizeof(struct scsi_per_res_in_keys) +
7319                         lun->pr_key_count *
7320                         sizeof(struct scsi_per_res_key);
7321                 break;
7322         case SPRI_RR: /* read reservation */
7323                 if (lun->flags & CTL_LUN_PR_RESERVED)
7324                         total_len = sizeof(struct scsi_per_res_in_rsrv);
7325                 else
7326                         total_len = sizeof(struct scsi_per_res_in_header);
7327                 break;
7328         case SPRI_RC: /* report capabilities */
7329                 total_len = sizeof(struct scsi_per_res_cap);
7330                 break;
7331         case SPRI_RS: /* read full status */
7332         default:
7333                 mtx_unlock(&softc->ctl_lock);
7334                 ctl_set_invalid_field(ctsio,
7335                                       /*sks_valid*/ 1,
7336                                       /*command*/ 1,
7337                                       /*field*/ 1,
7338                                       /*bit_valid*/ 1,
7339                                       /*bit*/ 0);
7340                 ctl_done((union ctl_io *)ctsio);
7341                 return (CTL_RETVAL_COMPLETE);
7342                 break; /* NOTREACHED */
7343         }
7344         mtx_unlock(&softc->ctl_lock);
7345
7346         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7347
7348         if (total_len < alloc_len) {
7349                 ctsio->residual = alloc_len - total_len;
7350                 ctsio->kern_data_len = total_len;
7351                 ctsio->kern_total_len = total_len;
7352         } else {
7353                 ctsio->residual = 0;
7354                 ctsio->kern_data_len = alloc_len;
7355                 ctsio->kern_total_len = alloc_len;
7356         }
7357
7358         ctsio->kern_data_resid = 0;
7359         ctsio->kern_rel_offset = 0;
7360         ctsio->kern_sg_entries = 0;
7361
7362         mtx_lock(&softc->ctl_lock);
7363         switch (cdb->action) {
7364         case SPRI_RK: { // read keys
7365         struct scsi_per_res_in_keys *res_keys;
7366                 int i, key_count;
7367
7368                 res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7369
7370                 /*
7371                  * We had to drop the lock to allocate our buffer, which
7372                  * leaves time for someone to come in with another
7373                  * persistent reservation.  (That is unlikely, though,
7374                  * since this should be the only persistent reservation
7375                  * command active right now.)
7376                  */
7377                 if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7378                     (lun->pr_key_count *
7379                      sizeof(struct scsi_per_res_key)))){
7380                         mtx_unlock(&softc->ctl_lock);
7381                         free(ctsio->kern_data_ptr, M_CTL);
7382                         printf("%s: reservation length changed, retrying\n",
7383                                __func__);
7384                         goto retry;
7385                 }
7386
7387                 scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7388
7389                 scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7390                              lun->pr_key_count, res_keys->header.length);
7391
7392                 for (i = 0, key_count = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7393                         if (!lun->per_res[i].registered)
7394                                 continue;
7395
7396                         /*
7397                          * We used lun->pr_key_count to calculate the
7398                          * size to allocate.  If it turns out the number of
7399                          * initiators with the registered flag set is
7400                          * larger than that (i.e. they haven't been kept in
7401                          * sync), we've got a problem.
7402                          */
7403                         if (key_count >= lun->pr_key_count) {
7404 #ifdef NEEDTOPORT
7405                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
7406                                             CTL_PR_ERROR,
7407                                             csevent_LogType_Fault,
7408                                             csevent_AlertLevel_Yellow,
7409                                             csevent_FRU_ShelfController,
7410                                             csevent_FRU_Firmware,
7411                                         csevent_FRU_Unknown,
7412                                             "registered keys %d >= key "
7413                                             "count %d", key_count,
7414                                             lun->pr_key_count);
7415 #endif
7416                                 key_count++;
7417                                 continue;
7418                         }
7419                         memcpy(res_keys->keys[key_count].key,
7420                                lun->per_res[i].res_key.key,
7421                                ctl_min(sizeof(res_keys->keys[key_count].key),
7422                                sizeof(lun->per_res[i].res_key)));
7423                         key_count++;
7424                 }
7425                 break;
7426         }
7427         case SPRI_RR: { // read reservation
7428                 struct scsi_per_res_in_rsrv *res;
7429                 int tmp_len, header_only;
7430
7431                 res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7432
7433                 scsi_ulto4b(lun->PRGeneration, res->header.generation);
7434
7435                 if (lun->flags & CTL_LUN_PR_RESERVED)
7436                 {
7437                         tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7438                         scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7439                                     res->header.length);
7440                         header_only = 0;
7441                 } else {
7442                         tmp_len = sizeof(struct scsi_per_res_in_header);
7443                         scsi_ulto4b(0, res->header.length);
7444                         header_only = 1;
7445                 }
7446
7447                 /*
7448                  * We had to drop the lock to allocate our buffer, which
7449                  * leaves time for someone to come in with another
7450                  * persistent reservation.  (That is unlikely, though,
7451                  * since this should be the only persistent reservation
7452                  * command active right now.)
7453                  */
7454                 if (tmp_len != total_len) {
7455                         mtx_unlock(&softc->ctl_lock);
7456                         free(ctsio->kern_data_ptr, M_CTL);
7457                         printf("%s: reservation status changed, retrying\n",
7458                                __func__);
7459                         goto retry;
7460                 }
7461
7462                 /*
7463                  * No reservation held, so we're done.
7464                  */
7465                 if (header_only != 0)
7466                         break;
7467
7468                 /*
7469                  * If the registration is an All Registrants type, the key
7470                  * is 0, since it doesn't really matter.
7471                  */
7472                 if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7473                         memcpy(res->data.reservation,
7474                                &lun->per_res[lun->pr_res_idx].res_key,
7475                                sizeof(struct scsi_per_res_key));
7476                 }
7477                 res->data.scopetype = lun->res_type;
7478                 break;
7479         }
7480         case SPRI_RC:     //report capabilities
7481         {
7482                 struct scsi_per_res_cap *res_cap;
7483                 uint16_t type_mask;
7484
7485                 res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7486                 scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7487                 res_cap->flags2 |= SPRI_TMV;
7488                 type_mask = SPRI_TM_WR_EX_AR |
7489                             SPRI_TM_EX_AC_RO |
7490                             SPRI_TM_WR_EX_RO |
7491                             SPRI_TM_EX_AC |
7492                             SPRI_TM_WR_EX |
7493                             SPRI_TM_EX_AC_AR;
7494                 scsi_ulto2b(type_mask, res_cap->type_mask);
7495                 break;
7496         }
7497         case SPRI_RS: //read full status
7498         default:
7499                 /*
7500                  * This is a bug, because we just checked for this above,
7501                  * and should have returned an error.
7502                  */
7503                 panic("Invalid PR type %x", cdb->action);
7504                 break; /* NOTREACHED */
7505         }
7506         mtx_unlock(&softc->ctl_lock);
7507
7508         ctsio->be_move_done = ctl_config_move_done;
7509
7510         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7511                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7512                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7513                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7514                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7515
7516         ctl_datamove((union ctl_io *)ctsio);
7517
7518         return (CTL_RETVAL_COMPLETE);
7519 }
7520
7521 /*
7522  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
7523  * it should return.
7524  */
7525 static int
7526 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
7527                 uint64_t sa_res_key, uint8_t type, uint32_t residx,
7528                 struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
7529                 struct scsi_per_res_out_parms* param)
7530 {
7531         union ctl_ha_msg persis_io;
7532         int retval, i;
7533         int isc_retval;
7534
7535         retval = 0;
7536
7537         if (sa_res_key == 0) {
7538                 mtx_lock(&softc->ctl_lock);
7539                 if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7540                         /* validate scope and type */
7541                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7542                              SPR_LU_SCOPE) {
7543                                 mtx_unlock(&softc->ctl_lock);
7544                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7545                                                       /*sks_valid*/ 1,
7546                                                       /*command*/ 1,
7547                                                       /*field*/ 2,
7548                                                       /*bit_valid*/ 1,
7549                                                       /*bit*/ 4);
7550                                 ctl_done((union ctl_io *)ctsio);
7551                                 return (1);
7552                         }
7553
7554                         if (type>8 || type==2 || type==4 || type==0) {
7555                                 mtx_unlock(&softc->ctl_lock);
7556                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7557                                                       /*sks_valid*/ 1,
7558                                                       /*command*/ 1,
7559                                                       /*field*/ 2,
7560                                                       /*bit_valid*/ 1,
7561                                                       /*bit*/ 0);
7562                                 ctl_done((union ctl_io *)ctsio);
7563                                 return (1);
7564                         }
7565
7566                         /* temporarily unregister this nexus */
7567                         lun->per_res[residx].registered = 0;
7568
7569                         /*
7570                          * Unregister everybody else and build UA for
7571                          * them
7572                          */
7573                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7574                                 if (lun->per_res[i].registered == 0)
7575                                         continue;
7576
7577                                 if (!persis_offset
7578                                  && i <CTL_MAX_INITIATORS)
7579                                         lun->pending_sense[i].ua_pending |=
7580                                                 CTL_UA_REG_PREEMPT;
7581                                 else if (persis_offset
7582                                       && i >= persis_offset)
7583                                         lun->pending_sense[i-persis_offset
7584                                                 ].ua_pending |=
7585                                                 CTL_UA_REG_PREEMPT;
7586                                 lun->per_res[i].registered = 0;
7587                                 memset(&lun->per_res[i].res_key, 0,
7588                                        sizeof(struct scsi_per_res_key));
7589                         }
7590                         lun->per_res[residx].registered = 1;
7591                         lun->pr_key_count = 1;
7592                         lun->res_type = type;
7593                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7594                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7595                                 lun->pr_res_idx = residx;
7596
7597                         mtx_unlock(&softc->ctl_lock);
7598                         /* send msg to other side */
7599                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7600                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7601                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7602                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7603                         persis_io.pr.pr_info.res_type = type;
7604                         memcpy(persis_io.pr.pr_info.sa_res_key,
7605                                param->serv_act_res_key,
7606                                sizeof(param->serv_act_res_key));
7607                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7608                              &persis_io, sizeof(persis_io), 0)) >
7609                              CTL_HA_STATUS_SUCCESS) {
7610                                 printf("CTL:Persis Out error returned "
7611                                        "from ctl_ha_msg_send %d\n",
7612                                        isc_retval);
7613                         }
7614                 } else {
7615                         /* not all registrants */
7616                         mtx_unlock(&softc->ctl_lock);
7617                         free(ctsio->kern_data_ptr, M_CTL);
7618                         ctl_set_invalid_field(ctsio,
7619                                               /*sks_valid*/ 1,
7620                                               /*command*/ 0,
7621                                               /*field*/ 8,
7622                                               /*bit_valid*/ 0,
7623                                               /*bit*/ 0);
7624                         ctl_done((union ctl_io *)ctsio);
7625                         return (1);
7626                 }
7627         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7628                 || !(lun->flags & CTL_LUN_PR_RESERVED)) {
7629                 int found = 0;
7630
7631                 mtx_lock(&softc->ctl_lock);
7632                 if (res_key == sa_res_key) {
7633                         /* special case */
7634                         /*
7635                          * The spec implies this is not good but doesn't
7636                          * say what to do. There are two choices either
7637                          * generate a res conflict or check condition
7638                          * with illegal field in parameter data. Since
7639                          * that is what is done when the sa_res_key is
7640                          * zero I'll take that approach since this has
7641                          * to do with the sa_res_key.
7642                          */
7643                         mtx_unlock(&softc->ctl_lock);
7644                         free(ctsio->kern_data_ptr, M_CTL);
7645                         ctl_set_invalid_field(ctsio,
7646                                               /*sks_valid*/ 1,
7647                                               /*command*/ 0,
7648                                               /*field*/ 8,
7649                                               /*bit_valid*/ 0,
7650                                               /*bit*/ 0);
7651                         ctl_done((union ctl_io *)ctsio);
7652                         return (1);
7653                 }
7654
7655                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7656                         if (lun->per_res[i].registered
7657                          && memcmp(param->serv_act_res_key,
7658                             lun->per_res[i].res_key.key,
7659                             sizeof(struct scsi_per_res_key)) != 0)
7660                                 continue;
7661
7662                         found = 1;
7663                         lun->per_res[i].registered = 0;
7664                         memset(&lun->per_res[i].res_key, 0,
7665                                sizeof(struct scsi_per_res_key));
7666                         lun->pr_key_count--;
7667
7668                         if (!persis_offset
7669                          && i < CTL_MAX_INITIATORS)
7670                                 lun->pending_sense[i].ua_pending |=
7671                                         CTL_UA_REG_PREEMPT;
7672                         else if (persis_offset
7673                               && i >= persis_offset)
7674                                 lun->pending_sense[i-persis_offset].ua_pending|=
7675                                         CTL_UA_REG_PREEMPT;
7676                 }
7677                 mtx_unlock(&softc->ctl_lock);
7678                 if (!found) {
7679                         free(ctsio->kern_data_ptr, M_CTL);
7680                         ctl_set_reservation_conflict(ctsio);
7681                         ctl_done((union ctl_io *)ctsio);
7682                         return (CTL_RETVAL_COMPLETE);
7683                 }
7684                 /* send msg to other side */
7685                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7686                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7687                 persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7688                 persis_io.pr.pr_info.residx = lun->pr_res_idx;
7689                 persis_io.pr.pr_info.res_type = type;
7690                 memcpy(persis_io.pr.pr_info.sa_res_key,
7691                        param->serv_act_res_key,
7692                        sizeof(param->serv_act_res_key));
7693                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7694                      &persis_io, sizeof(persis_io), 0)) >
7695                      CTL_HA_STATUS_SUCCESS) {
7696                         printf("CTL:Persis Out error returned from "
7697                                "ctl_ha_msg_send %d\n", isc_retval);
7698                 }
7699         } else {
7700                 /* Reserved but not all registrants */
7701                 /* sa_res_key is res holder */
7702                 if (memcmp(param->serv_act_res_key,
7703                    lun->per_res[lun->pr_res_idx].res_key.key,
7704                    sizeof(struct scsi_per_res_key)) == 0) {
7705                         /* validate scope and type */
7706                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7707                              SPR_LU_SCOPE) {
7708                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7709                                                       /*sks_valid*/ 1,
7710                                                       /*command*/ 1,
7711                                                       /*field*/ 2,
7712                                                       /*bit_valid*/ 1,
7713                                                       /*bit*/ 4);
7714                                 ctl_done((union ctl_io *)ctsio);
7715                                 return (1);
7716                         }
7717
7718                         if (type>8 || type==2 || type==4 || type==0) {
7719                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7720                                                       /*sks_valid*/ 1,
7721                                                       /*command*/ 1,
7722                                                       /*field*/ 2,
7723                                                       /*bit_valid*/ 1,
7724                                                       /*bit*/ 0);
7725                                 ctl_done((union ctl_io *)ctsio);
7726                                 return (1);
7727                         }
7728
7729                         /*
7730                          * Do the following:
7731                          * if sa_res_key != res_key remove all
7732                          * registrants w/sa_res_key and generate UA
7733                          * for these registrants(Registrations
7734                          * Preempted) if it wasn't an exclusive
7735                          * reservation generate UA(Reservations
7736                          * Preempted) for all other registered nexuses
7737                          * if the type has changed. Establish the new
7738                          * reservation and holder. If res_key and
7739                          * sa_res_key are the same do the above
7740                          * except don't unregister the res holder.
7741                          */
7742
7743                         /*
7744                          * Temporarily unregister so it won't get
7745                          * removed or UA generated
7746                          */
7747                         lun->per_res[residx].registered = 0;
7748                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7749                                 if (lun->per_res[i].registered == 0)
7750                                         continue;
7751
7752                                 if (memcmp(param->serv_act_res_key,
7753                                     lun->per_res[i].res_key.key,
7754                                     sizeof(struct scsi_per_res_key)) == 0) {
7755                                         lun->per_res[i].registered = 0;
7756                                         memset(&lun->per_res[i].res_key,
7757                                                0,
7758                                                sizeof(struct scsi_per_res_key));
7759                                         lun->pr_key_count--;
7760
7761                                         if (!persis_offset
7762                                          && i < CTL_MAX_INITIATORS)
7763                                                 lun->pending_sense[i
7764                                                         ].ua_pending |=
7765                                                         CTL_UA_REG_PREEMPT;
7766                                         else if (persis_offset
7767                                               && i >= persis_offset)
7768                                                 lun->pending_sense[
7769                                                   i-persis_offset].ua_pending |=
7770                                                   CTL_UA_REG_PREEMPT;
7771                                 } else if (type != lun->res_type
7772                                         && (lun->res_type == SPR_TYPE_WR_EX_RO
7773                                          || lun->res_type ==SPR_TYPE_EX_AC_RO)){
7774                                                 if (!persis_offset
7775                                                  && i < CTL_MAX_INITIATORS)
7776                                                         lun->pending_sense[i
7777                                                         ].ua_pending |=
7778                                                         CTL_UA_RES_RELEASE;
7779                                                 else if (persis_offset
7780                                                       && i >= persis_offset)
7781                                                         lun->pending_sense[
7782                                                         i-persis_offset
7783                                                         ].ua_pending |=
7784                                                         CTL_UA_RES_RELEASE;
7785                                 }
7786                         }
7787                         lun->per_res[residx].registered = 1;
7788                         lun->res_type = type;
7789                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7790                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7791                                 lun->pr_res_idx = residx;
7792                         else
7793                                 lun->pr_res_idx =
7794                                         CTL_PR_ALL_REGISTRANTS;
7795
7796                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7797                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7798                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7799                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7800                         persis_io.pr.pr_info.res_type = type;
7801                         memcpy(persis_io.pr.pr_info.sa_res_key,
7802                                param->serv_act_res_key,
7803                                sizeof(param->serv_act_res_key));
7804                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7805                              &persis_io, sizeof(persis_io), 0)) >
7806                              CTL_HA_STATUS_SUCCESS) {
7807                                 printf("CTL:Persis Out error returned "
7808                                        "from ctl_ha_msg_send %d\n",
7809                                        isc_retval);
7810                         }
7811                 } else {
7812                         /*
7813                          * sa_res_key is not the res holder just
7814                          * remove registrants
7815                          */
7816                         int found=0;
7817                         mtx_lock(&softc->ctl_lock);
7818
7819                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7820                                 if (memcmp(param->serv_act_res_key,
7821                                     lun->per_res[i].res_key.key,
7822                                     sizeof(struct scsi_per_res_key)) != 0)
7823                                         continue;
7824
7825                                 found = 1;
7826                                 lun->per_res[i].registered = 0;
7827                                 memset(&lun->per_res[i].res_key, 0,
7828                                        sizeof(struct scsi_per_res_key));
7829                                 lun->pr_key_count--;
7830
7831                                 if (!persis_offset
7832                                  && i < CTL_MAX_INITIATORS)
7833                                         lun->pending_sense[i].ua_pending |=
7834                                                 CTL_UA_REG_PREEMPT;
7835                                 else if (persis_offset
7836                                       && i >= persis_offset)
7837                                         lun->pending_sense[
7838                                                 i-persis_offset].ua_pending |=
7839                                                 CTL_UA_REG_PREEMPT;
7840                         }
7841
7842                         if (!found) {
7843                                 mtx_unlock(&softc->ctl_lock);
7844                                 free(ctsio->kern_data_ptr, M_CTL);
7845                                 ctl_set_reservation_conflict(ctsio);
7846                                 ctl_done((union ctl_io *)ctsio);
7847                                 return (1);
7848                         }
7849                         mtx_unlock(&softc->ctl_lock);
7850                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7851                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7852                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7853                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7854                         persis_io.pr.pr_info.res_type = type;
7855                         memcpy(persis_io.pr.pr_info.sa_res_key,
7856                                param->serv_act_res_key,
7857                                sizeof(param->serv_act_res_key));
7858                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7859                              &persis_io, sizeof(persis_io), 0)) >
7860                              CTL_HA_STATUS_SUCCESS) {
7861                                 printf("CTL:Persis Out error returned "
7862                                        "from ctl_ha_msg_send %d\n",
7863                                 isc_retval);
7864                         }
7865                 }
7866         }
7867
7868         lun->PRGeneration++;
7869
7870         return (retval);
7871 }
7872
7873 static void
7874 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
7875 {
7876         int i;
7877
7878         if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7879          || lun->pr_res_idx == CTL_PR_NO_RESERVATION
7880          || memcmp(&lun->per_res[lun->pr_res_idx].res_key,
7881                    msg->pr.pr_info.sa_res_key,
7882                    sizeof(struct scsi_per_res_key)) != 0) {
7883                 uint64_t sa_res_key;
7884                 sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
7885
7886                 if (sa_res_key == 0) {
7887                         /* temporarily unregister this nexus */
7888                         lun->per_res[msg->pr.pr_info.residx].registered = 0;
7889
7890                         /*
7891                          * Unregister everybody else and build UA for
7892                          * them
7893                          */
7894                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7895                                 if (lun->per_res[i].registered == 0)
7896                                         continue;
7897
7898                                 if (!persis_offset
7899                                  && i < CTL_MAX_INITIATORS)
7900                                         lun->pending_sense[i].ua_pending |=
7901                                                 CTL_UA_REG_PREEMPT;
7902                                 else if (persis_offset && i >= persis_offset)
7903                                         lun->pending_sense[i -
7904                                                 persis_offset].ua_pending |=
7905                                                 CTL_UA_REG_PREEMPT;
7906                                 lun->per_res[i].registered = 0;
7907                                 memset(&lun->per_res[i].res_key, 0,
7908                                        sizeof(struct scsi_per_res_key));
7909                         }
7910
7911                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
7912                         lun->pr_key_count = 1;
7913                         lun->res_type = msg->pr.pr_info.res_type;
7914                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7915                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7916                                 lun->pr_res_idx = msg->pr.pr_info.residx;
7917                 } else {
7918                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7919                                 if (memcmp(msg->pr.pr_info.sa_res_key,
7920                                    lun->per_res[i].res_key.key,
7921                                    sizeof(struct scsi_per_res_key)) != 0)
7922                                         continue;
7923
7924                                 lun->per_res[i].registered = 0;
7925                                 memset(&lun->per_res[i].res_key, 0,
7926                                        sizeof(struct scsi_per_res_key));
7927                                 lun->pr_key_count--;
7928
7929                                 if (!persis_offset
7930                                  && i < persis_offset)
7931                                         lun->pending_sense[i].ua_pending |=
7932                                                 CTL_UA_REG_PREEMPT;
7933                                 else if (persis_offset
7934                                       && i >= persis_offset)
7935                                         lun->pending_sense[i -
7936                                                 persis_offset].ua_pending |=
7937                                                 CTL_UA_REG_PREEMPT;
7938                         }
7939                 }
7940         } else {
7941                 /*
7942                  * Temporarily unregister so it won't get removed
7943                  * or UA generated
7944                  */
7945                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
7946                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7947                         if (lun->per_res[i].registered == 0)
7948                                 continue;
7949
7950                         if (memcmp(msg->pr.pr_info.sa_res_key,
7951                            lun->per_res[i].res_key.key,
7952                            sizeof(struct scsi_per_res_key)) == 0) {
7953                                 lun->per_res[i].registered = 0;
7954                                 memset(&lun->per_res[i].res_key, 0,
7955                                        sizeof(struct scsi_per_res_key));
7956                                 lun->pr_key_count--;
7957                                 if (!persis_offset
7958                                  && i < CTL_MAX_INITIATORS)
7959                                         lun->pending_sense[i].ua_pending |=
7960                                                 CTL_UA_REG_PREEMPT;
7961                                 else if (persis_offset
7962                                       && i >= persis_offset)
7963                                         lun->pending_sense[i -
7964                                                 persis_offset].ua_pending |=
7965                                                 CTL_UA_REG_PREEMPT;
7966                         } else if (msg->pr.pr_info.res_type != lun->res_type
7967                                 && (lun->res_type == SPR_TYPE_WR_EX_RO
7968                                  || lun->res_type == SPR_TYPE_EX_AC_RO)) {
7969                                         if (!persis_offset
7970                                          && i < persis_offset)
7971                                                 lun->pending_sense[i
7972                                                         ].ua_pending |=
7973                                                         CTL_UA_RES_RELEASE;
7974                                         else if (persis_offset
7975                                               && i >= persis_offset)
7976                                         lun->pending_sense[i -
7977                                                 persis_offset].ua_pending |=
7978                                                 CTL_UA_RES_RELEASE;
7979                         }
7980                 }
7981                 lun->per_res[msg->pr.pr_info.residx].registered = 1;
7982                 lun->res_type = msg->pr.pr_info.res_type;
7983                 if (lun->res_type != SPR_TYPE_WR_EX_AR
7984                  && lun->res_type != SPR_TYPE_EX_AC_AR)
7985                         lun->pr_res_idx = msg->pr.pr_info.residx;
7986                 else
7987                         lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
7988         }
7989         lun->PRGeneration++;
7990
7991 }
7992
7993
7994 int
7995 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
7996 {
7997         int retval;
7998         int isc_retval;
7999         u_int32_t param_len;
8000         struct scsi_per_res_out *cdb;
8001         struct ctl_lun *lun;
8002         struct scsi_per_res_out_parms* param;
8003         struct ctl_softc *softc;
8004         uint32_t residx;
8005         uint64_t res_key, sa_res_key;
8006         uint8_t type;
8007         union ctl_ha_msg persis_io;
8008         int    i;
8009
8010         CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
8011
8012         retval = CTL_RETVAL_COMPLETE;
8013
8014         softc = control_softc;
8015
8016         cdb = (struct scsi_per_res_out *)ctsio->cdb;
8017         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8018
8019         /*
8020          * We only support whole-LUN scope.  The scope & type are ignored for
8021          * register, register and ignore existing key and clear.
8022          * We sometimes ignore scope and type on preempts too!!
8023          * Verify reservation type here as well.
8024          */
8025         type = cdb->scope_type & SPR_TYPE_MASK;
8026         if ((cdb->action == SPRO_RESERVE)
8027          || (cdb->action == SPRO_RELEASE)) {
8028                 if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
8029                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8030                                               /*sks_valid*/ 1,
8031                                               /*command*/ 1,
8032                                               /*field*/ 2,
8033                                               /*bit_valid*/ 1,
8034                                               /*bit*/ 4);
8035                         ctl_done((union ctl_io *)ctsio);
8036                         return (CTL_RETVAL_COMPLETE);
8037                 }
8038
8039                 if (type>8 || type==2 || type==4 || type==0) {
8040                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8041                                               /*sks_valid*/ 1,
8042                                               /*command*/ 1,
8043                                               /*field*/ 2,
8044                                               /*bit_valid*/ 1,
8045                                               /*bit*/ 0);
8046                         ctl_done((union ctl_io *)ctsio);
8047                         return (CTL_RETVAL_COMPLETE);
8048                 }
8049         }
8050
8051         switch (cdb->action & SPRO_ACTION_MASK) {
8052         case SPRO_REGISTER:
8053         case SPRO_RESERVE:
8054         case SPRO_RELEASE:
8055         case SPRO_CLEAR:
8056         case SPRO_PREEMPT:
8057         case SPRO_REG_IGNO:
8058                 break;
8059         case SPRO_REG_MOVE:
8060         case SPRO_PRE_ABO:
8061         default:
8062                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8063                                       /*sks_valid*/ 1,
8064                                       /*command*/ 1,
8065                                       /*field*/ 1,
8066                                       /*bit_valid*/ 1,
8067                                       /*bit*/ 0);
8068                 ctl_done((union ctl_io *)ctsio);
8069                 return (CTL_RETVAL_COMPLETE);
8070                 break; /* NOTREACHED */
8071         }
8072
8073         param_len = scsi_4btoul(cdb->length);
8074
8075         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
8076                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
8077                 ctsio->kern_data_len = param_len;
8078                 ctsio->kern_total_len = param_len;
8079                 ctsio->kern_data_resid = 0;
8080                 ctsio->kern_rel_offset = 0;
8081                 ctsio->kern_sg_entries = 0;
8082                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8083                 ctsio->be_move_done = ctl_config_move_done;
8084                 ctl_datamove((union ctl_io *)ctsio);
8085
8086                 return (CTL_RETVAL_COMPLETE);
8087         }
8088
8089         param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
8090
8091         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8092         res_key = scsi_8btou64(param->res_key.key);
8093         sa_res_key = scsi_8btou64(param->serv_act_res_key);
8094
8095         /*
8096          * Validate the reservation key here except for SPRO_REG_IGNO
8097          * This must be done for all other service actions
8098          */
8099         if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
8100                 mtx_lock(&softc->ctl_lock);
8101                 if (lun->per_res[residx].registered) {
8102                     if (memcmp(param->res_key.key,
8103                                lun->per_res[residx].res_key.key,
8104                                ctl_min(sizeof(param->res_key),
8105                                sizeof(lun->per_res[residx].res_key))) != 0) {
8106                                 /*
8107                                  * The current key passed in doesn't match
8108                                  * the one the initiator previously
8109                                  * registered.
8110                                  */
8111                                 mtx_unlock(&softc->ctl_lock);
8112                                 free(ctsio->kern_data_ptr, M_CTL);
8113                                 ctl_set_reservation_conflict(ctsio);
8114                                 ctl_done((union ctl_io *)ctsio);
8115                                 return (CTL_RETVAL_COMPLETE);
8116                         }
8117                 } else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
8118                         /*
8119                          * We are not registered
8120                          */
8121                         mtx_unlock(&softc->ctl_lock);
8122                         free(ctsio->kern_data_ptr, M_CTL);
8123                         ctl_set_reservation_conflict(ctsio);
8124                         ctl_done((union ctl_io *)ctsio);
8125                         return (CTL_RETVAL_COMPLETE);
8126                 } else if (res_key != 0) {
8127                         /*
8128                          * We are not registered and trying to register but
8129                          * the register key isn't zero.
8130                          */
8131                         mtx_unlock(&softc->ctl_lock);
8132                         free(ctsio->kern_data_ptr, M_CTL);
8133                         ctl_set_reservation_conflict(ctsio);
8134                         ctl_done((union ctl_io *)ctsio);
8135                         return (CTL_RETVAL_COMPLETE);
8136                 }
8137                 mtx_unlock(&softc->ctl_lock);
8138         }
8139
8140         switch (cdb->action & SPRO_ACTION_MASK) {
8141         case SPRO_REGISTER:
8142         case SPRO_REG_IGNO: {
8143
8144 #if 0
8145                 printf("Registration received\n");
8146 #endif
8147
8148                 /*
8149                  * We don't support any of these options, as we report in
8150                  * the read capabilities request (see
8151                  * ctl_persistent_reserve_in(), above).
8152                  */
8153                 if ((param->flags & SPR_SPEC_I_PT)
8154                  || (param->flags & SPR_ALL_TG_PT)
8155                  || (param->flags & SPR_APTPL)) {
8156                         int bit_ptr;
8157
8158                         if (param->flags & SPR_APTPL)
8159                                 bit_ptr = 0;
8160                         else if (param->flags & SPR_ALL_TG_PT)
8161                                 bit_ptr = 2;
8162                         else /* SPR_SPEC_I_PT */
8163                                 bit_ptr = 3;
8164
8165                         free(ctsio->kern_data_ptr, M_CTL);
8166                         ctl_set_invalid_field(ctsio,
8167                                               /*sks_valid*/ 1,
8168                                               /*command*/ 0,
8169                                               /*field*/ 20,
8170                                               /*bit_valid*/ 1,
8171                                               /*bit*/ bit_ptr);
8172                         ctl_done((union ctl_io *)ctsio);
8173                         return (CTL_RETVAL_COMPLETE);
8174                 }
8175
8176                 mtx_lock(&softc->ctl_lock);
8177
8178                 /*
8179                  * The initiator wants to clear the
8180                  * key/unregister.
8181                  */
8182                 if (sa_res_key == 0) {
8183                         if ((res_key == 0
8184                           && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
8185                          || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
8186                           && !lun->per_res[residx].registered)) {
8187                                 mtx_unlock(&softc->ctl_lock);
8188                                 goto done;
8189                         }
8190
8191                         lun->per_res[residx].registered = 0;
8192                         memset(&lun->per_res[residx].res_key,
8193                                0, sizeof(lun->per_res[residx].res_key));
8194                         lun->pr_key_count--;
8195
8196                         if (residx == lun->pr_res_idx) {
8197                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8198                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8199
8200                                 if ((lun->res_type == SPR_TYPE_WR_EX_RO
8201                                   || lun->res_type == SPR_TYPE_EX_AC_RO)
8202                                  && lun->pr_key_count) {
8203                                         /*
8204                                          * If the reservation is a registrants
8205                                          * only type we need to generate a UA
8206                                          * for other registered inits.  The
8207                                          * sense code should be RESERVATIONS
8208                                          * RELEASED
8209                                          */
8210
8211                                         for (i = 0; i < CTL_MAX_INITIATORS;i++){
8212                                                 if (lun->per_res[
8213                                                     i+persis_offset].registered
8214                                                     == 0)
8215                                                         continue;
8216                                                 lun->pending_sense[i
8217                                                         ].ua_pending |=
8218                                                         CTL_UA_RES_RELEASE;
8219                                         }
8220                                 }
8221                                 lun->res_type = 0;
8222                         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8223                                 if (lun->pr_key_count==0) {
8224                                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8225                                         lun->res_type = 0;
8226                                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8227                                 }
8228                         }
8229                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8230                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8231                         persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8232                         persis_io.pr.pr_info.residx = residx;
8233                         if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8234                              &persis_io, sizeof(persis_io), 0 )) >
8235                              CTL_HA_STATUS_SUCCESS) {
8236                                 printf("CTL:Persis Out error returned from "
8237                                        "ctl_ha_msg_send %d\n", isc_retval);
8238                         }
8239                         mtx_unlock(&softc->ctl_lock);
8240                 } else /* sa_res_key != 0 */ {
8241
8242                         /*
8243                          * If we aren't registered currently then increment
8244                          * the key count and set the registered flag.
8245                          */
8246                         if (!lun->per_res[residx].registered) {
8247                                 lun->pr_key_count++;
8248                                 lun->per_res[residx].registered = 1;
8249                         }
8250
8251                         memcpy(&lun->per_res[residx].res_key,
8252                                param->serv_act_res_key,
8253                                ctl_min(sizeof(param->serv_act_res_key),
8254                                sizeof(lun->per_res[residx].res_key)));
8255
8256                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8257                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8258                         persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8259                         persis_io.pr.pr_info.residx = residx;
8260                         memcpy(persis_io.pr.pr_info.sa_res_key,
8261                                param->serv_act_res_key,
8262                                sizeof(param->serv_act_res_key));
8263                         mtx_unlock(&softc->ctl_lock);
8264                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8265                              &persis_io, sizeof(persis_io), 0)) >
8266                              CTL_HA_STATUS_SUCCESS) {
8267                                 printf("CTL:Persis Out error returned from "
8268                                        "ctl_ha_msg_send %d\n", isc_retval);
8269                         }
8270                 }
8271                 lun->PRGeneration++;
8272
8273                 break;
8274         }
8275         case SPRO_RESERVE:
8276 #if 0
8277                 printf("Reserve executed type %d\n", type);
8278 #endif
8279                 mtx_lock(&softc->ctl_lock);
8280                 if (lun->flags & CTL_LUN_PR_RESERVED) {
8281                         /*
8282                          * if this isn't the reservation holder and it's
8283                          * not a "all registrants" type or if the type is
8284                          * different then we have a conflict
8285                          */
8286                         if ((lun->pr_res_idx != residx
8287                           && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8288                          || lun->res_type != type) {
8289                                 mtx_unlock(&softc->ctl_lock);
8290                                 free(ctsio->kern_data_ptr, M_CTL);
8291                                 ctl_set_reservation_conflict(ctsio);
8292                                 ctl_done((union ctl_io *)ctsio);
8293                                 return (CTL_RETVAL_COMPLETE);
8294                         }
8295                         mtx_unlock(&softc->ctl_lock);
8296                 } else /* create a reservation */ {
8297                         /*
8298                          * If it's not an "all registrants" type record
8299                          * reservation holder
8300                          */
8301                         if (type != SPR_TYPE_WR_EX_AR
8302                          && type != SPR_TYPE_EX_AC_AR)
8303                                 lun->pr_res_idx = residx; /* Res holder */
8304                         else
8305                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8306
8307                         lun->flags |= CTL_LUN_PR_RESERVED;
8308                         lun->res_type = type;
8309
8310                         mtx_unlock(&softc->ctl_lock);
8311
8312                         /* send msg to other side */
8313                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8314                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8315                         persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8316                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8317                         persis_io.pr.pr_info.res_type = type;
8318                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8319                              &persis_io, sizeof(persis_io), 0)) >
8320                              CTL_HA_STATUS_SUCCESS) {
8321                                 printf("CTL:Persis Out error returned from "
8322                                        "ctl_ha_msg_send %d\n", isc_retval);
8323                         }
8324                 }
8325                 break;
8326
8327         case SPRO_RELEASE:
8328                 mtx_lock(&softc->ctl_lock);
8329                 if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8330                         /* No reservation exists return good status */
8331                         mtx_unlock(&softc->ctl_lock);
8332                         goto done;
8333                 }
8334                 /*
8335                  * Is this nexus a reservation holder?
8336                  */
8337                 if (lun->pr_res_idx != residx
8338                  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8339                         /*
8340                          * not a res holder return good status but
8341                          * do nothing
8342                          */
8343                         mtx_unlock(&softc->ctl_lock);
8344                         goto done;
8345                 }
8346
8347                 if (lun->res_type != type) {
8348                         mtx_unlock(&softc->ctl_lock);
8349                         free(ctsio->kern_data_ptr, M_CTL);
8350                         ctl_set_illegal_pr_release(ctsio);
8351                         ctl_done((union ctl_io *)ctsio);
8352                         return (CTL_RETVAL_COMPLETE);
8353                 }
8354
8355                 /* okay to release */
8356                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8357                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8358                 lun->res_type = 0;
8359
8360                 /*
8361                  * if this isn't an exclusive access
8362                  * res generate UA for all other
8363                  * registrants.
8364                  */
8365                 if (type != SPR_TYPE_EX_AC
8366                  && type != SPR_TYPE_WR_EX) {
8367                         /*
8368                          * temporarily unregister so we don't generate UA
8369                          */
8370                         lun->per_res[residx].registered = 0;
8371
8372                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8373                                 if (lun->per_res[i+persis_offset].registered
8374                                     == 0)
8375                                         continue;
8376                                 lun->pending_sense[i].ua_pending |=
8377                                         CTL_UA_RES_RELEASE;
8378                         }
8379
8380                         lun->per_res[residx].registered = 1;
8381                 }
8382                 mtx_unlock(&softc->ctl_lock);
8383                 /* Send msg to other side */
8384                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8385                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8386                 persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8387                 if ((isc_retval=ctl_ha_msg_send( CTL_HA_CHAN_CTL, &persis_io,
8388                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8389                         printf("CTL:Persis Out error returned from "
8390                                "ctl_ha_msg_send %d\n", isc_retval);
8391                 }
8392                 break;
8393
8394         case SPRO_CLEAR:
8395                 /* send msg to other side */
8396
8397                 mtx_lock(&softc->ctl_lock);
8398                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8399                 lun->res_type = 0;
8400                 lun->pr_key_count = 0;
8401                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8402
8403
8404                 memset(&lun->per_res[residx].res_key,
8405                        0, sizeof(lun->per_res[residx].res_key));
8406                 lun->per_res[residx].registered = 0;
8407
8408                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++)
8409                         if (lun->per_res[i].registered) {
8410                                 if (!persis_offset && i < CTL_MAX_INITIATORS)
8411                                         lun->pending_sense[i].ua_pending |=
8412                                                 CTL_UA_RES_PREEMPT;
8413                                 else if (persis_offset && i >= persis_offset)
8414                                         lun->pending_sense[i-persis_offset
8415                                             ].ua_pending |= CTL_UA_RES_PREEMPT;
8416
8417                                 memset(&lun->per_res[i].res_key,
8418                                        0, sizeof(struct scsi_per_res_key));
8419                                 lun->per_res[i].registered = 0;
8420                         }
8421                 lun->PRGeneration++;
8422                 mtx_unlock(&softc->ctl_lock);
8423                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8424                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8425                 persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8426                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8427                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8428                         printf("CTL:Persis Out error returned from "
8429                                "ctl_ha_msg_send %d\n", isc_retval);
8430                 }
8431                 break;
8432
8433         case SPRO_PREEMPT: {
8434                 int nretval;
8435
8436                 nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8437                                           residx, ctsio, cdb, param);
8438                 if (nretval != 0)
8439                         return (CTL_RETVAL_COMPLETE);
8440                 break;
8441         }
8442         case SPRO_REG_MOVE:
8443         case SPRO_PRE_ABO:
8444         default:
8445                 free(ctsio->kern_data_ptr, M_CTL);
8446                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8447                                       /*sks_valid*/ 1,
8448                                       /*command*/ 1,
8449                                       /*field*/ 1,
8450                                       /*bit_valid*/ 1,
8451                                       /*bit*/ 0);
8452                 ctl_done((union ctl_io *)ctsio);
8453                 return (CTL_RETVAL_COMPLETE);
8454                 break; /* NOTREACHED */
8455         }
8456
8457 done:
8458         free(ctsio->kern_data_ptr, M_CTL);
8459         ctl_set_success(ctsio);
8460         ctl_done((union ctl_io *)ctsio);
8461
8462         return (retval);
8463 }
8464
8465 /*
8466  * This routine is for handling a message from the other SC pertaining to
8467  * persistent reserve out. All the error checking will have been done
8468  * so only perorming the action need be done here to keep the two
8469  * in sync.
8470  */
8471 static void
8472 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8473 {
8474         struct ctl_lun *lun;
8475         struct ctl_softc *softc;
8476         int i;
8477         uint32_t targ_lun;
8478
8479         softc = control_softc;
8480
8481         mtx_lock(&softc->ctl_lock);
8482
8483         targ_lun = msg->hdr.nexus.targ_lun;
8484         if (msg->hdr.nexus.lun_map_fn != NULL)
8485                 targ_lun = msg->hdr.nexus.lun_map_fn(msg->hdr.nexus.lun_map_arg, targ_lun);
8486         lun = softc->ctl_luns[targ_lun];
8487         switch(msg->pr.pr_info.action) {
8488         case CTL_PR_REG_KEY:
8489                 if (!lun->per_res[msg->pr.pr_info.residx].registered) {
8490                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
8491                         lun->pr_key_count++;
8492                 }
8493                 lun->PRGeneration++;
8494                 memcpy(&lun->per_res[msg->pr.pr_info.residx].res_key,
8495                        msg->pr.pr_info.sa_res_key,
8496                        sizeof(struct scsi_per_res_key));
8497                 break;
8498
8499         case CTL_PR_UNREG_KEY:
8500                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
8501                 memset(&lun->per_res[msg->pr.pr_info.residx].res_key,
8502                        0, sizeof(struct scsi_per_res_key));
8503                 lun->pr_key_count--;
8504
8505                 /* XXX Need to see if the reservation has been released */
8506                 /* if so do we need to generate UA? */
8507                 if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8508                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8509                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8510
8511                         if ((lun->res_type == SPR_TYPE_WR_EX_RO
8512                           || lun->res_type == SPR_TYPE_EX_AC_RO)
8513                          && lun->pr_key_count) {
8514                                 /*
8515                                  * If the reservation is a registrants
8516                                  * only type we need to generate a UA
8517                                  * for other registered inits.  The
8518                                  * sense code should be RESERVATIONS
8519                                  * RELEASED
8520                                  */
8521
8522                                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8523                                         if (lun->per_res[i+
8524                                             persis_offset].registered == 0)
8525                                                 continue;
8526
8527                                         lun->pending_sense[i
8528                                                 ].ua_pending |=
8529                                                 CTL_UA_RES_RELEASE;
8530                                 }
8531                         }
8532                         lun->res_type = 0;
8533                 } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8534                         if (lun->pr_key_count==0) {
8535                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8536                                 lun->res_type = 0;
8537                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8538                         }
8539                 }
8540                 lun->PRGeneration++;
8541                 break;
8542
8543         case CTL_PR_RESERVE:
8544                 lun->flags |= CTL_LUN_PR_RESERVED;
8545                 lun->res_type = msg->pr.pr_info.res_type;
8546                 lun->pr_res_idx = msg->pr.pr_info.residx;
8547
8548                 break;
8549
8550         case CTL_PR_RELEASE:
8551                 /*
8552                  * if this isn't an exclusive access res generate UA for all
8553                  * other registrants.
8554                  */
8555                 if (lun->res_type != SPR_TYPE_EX_AC
8556                  && lun->res_type != SPR_TYPE_WR_EX) {
8557                         for (i = 0; i < CTL_MAX_INITIATORS; i++)
8558                                 if (lun->per_res[i+persis_offset].registered)
8559                                         lun->pending_sense[i].ua_pending |=
8560                                                 CTL_UA_RES_RELEASE;
8561                 }
8562
8563                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8564                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8565                 lun->res_type = 0;
8566                 break;
8567
8568         case CTL_PR_PREEMPT:
8569                 ctl_pro_preempt_other(lun, msg);
8570                 break;
8571         case CTL_PR_CLEAR:
8572                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8573                 lun->res_type = 0;
8574                 lun->pr_key_count = 0;
8575                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8576
8577                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8578                         if (lun->per_res[i].registered == 0)
8579                                 continue;
8580                         if (!persis_offset
8581                          && i < CTL_MAX_INITIATORS)
8582                                 lun->pending_sense[i].ua_pending |=
8583                                         CTL_UA_RES_PREEMPT;
8584                         else if (persis_offset
8585                               && i >= persis_offset)
8586                                 lun->pending_sense[i-persis_offset].ua_pending|=
8587                                         CTL_UA_RES_PREEMPT;
8588                         memset(&lun->per_res[i].res_key, 0,
8589                                sizeof(struct scsi_per_res_key));
8590                         lun->per_res[i].registered = 0;
8591                 }
8592                 lun->PRGeneration++;
8593                 break;
8594         }
8595
8596         mtx_unlock(&softc->ctl_lock);
8597 }
8598
8599 int
8600 ctl_read_write(struct ctl_scsiio *ctsio)
8601 {
8602         struct ctl_lun *lun;
8603         struct ctl_lba_len lbalen;
8604         uint64_t lba;
8605         uint32_t num_blocks;
8606         int reladdr, fua, dpo, ebp;
8607         int retval;
8608         int isread;
8609
8610         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8611
8612         CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8613
8614         reladdr = 0;
8615         fua = 0;
8616         dpo = 0;
8617         ebp = 0;
8618
8619         retval = CTL_RETVAL_COMPLETE;
8620
8621         isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8622               || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8623         if (lun->flags & CTL_LUN_PR_RESERVED && isread) {
8624                 uint32_t residx;
8625
8626                 /*
8627                  * XXX KDM need a lock here.
8628                  */
8629                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8630                 if ((lun->res_type == SPR_TYPE_EX_AC
8631                   && residx != lun->pr_res_idx)
8632                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
8633                    || lun->res_type == SPR_TYPE_EX_AC_AR)
8634                   && !lun->per_res[residx].registered)) {
8635                         ctl_set_reservation_conflict(ctsio);
8636                         ctl_done((union ctl_io *)ctsio);
8637                         return (CTL_RETVAL_COMPLETE);
8638                 }
8639         }
8640
8641         switch (ctsio->cdb[0]) {
8642         case READ_6:
8643         case WRITE_6: {
8644                 struct scsi_rw_6 *cdb;
8645
8646                 cdb = (struct scsi_rw_6 *)ctsio->cdb;
8647
8648                 lba = scsi_3btoul(cdb->addr);
8649                 /* only 5 bits are valid in the most significant address byte */
8650                 lba &= 0x1fffff;
8651                 num_blocks = cdb->length;
8652                 /*
8653                  * This is correct according to SBC-2.
8654                  */
8655                 if (num_blocks == 0)
8656                         num_blocks = 256;
8657                 break;
8658         }
8659         case READ_10:
8660         case WRITE_10: {
8661                 struct scsi_rw_10 *cdb;
8662
8663                 cdb = (struct scsi_rw_10 *)ctsio->cdb;
8664
8665                 if (cdb->byte2 & SRW10_RELADDR)
8666                         reladdr = 1;
8667                 if (cdb->byte2 & SRW10_FUA)
8668                         fua = 1;
8669                 if (cdb->byte2 & SRW10_DPO)
8670                         dpo = 1;
8671
8672                 if ((cdb->opcode == WRITE_10)
8673                  && (cdb->byte2 & SRW10_EBP))
8674                         ebp = 1;
8675
8676                 lba = scsi_4btoul(cdb->addr);
8677                 num_blocks = scsi_2btoul(cdb->length);
8678                 break;
8679         }
8680         case WRITE_VERIFY_10: {
8681                 struct scsi_write_verify_10 *cdb;
8682
8683                 cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
8684
8685                 /*
8686                  * XXX KDM we should do actual write verify support at some
8687                  * point.  This is obviously fake, we're just translating
8688                  * things to a write.  So we don't even bother checking the
8689                  * BYTCHK field, since we don't do any verification.  If
8690                  * the user asks for it, we'll just pretend we did it.
8691                  */
8692                 if (cdb->byte2 & SWV_DPO)
8693                         dpo = 1;
8694
8695                 lba = scsi_4btoul(cdb->addr);
8696                 num_blocks = scsi_2btoul(cdb->length);
8697                 break;
8698         }
8699         case READ_12:
8700         case WRITE_12: {
8701                 struct scsi_rw_12 *cdb;
8702
8703                 cdb = (struct scsi_rw_12 *)ctsio->cdb;
8704
8705                 if (cdb->byte2 & SRW12_RELADDR)
8706                         reladdr = 1;
8707                 if (cdb->byte2 & SRW12_FUA)
8708                         fua = 1;
8709                 if (cdb->byte2 & SRW12_DPO)
8710                         dpo = 1;
8711                 lba = scsi_4btoul(cdb->addr);
8712                 num_blocks = scsi_4btoul(cdb->length);
8713                 break;
8714         }
8715         case WRITE_VERIFY_12: {
8716                 struct scsi_write_verify_12 *cdb;
8717
8718                 cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
8719
8720                 if (cdb->byte2 & SWV_DPO)
8721                         dpo = 1;
8722
8723                 lba = scsi_4btoul(cdb->addr);
8724                 num_blocks = scsi_4btoul(cdb->length);
8725
8726                 break;
8727         }
8728         case READ_16:
8729         case WRITE_16: {
8730                 struct scsi_rw_16 *cdb;
8731
8732                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
8733
8734                 if (cdb->byte2 & SRW12_RELADDR)
8735                         reladdr = 1;
8736                 if (cdb->byte2 & SRW12_FUA)
8737                         fua = 1;
8738                 if (cdb->byte2 & SRW12_DPO)
8739                         dpo = 1;
8740
8741                 lba = scsi_8btou64(cdb->addr);
8742                 num_blocks = scsi_4btoul(cdb->length);
8743                 break;
8744         }
8745         case WRITE_VERIFY_16: {
8746                 struct scsi_write_verify_16 *cdb;
8747
8748                 cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
8749
8750                 if (cdb->byte2 & SWV_DPO)
8751                         dpo = 1;
8752
8753                 lba = scsi_8btou64(cdb->addr);
8754                 num_blocks = scsi_4btoul(cdb->length);
8755                 break;
8756         }
8757         default:
8758                 /*
8759                  * We got a command we don't support.  This shouldn't
8760                  * happen, commands should be filtered out above us.
8761                  */
8762                 ctl_set_invalid_opcode(ctsio);
8763                 ctl_done((union ctl_io *)ctsio);
8764
8765                 return (CTL_RETVAL_COMPLETE);
8766                 break; /* NOTREACHED */
8767         }
8768
8769         /*
8770          * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
8771          * interesting for us, but if RAIDCore is in write-back mode,
8772          * getting it to do write-through for a particular transaction may
8773          * not be possible.
8774          */
8775         /*
8776          * We don't support relative addressing.  That also requires
8777          * supporting linked commands, which we don't do.
8778          */
8779         if (reladdr != 0) {
8780                 ctl_set_invalid_field(ctsio,
8781                                       /*sks_valid*/ 1,
8782                                       /*command*/ 1,
8783                                       /*field*/ 1,
8784                                       /*bit_valid*/ 1,
8785                                       /*bit*/ 0);
8786                 ctl_done((union ctl_io *)ctsio);
8787                 return (CTL_RETVAL_COMPLETE);
8788         }
8789
8790         /*
8791          * The first check is to make sure we're in bounds, the second
8792          * check is to catch wrap-around problems.  If the lba + num blocks
8793          * is less than the lba, then we've wrapped around and the block
8794          * range is invalid anyway.
8795          */
8796         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
8797          || ((lba + num_blocks) < lba)) {
8798                 ctl_set_lba_out_of_range(ctsio);
8799                 ctl_done((union ctl_io *)ctsio);
8800                 return (CTL_RETVAL_COMPLETE);
8801         }
8802
8803         /*
8804          * According to SBC-3, a transfer length of 0 is not an error.
8805          * Note that this cannot happen with WRITE(6) or READ(6), since 0
8806          * translates to 256 blocks for those commands.
8807          */
8808         if (num_blocks == 0) {
8809                 ctl_set_success(ctsio);
8810                 ctl_done((union ctl_io *)ctsio);
8811                 return (CTL_RETVAL_COMPLETE);
8812         }
8813
8814         lbalen.lba = lba;
8815         lbalen.len = num_blocks;
8816         memcpy(ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes, &lbalen,
8817                sizeof(lbalen));
8818
8819         CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
8820
8821         retval = lun->backend->data_submit((union ctl_io *)ctsio);
8822
8823         return (retval);
8824 }
8825
8826 int
8827 ctl_report_luns(struct ctl_scsiio *ctsio)
8828 {
8829         struct scsi_report_luns *cdb;
8830         struct scsi_report_luns_data *lun_data;
8831         struct ctl_lun *lun, *request_lun;
8832         int num_luns, retval;
8833         uint32_t alloc_len, lun_datalen;
8834         int num_filled, well_known;
8835         uint32_t initidx, targ_lun_id, lun_id;
8836
8837         retval = CTL_RETVAL_COMPLETE;
8838         well_known = 0;
8839
8840         cdb = (struct scsi_report_luns *)ctsio->cdb;
8841
8842         CTL_DEBUG_PRINT(("ctl_report_luns\n"));
8843
8844         mtx_lock(&control_softc->ctl_lock);
8845         num_luns = control_softc->num_luns;
8846         mtx_unlock(&control_softc->ctl_lock);
8847
8848         switch (cdb->select_report) {
8849         case RPL_REPORT_DEFAULT:
8850         case RPL_REPORT_ALL:
8851                 break;
8852         case RPL_REPORT_WELLKNOWN:
8853                 well_known = 1;
8854                 num_luns = 0;
8855                 break;
8856         default:
8857                 ctl_set_invalid_field(ctsio,
8858                                       /*sks_valid*/ 1,
8859                                       /*command*/ 1,
8860                                       /*field*/ 2,
8861                                       /*bit_valid*/ 0,
8862                                       /*bit*/ 0);
8863                 ctl_done((union ctl_io *)ctsio);
8864                 return (retval);
8865                 break; /* NOTREACHED */
8866         }
8867
8868         alloc_len = scsi_4btoul(cdb->length);
8869         /*
8870          * The initiator has to allocate at least 16 bytes for this request,
8871          * so he can at least get the header and the first LUN.  Otherwise
8872          * we reject the request (per SPC-3 rev 14, section 6.21).
8873          */
8874         if (alloc_len < (sizeof(struct scsi_report_luns_data) +
8875             sizeof(struct scsi_report_luns_lundata))) {
8876                 ctl_set_invalid_field(ctsio,
8877                                       /*sks_valid*/ 1,
8878                                       /*command*/ 1,
8879                                       /*field*/ 6,
8880                                       /*bit_valid*/ 0,
8881                                       /*bit*/ 0);
8882                 ctl_done((union ctl_io *)ctsio);
8883                 return (retval);
8884         }
8885
8886         request_lun = (struct ctl_lun *)
8887                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8888
8889         lun_datalen = sizeof(*lun_data) +
8890                 (num_luns * sizeof(struct scsi_report_luns_lundata));
8891
8892         ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
8893         lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
8894         ctsio->kern_sg_entries = 0;
8895
8896         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
8897
8898         mtx_lock(&control_softc->ctl_lock);
8899         for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
8900                 lun_id = targ_lun_id;
8901                 if (ctsio->io_hdr.nexus.lun_map_fn != NULL)
8902                         lun_id = ctsio->io_hdr.nexus.lun_map_fn(ctsio->io_hdr.nexus.lun_map_arg, lun_id);
8903                 if (lun_id >= CTL_MAX_LUNS)
8904                         continue;
8905                 lun = control_softc->ctl_luns[lun_id];
8906                 if (lun == NULL)
8907                         continue;
8908
8909                 if (targ_lun_id <= 0xff) {
8910                         /*
8911                          * Peripheral addressing method, bus number 0.
8912                          */
8913                         lun_data->luns[num_filled].lundata[0] =
8914                                 RPL_LUNDATA_ATYP_PERIPH;
8915                         lun_data->luns[num_filled].lundata[1] = targ_lun_id;
8916                         num_filled++;
8917                 } else if (targ_lun_id <= 0x3fff) {
8918                         /*
8919                          * Flat addressing method.
8920                          */
8921                         lun_data->luns[num_filled].lundata[0] =
8922                                 RPL_LUNDATA_ATYP_FLAT |
8923                                 (targ_lun_id & RPL_LUNDATA_FLAT_LUN_MASK);
8924 #ifdef OLDCTLHEADERS
8925                                 (SRLD_ADDR_FLAT << SRLD_ADDR_SHIFT) |
8926                                 (targ_lun_id & SRLD_BUS_LUN_MASK);
8927 #endif
8928                         lun_data->luns[num_filled].lundata[1] =
8929 #ifdef OLDCTLHEADERS
8930                                 targ_lun_id >> SRLD_BUS_LUN_BITS;
8931 #endif
8932                                 targ_lun_id >> RPL_LUNDATA_FLAT_LUN_BITS;
8933                         num_filled++;
8934                 } else {
8935                         printf("ctl_report_luns: bogus LUN number %jd, "
8936                                "skipping\n", (intmax_t)targ_lun_id);
8937                 }
8938                 /*
8939                  * According to SPC-3, rev 14 section 6.21:
8940                  *
8941                  * "The execution of a REPORT LUNS command to any valid and
8942                  * installed logical unit shall clear the REPORTED LUNS DATA
8943                  * HAS CHANGED unit attention condition for all logical
8944                  * units of that target with respect to the requesting
8945                  * initiator. A valid and installed logical unit is one
8946                  * having a PERIPHERAL QUALIFIER of 000b in the standard
8947                  * INQUIRY data (see 6.4.2)."
8948                  *
8949                  * If request_lun is NULL, the LUN this report luns command
8950                  * was issued to is either disabled or doesn't exist. In that
8951                  * case, we shouldn't clear any pending lun change unit
8952                  * attention.
8953                  */
8954                 if (request_lun != NULL)
8955                         lun->pending_sense[initidx].ua_pending &=
8956                                 ~CTL_UA_LUN_CHANGE;
8957         }
8958         mtx_unlock(&control_softc->ctl_lock);
8959
8960         /*
8961          * It's quite possible that we've returned fewer LUNs than we allocated
8962          * space for.  Trim it.
8963          */
8964         lun_datalen = sizeof(*lun_data) +
8965                 (num_filled * sizeof(struct scsi_report_luns_lundata));
8966
8967         if (lun_datalen < alloc_len) {
8968                 ctsio->residual = alloc_len - lun_datalen;
8969                 ctsio->kern_data_len = lun_datalen;
8970                 ctsio->kern_total_len = lun_datalen;
8971         } else {
8972                 ctsio->residual = 0;
8973                 ctsio->kern_data_len = alloc_len;
8974                 ctsio->kern_total_len = alloc_len;
8975         }
8976         ctsio->kern_data_resid = 0;
8977         ctsio->kern_rel_offset = 0;
8978         ctsio->kern_sg_entries = 0;
8979
8980         /*
8981          * We set this to the actual data length, regardless of how much
8982          * space we actually have to return results.  If the user looks at
8983          * this value, he'll know whether or not he allocated enough space
8984          * and reissue the command if necessary.  We don't support well
8985          * known logical units, so if the user asks for that, return none.
8986          */
8987         scsi_ulto4b(lun_datalen - 8, lun_data->length);
8988
8989         /*
8990          * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
8991          * this request.
8992          */
8993         ctsio->scsi_status = SCSI_STATUS_OK;
8994
8995         ctsio->be_move_done = ctl_config_move_done;
8996         ctl_datamove((union ctl_io *)ctsio);
8997
8998         return (retval);
8999 }
9000
9001 int
9002 ctl_request_sense(struct ctl_scsiio *ctsio)
9003 {
9004         struct scsi_request_sense *cdb;
9005         struct scsi_sense_data *sense_ptr;
9006         struct ctl_lun *lun;
9007         uint32_t initidx;
9008         int have_error;
9009         scsi_sense_data_type sense_format;
9010
9011         cdb = (struct scsi_request_sense *)ctsio->cdb;
9012
9013         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9014
9015         CTL_DEBUG_PRINT(("ctl_request_sense\n"));
9016
9017         /*
9018          * Determine which sense format the user wants.
9019          */
9020         if (cdb->byte2 & SRS_DESC)
9021                 sense_format = SSD_TYPE_DESC;
9022         else
9023                 sense_format = SSD_TYPE_FIXED;
9024
9025         ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
9026         sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
9027         ctsio->kern_sg_entries = 0;
9028
9029         /*
9030          * struct scsi_sense_data, which is currently set to 256 bytes, is
9031          * larger than the largest allowed value for the length field in the
9032          * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
9033          */
9034         ctsio->residual = 0;
9035         ctsio->kern_data_len = cdb->length;
9036         ctsio->kern_total_len = cdb->length;
9037
9038         ctsio->kern_data_resid = 0;
9039         ctsio->kern_rel_offset = 0;
9040         ctsio->kern_sg_entries = 0;
9041
9042         /*
9043          * If we don't have a LUN, we don't have any pending sense.
9044          */
9045         if (lun == NULL)
9046                 goto no_sense;
9047
9048         have_error = 0;
9049         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9050         /*
9051          * Check for pending sense, and then for pending unit attentions.
9052          * Pending sense gets returned first, then pending unit attentions.
9053          */
9054         mtx_lock(&lun->ctl_softc->ctl_lock);
9055         if (ctl_is_set(lun->have_ca, initidx)) {
9056                 scsi_sense_data_type stored_format;
9057
9058                 /*
9059                  * Check to see which sense format was used for the stored
9060                  * sense data.
9061                  */
9062                 stored_format = scsi_sense_type(
9063                     &lun->pending_sense[initidx].sense);
9064
9065                 /*
9066                  * If the user requested a different sense format than the
9067                  * one we stored, then we need to convert it to the other
9068                  * format.  If we're going from descriptor to fixed format
9069                  * sense data, we may lose things in translation, depending
9070                  * on what options were used.
9071                  *
9072                  * If the stored format is SSD_TYPE_NONE (i.e. invalid),
9073                  * for some reason we'll just copy it out as-is.
9074                  */
9075                 if ((stored_format == SSD_TYPE_FIXED)
9076                  && (sense_format == SSD_TYPE_DESC))
9077                         ctl_sense_to_desc((struct scsi_sense_data_fixed *)
9078                             &lun->pending_sense[initidx].sense,
9079                             (struct scsi_sense_data_desc *)sense_ptr);
9080                 else if ((stored_format == SSD_TYPE_DESC)
9081                       && (sense_format == SSD_TYPE_FIXED))
9082                         ctl_sense_to_fixed((struct scsi_sense_data_desc *)
9083                             &lun->pending_sense[initidx].sense,
9084                             (struct scsi_sense_data_fixed *)sense_ptr);
9085                 else
9086                         memcpy(sense_ptr, &lun->pending_sense[initidx].sense,
9087                                ctl_min(sizeof(*sense_ptr),
9088                                sizeof(lun->pending_sense[initidx].sense)));
9089
9090                 ctl_clear_mask(lun->have_ca, initidx);
9091                 have_error = 1;
9092         } else if (lun->pending_sense[initidx].ua_pending != CTL_UA_NONE) {
9093                 ctl_ua_type ua_type;
9094
9095                 ua_type = ctl_build_ua(lun->pending_sense[initidx].ua_pending,
9096                                        sense_ptr, sense_format);
9097                 if (ua_type != CTL_UA_NONE) {
9098                         have_error = 1;
9099                         /* We're reporting this UA, so clear it */
9100                         lun->pending_sense[initidx].ua_pending &= ~ua_type;
9101                 }
9102         }
9103         mtx_unlock(&lun->ctl_softc->ctl_lock);
9104
9105         /*
9106          * We already have a pending error, return it.
9107          */
9108         if (have_error != 0) {
9109                 /*
9110                  * We report the SCSI status as OK, since the status of the
9111                  * request sense command itself is OK.
9112                  */
9113                 ctsio->scsi_status = SCSI_STATUS_OK;
9114
9115                 /*
9116                  * We report 0 for the sense length, because we aren't doing
9117                  * autosense in this case.  We're reporting sense as
9118                  * parameter data.
9119                  */
9120                 ctsio->sense_len = 0;
9121
9122                 ctsio->be_move_done = ctl_config_move_done;
9123                 ctl_datamove((union ctl_io *)ctsio);
9124
9125                 return (CTL_RETVAL_COMPLETE);
9126         }
9127
9128 no_sense:
9129
9130         /*
9131          * No sense information to report, so we report that everything is
9132          * okay.
9133          */
9134         ctl_set_sense_data(sense_ptr,
9135                            lun,
9136                            sense_format,
9137                            /*current_error*/ 1,
9138                            /*sense_key*/ SSD_KEY_NO_SENSE,
9139                            /*asc*/ 0x00,
9140                            /*ascq*/ 0x00,
9141                            SSD_ELEM_NONE);
9142
9143         ctsio->scsi_status = SCSI_STATUS_OK;
9144
9145         /*
9146          * We report 0 for the sense length, because we aren't doing
9147          * autosense in this case.  We're reporting sense as parameter data.
9148          */
9149         ctsio->sense_len = 0;
9150         ctsio->be_move_done = ctl_config_move_done;
9151         ctl_datamove((union ctl_io *)ctsio);
9152
9153         return (CTL_RETVAL_COMPLETE);
9154 }
9155
9156 int
9157 ctl_tur(struct ctl_scsiio *ctsio)
9158 {
9159         struct ctl_lun *lun;
9160
9161         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9162
9163         CTL_DEBUG_PRINT(("ctl_tur\n"));
9164
9165         if (lun == NULL)
9166                 return (-EINVAL);
9167
9168         ctsio->scsi_status = SCSI_STATUS_OK;
9169         ctsio->io_hdr.status = CTL_SUCCESS;
9170
9171         ctl_done((union ctl_io *)ctsio);
9172
9173         return (CTL_RETVAL_COMPLETE);
9174 }
9175
9176 #ifdef notyet
9177 static int
9178 ctl_cmddt_inquiry(struct ctl_scsiio *ctsio)
9179 {
9180
9181 }
9182 #endif
9183
9184 static int
9185 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
9186 {
9187         struct scsi_vpd_supported_pages *pages;
9188         int sup_page_size;
9189         struct ctl_lun *lun;
9190
9191         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9192
9193         sup_page_size = sizeof(struct scsi_vpd_supported_pages) *
9194             SCSI_EVPD_NUM_SUPPORTED_PAGES;
9195         ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9196         pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9197         ctsio->kern_sg_entries = 0;
9198
9199         if (sup_page_size < alloc_len) {
9200                 ctsio->residual = alloc_len - sup_page_size;
9201                 ctsio->kern_data_len = sup_page_size;
9202                 ctsio->kern_total_len = sup_page_size;
9203         } else {
9204                 ctsio->residual = 0;
9205                 ctsio->kern_data_len = alloc_len;
9206                 ctsio->kern_total_len = alloc_len;
9207         }
9208         ctsio->kern_data_resid = 0;
9209         ctsio->kern_rel_offset = 0;
9210         ctsio->kern_sg_entries = 0;
9211
9212         /*
9213          * The control device is always connected.  The disk device, on the
9214          * other hand, may not be online all the time.  Need to change this
9215          * to figure out whether the disk device is actually online or not.
9216          */
9217         if (lun != NULL)
9218                 pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9219                                 lun->be_lun->lun_type;
9220         else
9221                 pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9222
9223         pages->length = SCSI_EVPD_NUM_SUPPORTED_PAGES;
9224         /* Supported VPD pages */
9225         pages->page_list[0] = SVPD_SUPPORTED_PAGES;
9226         /* Serial Number */
9227         pages->page_list[1] = SVPD_UNIT_SERIAL_NUMBER;
9228         /* Device Identification */
9229         pages->page_list[2] = SVPD_DEVICE_ID;
9230         /* Block limits */
9231         pages->page_list[3] = SVPD_BLOCK_LIMITS;
9232         /* Logical Block Provisioning */
9233         pages->page_list[4] = SVPD_LBP;
9234
9235         ctsio->scsi_status = SCSI_STATUS_OK;
9236
9237         ctsio->be_move_done = ctl_config_move_done;
9238         ctl_datamove((union ctl_io *)ctsio);
9239
9240         return (CTL_RETVAL_COMPLETE);
9241 }
9242
9243 static int
9244 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9245 {
9246         struct scsi_vpd_unit_serial_number *sn_ptr;
9247         struct ctl_lun *lun;
9248 #ifndef CTL_USE_BACKEND_SN
9249         char tmpstr[32];
9250 #endif
9251
9252         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9253
9254         ctsio->kern_data_ptr = malloc(sizeof(*sn_ptr), M_CTL, M_WAITOK | M_ZERO);
9255         sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9256         ctsio->kern_sg_entries = 0;
9257
9258         if (sizeof(*sn_ptr) < alloc_len) {
9259                 ctsio->residual = alloc_len - sizeof(*sn_ptr);
9260                 ctsio->kern_data_len = sizeof(*sn_ptr);
9261                 ctsio->kern_total_len = sizeof(*sn_ptr);
9262         } else {
9263                 ctsio->residual = 0;
9264                 ctsio->kern_data_len = alloc_len;
9265                 ctsio->kern_total_len = alloc_len;
9266         }
9267         ctsio->kern_data_resid = 0;
9268         ctsio->kern_rel_offset = 0;
9269         ctsio->kern_sg_entries = 0;
9270
9271         /*
9272          * The control device is always connected.  The disk device, on the
9273          * other hand, may not be online all the time.  Need to change this
9274          * to figure out whether the disk device is actually online or not.
9275          */
9276         if (lun != NULL)
9277                 sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9278                                   lun->be_lun->lun_type;
9279         else
9280                 sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9281
9282         sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9283         sn_ptr->length = ctl_min(sizeof(*sn_ptr) - 4, CTL_SN_LEN);
9284 #ifdef CTL_USE_BACKEND_SN
9285         /*
9286          * If we don't have a LUN, we just leave the serial number as
9287          * all spaces.
9288          */
9289         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9290         if (lun != NULL) {
9291                 strncpy((char *)sn_ptr->serial_num,
9292                         (char *)lun->be_lun->serial_num, CTL_SN_LEN);
9293         }
9294 #else
9295         /*
9296          * Note that we're using a non-unique serial number here,
9297          */
9298         snprintf(tmpstr, sizeof(tmpstr), "MYSERIALNUMIS000");
9299         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9300         strncpy(sn_ptr->serial_num, tmpstr, ctl_min(CTL_SN_LEN,
9301                 ctl_min(sizeof(tmpstr), sizeof(*sn_ptr) - 4)));
9302 #endif
9303         ctsio->scsi_status = SCSI_STATUS_OK;
9304
9305         ctsio->be_move_done = ctl_config_move_done;
9306         ctl_datamove((union ctl_io *)ctsio);
9307
9308         return (CTL_RETVAL_COMPLETE);
9309 }
9310
9311
9312 static int
9313 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9314 {
9315         struct scsi_vpd_device_id *devid_ptr;
9316         struct scsi_vpd_id_descriptor *desc, *desc1;
9317         struct scsi_vpd_id_descriptor *desc2, *desc3; /* for types 4h and 5h */
9318         struct scsi_vpd_id_t10 *t10id;
9319         struct ctl_softc *ctl_softc;
9320         struct ctl_lun *lun;
9321         struct ctl_frontend *fe;
9322 #ifndef CTL_USE_BACKEND_SN
9323         char tmpstr[32];
9324 #endif /* CTL_USE_BACKEND_SN */
9325         int devid_len;
9326
9327         ctl_softc = control_softc;
9328
9329         mtx_lock(&ctl_softc->ctl_lock);
9330         fe = ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)];
9331         mtx_unlock(&ctl_softc->ctl_lock);
9332
9333         if (fe->devid != NULL)
9334                 return ((fe->devid)(ctsio, alloc_len));
9335
9336         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9337
9338         devid_len = sizeof(struct scsi_vpd_device_id) +
9339                 sizeof(struct scsi_vpd_id_descriptor) +
9340                 sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN +
9341                 sizeof(struct scsi_vpd_id_descriptor) + CTL_WWPN_LEN +
9342                 sizeof(struct scsi_vpd_id_descriptor) +
9343                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9344                 sizeof(struct scsi_vpd_id_descriptor) +
9345                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9346
9347         ctsio->kern_data_ptr = malloc(devid_len, M_CTL, M_WAITOK | M_ZERO);
9348         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9349         ctsio->kern_sg_entries = 0;
9350
9351         if (devid_len < alloc_len) {
9352                 ctsio->residual = alloc_len - devid_len;
9353                 ctsio->kern_data_len = devid_len;
9354                 ctsio->kern_total_len = devid_len;
9355         } else {
9356                 ctsio->residual = 0;
9357                 ctsio->kern_data_len = alloc_len;
9358                 ctsio->kern_total_len = alloc_len;
9359         }
9360         ctsio->kern_data_resid = 0;
9361         ctsio->kern_rel_offset = 0;
9362         ctsio->kern_sg_entries = 0;
9363
9364         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
9365         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
9366         desc1 = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9367                 sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN);
9368         desc2 = (struct scsi_vpd_id_descriptor *)(&desc1->identifier[0] +
9369                   CTL_WWPN_LEN);
9370         desc3 = (struct scsi_vpd_id_descriptor *)(&desc2->identifier[0] +
9371                  sizeof(struct scsi_vpd_id_rel_trgt_port_id));
9372
9373         /*
9374          * The control device is always connected.  The disk device, on the
9375          * other hand, may not be online all the time.
9376          */
9377         if (lun != NULL)
9378                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9379                                      lun->be_lun->lun_type;
9380         else
9381                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9382
9383         devid_ptr->page_code = SVPD_DEVICE_ID;
9384
9385         scsi_ulto2b(devid_len - 4, devid_ptr->length);
9386
9387         mtx_lock(&ctl_softc->ctl_lock);
9388
9389         /*
9390          * For Fibre channel,
9391          */
9392         if (fe->port_type == CTL_PORT_FC)
9393         {
9394                 desc->proto_codeset = (SCSI_PROTO_FC << 4) |
9395                                       SVPD_ID_CODESET_ASCII;
9396                 desc1->proto_codeset = (SCSI_PROTO_FC << 4) |
9397                               SVPD_ID_CODESET_BINARY;
9398         }
9399         else
9400         {
9401                 desc->proto_codeset = (SCSI_PROTO_SPI << 4) |
9402                                       SVPD_ID_CODESET_ASCII;
9403                 desc1->proto_codeset = (SCSI_PROTO_SPI << 4) |
9404                               SVPD_ID_CODESET_BINARY;
9405         }
9406         desc2->proto_codeset = desc3->proto_codeset = desc1->proto_codeset;
9407         mtx_unlock(&ctl_softc->ctl_lock);
9408
9409         /*
9410          * We're using a LUN association here.  i.e., this device ID is a
9411          * per-LUN identifier.
9412          */
9413         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
9414         desc->length = sizeof(*t10id) + CTL_DEVID_LEN;
9415         strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
9416
9417         /*
9418          * desc1 is for the WWPN which is a port asscociation.
9419          */
9420         desc1->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT | SVPD_ID_TYPE_NAA;
9421         desc1->length = CTL_WWPN_LEN;
9422         /* XXX Call Reggie's get_WWNN func here then add port # to the end */
9423         /* For testing just create the WWPN */
9424 #if 0
9425         ddb_GetWWNN((char *)desc1->identifier);
9426
9427         /* NOTE: if the port is 0 or 8 we don't want to subtract 1 */
9428         /* This is so Copancontrol will return something sane */
9429         if (ctsio->io_hdr.nexus.targ_port!=0 &&
9430             ctsio->io_hdr.nexus.targ_port!=8)
9431                 desc1->identifier[7] += ctsio->io_hdr.nexus.targ_port-1;
9432         else
9433                 desc1->identifier[7] += ctsio->io_hdr.nexus.targ_port;
9434 #endif
9435
9436         be64enc(desc1->identifier, fe->wwpn);
9437
9438         /*
9439          * desc2 is for the Relative Target Port(type 4h) identifier
9440          */
9441         desc2->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT
9442                          | SVPD_ID_TYPE_RELTARG;
9443         desc2->length = 4;
9444 //#if 0
9445         /* NOTE: if the port is 0 or 8 we don't want to subtract 1 */
9446         /* This is so Copancontrol will return something sane */
9447         if (ctsio->io_hdr.nexus.targ_port!=0 &&
9448             ctsio->io_hdr.nexus.targ_port!=8)
9449                 desc2->identifier[3] = ctsio->io_hdr.nexus.targ_port - 1;
9450         else
9451                 desc2->identifier[3] = ctsio->io_hdr.nexus.targ_port;
9452 //#endif
9453
9454         /*
9455          * desc3 is for the Target Port Group(type 5h) identifier
9456          */
9457         desc3->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT
9458                          | SVPD_ID_TYPE_TPORTGRP;
9459         desc3->length = 4;
9460         if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS || ctl_is_single)
9461                 desc3->identifier[3] = 1;
9462         else
9463                 desc3->identifier[3] = 2;
9464
9465 #ifdef CTL_USE_BACKEND_SN
9466         /*
9467          * If we've actually got a backend, copy the device id from the
9468          * per-LUN data.  Otherwise, set it to all spaces.
9469          */
9470         if (lun != NULL) {
9471                 /*
9472                  * Copy the backend's LUN ID.
9473                  */
9474                 strncpy((char *)t10id->vendor_spec_id,
9475                         (char *)lun->be_lun->device_id, CTL_DEVID_LEN);
9476         } else {
9477                 /*
9478                  * No backend, set this to spaces.
9479                  */
9480                 memset(t10id->vendor_spec_id, 0x20, CTL_DEVID_LEN);
9481         }
9482 #else
9483         snprintf(tmpstr, sizeof(tmpstr), "MYDEVICEIDIS%4d",
9484                  (lun != NULL) ?  (int)lun->lun : 0);
9485         strncpy(t10id->vendor_spec_id, tmpstr, ctl_min(CTL_DEVID_LEN,
9486                 sizeof(tmpstr)));
9487 #endif
9488
9489         ctsio->scsi_status = SCSI_STATUS_OK;
9490
9491         ctsio->be_move_done = ctl_config_move_done;
9492         ctl_datamove((union ctl_io *)ctsio);
9493
9494         return (CTL_RETVAL_COMPLETE);
9495 }
9496
9497 static int
9498 ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
9499 {
9500         struct scsi_vpd_block_limits *bl_ptr;
9501         struct ctl_lun *lun;
9502         int bs;
9503
9504         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9505         bs = lun->be_lun->blocksize;
9506
9507         ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
9508         bl_ptr = (struct scsi_vpd_block_limits *)ctsio->kern_data_ptr;
9509         ctsio->kern_sg_entries = 0;
9510
9511         if (sizeof(*bl_ptr) < alloc_len) {
9512                 ctsio->residual = alloc_len - sizeof(*bl_ptr);
9513                 ctsio->kern_data_len = sizeof(*bl_ptr);
9514                 ctsio->kern_total_len = sizeof(*bl_ptr);
9515         } else {
9516                 ctsio->residual = 0;
9517                 ctsio->kern_data_len = alloc_len;
9518                 ctsio->kern_total_len = alloc_len;
9519         }
9520         ctsio->kern_data_resid = 0;
9521         ctsio->kern_rel_offset = 0;
9522         ctsio->kern_sg_entries = 0;
9523
9524         /*
9525          * The control device is always connected.  The disk device, on the
9526          * other hand, may not be online all the time.  Need to change this
9527          * to figure out whether the disk device is actually online or not.
9528          */
9529         if (lun != NULL)
9530                 bl_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9531                                   lun->be_lun->lun_type;
9532         else
9533                 bl_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9534
9535         bl_ptr->page_code = SVPD_BLOCK_LIMITS;
9536         scsi_ulto2b(sizeof(*bl_ptr), bl_ptr->page_length);
9537         scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
9538         scsi_ulto4b(MAXPHYS / bs, bl_ptr->opt_txfer_len);
9539         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
9540                 scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
9541                 scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_blk_cnt);
9542         }
9543         scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
9544
9545         ctsio->scsi_status = SCSI_STATUS_OK;
9546         ctsio->be_move_done = ctl_config_move_done;
9547         ctl_datamove((union ctl_io *)ctsio);
9548
9549         return (CTL_RETVAL_COMPLETE);
9550 }
9551
9552 static int
9553 ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
9554 {
9555         struct scsi_vpd_logical_block_prov *lbp_ptr;
9556         struct ctl_lun *lun;
9557         int bs;
9558
9559         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9560         bs = lun->be_lun->blocksize;
9561
9562         ctsio->kern_data_ptr = malloc(sizeof(*lbp_ptr), M_CTL, M_WAITOK | M_ZERO);
9563         lbp_ptr = (struct scsi_vpd_logical_block_prov *)ctsio->kern_data_ptr;
9564         ctsio->kern_sg_entries = 0;
9565
9566         if (sizeof(*lbp_ptr) < alloc_len) {
9567                 ctsio->residual = alloc_len - sizeof(*lbp_ptr);
9568                 ctsio->kern_data_len = sizeof(*lbp_ptr);
9569                 ctsio->kern_total_len = sizeof(*lbp_ptr);
9570         } else {
9571                 ctsio->residual = 0;
9572                 ctsio->kern_data_len = alloc_len;
9573                 ctsio->kern_total_len = alloc_len;
9574         }
9575         ctsio->kern_data_resid = 0;
9576         ctsio->kern_rel_offset = 0;
9577         ctsio->kern_sg_entries = 0;
9578
9579         /*
9580          * The control device is always connected.  The disk device, on the
9581          * other hand, may not be online all the time.  Need to change this
9582          * to figure out whether the disk device is actually online or not.
9583          */
9584         if (lun != NULL)
9585                 lbp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9586                                   lun->be_lun->lun_type;
9587         else
9588                 lbp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9589
9590         lbp_ptr->page_code = SVPD_LBP;
9591         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
9592                 lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 | SVPD_LBP_WS10;
9593
9594         ctsio->scsi_status = SCSI_STATUS_OK;
9595         ctsio->be_move_done = ctl_config_move_done;
9596         ctl_datamove((union ctl_io *)ctsio);
9597
9598         return (CTL_RETVAL_COMPLETE);
9599 }
9600
9601 static int
9602 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
9603 {
9604         struct scsi_inquiry *cdb;
9605         struct ctl_lun *lun;
9606         int alloc_len, retval;
9607
9608         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9609         cdb = (struct scsi_inquiry *)ctsio->cdb;
9610
9611         retval = CTL_RETVAL_COMPLETE;
9612
9613         alloc_len = scsi_2btoul(cdb->length);
9614
9615         switch (cdb->page_code) {
9616         case SVPD_SUPPORTED_PAGES:
9617                 retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
9618                 break;
9619         case SVPD_UNIT_SERIAL_NUMBER:
9620                 retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
9621                 break;
9622         case SVPD_DEVICE_ID:
9623                 retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
9624                 break;
9625         case SVPD_BLOCK_LIMITS:
9626                 retval = ctl_inquiry_evpd_block_limits(ctsio, alloc_len);
9627                 break;
9628         case SVPD_LBP:
9629                 retval = ctl_inquiry_evpd_lbp(ctsio, alloc_len);
9630                 break;
9631         default:
9632                 ctl_set_invalid_field(ctsio,
9633                                       /*sks_valid*/ 1,
9634                                       /*command*/ 1,
9635                                       /*field*/ 2,
9636                                       /*bit_valid*/ 0,
9637                                       /*bit*/ 0);
9638                 ctl_done((union ctl_io *)ctsio);
9639                 retval = CTL_RETVAL_COMPLETE;
9640                 break;
9641         }
9642
9643         return (retval);
9644 }
9645
9646 static int
9647 ctl_inquiry_std(struct ctl_scsiio *ctsio)
9648 {
9649         struct scsi_inquiry_data *inq_ptr;
9650         struct scsi_inquiry *cdb;
9651         struct ctl_softc *ctl_softc;
9652         struct ctl_lun *lun;
9653         uint32_t alloc_len;
9654         int is_fc;
9655
9656         ctl_softc = control_softc;
9657
9658         /*
9659          * Figure out whether we're talking to a Fibre Channel port or not.
9660          * We treat the ioctl front end, and any SCSI adapters, as packetized
9661          * SCSI front ends.
9662          */
9663         mtx_lock(&ctl_softc->ctl_lock);
9664         if (ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)]->port_type !=
9665             CTL_PORT_FC)
9666                 is_fc = 0;
9667         else
9668                 is_fc = 1;
9669         mtx_unlock(&ctl_softc->ctl_lock);
9670
9671         lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9672         cdb = (struct scsi_inquiry *)ctsio->cdb;
9673         alloc_len = scsi_2btoul(cdb->length);
9674
9675         /*
9676          * We malloc the full inquiry data size here and fill it
9677          * in.  If the user only asks for less, we'll give him
9678          * that much.
9679          */
9680         ctsio->kern_data_ptr = malloc(sizeof(*inq_ptr), M_CTL, M_WAITOK | M_ZERO);
9681         inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
9682         ctsio->kern_sg_entries = 0;
9683         ctsio->kern_data_resid = 0;
9684         ctsio->kern_rel_offset = 0;
9685
9686         if (sizeof(*inq_ptr) < alloc_len) {
9687                 ctsio->residual = alloc_len - sizeof(*inq_ptr);
9688                 ctsio->kern_data_len = sizeof(*inq_ptr);
9689                 ctsio->kern_total_len = sizeof(*inq_ptr);
9690         } else {
9691                 ctsio->residual = 0;
9692                 ctsio->kern_data_len = alloc_len;
9693                 ctsio->kern_total_len = alloc_len;
9694         }
9695
9696         /*
9697          * If we have a LUN configured, report it as connected.  Otherwise,
9698          * report that it is offline or no device is supported, depending
9699          * on the value of inquiry_pq_no_lun.
9700          *
9701          * According to the spec (SPC-4 r34), the peripheral qualifier
9702          * SID_QUAL_LU_OFFLINE (001b) is used in the following scenario:
9703          *
9704          * "A peripheral device having the specified peripheral device type
9705          * is not connected to this logical unit. However, the device
9706          * server is capable of supporting the specified peripheral device
9707          * type on this logical unit."
9708          *
9709          * According to the same spec, the peripheral qualifier
9710          * SID_QUAL_BAD_LU (011b) is used in this scenario:
9711          *
9712          * "The device server is not capable of supporting a peripheral
9713          * device on this logical unit. For this peripheral qualifier the
9714          * peripheral device type shall be set to 1Fh. All other peripheral
9715          * device type values are reserved for this peripheral qualifier."
9716          *
9717          * Given the text, it would seem that we probably want to report that
9718          * the LUN is offline here.  There is no LUN connected, but we can
9719          * support a LUN at the given LUN number.
9720          *
9721          * In the real world, though, it sounds like things are a little
9722          * different:
9723          *
9724          * - Linux, when presented with a LUN with the offline peripheral
9725          *   qualifier, will create an sg driver instance for it.  So when
9726          *   you attach it to CTL, you wind up with a ton of sg driver
9727          *   instances.  (One for every LUN that Linux bothered to probe.)
9728          *   Linux does this despite the fact that it issues a REPORT LUNs
9729          *   to LUN 0 to get the inventory of supported LUNs.
9730          *
9731          * - There is other anecdotal evidence (from Emulex folks) about
9732          *   arrays that use the offline peripheral qualifier for LUNs that
9733          *   are on the "passive" path in an active/passive array.
9734          *
9735          * So the solution is provide a hopefully reasonable default
9736          * (return bad/no LUN) and allow the user to change the behavior
9737          * with a tunable/sysctl variable.
9738          */
9739         if (lun != NULL)
9740                 inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9741                                   lun->be_lun->lun_type;
9742         else if (ctl_softc->inquiry_pq_no_lun == 0)
9743                 inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9744         else
9745                 inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
9746
9747         /* RMB in byte 2 is 0 */
9748         inq_ptr->version = SCSI_REV_SPC3;
9749
9750         /*
9751          * According to SAM-3, even if a device only supports a single
9752          * level of LUN addressing, it should still set the HISUP bit:
9753          *
9754          * 4.9.1 Logical unit numbers overview
9755          *
9756          * All logical unit number formats described in this standard are
9757          * hierarchical in structure even when only a single level in that
9758          * hierarchy is used. The HISUP bit shall be set to one in the
9759          * standard INQUIRY data (see SPC-2) when any logical unit number
9760          * format described in this standard is used.  Non-hierarchical
9761          * formats are outside the scope of this standard.
9762          *
9763          * Therefore we set the HiSup bit here.
9764          *
9765          * The reponse format is 2, per SPC-3.
9766          */
9767         inq_ptr->response_format = SID_HiSup | 2;
9768
9769         inq_ptr->additional_length = sizeof(*inq_ptr) - 4;
9770         CTL_DEBUG_PRINT(("additional_length = %d\n",
9771                          inq_ptr->additional_length));
9772
9773         inq_ptr->spc3_flags = SPC3_SID_TPGS_IMPLICIT;
9774         /* 16 bit addressing */
9775         if (is_fc == 0)
9776                 inq_ptr->spc2_flags = SPC2_SID_ADDR16;
9777         /* XXX set the SID_MultiP bit here if we're actually going to
9778            respond on multiple ports */
9779         inq_ptr->spc2_flags |= SPC2_SID_MultiP;
9780
9781         /* 16 bit data bus, synchronous transfers */
9782         /* XXX these flags don't apply for FC */
9783         if (is_fc == 0)
9784                 inq_ptr->flags = SID_WBus16 | SID_Sync;
9785         /*
9786          * XXX KDM do we want to support tagged queueing on the control
9787          * device at all?
9788          */
9789         if ((lun == NULL)
9790          || (lun->be_lun->lun_type != T_PROCESSOR))
9791                 inq_ptr->flags |= SID_CmdQue;
9792         /*
9793          * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
9794          * We have 8 bytes for the vendor name, and 16 bytes for the device
9795          * name and 4 bytes for the revision.
9796          */
9797         strncpy(inq_ptr->vendor, CTL_VENDOR, sizeof(inq_ptr->vendor));
9798         if (lun == NULL) {
9799                 strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
9800         } else {
9801                 switch (lun->be_lun->lun_type) {
9802                 case T_DIRECT:
9803                         strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
9804                         break;
9805                 case T_PROCESSOR:
9806                         strcpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT);
9807                         break;
9808                 default:
9809                         strcpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT);
9810                         break;
9811                 }
9812         }
9813
9814         /*
9815          * XXX make this a macro somewhere so it automatically gets
9816          * incremented when we make changes.
9817          */
9818         strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
9819
9820         /*
9821          * For parallel SCSI, we support double transition and single
9822          * transition clocking.  We also support QAS (Quick Arbitration
9823          * and Selection) and Information Unit transfers on both the
9824          * control and array devices.
9825          */
9826         if (is_fc == 0)
9827                 inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
9828                                     SID_SPI_IUS;
9829
9830         /* SAM-3 */
9831         scsi_ulto2b(0x0060, inq_ptr->version1);
9832         /* SPC-3 (no version claimed) XXX should we claim a version? */
9833         scsi_ulto2b(0x0300, inq_ptr->version2);
9834         if (is_fc) {
9835                 /* FCP-2 ANSI INCITS.350:2003 */
9836                 scsi_ulto2b(0x0917, inq_ptr->version3);
9837         } else {
9838                 /* SPI-4 ANSI INCITS.362:200x */
9839                 scsi_ulto2b(0x0B56, inq_ptr->version3);
9840         }
9841
9842         if (lun == NULL) {
9843                 /* SBC-2 (no version claimed) XXX should we claim a version? */
9844                 scsi_ulto2b(0x0320, inq_ptr->version4);
9845         } else {
9846                 switch (lun->be_lun->lun_type) {
9847                 case T_DIRECT:
9848                         /*
9849                          * SBC-2 (no version claimed) XXX should we claim a
9850                          * version?
9851                          */
9852                         scsi_ulto2b(0x0320, inq_ptr->version4);
9853                         break;
9854                 case T_PROCESSOR:
9855                 default:
9856                         break;
9857                 }
9858         }
9859
9860         ctsio->scsi_status = SCSI_STATUS_OK;
9861         if (ctsio->kern_data_len > 0) {
9862                 ctsio->be_move_done = ctl_config_move_done;
9863                 ctl_datamove((union ctl_io *)ctsio);
9864         } else {
9865                 ctsio->io_hdr.status = CTL_SUCCESS;
9866                 ctl_done((union ctl_io *)ctsio);
9867         }
9868
9869         return (CTL_RETVAL_COMPLETE);
9870 }
9871
9872 int
9873 ctl_inquiry(struct ctl_scsiio *ctsio)
9874 {
9875         struct scsi_inquiry *cdb;
9876         int retval;
9877
9878         cdb = (struct scsi_inquiry *)ctsio->cdb;
9879
9880         retval = 0;
9881
9882         CTL_DEBUG_PRINT(("ctl_inquiry\n"));
9883
9884         /*
9885          * Right now, we don't support the CmdDt inquiry information.
9886          * This would be nice to support in the future.  When we do
9887          * support it, we should change this test so that it checks to make
9888          * sure SI_EVPD and SI_CMDDT aren't both set at the same time.
9889          */
9890 #ifdef notyet
9891         if (((cdb->byte2 & SI_EVPD)
9892          && (cdb->byte2 & SI_CMDDT)))
9893 #endif
9894         if (cdb->byte2 & SI_CMDDT) {
9895                 /*
9896                  * Point to the SI_CMDDT bit.  We might change this
9897                  * when we support SI_CMDDT, but since both bits would be
9898                  * "wrong", this should probably just stay as-is then.
9899                  */
9900                 ctl_set_invalid_field(ctsio,
9901                                       /*sks_valid*/ 1,
9902                                       /*command*/ 1,
9903                                       /*field*/ 1,
9904                                       /*bit_valid*/ 1,
9905                                       /*bit*/ 1);
9906                 ctl_done((union ctl_io *)ctsio);
9907                 return (CTL_RETVAL_COMPLETE);
9908         }
9909         if (cdb->byte2 & SI_EVPD)
9910                 retval = ctl_inquiry_evpd(ctsio);
9911 #ifdef notyet
9912         else if (cdb->byte2 & SI_CMDDT)
9913                 retval = ctl_inquiry_cmddt(ctsio);
9914 #endif
9915         else
9916                 retval = ctl_inquiry_std(ctsio);
9917
9918         return (retval);
9919 }
9920
9921 /*
9922  * For known CDB types, parse the LBA and length.
9923  */
9924 static int
9925 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len)
9926 {
9927         if (io->io_hdr.io_type != CTL_IO_SCSI)
9928                 return (1);
9929
9930         switch (io->scsiio.cdb[0]) {
9931         case READ_6:
9932         case WRITE_6: {
9933                 struct scsi_rw_6 *cdb;
9934
9935                 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
9936
9937                 *lba = scsi_3btoul(cdb->addr);
9938                 /* only 5 bits are valid in the most significant address byte */
9939                 *lba &= 0x1fffff;
9940                 *len = cdb->length;
9941                 break;
9942         }
9943         case READ_10:
9944         case WRITE_10: {
9945                 struct scsi_rw_10 *cdb;
9946
9947                 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
9948
9949                 *lba = scsi_4btoul(cdb->addr);
9950                 *len = scsi_2btoul(cdb->length);
9951                 break;
9952         }
9953         case WRITE_VERIFY_10: {
9954                 struct scsi_write_verify_10 *cdb;
9955
9956                 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
9957
9958                 *lba = scsi_4btoul(cdb->addr);
9959                 *len = scsi_2btoul(cdb->length);
9960                 break;
9961         }
9962         case READ_12:
9963         case WRITE_12: {
9964                 struct scsi_rw_12 *cdb;
9965
9966                 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
9967
9968                 *lba = scsi_4btoul(cdb->addr);
9969                 *len = scsi_4btoul(cdb->length);
9970                 break;
9971         }
9972         case WRITE_VERIFY_12: {
9973                 struct scsi_write_verify_12 *cdb;
9974
9975                 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
9976
9977                 *lba = scsi_4btoul(cdb->addr);
9978                 *len = scsi_4btoul(cdb->length);
9979                 break;
9980         }
9981         case READ_16:
9982         case WRITE_16: {
9983                 struct scsi_rw_16 *cdb;
9984
9985                 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
9986
9987                 *lba = scsi_8btou64(cdb->addr);
9988                 *len = scsi_4btoul(cdb->length);
9989                 break;
9990         }
9991         case WRITE_VERIFY_16: {
9992                 struct scsi_write_verify_16 *cdb;
9993
9994                 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
9995
9996
9997                 *lba = scsi_8btou64(cdb->addr);
9998                 *len = scsi_4btoul(cdb->length);
9999                 break;
10000         }
10001         case WRITE_SAME_10: {
10002                 struct scsi_write_same_10 *cdb;
10003
10004                 cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10005
10006                 *lba = scsi_4btoul(cdb->addr);
10007                 *len = scsi_2btoul(cdb->length);
10008                 break;
10009         }
10010         case WRITE_SAME_16: {
10011                 struct scsi_write_same_16 *cdb;
10012
10013                 cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10014
10015                 *lba = scsi_8btou64(cdb->addr);
10016                 *len = scsi_4btoul(cdb->length);
10017                 break;
10018         }
10019         default:
10020                 return (1);
10021                 break; /* NOTREACHED */
10022         }
10023
10024         return (0);
10025 }
10026
10027 static ctl_action
10028 ctl_extent_check_lba(uint64_t lba1, uint32_t len1, uint64_t lba2, uint32_t len2)
10029 {
10030         uint64_t endlba1, endlba2;
10031
10032         endlba1 = lba1 + len1 - 1;
10033         endlba2 = lba2 + len2 - 1;
10034
10035         if ((endlba1 < lba2)
10036          || (endlba2 < lba1))
10037                 return (CTL_ACTION_PASS);
10038         else
10039                 return (CTL_ACTION_BLOCK);
10040 }
10041
10042 static ctl_action
10043 ctl_extent_check(union ctl_io *io1, union ctl_io *io2)
10044 {
10045         uint64_t lba1, lba2;
10046         uint32_t len1, len2;
10047         int retval;
10048
10049         retval = ctl_get_lba_len(io1, &lba1, &len1);
10050         if (retval != 0)
10051                 return (CTL_ACTION_ERROR);
10052
10053         retval = ctl_get_lba_len(io2, &lba2, &len2);
10054         if (retval != 0)
10055                 return (CTL_ACTION_ERROR);
10056
10057         return (ctl_extent_check_lba(lba1, len1, lba2, len2));
10058 }
10059
10060 static ctl_action
10061 ctl_check_for_blockage(union ctl_io *pending_io, union ctl_io *ooa_io)
10062 {
10063         struct ctl_cmd_entry *pending_entry, *ooa_entry;
10064         ctl_serialize_action *serialize_row;
10065
10066         /*
10067          * The initiator attempted multiple untagged commands at the same
10068          * time.  Can't do that.
10069          */
10070         if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10071          && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10072          && ((pending_io->io_hdr.nexus.targ_port ==
10073               ooa_io->io_hdr.nexus.targ_port)
10074           && (pending_io->io_hdr.nexus.initid.id ==
10075               ooa_io->io_hdr.nexus.initid.id))
10076          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
10077                 return (CTL_ACTION_OVERLAP);
10078
10079         /*
10080          * The initiator attempted to send multiple tagged commands with
10081          * the same ID.  (It's fine if different initiators have the same
10082          * tag ID.)
10083          *
10084          * Even if all of those conditions are true, we don't kill the I/O
10085          * if the command ahead of us has been aborted.  We won't end up
10086          * sending it to the FETD, and it's perfectly legal to resend a
10087          * command with the same tag number as long as the previous
10088          * instance of this tag number has been aborted somehow.
10089          */
10090         if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10091          && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10092          && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
10093          && ((pending_io->io_hdr.nexus.targ_port ==
10094               ooa_io->io_hdr.nexus.targ_port)
10095           && (pending_io->io_hdr.nexus.initid.id ==
10096               ooa_io->io_hdr.nexus.initid.id))
10097          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
10098                 return (CTL_ACTION_OVERLAP_TAG);
10099
10100         /*
10101          * If we get a head of queue tag, SAM-3 says that we should
10102          * immediately execute it.
10103          *
10104          * What happens if this command would normally block for some other
10105          * reason?  e.g. a request sense with a head of queue tag
10106          * immediately after a write.  Normally that would block, but this
10107          * will result in its getting executed immediately...
10108          *
10109          * We currently return "pass" instead of "skip", so we'll end up
10110          * going through the rest of the queue to check for overlapped tags.
10111          *
10112          * XXX KDM check for other types of blockage first??
10113          */
10114         if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10115                 return (CTL_ACTION_PASS);
10116
10117         /*
10118          * Ordered tags have to block until all items ahead of them
10119          * have completed.  If we get called with an ordered tag, we always
10120          * block, if something else is ahead of us in the queue.
10121          */
10122         if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
10123                 return (CTL_ACTION_BLOCK);
10124
10125         /*
10126          * Simple tags get blocked until all head of queue and ordered tags
10127          * ahead of them have completed.  I'm lumping untagged commands in
10128          * with simple tags here.  XXX KDM is that the right thing to do?
10129          */
10130         if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10131           || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
10132          && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10133           || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
10134                 return (CTL_ACTION_BLOCK);
10135
10136         pending_entry = &ctl_cmd_table[pending_io->scsiio.cdb[0]];
10137         ooa_entry = &ctl_cmd_table[ooa_io->scsiio.cdb[0]];
10138
10139         serialize_row = ctl_serialize_table[ooa_entry->seridx];
10140
10141         switch (serialize_row[pending_entry->seridx]) {
10142         case CTL_SER_BLOCK:
10143                 return (CTL_ACTION_BLOCK);
10144                 break; /* NOTREACHED */
10145         case CTL_SER_EXTENT:
10146                 return (ctl_extent_check(pending_io, ooa_io));
10147                 break; /* NOTREACHED */
10148         case CTL_SER_PASS:
10149                 return (CTL_ACTION_PASS);
10150                 break; /* NOTREACHED */
10151         case CTL_SER_SKIP:
10152                 return (CTL_ACTION_SKIP);
10153                 break;
10154         default:
10155                 panic("invalid serialization value %d",
10156                       serialize_row[pending_entry->seridx]);
10157                 break; /* NOTREACHED */
10158         }
10159
10160         return (CTL_ACTION_ERROR);
10161 }
10162
10163 /*
10164  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
10165  * Assumptions:
10166  * - pending_io is generally either incoming, or on the blocked queue
10167  * - starting I/O is the I/O we want to start the check with.
10168  */
10169 static ctl_action
10170 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
10171               union ctl_io *starting_io)
10172 {
10173         union ctl_io *ooa_io;
10174         ctl_action action;
10175
10176         mtx_assert(&control_softc->ctl_lock, MA_OWNED);
10177
10178         /*
10179          * Run back along the OOA queue, starting with the current
10180          * blocked I/O and going through every I/O before it on the
10181          * queue.  If starting_io is NULL, we'll just end up returning
10182          * CTL_ACTION_PASS.
10183          */
10184         for (ooa_io = starting_io; ooa_io != NULL;
10185              ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
10186              ooa_links)){
10187
10188                 /*
10189                  * This routine just checks to see whether
10190                  * cur_blocked is blocked by ooa_io, which is ahead
10191                  * of it in the queue.  It doesn't queue/dequeue
10192                  * cur_blocked.
10193                  */
10194                 action = ctl_check_for_blockage(pending_io, ooa_io);
10195                 switch (action) {
10196                 case CTL_ACTION_BLOCK:
10197                 case CTL_ACTION_OVERLAP:
10198                 case CTL_ACTION_OVERLAP_TAG:
10199                 case CTL_ACTION_SKIP:
10200                 case CTL_ACTION_ERROR:
10201                         return (action);
10202                         break; /* NOTREACHED */
10203                 case CTL_ACTION_PASS:
10204                         break;
10205                 default:
10206                         panic("invalid action %d", action);
10207                         break;  /* NOTREACHED */
10208                 }
10209         }
10210
10211         return (CTL_ACTION_PASS);
10212 }
10213
10214 /*
10215  * Assumptions:
10216  * - An I/O has just completed, and has been removed from the per-LUN OOA
10217  *   queue, so some items on the blocked queue may now be unblocked.
10218  */
10219 static int
10220 ctl_check_blocked(struct ctl_lun *lun)
10221 {
10222         union ctl_io *cur_blocked, *next_blocked;
10223
10224         mtx_assert(&control_softc->ctl_lock, MA_OWNED);
10225
10226         /*
10227          * Run forward from the head of the blocked queue, checking each
10228          * entry against the I/Os prior to it on the OOA queue to see if
10229          * there is still any blockage.
10230          *
10231          * We cannot use the TAILQ_FOREACH() macro, because it can't deal
10232          * with our removing a variable on it while it is traversing the
10233          * list.
10234          */
10235         for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
10236              cur_blocked != NULL; cur_blocked = next_blocked) {
10237                 union ctl_io *prev_ooa;
10238                 ctl_action action;
10239
10240                 next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
10241                                                           blocked_links);
10242
10243                 prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
10244                                                       ctl_ooaq, ooa_links);
10245
10246                 /*
10247                  * If cur_blocked happens to be the first item in the OOA
10248                  * queue now, prev_ooa will be NULL, and the action
10249                  * returned will just be CTL_ACTION_PASS.
10250                  */
10251                 action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
10252
10253                 switch (action) {
10254                 case CTL_ACTION_BLOCK:
10255                         /* Nothing to do here, still blocked */
10256                         break;
10257                 case CTL_ACTION_OVERLAP:
10258                 case CTL_ACTION_OVERLAP_TAG:
10259                         /*
10260                          * This shouldn't happen!  In theory we've already
10261                          * checked this command for overlap...
10262                          */
10263                         break;
10264                 case CTL_ACTION_PASS:
10265                 case CTL_ACTION_SKIP: {
10266                         struct ctl_softc *softc;
10267                         struct ctl_cmd_entry *entry;
10268                         uint32_t initidx;
10269                         uint8_t opcode;
10270                         int isc_retval;
10271
10272                         /*
10273                          * The skip case shouldn't happen, this transaction
10274                          * should have never made it onto the blocked queue.
10275                          */
10276                         /*
10277                          * This I/O is no longer blocked, we can remove it
10278                          * from the blocked queue.  Since this is a TAILQ
10279                          * (doubly linked list), we can do O(1) removals
10280                          * from any place on the list.
10281                          */
10282                         TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
10283                                      blocked_links);
10284                         cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10285
10286                         if (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC){
10287                                 /*
10288                                  * Need to send IO back to original side to
10289                                  * run
10290                                  */
10291                                 union ctl_ha_msg msg_info;
10292
10293                                 msg_info.hdr.original_sc =
10294                                         cur_blocked->io_hdr.original_sc;
10295                                 msg_info.hdr.serializing_sc = cur_blocked;
10296                                 msg_info.hdr.msg_type = CTL_MSG_R2R;
10297                                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10298                                      &msg_info, sizeof(msg_info), 0)) >
10299                                      CTL_HA_STATUS_SUCCESS) {
10300                                         printf("CTL:Check Blocked error from "
10301                                                "ctl_ha_msg_send %d\n",
10302                                                isc_retval);
10303                                 }
10304                                 break;
10305                         }
10306                         opcode = cur_blocked->scsiio.cdb[0];
10307                         entry = &ctl_cmd_table[opcode];
10308                         softc = control_softc;
10309
10310                         initidx = ctl_get_initindex(&cur_blocked->io_hdr.nexus);
10311
10312                         /*
10313                          * Check this I/O for LUN state changes that may
10314                          * have happened while this command was blocked.
10315                          * The LUN state may have been changed by a command
10316                          * ahead of us in the queue, so we need to re-check
10317                          * for any states that can be caused by SCSI
10318                          * commands.
10319                          */
10320                         if (ctl_scsiio_lun_check(softc, lun, entry,
10321                                                  &cur_blocked->scsiio) == 0) {
10322                                 cur_blocked->io_hdr.flags |=
10323                                                       CTL_FLAG_IS_WAS_ON_RTR;
10324                                 STAILQ_INSERT_TAIL(&lun->ctl_softc->rtr_queue,
10325                                                    &cur_blocked->io_hdr, links);
10326                                 /*
10327                                  * In the non CTL_DONE_THREAD case, we need
10328                                  * to wake up the work thread here.  When
10329                                  * we're processing completed requests from
10330                                  * the work thread context, we'll pop back
10331                                  * around and end up pulling things off the
10332                                  * RtR queue.  When we aren't processing
10333                                  * things from the work thread context,
10334                                  * though, we won't ever check the RtR queue.
10335                                  * So we need to wake up the thread to clear
10336                                  * things off the queue.  Otherwise this
10337                                  * transaction will just sit on the RtR queue
10338                                  * until a new I/O comes in.  (Which may or
10339                                  * may not happen...)
10340                                  */
10341 #ifndef CTL_DONE_THREAD
10342                                 ctl_wakeup_thread();
10343 #endif
10344                         } else
10345                                 ctl_done_lock(cur_blocked, /*have_lock*/ 1);
10346                         break;
10347                 }
10348                 default:
10349                         /*
10350                          * This probably shouldn't happen -- we shouldn't
10351                          * get CTL_ACTION_ERROR, or anything else.
10352                          */
10353                         break;
10354                 }
10355         }
10356
10357         return (CTL_RETVAL_COMPLETE);
10358 }
10359
10360 /*
10361  * This routine (with one exception) checks LUN flags that can be set by
10362  * commands ahead of us in the OOA queue.  These flags have to be checked
10363  * when a command initially comes in, and when we pull a command off the
10364  * blocked queue and are preparing to execute it.  The reason we have to
10365  * check these flags for commands on the blocked queue is that the LUN
10366  * state may have been changed by a command ahead of us while we're on the
10367  * blocked queue.
10368  *
10369  * Ordering is somewhat important with these checks, so please pay
10370  * careful attention to the placement of any new checks.
10371  */
10372 static int
10373 ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
10374                      struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
10375 {
10376         int retval;
10377
10378         retval = 0;
10379
10380         /*
10381          * If this shelf is a secondary shelf controller, we have to reject
10382          * any media access commands.
10383          */
10384 #if 0
10385         /* No longer needed for HA */
10386         if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
10387          && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
10388                 ctl_set_lun_standby(ctsio);
10389                 retval = 1;
10390                 goto bailout;
10391         }
10392 #endif
10393
10394         /*
10395          * Check for a reservation conflict.  If this command isn't allowed
10396          * even on reserved LUNs, and if this initiator isn't the one who
10397          * reserved us, reject the command with a reservation conflict.
10398          */
10399         if ((lun->flags & CTL_LUN_RESERVED)
10400          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
10401                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
10402                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
10403                  || (ctsio->io_hdr.nexus.targ_target.id !=
10404                      lun->rsv_nexus.targ_target.id)) {
10405                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10406                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
10407                         retval = 1;
10408                         goto bailout;
10409                 }
10410         }
10411
10412         if ( (lun->flags & CTL_LUN_PR_RESERVED)
10413          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV) == 0)) {
10414                 uint32_t residx;
10415
10416                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
10417                 /*
10418                  * if we aren't registered or it's a res holder type
10419                  * reservation and this isn't the res holder then set a
10420                  * conflict.
10421                  * NOTE: Commands which might be allowed on write exclusive
10422                  * type reservations are checked in the particular command
10423                  * for a conflict. Read and SSU are the only ones.
10424                  */
10425                 if (!lun->per_res[residx].registered
10426                  || (residx != lun->pr_res_idx && lun->res_type < 4)) {
10427                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10428                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
10429                         retval = 1;
10430                         goto bailout;
10431                 }
10432
10433         }
10434
10435         if ((lun->flags & CTL_LUN_OFFLINE)
10436          && ((entry->flags & CTL_CMD_FLAG_OK_ON_OFFLINE) == 0)) {
10437                 ctl_set_lun_not_ready(ctsio);
10438                 retval = 1;
10439                 goto bailout;
10440         }
10441
10442         /*
10443          * If the LUN is stopped, see if this particular command is allowed
10444          * for a stopped lun.  Otherwise, reject it with 0x04,0x02.
10445          */
10446         if ((lun->flags & CTL_LUN_STOPPED)
10447          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
10448                 /* "Logical unit not ready, initializing cmd. required" */
10449                 ctl_set_lun_stopped(ctsio);
10450                 retval = 1;
10451                 goto bailout;
10452         }
10453
10454         if ((lun->flags & CTL_LUN_INOPERABLE)
10455          && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
10456                 /* "Medium format corrupted" */
10457                 ctl_set_medium_format_corrupted(ctsio);
10458                 retval = 1;
10459                 goto bailout;
10460         }
10461
10462 bailout:
10463         return (retval);
10464
10465 }
10466
10467 static void
10468 ctl_failover_io(union ctl_io *io, int have_lock)
10469 {
10470         ctl_set_busy(&io->scsiio);
10471         ctl_done_lock(io, have_lock);
10472 }
10473
10474 static void
10475 ctl_failover(void)
10476 {
10477         struct ctl_lun *lun;
10478         struct ctl_softc *ctl_softc;
10479         union ctl_io *next_io, *pending_io;
10480         union ctl_io *io;
10481         int lun_idx;
10482         int i;
10483
10484         ctl_softc = control_softc;
10485
10486         mtx_lock(&ctl_softc->ctl_lock);
10487         /*
10488          * Remove any cmds from the other SC from the rtr queue.  These
10489          * will obviously only be for LUNs for which we're the primary.
10490          * We can't send status or get/send data for these commands.
10491          * Since they haven't been executed yet, we can just remove them.
10492          * We'll either abort them or delete them below, depending on
10493          * which HA mode we're in.
10494          */
10495         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->rtr_queue);
10496              io != NULL; io = next_io) {
10497                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
10498                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10499                         STAILQ_REMOVE(&ctl_softc->rtr_queue, &io->io_hdr,
10500                                       ctl_io_hdr, links);
10501         }
10502
10503         for (lun_idx=0; lun_idx < ctl_softc->num_luns; lun_idx++) {
10504                 lun = ctl_softc->ctl_luns[lun_idx];
10505                 if (lun==NULL)
10506                         continue;
10507
10508                 /*
10509                  * Processor LUNs are primary on both sides.
10510                  * XXX will this always be true?
10511                  */
10512                 if (lun->be_lun->lun_type == T_PROCESSOR)
10513                         continue;
10514
10515                 if ((lun->flags & CTL_LUN_PRIMARY_SC)
10516                  && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10517                         printf("FAILOVER: primary lun %d\n", lun_idx);
10518                         /*
10519                          * Remove all commands from the other SC. First from the
10520                          * blocked queue then from the ooa queue. Once we have
10521                          * removed them. Call ctl_check_blocked to see if there
10522                          * is anything that can run.
10523                          */
10524                         for (io = (union ctl_io *)TAILQ_FIRST(
10525                              &lun->blocked_queue); io != NULL; io = next_io) {
10526
10527                                 next_io = (union ctl_io *)TAILQ_NEXT(
10528                                     &io->io_hdr, blocked_links);
10529
10530                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10531                                         TAILQ_REMOVE(&lun->blocked_queue,
10532                                                      &io->io_hdr,blocked_links);
10533                                         io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10534                                         TAILQ_REMOVE(&lun->ooa_queue,
10535                                                      &io->io_hdr, ooa_links);
10536
10537                                         ctl_free_io(io);
10538                                 }
10539                         }
10540
10541                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10542                              io != NULL; io = next_io) {
10543
10544                                 next_io = (union ctl_io *)TAILQ_NEXT(
10545                                     &io->io_hdr, ooa_links);
10546
10547                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10548
10549                                         TAILQ_REMOVE(&lun->ooa_queue,
10550                                                 &io->io_hdr,
10551                                                 ooa_links);
10552
10553                                         ctl_free_io(io);
10554                                 }
10555                         }
10556                         ctl_check_blocked(lun);
10557                 } else if ((lun->flags & CTL_LUN_PRIMARY_SC)
10558                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10559
10560                         printf("FAILOVER: primary lun %d\n", lun_idx);
10561                         /*
10562                          * Abort all commands from the other SC.  We can't
10563                          * send status back for them now.  These should get
10564                          * cleaned up when they are completed or come out
10565                          * for a datamove operation.
10566                          */
10567                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10568                              io != NULL; io = next_io) {
10569                                 next_io = (union ctl_io *)TAILQ_NEXT(
10570                                         &io->io_hdr, ooa_links);
10571
10572                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10573                                         io->io_hdr.flags |= CTL_FLAG_ABORT;
10574                         }
10575                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10576                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10577
10578                         printf("FAILOVER: secondary lun %d\n", lun_idx);
10579
10580                         lun->flags |= CTL_LUN_PRIMARY_SC;
10581
10582                         /*
10583                          * We send all I/O that was sent to this controller
10584                          * and redirected to the other side back with
10585                          * busy status, and have the initiator retry it.
10586                          * Figuring out how much data has been transferred,
10587                          * etc. and picking up where we left off would be
10588                          * very tricky.
10589                          *
10590                          * XXX KDM need to remove I/O from the blocked
10591                          * queue as well!
10592                          */
10593                         for (pending_io = (union ctl_io *)TAILQ_FIRST(
10594                              &lun->ooa_queue); pending_io != NULL;
10595                              pending_io = next_io) {
10596
10597                                 next_io =  (union ctl_io *)TAILQ_NEXT(
10598                                         &pending_io->io_hdr, ooa_links);
10599
10600                                 pending_io->io_hdr.flags &=
10601                                         ~CTL_FLAG_SENT_2OTHER_SC;
10602
10603                                 if (pending_io->io_hdr.flags &
10604                                     CTL_FLAG_IO_ACTIVE) {
10605                                         pending_io->io_hdr.flags |=
10606                                                 CTL_FLAG_FAILOVER;
10607                                 } else {
10608                                         ctl_set_busy(&pending_io->scsiio);
10609                                         ctl_done_lock(pending_io,
10610                                                       /*have_lock*/1);
10611                                 }
10612                         }
10613
10614                         /*
10615                          * Build Unit Attention
10616                          */
10617                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10618                                 lun->pending_sense[i].ua_pending |=
10619                                                      CTL_UA_ASYM_ACC_CHANGE;
10620                         }
10621                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10622                         && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10623                         printf("FAILOVER: secondary lun %d\n", lun_idx);
10624                         /*
10625                          * if the first io on the OOA is not on the RtR queue
10626                          * add it.
10627                          */
10628                         lun->flags |= CTL_LUN_PRIMARY_SC;
10629
10630                         pending_io = (union ctl_io *)TAILQ_FIRST(
10631                             &lun->ooa_queue);
10632                         if (pending_io==NULL) {
10633                                 printf("Nothing on OOA queue\n");
10634                                 continue;
10635                         }
10636
10637                         pending_io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
10638                         if ((pending_io->io_hdr.flags &
10639                              CTL_FLAG_IS_WAS_ON_RTR) == 0) {
10640                                 pending_io->io_hdr.flags |=
10641                                     CTL_FLAG_IS_WAS_ON_RTR;
10642                                 STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
10643                                                    &pending_io->io_hdr, links);
10644                         }
10645 #if 0
10646                         else
10647                         {
10648                                 printf("Tag 0x%04x is running\n",
10649                                       pending_io->scsiio.tag_num);
10650                         }
10651 #endif
10652
10653                         next_io = (union ctl_io *)TAILQ_NEXT(
10654                             &pending_io->io_hdr, ooa_links);
10655                         for (pending_io=next_io; pending_io != NULL;
10656                              pending_io = next_io) {
10657                                 pending_io->io_hdr.flags &=
10658                                     ~CTL_FLAG_SENT_2OTHER_SC;
10659                                 next_io = (union ctl_io *)TAILQ_NEXT(
10660                                         &pending_io->io_hdr, ooa_links);
10661                                 if (pending_io->io_hdr.flags &
10662                                     CTL_FLAG_IS_WAS_ON_RTR) {
10663 #if 0
10664                                         printf("Tag 0x%04x is running\n",
10665                                                 pending_io->scsiio.tag_num);
10666 #endif
10667                                         continue;
10668                                 }
10669
10670                                 switch (ctl_check_ooa(lun, pending_io,
10671                                     (union ctl_io *)TAILQ_PREV(
10672                                     &pending_io->io_hdr, ctl_ooaq,
10673                                     ooa_links))) {
10674
10675                                 case CTL_ACTION_BLOCK:
10676                                         TAILQ_INSERT_TAIL(&lun->blocked_queue,
10677                                                           &pending_io->io_hdr,
10678                                                           blocked_links);
10679                                         pending_io->io_hdr.flags |=
10680                                             CTL_FLAG_BLOCKED;
10681                                         break;
10682                                 case CTL_ACTION_PASS:
10683                                 case CTL_ACTION_SKIP:
10684                                         pending_io->io_hdr.flags |=
10685                                             CTL_FLAG_IS_WAS_ON_RTR;
10686                                         STAILQ_INSERT_TAIL(
10687                                             &ctl_softc->rtr_queue,
10688                                             &pending_io->io_hdr, links);
10689                                         break;
10690                                 case CTL_ACTION_OVERLAP:
10691                                         ctl_set_overlapped_cmd(
10692                                             (struct ctl_scsiio *)pending_io);
10693                                         ctl_done_lock(pending_io,
10694                                                       /*have_lock*/ 1);
10695                                         break;
10696                                 case CTL_ACTION_OVERLAP_TAG:
10697                                         ctl_set_overlapped_tag(
10698                                             (struct ctl_scsiio *)pending_io,
10699                                             pending_io->scsiio.tag_num & 0xff);
10700                                         ctl_done_lock(pending_io,
10701                                                       /*have_lock*/ 1);
10702                                         break;
10703                                 case CTL_ACTION_ERROR:
10704                                 default:
10705                                         ctl_set_internal_failure(
10706                                                 (struct ctl_scsiio *)pending_io,
10707                                                 0,  // sks_valid
10708                                                 0); //retry count
10709                                         ctl_done_lock(pending_io,
10710                                                       /*have_lock*/ 1);
10711                                         break;
10712                                 }
10713                         }
10714
10715                         /*
10716                          * Build Unit Attention
10717                          */
10718                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10719                                 lun->pending_sense[i].ua_pending |=
10720                                                      CTL_UA_ASYM_ACC_CHANGE;
10721                         }
10722                 } else {
10723                         panic("Unhandled HA mode failover, LUN flags = %#x, "
10724                               "ha_mode = #%x", lun->flags, ctl_softc->ha_mode);
10725                 }
10726         }
10727         ctl_pause_rtr = 0;
10728         mtx_unlock(&ctl_softc->ctl_lock);
10729 }
10730
10731 static int
10732 ctl_scsiio_precheck(struct ctl_softc *ctl_softc, struct ctl_scsiio *ctsio)
10733 {
10734         struct ctl_lun *lun;
10735         struct ctl_cmd_entry *entry;
10736         uint8_t opcode;
10737         uint32_t initidx, targ_lun;
10738         int retval;
10739
10740         retval = 0;
10741
10742         lun = NULL;
10743
10744         opcode = ctsio->cdb[0];
10745
10746         mtx_lock(&ctl_softc->ctl_lock);
10747
10748         targ_lun = ctsio->io_hdr.nexus.targ_lun;
10749         if (ctsio->io_hdr.nexus.lun_map_fn != NULL)
10750                 targ_lun = ctsio->io_hdr.nexus.lun_map_fn(ctsio->io_hdr.nexus.lun_map_arg, targ_lun);
10751         if ((targ_lun < CTL_MAX_LUNS)
10752          && (ctl_softc->ctl_luns[targ_lun] != NULL)) {
10753                 lun = ctl_softc->ctl_luns[targ_lun];
10754                 /*
10755                  * If the LUN is invalid, pretend that it doesn't exist.
10756                  * It will go away as soon as all pending I/O has been
10757                  * completed.
10758                  */
10759                 if (lun->flags & CTL_LUN_DISABLED) {
10760                         lun = NULL;
10761                 } else {
10762                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
10763                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
10764                                 lun->be_lun;
10765                         if (lun->be_lun->lun_type == T_PROCESSOR) {
10766                                 ctsio->io_hdr.flags |= CTL_FLAG_CONTROL_DEV;
10767                         }
10768                 }
10769         } else {
10770                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
10771                 ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
10772         }
10773
10774         entry = &ctl_cmd_table[opcode];
10775
10776         ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
10777         ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
10778
10779         /*
10780          * Check to see whether we can send this command to LUNs that don't
10781          * exist.  This should pretty much only be the case for inquiry
10782          * and request sense.  Further checks, below, really require having
10783          * a LUN, so we can't really check the command anymore.  Just put
10784          * it on the rtr queue.
10785          */
10786         if (lun == NULL) {
10787                 if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10788                         goto queue_rtr;
10789
10790                 ctl_set_unsupported_lun(ctsio);
10791                 mtx_unlock(&ctl_softc->ctl_lock);
10792                 ctl_done((union ctl_io *)ctsio);
10793                 CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
10794                 goto bailout;
10795         } else {
10796                 /*
10797                  * Every I/O goes into the OOA queue for a particular LUN, and
10798                  * stays there until completion.
10799                  */
10800                 TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
10801
10802                 /*
10803                  * Make sure we support this particular command on this LUN.
10804                  * e.g., we don't support writes to the control LUN.
10805                  */
10806                 switch (lun->be_lun->lun_type) {
10807                 case T_PROCESSOR:
10808                         if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0)
10809                          && ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10810                               == 0)) {
10811                                 ctl_set_invalid_opcode(ctsio);
10812                                 mtx_unlock(&ctl_softc->ctl_lock);
10813                                 ctl_done((union ctl_io *)ctsio);
10814                                 goto bailout;
10815                         }
10816                         break;
10817                 case T_DIRECT:
10818                         if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0)
10819                          && ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10820                               == 0)){
10821                                 ctl_set_invalid_opcode(ctsio);
10822                                 mtx_unlock(&ctl_softc->ctl_lock);
10823                                 ctl_done((union ctl_io *)ctsio);
10824                                 goto bailout;
10825                         }
10826                         break;
10827                 default:
10828                         printf("Unsupported CTL LUN type %d\n",
10829                                lun->be_lun->lun_type);
10830                         panic("Unsupported CTL LUN type %d\n",
10831                               lun->be_lun->lun_type);
10832                         break; /* NOTREACHED */
10833                 }
10834         }
10835
10836         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
10837
10838         /*
10839          * If we've got a request sense, it'll clear the contingent
10840          * allegiance condition.  Otherwise, if we have a CA condition for
10841          * this initiator, clear it, because it sent down a command other
10842          * than request sense.
10843          */
10844         if ((opcode != REQUEST_SENSE)
10845          && (ctl_is_set(lun->have_ca, initidx)))
10846                 ctl_clear_mask(lun->have_ca, initidx);
10847
10848         /*
10849          * If the command has this flag set, it handles its own unit
10850          * attention reporting, we shouldn't do anything.  Otherwise we
10851          * check for any pending unit attentions, and send them back to the
10852          * initiator.  We only do this when a command initially comes in,
10853          * not when we pull it off the blocked queue.
10854          *
10855          * According to SAM-3, section 5.3.2, the order that things get
10856          * presented back to the host is basically unit attentions caused
10857          * by some sort of reset event, busy status, reservation conflicts
10858          * or task set full, and finally any other status.
10859          *
10860          * One issue here is that some of the unit attentions we report
10861          * don't fall into the "reset" category (e.g. "reported luns data
10862          * has changed").  So reporting it here, before the reservation
10863          * check, may be technically wrong.  I guess the only thing to do
10864          * would be to check for and report the reset events here, and then
10865          * check for the other unit attention types after we check for a
10866          * reservation conflict.
10867          *
10868          * XXX KDM need to fix this
10869          */
10870         if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
10871                 ctl_ua_type ua_type;
10872
10873                 ua_type = lun->pending_sense[initidx].ua_pending;
10874                 if (ua_type != CTL_UA_NONE) {
10875                         scsi_sense_data_type sense_format;
10876
10877                         if (lun != NULL)
10878                                 sense_format = (lun->flags &
10879                                     CTL_LUN_SENSE_DESC) ? SSD_TYPE_DESC :
10880                                     SSD_TYPE_FIXED;
10881                         else
10882                                 sense_format = SSD_TYPE_FIXED;
10883
10884                         ua_type = ctl_build_ua(ua_type, &ctsio->sense_data,
10885                                                sense_format);
10886                         if (ua_type != CTL_UA_NONE) {
10887                                 ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
10888                                 ctsio->io_hdr.status = CTL_SCSI_ERROR |
10889                                                        CTL_AUTOSENSE;
10890                                 ctsio->sense_len = SSD_FULL_SIZE;
10891                                 lun->pending_sense[initidx].ua_pending &=
10892                                         ~ua_type;
10893                                 mtx_unlock(&ctl_softc->ctl_lock);
10894                                 ctl_done((union ctl_io *)ctsio);
10895                                 goto bailout;
10896                         }
10897                 }
10898         }
10899
10900
10901         if (ctl_scsiio_lun_check(ctl_softc, lun, entry, ctsio) != 0) {
10902                 mtx_unlock(&ctl_softc->ctl_lock);
10903                 ctl_done((union ctl_io *)ctsio);
10904                 goto bailout;
10905         }
10906
10907         /*
10908          * XXX CHD this is where we want to send IO to other side if
10909          * this LUN is secondary on this SC. We will need to make a copy
10910          * of the IO and flag the IO on this side as SENT_2OTHER and the flag
10911          * the copy we send as FROM_OTHER.
10912          * We also need to stuff the address of the original IO so we can
10913          * find it easily. Something similar will need be done on the other
10914          * side so when we are done we can find the copy.
10915          */
10916         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
10917                 union ctl_ha_msg msg_info;
10918                 int isc_retval;
10919
10920                 ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
10921
10922                 msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
10923                 msg_info.hdr.original_sc = (union ctl_io *)ctsio;
10924 #if 0
10925                 printf("1. ctsio %p\n", ctsio);
10926 #endif
10927                 msg_info.hdr.serializing_sc = NULL;
10928                 msg_info.hdr.nexus = ctsio->io_hdr.nexus;
10929                 msg_info.scsi.tag_num = ctsio->tag_num;
10930                 msg_info.scsi.tag_type = ctsio->tag_type;
10931                 memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
10932
10933                 ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
10934
10935                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10936                     (void *)&msg_info, sizeof(msg_info), 0)) >
10937                     CTL_HA_STATUS_SUCCESS) {
10938                         printf("CTL:precheck, ctl_ha_msg_send returned %d\n",
10939                                isc_retval);
10940                         printf("CTL:opcode is %x\n",opcode);
10941                 } else {
10942 #if 0
10943                         printf("CTL:Precheck sent msg, opcode is %x\n",opcode);
10944 #endif
10945                 }
10946
10947                 /*
10948                  * XXX KDM this I/O is off the incoming queue, but hasn't
10949                  * been inserted on any other queue.  We may need to come
10950                  * up with a holding queue while we wait for serialization
10951                  * so that we have an idea of what we're waiting for from
10952                  * the other side.
10953                  */
10954                 goto bailout_unlock;
10955         }
10956
10957         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
10958                               (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
10959                               ctl_ooaq, ooa_links))) {
10960         case CTL_ACTION_BLOCK:
10961                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
10962                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
10963                                   blocked_links);
10964                 goto bailout_unlock;
10965                 break; /* NOTREACHED */
10966         case CTL_ACTION_PASS:
10967         case CTL_ACTION_SKIP:
10968                 goto queue_rtr;
10969                 break; /* NOTREACHED */
10970         case CTL_ACTION_OVERLAP:
10971                 ctl_set_overlapped_cmd(ctsio);
10972                 mtx_unlock(&ctl_softc->ctl_lock);
10973                 ctl_done((union ctl_io *)ctsio);
10974                 goto bailout;
10975                 break; /* NOTREACHED */
10976         case CTL_ACTION_OVERLAP_TAG:
10977                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
10978                 mtx_unlock(&ctl_softc->ctl_lock);
10979                 ctl_done((union ctl_io *)ctsio);
10980                 goto bailout;
10981                 break; /* NOTREACHED */
10982         case CTL_ACTION_ERROR:
10983         default:
10984                 ctl_set_internal_failure(ctsio,
10985                                          /*sks_valid*/ 0,
10986                                          /*retry_count*/ 0);
10987                 mtx_unlock(&ctl_softc->ctl_lock);
10988                 ctl_done((union ctl_io *)ctsio);
10989                 goto bailout;
10990                 break; /* NOTREACHED */
10991         }
10992
10993         goto bailout_unlock;
10994
10995 queue_rtr:
10996         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
10997         STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue, &ctsio->io_hdr, links);
10998
10999 bailout_unlock:
11000         mtx_unlock(&ctl_softc->ctl_lock);
11001
11002 bailout:
11003         return (retval);
11004 }
11005
11006 static int
11007 ctl_scsiio(struct ctl_scsiio *ctsio)
11008 {
11009         int retval;
11010         struct ctl_cmd_entry *entry;
11011
11012         retval = CTL_RETVAL_COMPLETE;
11013
11014         CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
11015
11016         entry = &ctl_cmd_table[ctsio->cdb[0]];
11017
11018         /*
11019          * If this I/O has been aborted, just send it straight to
11020          * ctl_done() without executing it.
11021          */
11022         if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
11023                 ctl_done((union ctl_io *)ctsio);
11024                 goto bailout;
11025         }
11026
11027         /*
11028          * All the checks should have been handled by ctl_scsiio_precheck().
11029          * We should be clear now to just execute the I/O.
11030          */
11031         retval = entry->execute(ctsio);
11032
11033 bailout:
11034         return (retval);
11035 }
11036
11037 /*
11038  * Since we only implement one target right now, a bus reset simply resets
11039  * our single target.
11040  */
11041 static int
11042 ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io)
11043 {
11044         return(ctl_target_reset(ctl_softc, io, CTL_UA_BUS_RESET));
11045 }
11046
11047 static int
11048 ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
11049                  ctl_ua_type ua_type)
11050 {
11051         struct ctl_lun *lun;
11052         int retval;
11053
11054         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11055                 union ctl_ha_msg msg_info;
11056
11057                 io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11058                 msg_info.hdr.nexus = io->io_hdr.nexus;
11059                 if (ua_type==CTL_UA_TARG_RESET)
11060                         msg_info.task.task_action = CTL_TASK_TARGET_RESET;
11061                 else
11062                         msg_info.task.task_action = CTL_TASK_BUS_RESET;
11063                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11064                 msg_info.hdr.original_sc = NULL;
11065                 msg_info.hdr.serializing_sc = NULL;
11066                 if (CTL_HA_STATUS_SUCCESS != ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11067                     (void *)&msg_info, sizeof(msg_info), 0)) {
11068                 }
11069         }
11070         retval = 0;
11071
11072         STAILQ_FOREACH(lun, &ctl_softc->lun_list, links)
11073                 retval += ctl_lun_reset(lun, io, ua_type);
11074
11075         return (retval);
11076 }
11077
11078 /*
11079  * The LUN should always be set.  The I/O is optional, and is used to
11080  * distinguish between I/Os sent by this initiator, and by other
11081  * initiators.  We set unit attention for initiators other than this one.
11082  * SAM-3 is vague on this point.  It does say that a unit attention should
11083  * be established for other initiators when a LUN is reset (see section
11084  * 5.7.3), but it doesn't specifically say that the unit attention should
11085  * be established for this particular initiator when a LUN is reset.  Here
11086  * is the relevant text, from SAM-3 rev 8:
11087  *
11088  * 5.7.2 When a SCSI initiator port aborts its own tasks
11089  *
11090  * When a SCSI initiator port causes its own task(s) to be aborted, no
11091  * notification that the task(s) have been aborted shall be returned to
11092  * the SCSI initiator port other than the completion response for the
11093  * command or task management function action that caused the task(s) to
11094  * be aborted and notification(s) associated with related effects of the
11095  * action (e.g., a reset unit attention condition).
11096  *
11097  * XXX KDM for now, we're setting unit attention for all initiators.
11098  */
11099 static int
11100 ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
11101 {
11102         union ctl_io *xio;
11103 #if 0
11104         uint32_t initindex;
11105 #endif
11106         int i;
11107
11108         /*
11109          * Run through the OOA queue and abort each I/O.
11110          */
11111 #if 0
11112         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
11113 #endif
11114         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11115              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11116                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
11117         }
11118
11119         /*
11120          * This version sets unit attention for every
11121          */
11122 #if 0
11123         initindex = ctl_get_initindex(&io->io_hdr.nexus);
11124         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11125                 if (initindex == i)
11126                         continue;
11127                 lun->pending_sense[i].ua_pending |= ua_type;
11128         }
11129 #endif
11130
11131         /*
11132          * A reset (any kind, really) clears reservations established with
11133          * RESERVE/RELEASE.  It does not clear reservations established
11134          * with PERSISTENT RESERVE OUT, but we don't support that at the
11135          * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
11136          * reservations made with the RESERVE/RELEASE commands, because
11137          * those commands are obsolete in SPC-3.
11138          */
11139         lun->flags &= ~CTL_LUN_RESERVED;
11140
11141         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11142                 ctl_clear_mask(lun->have_ca, i);
11143                 lun->pending_sense[i].ua_pending |= ua_type;
11144         }
11145
11146         return (0);
11147 }
11148
11149 static int
11150 ctl_abort_task(union ctl_io *io)
11151 {
11152         union ctl_io *xio;
11153         struct ctl_lun *lun;
11154         struct ctl_softc *ctl_softc;
11155 #if 0
11156         struct sbuf sb;
11157         char printbuf[128];
11158 #endif
11159         int found;
11160         uint32_t targ_lun;
11161
11162         ctl_softc = control_softc;
11163         found = 0;
11164
11165         /*
11166          * Look up the LUN.
11167          */
11168         targ_lun = io->io_hdr.nexus.targ_lun;
11169         if (io->io_hdr.nexus.lun_map_fn != NULL)
11170                 targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
11171         if ((targ_lun < CTL_MAX_LUNS)
11172          && (ctl_softc->ctl_luns[targ_lun] != NULL))
11173                 lun = ctl_softc->ctl_luns[targ_lun];
11174         else
11175                 goto bailout;
11176
11177 #if 0
11178         printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
11179                lun->lun, io->taskio.tag_num, io->taskio.tag_type);
11180 #endif
11181
11182         /*
11183          * Run through the OOA queue and attempt to find the given I/O.
11184          * The target port, initiator ID, tag type and tag number have to
11185          * match the values that we got from the initiator.  If we have an
11186          * untagged command to abort, simply abort the first untagged command
11187          * we come to.  We only allow one untagged command at a time of course.
11188          */
11189 #if 0
11190         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
11191 #endif
11192         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11193              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11194 #if 0
11195                 sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
11196
11197                 sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
11198                             lun->lun, xio->scsiio.tag_num,
11199                             xio->scsiio.tag_type,
11200                             (xio->io_hdr.blocked_links.tqe_prev
11201                             == NULL) ? "" : " BLOCKED",
11202                             (xio->io_hdr.flags &
11203                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
11204                             (xio->io_hdr.flags &
11205                             CTL_FLAG_ABORT) ? " ABORT" : "",
11206                             (xio->io_hdr.flags &
11207                             CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
11208                 ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
11209                 sbuf_finish(&sb);
11210                 printf("%s\n", sbuf_data(&sb));
11211 #endif
11212
11213                 if ((xio->io_hdr.nexus.targ_port == io->io_hdr.nexus.targ_port)
11214                  && (xio->io_hdr.nexus.initid.id ==
11215                      io->io_hdr.nexus.initid.id)) {
11216                         /*
11217                          * If the abort says that the task is untagged, the
11218                          * task in the queue must be untagged.  Otherwise,
11219                          * we just check to see whether the tag numbers
11220                          * match.  This is because the QLogic firmware
11221                          * doesn't pass back the tag type in an abort
11222                          * request.
11223                          */
11224 #if 0
11225                         if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
11226                           && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
11227                          || (xio->scsiio.tag_num == io->taskio.tag_num)) {
11228 #endif
11229                         /*
11230                          * XXX KDM we've got problems with FC, because it
11231                          * doesn't send down a tag type with aborts.  So we
11232                          * can only really go by the tag number...
11233                          * This may cause problems with parallel SCSI.
11234                          * Need to figure that out!!
11235                          */
11236                         if (xio->scsiio.tag_num == io->taskio.tag_num) {
11237                                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
11238                                 found = 1;
11239                                 if ((io->io_hdr.flags &
11240                                      CTL_FLAG_FROM_OTHER_SC) == 0 &&
11241                                     !(lun->flags & CTL_LUN_PRIMARY_SC)) {
11242                                         union ctl_ha_msg msg_info;
11243
11244                                         io->io_hdr.flags |=
11245                                                         CTL_FLAG_SENT_2OTHER_SC;
11246                                         msg_info.hdr.nexus = io->io_hdr.nexus;
11247                                         msg_info.task.task_action =
11248                                                 CTL_TASK_ABORT_TASK;
11249                                         msg_info.task.tag_num =
11250                                                 io->taskio.tag_num;
11251                                         msg_info.task.tag_type =
11252                                                 io->taskio.tag_type;
11253                                         msg_info.hdr.msg_type =
11254                                                 CTL_MSG_MANAGE_TASKS;
11255                                         msg_info.hdr.original_sc = NULL;
11256                                         msg_info.hdr.serializing_sc = NULL;
11257 #if 0
11258                                         printf("Sent Abort to other side\n");
11259 #endif
11260                                         if (CTL_HA_STATUS_SUCCESS !=
11261                                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11262                                                 (void *)&msg_info,
11263                                                 sizeof(msg_info), 0)) {
11264                                         }
11265                                 }
11266 #if 0
11267                                 printf("ctl_abort_task: found I/O to abort\n");
11268 #endif
11269                                 break;
11270                         }
11271                 }
11272         }
11273
11274 bailout:
11275
11276         if (found == 0) {
11277                 /*
11278                  * This isn't really an error.  It's entirely possible for
11279                  * the abort and command completion to cross on the wire.
11280                  * This is more of an informative/diagnostic error.
11281                  */
11282 #if 0
11283                 printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
11284                        "%d:%d:%d:%d tag %d type %d\n",
11285                        io->io_hdr.nexus.initid.id,
11286                        io->io_hdr.nexus.targ_port,
11287                        io->io_hdr.nexus.targ_target.id,
11288                        io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
11289                        io->taskio.tag_type);
11290 #endif
11291                 return (1);
11292         } else
11293                 return (0);
11294 }
11295
11296 /*
11297  * This routine cannot block!  It must be callable from an interrupt
11298  * handler as well as from the work thread.
11299  */
11300 static void
11301 ctl_run_task_queue(struct ctl_softc *ctl_softc)
11302 {
11303         union ctl_io *io, *next_io;
11304
11305         mtx_assert(&ctl_softc->ctl_lock, MA_OWNED);
11306
11307         CTL_DEBUG_PRINT(("ctl_run_task_queue\n"));
11308
11309         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->task_queue);
11310              io != NULL; io = next_io) {
11311                 int retval;
11312                 const char *task_desc;
11313
11314                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
11315
11316                 retval = 0;
11317
11318                 switch (io->io_hdr.io_type) {
11319                 case CTL_IO_TASK: {
11320                         task_desc = ctl_scsi_task_string(&io->taskio);
11321                         if (task_desc != NULL) {
11322 #ifdef NEEDTOPORT
11323                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
11324                                             CTL_TASK_REPORT,
11325                                             csevent_LogType_Trace,
11326                                             csevent_Severity_Information,
11327                                             csevent_AlertLevel_Green,
11328                                             csevent_FRU_Firmware,
11329                                             csevent_FRU_Unknown,
11330                                             "CTL: received task: %s",task_desc);
11331 #endif
11332                         } else {
11333 #ifdef NEEDTOPORT
11334                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
11335                                             CTL_TASK_REPORT,
11336                                             csevent_LogType_Trace,
11337                                             csevent_Severity_Information,
11338                                             csevent_AlertLevel_Green,
11339                                             csevent_FRU_Firmware,
11340                                             csevent_FRU_Unknown,
11341                                             "CTL: received unknown task "
11342                                             "type: %d (%#x)",
11343                                             io->taskio.task_action,
11344                                             io->taskio.task_action);
11345 #endif
11346                         }
11347                         switch (io->taskio.task_action) {
11348                         case CTL_TASK_ABORT_TASK:
11349                                 retval = ctl_abort_task(io);
11350                                 break;
11351                         case CTL_TASK_ABORT_TASK_SET:
11352                                 break;
11353                         case CTL_TASK_CLEAR_ACA:
11354                                 break;
11355                         case CTL_TASK_CLEAR_TASK_SET:
11356                                 break;
11357                         case CTL_TASK_LUN_RESET: {
11358                                 struct ctl_lun *lun;
11359                                 uint32_t targ_lun;
11360                                 int retval;
11361
11362                                 targ_lun = io->io_hdr.nexus.targ_lun;
11363                                 if (io->io_hdr.nexus.lun_map_fn != NULL)
11364                                         targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
11365
11366                                 if ((targ_lun < CTL_MAX_LUNS)
11367                                  && (ctl_softc->ctl_luns[targ_lun] != NULL))
11368                                         lun = ctl_softc->ctl_luns[targ_lun];
11369                                 else {
11370                                         retval = 1;
11371                                         break;
11372                                 }
11373
11374                                 if (!(io->io_hdr.flags &
11375                                     CTL_FLAG_FROM_OTHER_SC)) {
11376                                         union ctl_ha_msg msg_info;
11377
11378                                         io->io_hdr.flags |=
11379                                                 CTL_FLAG_SENT_2OTHER_SC;
11380                                         msg_info.hdr.msg_type =
11381                                                 CTL_MSG_MANAGE_TASKS;
11382                                         msg_info.hdr.nexus = io->io_hdr.nexus;
11383                                         msg_info.task.task_action =
11384                                                 CTL_TASK_LUN_RESET;
11385                                         msg_info.hdr.original_sc = NULL;
11386                                         msg_info.hdr.serializing_sc = NULL;
11387                                         if (CTL_HA_STATUS_SUCCESS !=
11388                                             ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11389                                             (void *)&msg_info,
11390                                             sizeof(msg_info), 0)) {
11391                                         }
11392                                 }
11393
11394                                 retval = ctl_lun_reset(lun, io,
11395                                                        CTL_UA_LUN_RESET);
11396                                 break;
11397                         }
11398                         case CTL_TASK_TARGET_RESET:
11399                                 retval = ctl_target_reset(ctl_softc, io,
11400                                                           CTL_UA_TARG_RESET);
11401                                 break;
11402                         case CTL_TASK_BUS_RESET:
11403                                 retval = ctl_bus_reset(ctl_softc, io);
11404                                 break;
11405                         case CTL_TASK_PORT_LOGIN:
11406                                 break;
11407                         case CTL_TASK_PORT_LOGOUT:
11408                                 break;
11409                         default:
11410                                 printf("ctl_run_task_queue: got unknown task "
11411                                        "management event %d\n",
11412                                        io->taskio.task_action);
11413                                 break;
11414                         }
11415                         if (retval == 0)
11416                                 io->io_hdr.status = CTL_SUCCESS;
11417                         else
11418                                 io->io_hdr.status = CTL_ERROR;
11419
11420                         STAILQ_REMOVE(&ctl_softc->task_queue, &io->io_hdr,
11421                                       ctl_io_hdr, links);
11422                         /*
11423                          * This will queue this I/O to the done queue, but the
11424                          * work thread won't be able to process it until we
11425                          * return and the lock is released.
11426                          */
11427                         ctl_done_lock(io, /*have_lock*/ 1);
11428                         break;
11429                 }
11430                 default: {
11431
11432                         printf("%s: invalid I/O type %d msg %d cdb %x"
11433                                " iptl: %ju:%d:%ju:%d tag 0x%04x\n",
11434                                __func__, io->io_hdr.io_type,
11435                                io->io_hdr.msg_type, io->scsiio.cdb[0],
11436                                (uintmax_t)io->io_hdr.nexus.initid.id,
11437                                io->io_hdr.nexus.targ_port,
11438                                (uintmax_t)io->io_hdr.nexus.targ_target.id,
11439                                io->io_hdr.nexus.targ_lun /* XXX */,
11440                                (io->io_hdr.io_type == CTL_IO_TASK) ?
11441                                io->taskio.tag_num : io->scsiio.tag_num);
11442                         STAILQ_REMOVE(&ctl_softc->task_queue, &io->io_hdr,
11443                                       ctl_io_hdr, links);
11444                         ctl_free_io(io);
11445                         break;
11446                 }
11447                 }
11448         }
11449
11450         ctl_softc->flags &= ~CTL_FLAG_TASK_PENDING;
11451 }
11452
11453 /*
11454  * For HA operation.  Handle commands that come in from the other
11455  * controller.
11456  */
11457 static void
11458 ctl_handle_isc(union ctl_io *io)
11459 {
11460         int free_io;
11461         struct ctl_lun *lun;
11462         struct ctl_softc *ctl_softc;
11463         uint32_t targ_lun;
11464
11465         ctl_softc = control_softc;
11466
11467         targ_lun = io->io_hdr.nexus.targ_lun;
11468         if (io->io_hdr.nexus.lun_map_fn != NULL)
11469                 targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
11470         lun = ctl_softc->ctl_luns[targ_lun];
11471
11472         switch (io->io_hdr.msg_type) {
11473         case CTL_MSG_SERIALIZE:
11474                 free_io = ctl_serialize_other_sc_cmd(&io->scsiio,
11475                                                      /*have_lock*/ 0);
11476                 break;
11477         case CTL_MSG_R2R: {
11478                 uint8_t opcode;
11479                 struct ctl_cmd_entry *entry;
11480
11481                 /*
11482                  * This is only used in SER_ONLY mode.
11483                  */
11484                 free_io = 0;
11485                 opcode = io->scsiio.cdb[0];
11486                 entry = &ctl_cmd_table[opcode];
11487                 mtx_lock(&ctl_softc->ctl_lock);
11488                 if (ctl_scsiio_lun_check(ctl_softc, lun,
11489                     entry, (struct ctl_scsiio *)io) != 0) {
11490                         ctl_done_lock(io, /*have_lock*/ 1);
11491                         mtx_unlock(&ctl_softc->ctl_lock);
11492                         break;
11493                 }
11494                 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11495                 STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
11496                                    &io->io_hdr, links);
11497                 mtx_unlock(&ctl_softc->ctl_lock);
11498                 break;
11499         }
11500         case CTL_MSG_FINISH_IO:
11501                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
11502                         free_io = 0;
11503                         ctl_done_lock(io, /*have_lock*/ 0);
11504                 } else {
11505                         free_io = 1;
11506                         mtx_lock(&ctl_softc->ctl_lock);
11507                         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
11508                                      ooa_links);
11509                         STAILQ_REMOVE(&ctl_softc->task_queue,
11510                                       &io->io_hdr, ctl_io_hdr, links);
11511                         ctl_check_blocked(lun);
11512                         mtx_unlock(&ctl_softc->ctl_lock);
11513                 }
11514                 break;
11515         case CTL_MSG_PERS_ACTION:
11516                 ctl_hndl_per_res_out_on_other_sc(
11517                         (union ctl_ha_msg *)&io->presio.pr_msg);
11518                 free_io = 1;
11519                 break;
11520         case CTL_MSG_BAD_JUJU:
11521                 free_io = 0;
11522                 ctl_done_lock(io, /*have_lock*/ 0);
11523                 break;
11524         case CTL_MSG_DATAMOVE:
11525                 /* Only used in XFER mode */
11526                 free_io = 0;
11527                 ctl_datamove_remote(io);
11528                 break;
11529         case CTL_MSG_DATAMOVE_DONE:
11530                 /* Only used in XFER mode */
11531                 free_io = 0;
11532                 io->scsiio.be_move_done(io);
11533                 break;
11534         default:
11535                 free_io = 1;
11536                 printf("%s: Invalid message type %d\n",
11537                        __func__, io->io_hdr.msg_type);
11538                 break;
11539         }
11540         if (free_io)
11541                 ctl_free_io(io);
11542
11543 }
11544
11545
11546 /*
11547  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
11548  * there is no match.
11549  */
11550 static ctl_lun_error_pattern
11551 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
11552 {
11553         struct ctl_cmd_entry *entry;
11554         ctl_lun_error_pattern filtered_pattern, pattern;
11555         uint8_t opcode;
11556
11557         pattern = desc->error_pattern;
11558
11559         /*
11560          * XXX KDM we need more data passed into this function to match a
11561          * custom pattern, and we actually need to implement custom pattern
11562          * matching.
11563          */
11564         if (pattern & CTL_LUN_PAT_CMD)
11565                 return (CTL_LUN_PAT_CMD);
11566
11567         if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
11568                 return (CTL_LUN_PAT_ANY);
11569
11570         opcode = ctsio->cdb[0];
11571         entry = &ctl_cmd_table[opcode];
11572
11573         filtered_pattern = entry->pattern & pattern;
11574
11575         /*
11576          * If the user requested specific flags in the pattern (e.g.
11577          * CTL_LUN_PAT_RANGE), make sure the command supports all of those
11578          * flags.
11579          *
11580          * If the user did not specify any flags, it doesn't matter whether
11581          * or not the command supports the flags.
11582          */
11583         if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
11584              (pattern & ~CTL_LUN_PAT_MASK))
11585                 return (CTL_LUN_PAT_NONE);
11586
11587         /*
11588          * If the user asked for a range check, see if the requested LBA
11589          * range overlaps with this command's LBA range.
11590          */
11591         if (filtered_pattern & CTL_LUN_PAT_RANGE) {
11592                 uint64_t lba1;
11593                 uint32_t len1;
11594                 ctl_action action;
11595                 int retval;
11596
11597                 retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
11598                 if (retval != 0)
11599                         return (CTL_LUN_PAT_NONE);
11600
11601                 action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
11602                                               desc->lba_range.len);
11603                 /*
11604                  * A "pass" means that the LBA ranges don't overlap, so
11605                  * this doesn't match the user's range criteria.
11606                  */
11607                 if (action == CTL_ACTION_PASS)
11608                         return (CTL_LUN_PAT_NONE);
11609         }
11610
11611         return (filtered_pattern);
11612 }
11613
11614 static void
11615 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
11616 {
11617         struct ctl_error_desc *desc, *desc2;
11618
11619         mtx_assert(&control_softc->ctl_lock, MA_OWNED);
11620
11621         STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
11622                 ctl_lun_error_pattern pattern;
11623                 /*
11624                  * Check to see whether this particular command matches
11625                  * the pattern in the descriptor.
11626                  */
11627                 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
11628                 if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
11629                         continue;
11630
11631                 switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
11632                 case CTL_LUN_INJ_ABORTED:
11633                         ctl_set_aborted(&io->scsiio);
11634                         break;
11635                 case CTL_LUN_INJ_MEDIUM_ERR:
11636                         ctl_set_medium_error(&io->scsiio);
11637                         break;
11638                 case CTL_LUN_INJ_UA:
11639                         /* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
11640                          * OCCURRED */
11641                         ctl_set_ua(&io->scsiio, 0x29, 0x00);
11642                         break;
11643                 case CTL_LUN_INJ_CUSTOM:
11644                         /*
11645                          * We're assuming the user knows what he is doing.
11646                          * Just copy the sense information without doing
11647                          * checks.
11648                          */
11649                         bcopy(&desc->custom_sense, &io->scsiio.sense_data,
11650                               ctl_min(sizeof(desc->custom_sense),
11651                                       sizeof(io->scsiio.sense_data)));
11652                         io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
11653                         io->scsiio.sense_len = SSD_FULL_SIZE;
11654                         io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
11655                         break;
11656                 case CTL_LUN_INJ_NONE:
11657                 default:
11658                         /*
11659                          * If this is an error injection type we don't know
11660                          * about, clear the continuous flag (if it is set)
11661                          * so it will get deleted below.
11662                          */
11663                         desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
11664                         break;
11665                 }
11666                 /*
11667                  * By default, each error injection action is a one-shot
11668                  */
11669                 if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
11670                         continue;
11671
11672                 STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
11673
11674                 free(desc, M_CTL);
11675         }
11676 }
11677
11678 #ifdef CTL_IO_DELAY
11679 static void
11680 ctl_datamove_timer_wakeup(void *arg)
11681 {
11682         union ctl_io *io;
11683
11684         io = (union ctl_io *)arg;
11685
11686         ctl_datamove(io);
11687 }
11688 #endif /* CTL_IO_DELAY */
11689
11690 void
11691 ctl_datamove(union ctl_io *io)
11692 {
11693         void (*fe_datamove)(union ctl_io *io);
11694
11695         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
11696
11697         CTL_DEBUG_PRINT(("ctl_datamove\n"));
11698
11699 #ifdef CTL_TIME_IO
11700         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
11701                 char str[256];
11702                 char path_str[64];
11703                 struct sbuf sb;
11704
11705                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
11706                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
11707
11708                 sbuf_cat(&sb, path_str);
11709                 switch (io->io_hdr.io_type) {
11710                 case CTL_IO_SCSI:
11711                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
11712                         sbuf_printf(&sb, "\n");
11713                         sbuf_cat(&sb, path_str);
11714                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
11715                                     io->scsiio.tag_num, io->scsiio.tag_type);
11716                         break;
11717                 case CTL_IO_TASK:
11718                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
11719                                     "Tag Type: %d\n", io->taskio.task_action,
11720                                     io->taskio.tag_num, io->taskio.tag_type);
11721                         break;
11722                 default:
11723                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
11724                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
11725                         break;
11726                 }
11727                 sbuf_cat(&sb, path_str);
11728                 sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
11729                             (intmax_t)time_uptime - io->io_hdr.start_time);
11730                 sbuf_finish(&sb);
11731                 printf("%s", sbuf_data(&sb));
11732         }
11733 #endif /* CTL_TIME_IO */
11734
11735         mtx_lock(&control_softc->ctl_lock);
11736 #ifdef CTL_IO_DELAY
11737         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
11738                 struct ctl_lun *lun;
11739
11740                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
11741
11742                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
11743         } else {
11744                 struct ctl_lun *lun;
11745
11746                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
11747                 if ((lun != NULL)
11748                  && (lun->delay_info.datamove_delay > 0)) {
11749                         struct callout *callout;
11750
11751                         callout = (struct callout *)&io->io_hdr.timer_bytes;
11752                         callout_init(callout, /*mpsafe*/ 1);
11753                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
11754                         callout_reset(callout,
11755                                       lun->delay_info.datamove_delay * hz,
11756                                       ctl_datamove_timer_wakeup, io);
11757                         if (lun->delay_info.datamove_type ==
11758                             CTL_DELAY_TYPE_ONESHOT)
11759                                 lun->delay_info.datamove_delay = 0;
11760                         mtx_unlock(&control_softc->ctl_lock);
11761                         return;
11762                 }
11763         }
11764 #endif
11765         /*
11766          * If we have any pending task management commands, process them
11767          * first.  This is necessary to eliminate a race condition with the
11768          * FETD:
11769          *
11770          * - FETD submits a task management command, like an abort.
11771          * - Back end calls fe_datamove() to move the data for the aborted
11772          *   command.  The FETD can't really accept it, but if it did, it
11773          *   would end up transmitting data for a command that the initiator
11774          *   told us to abort.
11775          *
11776          * We close the race by processing all pending task management
11777          * commands here (we can't block!), and then check this I/O to see
11778          * if it has been aborted.  If so, return it to the back end with
11779          * bad status, so the back end can say return an error to the back end
11780          * and then when the back end returns an error, we can return the
11781          * aborted command to the FETD, so it can clean up its resources.
11782          */
11783         if (control_softc->flags & CTL_FLAG_TASK_PENDING)
11784                 ctl_run_task_queue(control_softc);
11785
11786         /*
11787          * This command has been aborted.  Set the port status, so we fail
11788          * the data move.
11789          */
11790         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
11791                 printf("ctl_datamove: tag 0x%04x on (%ju:%d:%ju:%d) aborted\n",
11792                        io->scsiio.tag_num,(uintmax_t)io->io_hdr.nexus.initid.id,
11793                        io->io_hdr.nexus.targ_port,
11794                        (uintmax_t)io->io_hdr.nexus.targ_target.id,
11795                        io->io_hdr.nexus.targ_lun);
11796                 io->io_hdr.status = CTL_CMD_ABORTED;
11797                 io->io_hdr.port_status = 31337;
11798                 mtx_unlock(&control_softc->ctl_lock);
11799                 /*
11800                  * Note that the backend, in this case, will get the
11801                  * callback in its context.  In other cases it may get
11802                  * called in the frontend's interrupt thread context.
11803                  */
11804                 io->scsiio.be_move_done(io);
11805                 return;
11806         }
11807
11808         /*
11809          * If we're in XFER mode and this I/O is from the other shelf
11810          * controller, we need to send the DMA to the other side to
11811          * actually transfer the data to/from the host.  In serialize only
11812          * mode the transfer happens below CTL and ctl_datamove() is only
11813          * called on the machine that originally received the I/O.
11814          */
11815         if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
11816          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11817                 union ctl_ha_msg msg;
11818                 uint32_t sg_entries_sent;
11819                 int do_sg_copy;
11820                 int i;
11821
11822                 memset(&msg, 0, sizeof(msg));
11823                 msg.hdr.msg_type = CTL_MSG_DATAMOVE;
11824                 msg.hdr.original_sc = io->io_hdr.original_sc;
11825                 msg.hdr.serializing_sc = io;
11826                 msg.hdr.nexus = io->io_hdr.nexus;
11827                 msg.dt.flags = io->io_hdr.flags;
11828                 /*
11829                  * We convert everything into a S/G list here.  We can't
11830                  * pass by reference, only by value between controllers.
11831                  * So we can't pass a pointer to the S/G list, only as many
11832                  * S/G entries as we can fit in here.  If it's possible for
11833                  * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
11834                  * then we need to break this up into multiple transfers.
11835                  */
11836                 if (io->scsiio.kern_sg_entries == 0) {
11837                         msg.dt.kern_sg_entries = 1;
11838                         /*
11839                          * If this is in cached memory, flush the cache
11840                          * before we send the DMA request to the other
11841                          * controller.  We want to do this in either the
11842                          * read or the write case.  The read case is
11843                          * straightforward.  In the write case, we want to
11844                          * make sure nothing is in the local cache that
11845                          * could overwrite the DMAed data.
11846                          */
11847                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
11848                                 /*
11849                                  * XXX KDM use bus_dmamap_sync() here.
11850                                  */
11851                         }
11852
11853                         /*
11854                          * Convert to a physical address if this is a
11855                          * virtual address.
11856                          */
11857                         if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
11858                                 msg.dt.sg_list[0].addr =
11859                                         io->scsiio.kern_data_ptr;
11860                         } else {
11861                                 /*
11862                                  * XXX KDM use busdma here!
11863                                  */
11864 #if 0
11865                                 msg.dt.sg_list[0].addr = (void *)
11866                                         vtophys(io->scsiio.kern_data_ptr);
11867 #endif
11868                         }
11869
11870                         msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
11871                         do_sg_copy = 0;
11872                 } else {
11873                         struct ctl_sg_entry *sgl;
11874
11875                         do_sg_copy = 1;
11876                         msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
11877                         sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
11878                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
11879                                 /*
11880                                  * XXX KDM use bus_dmamap_sync() here.
11881                                  */
11882                         }
11883                 }
11884
11885                 msg.dt.kern_data_len = io->scsiio.kern_data_len;
11886                 msg.dt.kern_total_len = io->scsiio.kern_total_len;
11887                 msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
11888                 msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
11889                 msg.dt.sg_sequence = 0;
11890
11891                 /*
11892                  * Loop until we've sent all of the S/G entries.  On the
11893                  * other end, we'll recompose these S/G entries into one
11894                  * contiguous list before passing it to the
11895                  */
11896                 for (sg_entries_sent = 0; sg_entries_sent <
11897                      msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
11898                         msg.dt.cur_sg_entries = ctl_min((sizeof(msg.dt.sg_list)/
11899                                 sizeof(msg.dt.sg_list[0])),
11900                                 msg.dt.kern_sg_entries - sg_entries_sent);
11901
11902                         if (do_sg_copy != 0) {
11903                                 struct ctl_sg_entry *sgl;
11904                                 int j;
11905
11906                                 sgl = (struct ctl_sg_entry *)
11907                                         io->scsiio.kern_data_ptr;
11908                                 /*
11909                                  * If this is in cached memory, flush the cache
11910                                  * before we send the DMA request to the other
11911                                  * controller.  We want to do this in either
11912                                  * the * read or the write case.  The read
11913                                  * case is straightforward.  In the write
11914                                  * case, we want to make sure nothing is
11915                                  * in the local cache that could overwrite
11916                                  * the DMAed data.
11917                                  */
11918
11919                                 for (i = sg_entries_sent, j = 0;
11920                                      i < msg.dt.cur_sg_entries; i++, j++) {
11921                                         if ((io->io_hdr.flags &
11922                                              CTL_FLAG_NO_DATASYNC) == 0) {
11923                                                 /*
11924                                                  * XXX KDM use bus_dmamap_sync()
11925                                                  */
11926                                         }
11927                                         if ((io->io_hdr.flags &
11928                                              CTL_FLAG_BUS_ADDR) == 0) {
11929                                                 /*
11930                                                  * XXX KDM use busdma.
11931                                                  */
11932 #if 0
11933                                                 msg.dt.sg_list[j].addr =(void *)
11934                                                        vtophys(sgl[i].addr);
11935 #endif
11936                                         } else {
11937                                                 msg.dt.sg_list[j].addr =
11938                                                         sgl[i].addr;
11939                                         }
11940                                         msg.dt.sg_list[j].len = sgl[i].len;
11941                                 }
11942                         }
11943
11944                         sg_entries_sent += msg.dt.cur_sg_entries;
11945                         if (sg_entries_sent >= msg.dt.kern_sg_entries)
11946                                 msg.dt.sg_last = 1;
11947                         else
11948                                 msg.dt.sg_last = 0;
11949
11950                         /*
11951                          * XXX KDM drop and reacquire the lock here?
11952                          */
11953                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
11954                             sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
11955                                 /*
11956                                  * XXX do something here.
11957                                  */
11958                         }
11959
11960                         msg.dt.sent_sg_entries = sg_entries_sent;
11961                 }
11962                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11963                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER)
11964                         ctl_failover_io(io, /*have_lock*/ 1);
11965
11966         } else {
11967
11968                 /*
11969                  * Lookup the fe_datamove() function for this particular
11970                  * front end.
11971                  */
11972                 fe_datamove =
11973                     control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
11974                 mtx_unlock(&control_softc->ctl_lock);
11975
11976                 fe_datamove(io);
11977         }
11978 }
11979
11980 static void
11981 ctl_send_datamove_done(union ctl_io *io, int have_lock)
11982 {
11983         union ctl_ha_msg msg;
11984         int isc_status;
11985
11986         memset(&msg, 0, sizeof(msg));
11987
11988         msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
11989         msg.hdr.original_sc = io;
11990         msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
11991         msg.hdr.nexus = io->io_hdr.nexus;
11992         msg.hdr.status = io->io_hdr.status;
11993         msg.scsi.tag_num = io->scsiio.tag_num;
11994         msg.scsi.tag_type = io->scsiio.tag_type;
11995         msg.scsi.scsi_status = io->scsiio.scsi_status;
11996         memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
11997                sizeof(io->scsiio.sense_data));
11998         msg.scsi.sense_len = io->scsiio.sense_len;
11999         msg.scsi.sense_residual = io->scsiio.sense_residual;
12000         msg.scsi.fetd_status = io->io_hdr.port_status;
12001         msg.scsi.residual = io->scsiio.residual;
12002         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12003
12004         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12005                 ctl_failover_io(io, /*have_lock*/ have_lock);
12006                 return;
12007         }
12008
12009         isc_status = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0);
12010         if (isc_status > CTL_HA_STATUS_SUCCESS) {
12011                 /* XXX do something if this fails */
12012         }
12013
12014 }
12015
12016 /*
12017  * The DMA to the remote side is done, now we need to tell the other side
12018  * we're done so it can continue with its data movement.
12019  */
12020 static void
12021 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
12022 {
12023         union ctl_io *io;
12024
12025         io = rq->context;
12026
12027         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12028                 printf("%s: ISC DMA write failed with error %d", __func__,
12029                        rq->ret);
12030                 ctl_set_internal_failure(&io->scsiio,
12031                                          /*sks_valid*/ 1,
12032                                          /*retry_count*/ rq->ret);
12033         }
12034
12035         ctl_dt_req_free(rq);
12036
12037         /*
12038          * In this case, we had to malloc the memory locally.  Free it.
12039          */
12040         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
12041                 int i;
12042                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12043                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12044         }
12045         /*
12046          * The data is in local and remote memory, so now we need to send
12047          * status (good or back) back to the other side.
12048          */
12049         ctl_send_datamove_done(io, /*have_lock*/ 0);
12050 }
12051
12052 /*
12053  * We've moved the data from the host/controller into local memory.  Now we
12054  * need to push it over to the remote controller's memory.
12055  */
12056 static int
12057 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
12058 {
12059         int retval;
12060
12061         retval = 0;
12062
12063         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
12064                                           ctl_datamove_remote_write_cb);
12065
12066         return (retval);
12067 }
12068
12069 static void
12070 ctl_datamove_remote_write(union ctl_io *io)
12071 {
12072         int retval;
12073         void (*fe_datamove)(union ctl_io *io);
12074
12075         /*
12076          * - Get the data from the host/HBA into local memory.
12077          * - DMA memory from the local controller to the remote controller.
12078          * - Send status back to the remote controller.
12079          */
12080
12081         retval = ctl_datamove_remote_sgl_setup(io);
12082         if (retval != 0)
12083                 return;
12084
12085         /* Switch the pointer over so the FETD knows what to do */
12086         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12087
12088         /*
12089          * Use a custom move done callback, since we need to send completion
12090          * back to the other controller, not to the backend on this side.
12091          */
12092         io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
12093
12094         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12095
12096         fe_datamove(io);
12097
12098         return;
12099
12100 }
12101
12102 static int
12103 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
12104 {
12105 #if 0
12106         char str[256];
12107         char path_str[64];
12108         struct sbuf sb;
12109 #endif
12110
12111         /*
12112          * In this case, we had to malloc the memory locally.  Free it.
12113          */
12114         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
12115                 int i;
12116                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12117                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12118         }
12119
12120 #if 0
12121         scsi_path_string(io, path_str, sizeof(path_str));
12122         sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12123         sbuf_cat(&sb, path_str);
12124         scsi_command_string(&io->scsiio, NULL, &sb);
12125         sbuf_printf(&sb, "\n");
12126         sbuf_cat(&sb, path_str);
12127         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12128                     io->scsiio.tag_num, io->scsiio.tag_type);
12129         sbuf_cat(&sb, path_str);
12130         sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
12131                     io->io_hdr.flags, io->io_hdr.status);
12132         sbuf_finish(&sb);
12133         printk("%s", sbuf_data(&sb));
12134 #endif
12135
12136
12137         /*
12138          * The read is done, now we need to send status (good or bad) back
12139          * to the other side.
12140          */
12141         ctl_send_datamove_done(io, /*have_lock*/ 0);
12142
12143         return (0);
12144 }
12145
12146 static void
12147 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
12148 {
12149         union ctl_io *io;
12150         void (*fe_datamove)(union ctl_io *io);
12151
12152         io = rq->context;
12153
12154         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12155                 printf("%s: ISC DMA read failed with error %d", __func__,
12156                        rq->ret);
12157                 ctl_set_internal_failure(&io->scsiio,
12158                                          /*sks_valid*/ 1,
12159                                          /*retry_count*/ rq->ret);
12160         }
12161
12162         ctl_dt_req_free(rq);
12163
12164         /* Switch the pointer over so the FETD knows what to do */
12165         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12166
12167         /*
12168          * Use a custom move done callback, since we need to send completion
12169          * back to the other controller, not to the backend on this side.
12170          */
12171         io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
12172
12173         /* XXX KDM add checks like the ones in ctl_datamove? */
12174
12175         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12176
12177         fe_datamove(io);
12178 }
12179
12180 static int
12181 ctl_datamove_remote_sgl_setup(union ctl_io *io)
12182 {
12183         struct ctl_sg_entry *local_sglist, *remote_sglist;
12184         struct ctl_sg_entry *local_dma_sglist, *remote_dma_sglist;
12185         struct ctl_softc *softc;
12186         int retval;
12187         int i;
12188
12189         retval = 0;
12190         softc = control_softc;
12191
12192         local_sglist = io->io_hdr.local_sglist;
12193         local_dma_sglist = io->io_hdr.local_dma_sglist;
12194         remote_sglist = io->io_hdr.remote_sglist;
12195         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
12196
12197         if (io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) {
12198                 for (i = 0; i < io->scsiio.kern_sg_entries; i++) {
12199                         local_sglist[i].len = remote_sglist[i].len;
12200
12201                         /*
12202                          * XXX Detect the situation where the RS-level I/O
12203                          * redirector on the other side has already read the
12204                          * data off of the AOR RS on this side, and
12205                          * transferred it to remote (mirror) memory on the
12206                          * other side.  Since we already have the data in
12207                          * memory here, we just need to use it.
12208                          *
12209                          * XXX KDM this can probably be removed once we
12210                          * get the cache device code in and take the
12211                          * current AOR implementation out.
12212                          */
12213 #ifdef NEEDTOPORT
12214                         if ((remote_sglist[i].addr >=
12215                              (void *)vtophys(softc->mirr->addr))
12216                          && (remote_sglist[i].addr <
12217                              ((void *)vtophys(softc->mirr->addr) +
12218                              CacheMirrorOffset))) {
12219                                 local_sglist[i].addr = remote_sglist[i].addr -
12220                                         CacheMirrorOffset;
12221                                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
12222                                      CTL_FLAG_DATA_IN)
12223                                         io->io_hdr.flags |= CTL_FLAG_REDIR_DONE;
12224                         } else {
12225                                 local_sglist[i].addr = remote_sglist[i].addr +
12226                                         CacheMirrorOffset;
12227                         }
12228 #endif
12229 #if 0
12230                         printf("%s: local %p, remote %p, len %d\n",
12231                                __func__, local_sglist[i].addr,
12232                                remote_sglist[i].addr, local_sglist[i].len);
12233 #endif
12234                 }
12235         } else {
12236                 uint32_t len_to_go;
12237
12238                 /*
12239                  * In this case, we don't have automatically allocated
12240                  * memory for this I/O on this controller.  This typically
12241                  * happens with internal CTL I/O -- e.g. inquiry, mode
12242                  * sense, etc.  Anything coming from RAIDCore will have
12243                  * a mirror area available.
12244                  */
12245                 len_to_go = io->scsiio.kern_data_len;
12246
12247                 /*
12248                  * Clear the no datasync flag, we have to use malloced
12249                  * buffers.
12250                  */
12251                 io->io_hdr.flags &= ~CTL_FLAG_NO_DATASYNC;
12252
12253                 /*
12254                  * The difficult thing here is that the size of the various
12255                  * S/G segments may be different than the size from the
12256                  * remote controller.  That'll make it harder when DMAing
12257                  * the data back to the other side.
12258                  */
12259                 for (i = 0; (i < sizeof(io->io_hdr.remote_sglist) /
12260                      sizeof(io->io_hdr.remote_sglist[0])) &&
12261                      (len_to_go > 0); i++) {
12262                         local_sglist[i].len = ctl_min(len_to_go, 131072);
12263                         CTL_SIZE_8B(local_dma_sglist[i].len,
12264                                     local_sglist[i].len);
12265                         local_sglist[i].addr =
12266                                 malloc(local_dma_sglist[i].len, M_CTL,M_WAITOK);
12267
12268                         local_dma_sglist[i].addr = local_sglist[i].addr;
12269
12270                         if (local_sglist[i].addr == NULL) {
12271                                 int j;
12272
12273                                 printf("malloc failed for %zd bytes!",
12274                                        local_dma_sglist[i].len);
12275                                 for (j = 0; j < i; j++) {
12276                                         free(local_sglist[j].addr, M_CTL);
12277                                 }
12278                                 ctl_set_internal_failure(&io->scsiio,
12279                                                          /*sks_valid*/ 1,
12280                                                          /*retry_count*/ 4857);
12281                                 retval = 1;
12282                                 goto bailout_error;
12283
12284                         }
12285                         /* XXX KDM do we need a sync here? */
12286
12287                         len_to_go -= local_sglist[i].len;
12288                 }
12289                 /*
12290                  * Reset the number of S/G entries accordingly.  The
12291                  * original number of S/G entries is available in
12292                  * rem_sg_entries.
12293                  */
12294                 io->scsiio.kern_sg_entries = i;
12295
12296 #if 0
12297                 printf("%s: kern_sg_entries = %d\n", __func__,
12298                        io->scsiio.kern_sg_entries);
12299                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12300                         printf("%s: sg[%d] = %p, %d (DMA: %d)\n", __func__, i,
12301                                local_sglist[i].addr, local_sglist[i].len,
12302                                local_dma_sglist[i].len);
12303 #endif
12304         }
12305
12306
12307         return (retval);
12308
12309 bailout_error:
12310
12311         ctl_send_datamove_done(io, /*have_lock*/ 0);
12312
12313         return (retval);
12314 }
12315
12316 static int
12317 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
12318                          ctl_ha_dt_cb callback)
12319 {
12320         struct ctl_ha_dt_req *rq;
12321         struct ctl_sg_entry *remote_sglist, *local_sglist;
12322         struct ctl_sg_entry *remote_dma_sglist, *local_dma_sglist;
12323         uint32_t local_used, remote_used, total_used;
12324         int retval;
12325         int i, j;
12326
12327         retval = 0;
12328
12329         rq = ctl_dt_req_alloc();
12330
12331         /*
12332          * If we failed to allocate the request, and if the DMA didn't fail
12333          * anyway, set busy status.  This is just a resource allocation
12334          * failure.
12335          */
12336         if ((rq == NULL)
12337          && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
12338                 ctl_set_busy(&io->scsiio);
12339
12340         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
12341
12342                 if (rq != NULL)
12343                         ctl_dt_req_free(rq);
12344
12345                 /*
12346                  * The data move failed.  We need to return status back
12347                  * to the other controller.  No point in trying to DMA
12348                  * data to the remote controller.
12349                  */
12350
12351                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12352
12353                 retval = 1;
12354
12355                 goto bailout;
12356         }
12357
12358         local_sglist = io->io_hdr.local_sglist;
12359         local_dma_sglist = io->io_hdr.local_dma_sglist;
12360         remote_sglist = io->io_hdr.remote_sglist;
12361         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
12362         local_used = 0;
12363         remote_used = 0;
12364         total_used = 0;
12365
12366         if (io->io_hdr.flags & CTL_FLAG_REDIR_DONE) {
12367                 rq->ret = CTL_HA_STATUS_SUCCESS;
12368                 rq->context = io;
12369                 callback(rq);
12370                 goto bailout;
12371         }
12372
12373         /*
12374          * Pull/push the data over the wire from/to the other controller.
12375          * This takes into account the possibility that the local and
12376          * remote sglists may not be identical in terms of the size of
12377          * the elements and the number of elements.
12378          *
12379          * One fundamental assumption here is that the length allocated for
12380          * both the local and remote sglists is identical.  Otherwise, we've
12381          * essentially got a coding error of some sort.
12382          */
12383         for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
12384                 int isc_ret;
12385                 uint32_t cur_len, dma_length;
12386                 uint8_t *tmp_ptr;
12387
12388                 rq->id = CTL_HA_DATA_CTL;
12389                 rq->command = command;
12390                 rq->context = io;
12391
12392                 /*
12393                  * Both pointers should be aligned.  But it is possible
12394                  * that the allocation length is not.  They should both
12395                  * also have enough slack left over at the end, though,
12396                  * to round up to the next 8 byte boundary.
12397                  */
12398                 cur_len = ctl_min(local_sglist[i].len - local_used,
12399                                   remote_sglist[j].len - remote_used);
12400
12401                 /*
12402                  * In this case, we have a size issue and need to decrease
12403                  * the size, except in the case where we actually have less
12404                  * than 8 bytes left.  In that case, we need to increase
12405                  * the DMA length to get the last bit.
12406                  */
12407                 if ((cur_len & 0x7) != 0) {
12408                         if (cur_len > 0x7) {
12409                                 cur_len = cur_len - (cur_len & 0x7);
12410                                 dma_length = cur_len;
12411                         } else {
12412                                 CTL_SIZE_8B(dma_length, cur_len);
12413                         }
12414
12415                 } else
12416                         dma_length = cur_len;
12417
12418                 /*
12419                  * If we had to allocate memory for this I/O, instead of using
12420                  * the non-cached mirror memory, we'll need to flush the cache
12421                  * before trying to DMA to the other controller.
12422                  *
12423                  * We could end up doing this multiple times for the same
12424                  * segment if we have a larger local segment than remote
12425                  * segment.  That shouldn't be an issue.
12426                  */
12427                 if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12428                         /*
12429                          * XXX KDM use bus_dmamap_sync() here.
12430                          */
12431                 }
12432
12433                 rq->size = dma_length;
12434
12435                 tmp_ptr = (uint8_t *)local_sglist[i].addr;
12436                 tmp_ptr += local_used;
12437
12438                 /* Use physical addresses when talking to ISC hardware */
12439                 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
12440                         /* XXX KDM use busdma */
12441 #if 0
12442                         rq->local = vtophys(tmp_ptr);
12443 #endif
12444                 } else
12445                         rq->local = tmp_ptr;
12446
12447                 tmp_ptr = (uint8_t *)remote_sglist[j].addr;
12448                 tmp_ptr += remote_used;
12449                 rq->remote = tmp_ptr;
12450
12451                 rq->callback = NULL;
12452
12453                 local_used += cur_len;
12454                 if (local_used >= local_sglist[i].len) {
12455                         i++;
12456                         local_used = 0;
12457                 }
12458
12459                 remote_used += cur_len;
12460                 if (remote_used >= remote_sglist[j].len) {
12461                         j++;
12462                         remote_used = 0;
12463                 }
12464                 total_used += cur_len;
12465
12466                 if (total_used >= io->scsiio.kern_data_len)
12467                         rq->callback = callback;
12468
12469                 if ((rq->size & 0x7) != 0) {
12470                         printf("%s: warning: size %d is not on 8b boundary\n",
12471                                __func__, rq->size);
12472                 }
12473                 if (((uintptr_t)rq->local & 0x7) != 0) {
12474                         printf("%s: warning: local %p not on 8b boundary\n",
12475                                __func__, rq->local);
12476                 }
12477                 if (((uintptr_t)rq->remote & 0x7) != 0) {
12478                         printf("%s: warning: remote %p not on 8b boundary\n",
12479                                __func__, rq->local);
12480                 }
12481 #if 0
12482                 printf("%s: %s: local %#x remote %#x size %d\n", __func__,
12483                        (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
12484                        rq->local, rq->remote, rq->size);
12485 #endif
12486
12487                 isc_ret = ctl_dt_single(rq);
12488                 if (isc_ret == CTL_HA_STATUS_WAIT)
12489                         continue;
12490
12491                 if (isc_ret == CTL_HA_STATUS_DISCONNECT) {
12492                         rq->ret = CTL_HA_STATUS_SUCCESS;
12493                 } else {
12494                         rq->ret = isc_ret;
12495                 }
12496                 callback(rq);
12497                 goto bailout;
12498         }
12499
12500 bailout:
12501         return (retval);
12502
12503 }
12504
12505 static void
12506 ctl_datamove_remote_read(union ctl_io *io)
12507 {
12508         int retval;
12509         int i;
12510
12511         /*
12512          * This will send an error to the other controller in the case of a
12513          * failure.
12514          */
12515         retval = ctl_datamove_remote_sgl_setup(io);
12516         if (retval != 0)
12517                 return;
12518
12519         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
12520                                           ctl_datamove_remote_read_cb);
12521         if ((retval != 0)
12522          && ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0)) {
12523                 /*
12524                  * Make sure we free memory if there was an error..  The
12525                  * ctl_datamove_remote_xfer() function will send the
12526                  * datamove done message, or call the callback with an
12527                  * error if there is a problem.
12528                  */
12529                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12530                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12531         }
12532
12533         return;
12534 }
12535
12536 /*
12537  * Process a datamove request from the other controller.  This is used for
12538  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
12539  * first.  Once that is complete, the data gets DMAed into the remote
12540  * controller's memory.  For reads, we DMA from the remote controller's
12541  * memory into our memory first, and then move it out to the FETD.
12542  */
12543 static void
12544 ctl_datamove_remote(union ctl_io *io)
12545 {
12546         struct ctl_softc *softc;
12547
12548         softc = control_softc;
12549
12550         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
12551
12552         /*
12553          * Note that we look for an aborted I/O here, but don't do some of
12554          * the other checks that ctl_datamove() normally does.  We don't
12555          * need to run the task queue, because this I/O is on the ISC
12556          * queue, which is executed by the work thread after the task queue.
12557          * We don't need to run the datamove delay code, since that should
12558          * have been done if need be on the other controller.
12559          */
12560         mtx_lock(&softc->ctl_lock);
12561
12562         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12563
12564                 printf("%s: tag 0x%04x on (%d:%d:%d:%d) aborted\n", __func__,
12565                        io->scsiio.tag_num, io->io_hdr.nexus.initid.id,
12566                        io->io_hdr.nexus.targ_port,
12567                        io->io_hdr.nexus.targ_target.id,
12568                        io->io_hdr.nexus.targ_lun);
12569                 io->io_hdr.status = CTL_CMD_ABORTED;
12570                 io->io_hdr.port_status = 31338;
12571
12572                 mtx_unlock(&softc->ctl_lock);
12573
12574                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12575
12576                 return;
12577         }
12578
12579         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT) {
12580                 mtx_unlock(&softc->ctl_lock);
12581                 ctl_datamove_remote_write(io);
12582         } else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN){
12583                 mtx_unlock(&softc->ctl_lock);
12584                 ctl_datamove_remote_read(io);
12585         } else {
12586                 union ctl_ha_msg msg;
12587                 struct scsi_sense_data *sense;
12588                 uint8_t sks[3];
12589                 int retry_count;
12590
12591                 memset(&msg, 0, sizeof(msg));
12592
12593                 msg.hdr.msg_type = CTL_MSG_BAD_JUJU;
12594                 msg.hdr.status = CTL_SCSI_ERROR;
12595                 msg.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
12596
12597                 retry_count = 4243;
12598
12599                 sense = &msg.scsi.sense_data;
12600                 sks[0] = SSD_SCS_VALID;
12601                 sks[1] = (retry_count >> 8) & 0xff;
12602                 sks[2] = retry_count & 0xff;
12603
12604                 /* "Internal target failure" */
12605                 scsi_set_sense_data(sense,
12606                                     /*sense_format*/ SSD_TYPE_NONE,
12607                                     /*current_error*/ 1,
12608                                     /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
12609                                     /*asc*/ 0x44,
12610                                     /*ascq*/ 0x00,
12611                                     /*type*/ SSD_ELEM_SKS,
12612                                     /*size*/ sizeof(sks),
12613                                     /*data*/ sks,
12614                                     SSD_ELEM_NONE);
12615
12616                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12617                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12618                         ctl_failover_io(io, /*have_lock*/ 1);
12619                         mtx_unlock(&softc->ctl_lock);
12620                         return;
12621                 }
12622
12623                 mtx_unlock(&softc->ctl_lock);
12624
12625                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0) >
12626                     CTL_HA_STATUS_SUCCESS) {
12627                         /* XXX KDM what to do if this fails? */
12628                 }
12629                 return;
12630         }
12631
12632 }
12633
12634 static int
12635 ctl_process_done(union ctl_io *io, int have_lock)
12636 {
12637         struct ctl_lun *lun;
12638         struct ctl_softc *ctl_softc;
12639         void (*fe_done)(union ctl_io *io);
12640         uint32_t targ_port = ctl_port_idx(io->io_hdr.nexus.targ_port);
12641
12642         CTL_DEBUG_PRINT(("ctl_process_done\n"));
12643
12644         fe_done =
12645             control_softc->ctl_ports[targ_port]->fe_done;
12646
12647 #ifdef CTL_TIME_IO
12648         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12649                 char str[256];
12650                 char path_str[64];
12651                 struct sbuf sb;
12652
12653                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12654                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12655
12656                 sbuf_cat(&sb, path_str);
12657                 switch (io->io_hdr.io_type) {
12658                 case CTL_IO_SCSI:
12659                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12660                         sbuf_printf(&sb, "\n");
12661                         sbuf_cat(&sb, path_str);
12662                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12663                                     io->scsiio.tag_num, io->scsiio.tag_type);
12664                         break;
12665                 case CTL_IO_TASK:
12666                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12667                                     "Tag Type: %d\n", io->taskio.task_action,
12668                                     io->taskio.tag_num, io->taskio.tag_type);
12669                         break;
12670                 default:
12671                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12672                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12673                         break;
12674                 }
12675                 sbuf_cat(&sb, path_str);
12676                 sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
12677                             (intmax_t)time_uptime - io->io_hdr.start_time);
12678                 sbuf_finish(&sb);
12679                 printf("%s", sbuf_data(&sb));
12680         }
12681 #endif /* CTL_TIME_IO */
12682
12683         switch (io->io_hdr.io_type) {
12684         case CTL_IO_SCSI:
12685                 break;
12686         case CTL_IO_TASK:
12687                 if (bootverbose || verbose > 0)
12688                         ctl_io_error_print(io, NULL);
12689                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
12690                         ctl_free_io(io);
12691                 else
12692                         fe_done(io);
12693                 return (CTL_RETVAL_COMPLETE);
12694                 break;
12695         default:
12696                 printf("ctl_process_done: invalid io type %d\n",
12697                        io->io_hdr.io_type);
12698                 panic("ctl_process_done: invalid io type %d\n",
12699                       io->io_hdr.io_type);
12700                 break; /* NOTREACHED */
12701         }
12702
12703         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12704         if (lun == NULL) {
12705                 CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
12706                                  io->io_hdr.nexus.targ_lun));
12707                 fe_done(io);
12708                 goto bailout;
12709         }
12710         ctl_softc = lun->ctl_softc;
12711
12712         /*
12713          * Remove this from the OOA queue.
12714          */
12715         if (have_lock == 0)
12716                 mtx_lock(&ctl_softc->ctl_lock);
12717
12718         /*
12719          * Check to see if we have any errors to inject here.  We only
12720          * inject errors for commands that don't already have errors set.
12721          */
12722         if ((STAILQ_FIRST(&lun->error_list) != NULL)
12723          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))
12724                 ctl_inject_error(lun, io);
12725
12726         /*
12727          * XXX KDM how do we treat commands that aren't completed
12728          * successfully?
12729          *
12730          * XXX KDM should we also track I/O latency?
12731          */
12732         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
12733                 uint32_t blocksize;
12734 #ifdef CTL_TIME_IO
12735                 struct bintime cur_bt;
12736 #endif
12737
12738                 if ((lun->be_lun != NULL)
12739                  && (lun->be_lun->blocksize != 0))
12740                         blocksize = lun->be_lun->blocksize;
12741                 else
12742                         blocksize = 512;
12743
12744                 switch (io->io_hdr.io_type) {
12745                 case CTL_IO_SCSI: {
12746                         int isread;
12747                         struct ctl_lba_len lbalen;
12748
12749                         isread = 0;
12750                         switch (io->scsiio.cdb[0]) {
12751                         case READ_6:
12752                         case READ_10:
12753                         case READ_12:
12754                         case READ_16:
12755                                 isread = 1;
12756                                 /* FALLTHROUGH */
12757                         case WRITE_6:
12758                         case WRITE_10:
12759                         case WRITE_12:
12760                         case WRITE_16:
12761                         case WRITE_VERIFY_10:
12762                         case WRITE_VERIFY_12:
12763                         case WRITE_VERIFY_16:
12764                                 memcpy(&lbalen, io->io_hdr.ctl_private[
12765                                        CTL_PRIV_LBA_LEN].bytes, sizeof(lbalen));
12766
12767                                 if (isread) {
12768                                         lun->stats.ports[targ_port].bytes[CTL_STATS_READ] +=
12769                                                 lbalen.len * blocksize;
12770                                         lun->stats.ports[targ_port].operations[CTL_STATS_READ]++;
12771
12772 #ifdef CTL_TIME_IO
12773                                         bintime_add(
12774                                            &lun->stats.ports[targ_port].dma_time[CTL_STATS_READ],
12775                                            &io->io_hdr.dma_bt);
12776                                         lun->stats.ports[targ_port].num_dmas[CTL_STATS_READ] +=
12777                                                 io->io_hdr.num_dmas;
12778                                         getbintime(&cur_bt);
12779                                         bintime_sub(&cur_bt,
12780                                                     &io->io_hdr.start_bt);
12781
12782                                         bintime_add(
12783                                             &lun->stats.ports[targ_port].time[CTL_STATS_READ],
12784                                             &cur_bt);
12785
12786 #if 0
12787                                         cs_prof_gettime(&cur_ticks);
12788                                         lun->stats.time[CTL_STATS_READ] +=
12789                                                 cur_ticks -
12790                                                 io->io_hdr.start_ticks;
12791 #endif
12792 #if 0
12793                                         lun->stats.time[CTL_STATS_READ] +=
12794                                                 jiffies - io->io_hdr.start_time;
12795 #endif
12796 #endif /* CTL_TIME_IO */
12797                                 } else {
12798                                         lun->stats.ports[targ_port].bytes[CTL_STATS_WRITE] +=
12799                                                 lbalen.len * blocksize;
12800                                         lun->stats.ports[targ_port].operations[
12801                                                 CTL_STATS_WRITE]++;
12802
12803 #ifdef CTL_TIME_IO
12804                                         bintime_add(
12805                                           &lun->stats.ports[targ_port].dma_time[CTL_STATS_WRITE],
12806                                           &io->io_hdr.dma_bt);
12807                                         lun->stats.ports[targ_port].num_dmas[CTL_STATS_WRITE] +=
12808                                                 io->io_hdr.num_dmas;
12809                                         getbintime(&cur_bt);
12810                                         bintime_sub(&cur_bt,
12811                                                     &io->io_hdr.start_bt);
12812
12813                                         bintime_add(
12814                                             &lun->stats.ports[targ_port].time[CTL_STATS_WRITE],
12815                                             &cur_bt);
12816 #if 0
12817                                         cs_prof_gettime(&cur_ticks);
12818                                         lun->stats.ports[targ_port].time[CTL_STATS_WRITE] +=
12819                                                 cur_ticks -
12820                                                 io->io_hdr.start_ticks;
12821                                         lun->stats.ports[targ_port].time[CTL_STATS_WRITE] +=
12822                                                 jiffies - io->io_hdr.start_time;
12823 #endif
12824 #endif /* CTL_TIME_IO */
12825                                 }
12826                                 break;
12827                         default:
12828                                 lun->stats.ports[targ_port].operations[CTL_STATS_NO_IO]++;
12829
12830 #ifdef CTL_TIME_IO
12831                                 bintime_add(
12832                                   &lun->stats.ports[targ_port].dma_time[CTL_STATS_NO_IO],
12833                                   &io->io_hdr.dma_bt);
12834                                 lun->stats.ports[targ_port].num_dmas[CTL_STATS_NO_IO] +=
12835                                         io->io_hdr.num_dmas;
12836                                 getbintime(&cur_bt);
12837                                 bintime_sub(&cur_bt, &io->io_hdr.start_bt);
12838
12839                                 bintime_add(&lun->stats.ports[targ_port].time[CTL_STATS_NO_IO],
12840                                             &cur_bt);
12841
12842 #if 0
12843                                 cs_prof_gettime(&cur_ticks);
12844                                 lun->stats.ports[targ_port].time[CTL_STATS_NO_IO] +=
12845                                         cur_ticks -
12846                                         io->io_hdr.start_ticks;
12847                                 lun->stats.ports[targ_port].time[CTL_STATS_NO_IO] +=
12848                                         jiffies - io->io_hdr.start_time;
12849 #endif
12850 #endif /* CTL_TIME_IO */
12851                                 break;
12852                         }
12853                         break;
12854                 }
12855                 default:
12856                         break;
12857                 }
12858         }
12859
12860         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
12861
12862         /*
12863          * Run through the blocked queue on this LUN and see if anything
12864          * has become unblocked, now that this transaction is done.
12865          */
12866         ctl_check_blocked(lun);
12867
12868         /*
12869          * If the LUN has been invalidated, free it if there is nothing
12870          * left on its OOA queue.
12871          */
12872         if ((lun->flags & CTL_LUN_INVALID)
12873          && (TAILQ_FIRST(&lun->ooa_queue) == NULL))
12874                 ctl_free_lun(lun);
12875
12876         /*
12877          * If this command has been aborted, make sure we set the status
12878          * properly.  The FETD is responsible for freeing the I/O and doing
12879          * whatever it needs to do to clean up its state.
12880          */
12881         if (io->io_hdr.flags & CTL_FLAG_ABORT)
12882                 io->io_hdr.status = CTL_CMD_ABORTED;
12883
12884         /*
12885          * We print out status for every task management command.  For SCSI
12886          * commands, we filter out any unit attention errors; they happen
12887          * on every boot, and would clutter up the log.  Note:  task
12888          * management commands aren't printed here, they are printed above,
12889          * since they should never even make it down here.
12890          */
12891         switch (io->io_hdr.io_type) {
12892         case CTL_IO_SCSI: {
12893                 int error_code, sense_key, asc, ascq;
12894
12895                 sense_key = 0;
12896
12897                 if (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR)
12898                  && (io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)) {
12899                         /*
12900                          * Since this is just for printing, no need to
12901                          * show errors here.
12902                          */
12903                         scsi_extract_sense_len(&io->scsiio.sense_data,
12904                                                io->scsiio.sense_len,
12905                                                &error_code,
12906                                                &sense_key,
12907                                                &asc,
12908                                                &ascq,
12909                                                /*show_errors*/ 0);
12910                 }
12911
12912                 if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
12913                  && (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SCSI_ERROR)
12914                   || (io->scsiio.scsi_status != SCSI_STATUS_CHECK_COND)
12915                   || (sense_key != SSD_KEY_UNIT_ATTENTION))) {
12916
12917                         if ((time_uptime - ctl_softc->last_print_jiffies) <= 0){
12918                                 ctl_softc->skipped_prints++;
12919                                 if (have_lock == 0)
12920                                         mtx_unlock(&ctl_softc->ctl_lock);
12921                         } else {
12922                                 uint32_t skipped_prints;
12923
12924                                 skipped_prints = ctl_softc->skipped_prints;
12925
12926                                 ctl_softc->skipped_prints = 0;
12927                                 ctl_softc->last_print_jiffies = time_uptime;
12928
12929                                 if (have_lock == 0)
12930                                         mtx_unlock(&ctl_softc->ctl_lock);
12931                                 if (skipped_prints > 0) {
12932 #ifdef NEEDTOPORT
12933                                         csevent_log(CSC_CTL | CSC_SHELF_SW |
12934                                             CTL_ERROR_REPORT,
12935                                             csevent_LogType_Trace,
12936                                             csevent_Severity_Information,
12937                                             csevent_AlertLevel_Green,
12938                                             csevent_FRU_Firmware,
12939                                             csevent_FRU_Unknown,
12940                                             "High CTL error volume, %d prints "
12941                                             "skipped", skipped_prints);
12942 #endif
12943                                 }
12944                                 if (bootverbose || verbose > 0)
12945                                         ctl_io_error_print(io, NULL);
12946                         }
12947                 } else {
12948                         if (have_lock == 0)
12949                                 mtx_unlock(&ctl_softc->ctl_lock);
12950                 }
12951                 break;
12952         }
12953         case CTL_IO_TASK:
12954                 if (have_lock == 0)
12955                         mtx_unlock(&ctl_softc->ctl_lock);
12956                 if (bootverbose || verbose > 0)
12957                         ctl_io_error_print(io, NULL);
12958                 break;
12959         default:
12960                 if (have_lock == 0)
12961                         mtx_unlock(&ctl_softc->ctl_lock);
12962                 break;
12963         }
12964
12965         /*
12966          * Tell the FETD or the other shelf controller we're done with this
12967          * command.  Note that only SCSI commands get to this point.  Task
12968          * management commands are completed above.
12969          *
12970          * We only send status to the other controller if we're in XFER
12971          * mode.  In SER_ONLY mode, the I/O is done on the controller that
12972          * received the I/O (from CTL's perspective), and so the status is
12973          * generated there.
12974          *
12975          * XXX KDM if we hold the lock here, we could cause a deadlock
12976          * if the frontend comes back in in this context to queue
12977          * something.
12978          */
12979         if ((ctl_softc->ha_mode == CTL_HA_MODE_XFER)
12980          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12981                 union ctl_ha_msg msg;
12982
12983                 memset(&msg, 0, sizeof(msg));
12984                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
12985                 msg.hdr.original_sc = io->io_hdr.original_sc;
12986                 msg.hdr.nexus = io->io_hdr.nexus;
12987                 msg.hdr.status = io->io_hdr.status;
12988                 msg.scsi.scsi_status = io->scsiio.scsi_status;
12989                 msg.scsi.tag_num = io->scsiio.tag_num;
12990                 msg.scsi.tag_type = io->scsiio.tag_type;
12991                 msg.scsi.sense_len = io->scsiio.sense_len;
12992                 msg.scsi.sense_residual = io->scsiio.sense_residual;
12993                 msg.scsi.residual = io->scsiio.residual;
12994                 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12995                        sizeof(io->scsiio.sense_data));
12996                 /*
12997                  * We copy this whether or not this is an I/O-related
12998                  * command.  Otherwise, we'd have to go and check to see
12999                  * whether it's a read/write command, and it really isn't
13000                  * worth it.
13001                  */
13002                 memcpy(&msg.scsi.lbalen,
13003                        &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
13004                        sizeof(msg.scsi.lbalen));
13005
13006                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13007                                 sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
13008                         /* XXX do something here */
13009                 }
13010
13011                 ctl_free_io(io);
13012         } else
13013                 fe_done(io);
13014
13015 bailout:
13016
13017         return (CTL_RETVAL_COMPLETE);
13018 }
13019
13020 /*
13021  * Front end should call this if it doesn't do autosense.  When the request
13022  * sense comes back in from the initiator, we'll dequeue this and send it.
13023  */
13024 int
13025 ctl_queue_sense(union ctl_io *io)
13026 {
13027         struct ctl_lun *lun;
13028         struct ctl_softc *ctl_softc;
13029         uint32_t initidx, targ_lun;
13030
13031         ctl_softc = control_softc;
13032
13033         CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
13034
13035         /*
13036          * LUN lookup will likely move to the ctl_work_thread() once we
13037          * have our new queueing infrastructure (that doesn't put things on
13038          * a per-LUN queue initially).  That is so that we can handle
13039          * things like an INQUIRY to a LUN that we don't have enabled.  We
13040          * can't deal with that right now.
13041          */
13042         mtx_lock(&ctl_softc->ctl_lock);
13043
13044         /*
13045          * If we don't have a LUN for this, just toss the sense
13046          * information.
13047          */
13048         targ_lun = io->io_hdr.nexus.targ_lun;
13049         if (io->io_hdr.nexus.lun_map_fn != NULL)
13050                 targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
13051         if ((targ_lun < CTL_MAX_LUNS)
13052          && (ctl_softc->ctl_luns[targ_lun] != NULL))
13053                 lun = ctl_softc->ctl_luns[targ_lun];
13054         else
13055                 goto bailout;
13056
13057         initidx = ctl_get_initindex(&io->io_hdr.nexus);
13058
13059         /*
13060          * Already have CA set for this LUN...toss the sense information.
13061          */
13062         if (ctl_is_set(lun->have_ca, initidx))
13063                 goto bailout;
13064
13065         memcpy(&lun->pending_sense[initidx].sense, &io->scsiio.sense_data,
13066                ctl_min(sizeof(lun->pending_sense[initidx].sense),
13067                sizeof(io->scsiio.sense_data)));
13068         ctl_set_mask(lun->have_ca, initidx);
13069
13070 bailout:
13071         mtx_unlock(&ctl_softc->ctl_lock);
13072
13073         ctl_free_io(io);
13074
13075         return (CTL_RETVAL_COMPLETE);
13076 }
13077
13078 /*
13079  * Primary command inlet from frontend ports.  All SCSI and task I/O
13080  * requests must go through this function.
13081  */
13082 int
13083 ctl_queue(union ctl_io *io)
13084 {
13085         struct ctl_softc *ctl_softc;
13086
13087         CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
13088
13089         ctl_softc = control_softc;
13090
13091 #ifdef CTL_TIME_IO
13092         io->io_hdr.start_time = time_uptime;
13093         getbintime(&io->io_hdr.start_bt);
13094 #endif /* CTL_TIME_IO */
13095
13096         mtx_lock(&ctl_softc->ctl_lock);
13097
13098         switch (io->io_hdr.io_type) {
13099         case CTL_IO_SCSI:
13100                 STAILQ_INSERT_TAIL(&ctl_softc->incoming_queue, &io->io_hdr,
13101                                    links);
13102                 break;
13103         case CTL_IO_TASK:
13104                 STAILQ_INSERT_TAIL(&ctl_softc->task_queue, &io->io_hdr, links);
13105                 /*
13106                  * Set the task pending flag.  This is necessary to close a
13107                  * race condition with the FETD:
13108                  *
13109                  * - FETD submits a task management command, like an abort.
13110                  * - Back end calls fe_datamove() to move the data for the
13111                  *   aborted command.  The FETD can't really accept it, but
13112                  *   if it did, it would end up transmitting data for a
13113                  *   command that the initiator told us to abort.
13114                  *
13115                  * We close the race condition by setting the flag here,
13116                  * and checking it in ctl_datamove(), before calling the
13117                  * FETD's fe_datamove routine.  If we've got a task
13118                  * pending, we run the task queue and then check to see
13119                  * whether our particular I/O has been aborted.
13120                  */
13121                 ctl_softc->flags |= CTL_FLAG_TASK_PENDING;
13122                 break;
13123         default:
13124                 mtx_unlock(&ctl_softc->ctl_lock);
13125                 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
13126                 return (-EINVAL);
13127                 break; /* NOTREACHED */
13128         }
13129         mtx_unlock(&ctl_softc->ctl_lock);
13130
13131         ctl_wakeup_thread();
13132
13133         return (CTL_RETVAL_COMPLETE);
13134 }
13135
13136 #ifdef CTL_IO_DELAY
13137 static void
13138 ctl_done_timer_wakeup(void *arg)
13139 {
13140         union ctl_io *io;
13141
13142         io = (union ctl_io *)arg;
13143         ctl_done_lock(io, /*have_lock*/ 0);
13144 }
13145 #endif /* CTL_IO_DELAY */
13146
13147 void
13148 ctl_done_lock(union ctl_io *io, int have_lock)
13149 {
13150         struct ctl_softc *ctl_softc;
13151 #ifndef CTL_DONE_THREAD
13152         union ctl_io *xio;
13153 #endif /* !CTL_DONE_THREAD */
13154
13155         ctl_softc = control_softc;
13156
13157         if (have_lock == 0)
13158                 mtx_lock(&ctl_softc->ctl_lock);
13159
13160         /*
13161          * Enable this to catch duplicate completion issues.
13162          */
13163 #if 0
13164         if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
13165                 printf("%s: type %d msg %d cdb %x iptl: "
13166                        "%d:%d:%d:%d tag 0x%04x "
13167                        "flag %#x status %x\n",
13168                         __func__,
13169                         io->io_hdr.io_type,
13170                         io->io_hdr.msg_type,
13171                         io->scsiio.cdb[0],
13172                         io->io_hdr.nexus.initid.id,
13173                         io->io_hdr.nexus.targ_port,
13174                         io->io_hdr.nexus.targ_target.id,
13175                         io->io_hdr.nexus.targ_lun,
13176                         (io->io_hdr.io_type ==
13177                         CTL_IO_TASK) ?
13178                         io->taskio.tag_num :
13179                         io->scsiio.tag_num,
13180                         io->io_hdr.flags,
13181                         io->io_hdr.status);
13182         } else
13183                 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
13184 #endif
13185
13186         /*
13187          * This is an internal copy of an I/O, and should not go through
13188          * the normal done processing logic.
13189          */
13190         if (io->io_hdr.flags & CTL_FLAG_INT_COPY) {
13191                 if (have_lock == 0)
13192                         mtx_unlock(&ctl_softc->ctl_lock);
13193                 return;
13194         }
13195
13196         /*
13197          * We need to send a msg to the serializing shelf to finish the IO
13198          * as well.  We don't send a finish message to the other shelf if
13199          * this is a task management command.  Task management commands
13200          * aren't serialized in the OOA queue, but rather just executed on
13201          * both shelf controllers for commands that originated on that
13202          * controller.
13203          */
13204         if ((io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)
13205          && (io->io_hdr.io_type != CTL_IO_TASK)) {
13206                 union ctl_ha_msg msg_io;
13207
13208                 msg_io.hdr.msg_type = CTL_MSG_FINISH_IO;
13209                 msg_io.hdr.serializing_sc = io->io_hdr.serializing_sc;
13210                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_io,
13211                     sizeof(msg_io), 0 ) != CTL_HA_STATUS_SUCCESS) {
13212                 }
13213                 /* continue on to finish IO */
13214         }
13215 #ifdef CTL_IO_DELAY
13216         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
13217                 struct ctl_lun *lun;
13218
13219                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13220
13221                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
13222         } else {
13223                 struct ctl_lun *lun;
13224
13225                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13226
13227                 if ((lun != NULL)
13228                  && (lun->delay_info.done_delay > 0)) {
13229                         struct callout *callout;
13230
13231                         callout = (struct callout *)&io->io_hdr.timer_bytes;
13232                         callout_init(callout, /*mpsafe*/ 1);
13233                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
13234                         callout_reset(callout,
13235                                       lun->delay_info.done_delay * hz,
13236                                       ctl_done_timer_wakeup, io);
13237                         if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
13238                                 lun->delay_info.done_delay = 0;
13239                         if (have_lock == 0)
13240                                 mtx_unlock(&ctl_softc->ctl_lock);
13241                         return;
13242                 }
13243         }
13244 #endif /* CTL_IO_DELAY */
13245
13246         STAILQ_INSERT_TAIL(&ctl_softc->done_queue, &io->io_hdr, links);
13247
13248 #ifdef CTL_DONE_THREAD
13249         if (have_lock == 0)
13250                 mtx_unlock(&ctl_softc->ctl_lock);
13251
13252         ctl_wakeup_thread();
13253 #else /* CTL_DONE_THREAD */
13254         for (xio = (union ctl_io *)STAILQ_FIRST(&ctl_softc->done_queue);
13255              xio != NULL;
13256              xio =(union ctl_io *)STAILQ_FIRST(&ctl_softc->done_queue)) {
13257
13258                 STAILQ_REMOVE_HEAD(&ctl_softc->done_queue, links);
13259
13260                 ctl_process_done(xio, /*have_lock*/ 1);
13261         }
13262         if (have_lock == 0)
13263                 mtx_unlock(&ctl_softc->ctl_lock);
13264 #endif /* CTL_DONE_THREAD */
13265 }
13266
13267 void
13268 ctl_done(union ctl_io *io)
13269 {
13270         ctl_done_lock(io, /*have_lock*/ 0);
13271 }
13272
13273 int
13274 ctl_isc(struct ctl_scsiio *ctsio)
13275 {
13276         struct ctl_lun *lun;
13277         int retval;
13278
13279         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13280
13281         CTL_DEBUG_PRINT(("ctl_isc: command: %02x\n", ctsio->cdb[0]));
13282
13283         CTL_DEBUG_PRINT(("ctl_isc: calling data_submit()\n"));
13284
13285         retval = lun->backend->data_submit((union ctl_io *)ctsio);
13286
13287         return (retval);
13288 }
13289
13290
13291 static void
13292 ctl_work_thread(void *arg)
13293 {
13294         struct ctl_softc *softc;
13295         union ctl_io *io;
13296         struct ctl_be_lun *be_lun;
13297         int retval;
13298
13299         CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
13300
13301         softc = (struct ctl_softc *)arg;
13302         if (softc == NULL)
13303                 return;
13304
13305         mtx_lock(&softc->ctl_lock);
13306         for (;;) {
13307                 retval = 0;
13308
13309                 /*
13310                  * We handle the queues in this order:
13311                  * - task management
13312                  * - ISC
13313                  * - done queue (to free up resources, unblock other commands)
13314                  * - RtR queue
13315                  * - incoming queue
13316                  *
13317                  * If those queues are empty, we break out of the loop and
13318                  * go to sleep.
13319                  */
13320                 io = (union ctl_io *)STAILQ_FIRST(&softc->task_queue);
13321                 if (io != NULL) {
13322                         ctl_run_task_queue(softc);
13323                         continue;
13324                 }
13325                 io = (union ctl_io *)STAILQ_FIRST(&softc->isc_queue);
13326                 if (io != NULL) {
13327                         STAILQ_REMOVE_HEAD(&softc->isc_queue, links);
13328                         ctl_handle_isc(io);
13329                         continue;
13330                 }
13331                 io = (union ctl_io *)STAILQ_FIRST(&softc->done_queue);
13332                 if (io != NULL) {
13333                         STAILQ_REMOVE_HEAD(&softc->done_queue, links);
13334                         /* clear any blocked commands, call fe_done */
13335                         mtx_unlock(&softc->ctl_lock);
13336                         /*
13337                          * XXX KDM
13338                          * Call this without a lock for now.  This will
13339                          * depend on whether there is any way the FETD can
13340                          * sleep or deadlock if called with the CTL lock
13341                          * held.
13342                          */
13343                         retval = ctl_process_done(io, /*have_lock*/ 0);
13344                         mtx_lock(&softc->ctl_lock);
13345                         continue;
13346                 }
13347                 if (!ctl_pause_rtr) {
13348                         io = (union ctl_io *)STAILQ_FIRST(&softc->rtr_queue);
13349                         if (io != NULL) {
13350                                 STAILQ_REMOVE_HEAD(&softc->rtr_queue, links);
13351                                 mtx_unlock(&softc->ctl_lock);
13352                                 retval = ctl_scsiio(&io->scsiio);
13353                                 if (retval != CTL_RETVAL_COMPLETE)
13354                                         CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
13355                                 mtx_lock(&softc->ctl_lock);
13356                                 continue;
13357                         }
13358                 }
13359                 io = (union ctl_io *)STAILQ_FIRST(&softc->incoming_queue);
13360                 if (io != NULL) {
13361                         STAILQ_REMOVE_HEAD(&softc->incoming_queue, links);
13362                         mtx_unlock(&softc->ctl_lock);
13363                         ctl_scsiio_precheck(softc, &io->scsiio);
13364                         mtx_lock(&softc->ctl_lock);
13365                         continue;
13366                 }
13367                 /*
13368                  * We might want to move this to a separate thread, so that
13369                  * configuration requests (in this case LUN creations)
13370                  * won't impact the I/O path.
13371                  */
13372                 be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
13373                 if (be_lun != NULL) {
13374                         STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
13375                         mtx_unlock(&softc->ctl_lock);
13376                         ctl_create_lun(be_lun);
13377                         mtx_lock(&softc->ctl_lock);
13378                         continue;
13379                 }
13380
13381                 /* XXX KDM use the PDROP flag?? */
13382                 /* Sleep until we have something to do. */
13383                 mtx_sleep(softc, &softc->ctl_lock, PRIBIO, "-", 0);
13384
13385                 /* Back to the top of the loop to see what woke us up. */
13386                 continue;
13387         }
13388 }
13389
13390 void
13391 ctl_wakeup_thread()
13392 {
13393         struct ctl_softc *softc;
13394
13395         softc = control_softc;
13396
13397         wakeup_one(softc);
13398 }
13399
13400 /* Initialization and failover */
13401
13402 void
13403 ctl_init_isc_msg(void)
13404 {
13405         printf("CTL: Still calling this thing\n");
13406 }
13407
13408 /*
13409  * Init component
13410  *      Initializes component into configuration defined by bootMode
13411  *      (see hasc-sv.c)
13412  *      returns hasc_Status:
13413  *              OK
13414  *              ERROR - fatal error
13415  */
13416 static ctl_ha_comp_status
13417 ctl_isc_init(struct ctl_ha_component *c)
13418 {
13419         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13420
13421         c->status = ret;
13422         return ret;
13423 }
13424
13425 /* Start component
13426  *      Starts component in state requested. If component starts successfully,
13427  *      it must set its own state to the requestrd state
13428  *      When requested state is HASC_STATE_HA, the component may refine it
13429  *      by adding _SLAVE or _MASTER flags.
13430  *      Currently allowed state transitions are:
13431  *      UNKNOWN->HA             - initial startup
13432  *      UNKNOWN->SINGLE - initial startup when no parter detected
13433  *      HA->SINGLE              - failover
13434  * returns ctl_ha_comp_status:
13435  *              OK      - component successfully started in requested state
13436  *              FAILED  - could not start the requested state, failover may
13437  *                        be possible
13438  *              ERROR   - fatal error detected, no future startup possible
13439  */
13440 static ctl_ha_comp_status
13441 ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
13442 {
13443         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13444
13445         printf("%s: go\n", __func__);
13446
13447         // UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
13448         if (c->state == CTL_HA_STATE_UNKNOWN ) {
13449                 ctl_is_single = 0;
13450                 if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
13451                     != CTL_HA_STATUS_SUCCESS) {
13452                         printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
13453                         ret = CTL_HA_COMP_STATUS_ERROR;
13454                 }
13455         } else if (CTL_HA_STATE_IS_HA(c->state)
13456                 && CTL_HA_STATE_IS_SINGLE(state)){
13457                 // HA->SINGLE transition
13458                 ctl_failover();
13459                 ctl_is_single = 1;
13460         } else {
13461                 printf("ctl_isc_start:Invalid state transition %X->%X\n",
13462                        c->state, state);
13463                 ret = CTL_HA_COMP_STATUS_ERROR;
13464         }
13465         if (CTL_HA_STATE_IS_SINGLE(state))
13466                 ctl_is_single = 1;
13467
13468         c->state = state;
13469         c->status = ret;
13470         return ret;
13471 }
13472
13473 /*
13474  * Quiesce component
13475  * The component must clear any error conditions (set status to OK) and
13476  * prepare itself to another Start call
13477  * returns ctl_ha_comp_status:
13478  *      OK
13479  *      ERROR
13480  */
13481 static ctl_ha_comp_status
13482 ctl_isc_quiesce(struct ctl_ha_component *c)
13483 {
13484         int ret = CTL_HA_COMP_STATUS_OK;
13485
13486         ctl_pause_rtr = 1;
13487         c->status = ret;
13488         return ret;
13489 }
13490
13491 struct ctl_ha_component ctl_ha_component_ctlisc =
13492 {
13493         .name = "CTL ISC",
13494         .state = CTL_HA_STATE_UNKNOWN,
13495         .init = ctl_isc_init,
13496         .start = ctl_isc_start,
13497         .quiesce = ctl_isc_quiesce
13498 };
13499
13500 /*
13501  *  vim: ts=8
13502  */