sys/cam/ctl/ctl.c

   1 /*-
   2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
   3  * Copyright (c) 2012 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Edward Tomasz Napierala
   7  * under sponsorship from the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  16  *    substantially similar to the "NO WARRANTY" disclaimer below
  17  *    ("Disclaimer") and any redistribution must be conditioned upon
  18  *    including a substantially similar Disclaimer requirement for further
  19  *    binary redistribution.
  20  *
  21  * NO WARRANTY
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  30  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  31  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGES.
  33  *
  34  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl.c#8 $
  35  */
  36 /*
  37  * CAM Target Layer, a SCSI device emulation subsystem.
  38  *
  39  * Author: Ken Merry <ken@FreeBSD.org>
  40  */
  41
  42 #define _CTL_C
  43
  44 #include <sys/cdefs.h>
  45 __FBSDID("$FreeBSD$");
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/types.h>
  51 #include <sys/kthread.h>
  52 #include <sys/bio.h>
  53 #include <sys/fcntl.h>
  54 #include <sys/lock.h>
  55 #include <sys/module.h>
  56 #include <sys/mutex.h>
  57 #include <sys/condvar.h>
  58 #include <sys/malloc.h>
  59 #include <sys/conf.h>
  60 #include <sys/ioccom.h>
  61 #include <sys/queue.h>
  62 #include <sys/sbuf.h>
  63 #include <sys/smp.h>
  64 #include <sys/endian.h>
  65 #include <sys/sysctl.h>
  66
  67 #include <cam/cam.h>
  68 #include <cam/scsi/scsi_all.h>
  69 #include <cam/scsi/scsi_da.h>
  70 #include <cam/ctl/ctl_io.h>
  71 #include <cam/ctl/ctl.h>
  72 #include <cam/ctl/ctl_frontend.h>
  73 #include <cam/ctl/ctl_frontend_internal.h>
  74 #include <cam/ctl/ctl_util.h>
  75 #include <cam/ctl/ctl_backend.h>
  76 #include <cam/ctl/ctl_ioctl.h>
  77 #include <cam/ctl/ctl_ha.h>
  78 #include <cam/ctl/ctl_private.h>
  79 #include <cam/ctl/ctl_debug.h>
  80 #include <cam/ctl/ctl_scsi_all.h>
  81 #include <cam/ctl/ctl_error.h>
  82
  83 struct ctl_softc *control_softc = NULL;
  84
  85 /*
  86  * The default is to run with CTL_DONE_THREAD turned on.  Completed
  87  * transactions are queued for processing by the CTL work thread.  When
  88  * CTL_DONE_THREAD is not defined, completed transactions are processed in
  89  * the caller's context.
  90  */
  91 #define CTL_DONE_THREAD
  92
  93 /*
  94  * Use the serial number and device ID provided by the backend, rather than
  95  * making up our own.
  96  */
  97 #define CTL_USE_BACKEND_SN
  98
  99 /*
 100  * Size and alignment macros needed for Copan-specific HA hardware.  These
 101  * can go away when the HA code is re-written, and uses busdma for any
 102  * hardware.
 103  */
 104 #define CTL_ALIGN_8B(target, source, type)                              \
 105         if (((uint32_t)source & 0x7) != 0)                              \
 106                 target = (type)(source + (0x8 - ((uint32_t)source & 0x7)));\
 107         else                                                            \
 108                 target = (type)source;
 109
 110 #define CTL_SIZE_8B(target, size)                                       \
 111         if ((size & 0x7) != 0)                                          \
 112                 target = size + (0x8 - (size & 0x7));                   \
 113         else                                                            \
 114                 target = size;
 115
 116 #define CTL_ALIGN_8B_MARGIN     16
 117
 118 /*
 119  * Template mode pages.
 120  */
 121
 122 /*
 123  * Note that these are default values only.  The actual values will be
 124  * filled in when the user does a mode sense.
 125  */
 126 static struct copan_power_subpage power_page_default = {
 127         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 128         /*subpage*/ PWR_SUBPAGE_CODE,
 129         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 130                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 131         /*page_version*/ PWR_VERSION,
 132         /* total_luns */ 26,
 133         /* max_active_luns*/ PWR_DFLT_MAX_LUNS,
 134         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 135                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 136                       0, 0, 0, 0, 0, 0}
 137 };
 138
 139 static struct copan_power_subpage power_page_changeable = {
 140         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 141         /*subpage*/ PWR_SUBPAGE_CODE,
 142         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 143                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 144         /*page_version*/ 0,
 145         /* total_luns */ 0,
 146         /* max_active_luns*/ 0,
 147         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 148                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 149                       0, 0, 0, 0, 0, 0}
 150 };
 151
 152 static struct copan_aps_subpage aps_page_default = {
 153         APS_PAGE_CODE | SMPH_SPF, //page_code
 154         APS_SUBPAGE_CODE, //subpage
 155         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 156          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 157         APS_VERSION, //page_version
 158         0, //lock_active
 159         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 160         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 161         0, 0, 0, 0, 0} //reserved
 162 };
 163
 164 static struct copan_aps_subpage aps_page_changeable = {
 165         APS_PAGE_CODE | SMPH_SPF, //page_code
 166         APS_SUBPAGE_CODE, //subpage
 167         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 168          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 169         0, //page_version
 170         0, //lock_active
 171         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 172         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 173         0, 0, 0, 0, 0} //reserved
 174 };
 175
 176 static struct copan_debugconf_subpage debugconf_page_default = {
 177         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 178         DBGCNF_SUBPAGE_CODE,            /* subpage */
 179         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 180          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 181         DBGCNF_VERSION,                 /* page_version */
 182         {CTL_TIME_IO_DEFAULT_SECS>>8,
 183          CTL_TIME_IO_DEFAULT_SECS>>0},  /* ctl_time_io_secs */
 184 };
 185
 186 static struct copan_debugconf_subpage debugconf_page_changeable = {
 187         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 188         DBGCNF_SUBPAGE_CODE,            /* subpage */
 189         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 190          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 191         0,                              /* page_version */
 192         {0xff,0xff},                    /* ctl_time_io_secs */
 193 };
 194
 195 static struct scsi_format_page format_page_default = {
 196         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 197         /*page_length*/sizeof(struct scsi_format_page) - 2,
 198         /*tracks_per_zone*/ {0, 0},
 199         /*alt_sectors_per_zone*/ {0, 0},
 200         /*alt_tracks_per_zone*/ {0, 0},
 201         /*alt_tracks_per_lun*/ {0, 0},
 202         /*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
 203                                 CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
 204         /*bytes_per_sector*/ {0, 0},
 205         /*interleave*/ {0, 0},
 206         /*track_skew*/ {0, 0},
 207         /*cylinder_skew*/ {0, 0},
 208         /*flags*/ SFP_HSEC,
 209         /*reserved*/ {0, 0, 0}
 210 };
 211
 212 static struct scsi_format_page format_page_changeable = {
 213         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 214         /*page_length*/sizeof(struct scsi_format_page) - 2,
 215         /*tracks_per_zone*/ {0, 0},
 216         /*alt_sectors_per_zone*/ {0, 0},
 217         /*alt_tracks_per_zone*/ {0, 0},
 218         /*alt_tracks_per_lun*/ {0, 0},
 219         /*sectors_per_track*/ {0, 0},
 220         /*bytes_per_sector*/ {0, 0},
 221         /*interleave*/ {0, 0},
 222         /*track_skew*/ {0, 0},
 223         /*cylinder_skew*/ {0, 0},
 224         /*flags*/ 0,
 225         /*reserved*/ {0, 0, 0}
 226 };
 227
 228 static struct scsi_rigid_disk_page rigid_disk_page_default = {
 229         /*page_code*/SMS_RIGID_DISK_PAGE,
 230         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 231         /*cylinders*/ {0, 0, 0},
 232         /*heads*/ CTL_DEFAULT_HEADS,
 233         /*start_write_precomp*/ {0, 0, 0},
 234         /*start_reduced_current*/ {0, 0, 0},
 235         /*step_rate*/ {0, 0},
 236         /*landing_zone_cylinder*/ {0, 0, 0},
 237         /*rpl*/ SRDP_RPL_DISABLED,
 238         /*rotational_offset*/ 0,
 239         /*reserved1*/ 0,
 240         /*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
 241                            CTL_DEFAULT_ROTATION_RATE & 0xff},
 242         /*reserved2*/ {0, 0}
 243 };
 244
 245 static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
 246         /*page_code*/SMS_RIGID_DISK_PAGE,
 247         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 248         /*cylinders*/ {0, 0, 0},
 249         /*heads*/ 0,
 250         /*start_write_precomp*/ {0, 0, 0},
 251         /*start_reduced_current*/ {0, 0, 0},
 252         /*step_rate*/ {0, 0},
 253         /*landing_zone_cylinder*/ {0, 0, 0},
 254         /*rpl*/ 0,
 255         /*rotational_offset*/ 0,
 256         /*reserved1*/ 0,
 257         /*rotation_rate*/ {0, 0},
 258         /*reserved2*/ {0, 0}
 259 };
 260
 261 static struct scsi_caching_page caching_page_default = {
 262         /*page_code*/SMS_CACHING_PAGE,
 263         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 264         /*flags1*/ SCP_DISC | SCP_WCE,
 265         /*ret_priority*/ 0,
 266         /*disable_pf_transfer_len*/ {0xff, 0xff},
 267         /*min_prefetch*/ {0, 0},
 268         /*max_prefetch*/ {0xff, 0xff},
 269         /*max_pf_ceiling*/ {0xff, 0xff},
 270         /*flags2*/ 0,
 271         /*cache_segments*/ 0,
 272         /*cache_seg_size*/ {0, 0},
 273         /*reserved*/ 0,
 274         /*non_cache_seg_size*/ {0, 0, 0}
 275 };
 276
 277 static struct scsi_caching_page caching_page_changeable = {
 278         /*page_code*/SMS_CACHING_PAGE,
 279         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 280         /*flags1*/ 0,
 281         /*ret_priority*/ 0,
 282         /*disable_pf_transfer_len*/ {0, 0},
 283         /*min_prefetch*/ {0, 0},
 284         /*max_prefetch*/ {0, 0},
 285         /*max_pf_ceiling*/ {0, 0},
 286         /*flags2*/ 0,
 287         /*cache_segments*/ 0,
 288         /*cache_seg_size*/ {0, 0},
 289         /*reserved*/ 0,
 290         /*non_cache_seg_size*/ {0, 0, 0}
 291 };
 292
 293 static struct scsi_control_page control_page_default = {
 294         /*page_code*/SMS_CONTROL_MODE_PAGE,
 295         /*page_length*/sizeof(struct scsi_control_page) - 2,
 296         /*rlec*/0,
 297         /*queue_flags*/0,
 298         /*eca_and_aen*/0,
 299         /*reserved*/0,
 300         /*aen_holdoff_period*/{0, 0}
 301 };
 302
 303 static struct scsi_control_page control_page_changeable = {
 304         /*page_code*/SMS_CONTROL_MODE_PAGE,
 305         /*page_length*/sizeof(struct scsi_control_page) - 2,
 306         /*rlec*/SCP_DSENSE,
 307         /*queue_flags*/0,
 308         /*eca_and_aen*/0,
 309         /*reserved*/0,
 310         /*aen_holdoff_period*/{0, 0}
 311 };
 312
 313
 314 /*
 315  * XXX KDM move these into the softc.
 316  */
 317 static int rcv_sync_msg;
 318 static int persis_offset;
 319 static uint8_t ctl_pause_rtr;
 320 static int     ctl_is_single = 1;
 321 static int     index_to_aps_page;
 322
 323 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 324 static int worker_threads = 1;
 325 TUNABLE_INT("kern.cam.ctl.worker_threads", &worker_threads);
 326 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
 327     &worker_threads, 1, "Number of worker threads");
 328 static int verbose = 0;
 329 TUNABLE_INT("kern.cam.ctl.verbose", &verbose);
 330 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, verbose, CTLFLAG_RWTUN,
 331     &verbose, 0, "Show SCSI errors returned to initiator");
 332
 333 /*
 334  * Serial number (0x80), device id (0x83), and supported pages (0x00)
 335  */
 336 #define SCSI_EVPD_NUM_SUPPORTED_PAGES   3
 337
 338 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 339                                   int param);
 340 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 341 static int ctl_init(void);
 342 void ctl_shutdown(void);
 343 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 344 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 345 static void ctl_ioctl_online(void *arg);
 346 static void ctl_ioctl_offline(void *arg);
 347 static int ctl_ioctl_targ_enable(void *arg, struct ctl_id targ_id);
 348 static int ctl_ioctl_targ_disable(void *arg, struct ctl_id targ_id);
 349 static int ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id);
 350 static int ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id);
 351 static int ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio);
 352 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio, int have_lock);
 353 static int ctl_ioctl_submit_wait(union ctl_io *io);
 354 static void ctl_ioctl_datamove(union ctl_io *io);
 355 static void ctl_ioctl_done(union ctl_io *io);
 356 static void ctl_ioctl_hard_startstop_callback(void *arg,
 357                                               struct cfi_metatask *metatask);
 358 static void ctl_ioctl_bbrread_callback(void *arg,struct cfi_metatask *metatask);
 359 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 360                               struct ctl_ooa *ooa_hdr,
 361                               struct ctl_ooa_entry *kern_entries);
 362 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 363                      struct thread *td);
 364 uint32_t ctl_get_resindex(struct ctl_nexus *nexus);
 365 uint32_t ctl_port_idx(int port_num);
 366 #ifdef unused
 367 static union ctl_io *ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port,
 368                                    uint32_t targ_target, uint32_t targ_lun,
 369                                    int can_wait);
 370 static void ctl_kfree_io(union ctl_io *io);
 371 #endif /* unused */
 372 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
 373                          struct ctl_be_lun *be_lun, struct ctl_id target_id);
 374 static int ctl_free_lun(struct ctl_lun *lun);
 375 static void ctl_create_lun(struct ctl_be_lun *be_lun);
 376 /**
 377 static void ctl_failover_change_pages(struct ctl_softc *softc,
 378                                       struct ctl_scsiio *ctsio, int master);
 379 **/
 380
 381 static int ctl_do_mode_select(union ctl_io *io);
 382 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
 383                            uint64_t res_key, uint64_t sa_res_key,
 384                            uint8_t type, uint32_t residx,
 385                            struct ctl_scsiio *ctsio,
 386                            struct scsi_per_res_out *cdb,
 387                            struct scsi_per_res_out_parms* param);
 388 static void ctl_pro_preempt_other(struct ctl_lun *lun,
 389                                   union ctl_ha_msg *msg);
 390 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
 391 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
 392 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
 393 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
 394 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
 395 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
 396 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len);
 397 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2);
 398 static ctl_action ctl_check_for_blockage(union ctl_io *pending_io,
 399                                          union ctl_io *ooa_io);
 400 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
 401                                 union ctl_io *starting_io);
 402 static int ctl_check_blocked(struct ctl_lun *lun);
 403 static int ctl_scsiio_lun_check(struct ctl_softc *ctl_softc,
 404                                 struct ctl_lun *lun,
 405                                 struct ctl_cmd_entry *entry,
 406                                 struct ctl_scsiio *ctsio);
 407 //static int ctl_check_rtr(union ctl_io *pending_io, struct ctl_softc *softc);
 408 static void ctl_failover(void);
 409 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 410                                struct ctl_scsiio *ctsio);
 411 static int ctl_scsiio(struct ctl_scsiio *ctsio);
 412
 413 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 414 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 415                             ctl_ua_type ua_type);
 416 static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 417                          ctl_ua_type ua_type);
 418 static int ctl_abort_task(union ctl_io *io);
 419 static void ctl_run_task_queue(struct ctl_softc *ctl_softc);
 420 #ifdef CTL_IO_DELAY
 421 static void ctl_datamove_timer_wakeup(void *arg);
 422 static void ctl_done_timer_wakeup(void *arg);
 423 #endif /* CTL_IO_DELAY */
 424
 425 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 426 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
 427 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
 428 static void ctl_datamove_remote_write(union ctl_io *io);
 429 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
 430 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 431 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 432 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 433                                     ctl_ha_dt_cb callback);
 434 static void ctl_datamove_remote_read(union ctl_io *io);
 435 static void ctl_datamove_remote(union ctl_io *io);
 436 static int ctl_process_done(union ctl_io *io, int have_lock);
 437 static void ctl_work_thread(void *arg);
 438
 439 /*
 440  * Load the serialization table.  This isn't very pretty, but is probably
 441  * the easiest way to do it.
 442  */
 443 #include "ctl_ser_table.c"
 444
 445 /*
 446  * We only need to define open, close and ioctl routines for this driver.
 447  */
 448 static struct cdevsw ctl_cdevsw = {
 449         .d_version =    D_VERSION,
 450         .d_flags =      0,
 451         .d_open =       ctl_open,
 452         .d_close =      ctl_close,
 453         .d_ioctl =      ctl_ioctl,
 454         .d_name =       "ctl",
 455 };
 456
 457
 458 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 459
 460 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 461
 462 static moduledata_t ctl_moduledata = {
 463         "ctl",
 464         ctl_module_event_handler,
 465         NULL
 466 };
 467
 468 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
 469 MODULE_VERSION(ctl, 1);
 470
 471 static void
 472 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 473                             union ctl_ha_msg *msg_info)
 474 {
 475         struct ctl_scsiio *ctsio;
 476
 477         if (msg_info->hdr.original_sc == NULL) {
 478                 printf("%s: original_sc == NULL!\n", __func__);
 479                 /* XXX KDM now what? */
 480                 return;
 481         }
 482
 483         ctsio = &msg_info->hdr.original_sc->scsiio;
 484         ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 485         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 486         ctsio->io_hdr.status = msg_info->hdr.status;
 487         ctsio->scsi_status = msg_info->scsi.scsi_status;
 488         ctsio->sense_len = msg_info->scsi.sense_len;
 489         ctsio->sense_residual = msg_info->scsi.sense_residual;
 490         ctsio->residual = msg_info->scsi.residual;
 491         memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 492                sizeof(ctsio->sense_data));
 493         memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
 494                &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));
 495         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue, &ctsio->io_hdr, links);
 496         ctl_wakeup_thread();
 497 }
 498
 499 static void
 500 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
 501                                 union ctl_ha_msg *msg_info)
 502 {
 503         struct ctl_scsiio *ctsio;
 504
 505         if (msg_info->hdr.serializing_sc == NULL) {
 506                 printf("%s: serializing_sc == NULL!\n", __func__);
 507                 /* XXX KDM now what? */
 508                 return;
 509         }
 510
 511         ctsio = &msg_info->hdr.serializing_sc->scsiio;
 512 #if 0
 513         /*
 514          * Attempt to catch the situation where an I/O has
 515          * been freed, and we're using it again.
 516          */
 517         if (ctsio->io_hdr.io_type == 0xff) {
 518                 union ctl_io *tmp_io;
 519                 tmp_io = (union ctl_io *)ctsio;
 520                 printf("%s: %p use after free!\n", __func__,
 521                        ctsio);
 522                 printf("%s: type %d msg %d cdb %x iptl: "
 523                        "%d:%d:%d:%d tag 0x%04x "
 524                        "flag %#x status %x\n",
 525                         __func__,
 526                         tmp_io->io_hdr.io_type,
 527                         tmp_io->io_hdr.msg_type,
 528                         tmp_io->scsiio.cdb[0],
 529                         tmp_io->io_hdr.nexus.initid.id,
 530                         tmp_io->io_hdr.nexus.targ_port,
 531                         tmp_io->io_hdr.nexus.targ_target.id,
 532                         tmp_io->io_hdr.nexus.targ_lun,
 533                         (tmp_io->io_hdr.io_type ==
 534                         CTL_IO_TASK) ?
 535                         tmp_io->taskio.tag_num :
 536                         tmp_io->scsiio.tag_num,
 537                         tmp_io->io_hdr.flags,
 538                         tmp_io->io_hdr.status);
 539         }
 540 #endif
 541         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 542         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue, &ctsio->io_hdr, links);
 543         ctl_wakeup_thread();
 544 }
 545
 546 /*
 547  * ISC (Inter Shelf Communication) event handler.  Events from the HA
 548  * subsystem come in here.
 549  */
 550 static void
 551 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 552 {
 553         struct ctl_softc *ctl_softc;
 554         union ctl_io *io;
 555         struct ctl_prio *presio;
 556         ctl_ha_status isc_status;
 557
 558         ctl_softc = control_softc;
 559         io = NULL;
 560
 561
 562 #if 0
 563         printf("CTL: Isc Msg event %d\n", event);
 564 #endif
 565         if (event == CTL_HA_EVT_MSG_RECV) {
 566                 union ctl_ha_msg msg_info;
 567
 568                 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
 569                                              sizeof(msg_info), /*wait*/ 0);
 570 #if 0
 571                 printf("CTL: msg_type %d\n", msg_info.msg_type);
 572 #endif
 573                 if (isc_status != 0) {
 574                         printf("Error receiving message, status = %d\n",
 575                                isc_status);
 576                         return;
 577                 }
 578                 mtx_lock(&ctl_softc->ctl_lock);
 579
 580                 switch (msg_info.hdr.msg_type) {
 581                 case CTL_MSG_SERIALIZE:
 582 #if 0
 583                         printf("Serialize\n");
 584 #endif
 585                         io = ctl_alloc_io((void *)ctl_softc->othersc_pool);
 586                         if (io == NULL) {
 587                                 printf("ctl_isc_event_handler: can't allocate "
 588                                        "ctl_io!\n");
 589                                 /* Bad Juju */
 590                                 /* Need to set busy and send msg back */
 591                                 mtx_unlock(&ctl_softc->ctl_lock);
 592                                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
 593                                 msg_info.hdr.status = CTL_SCSI_ERROR;
 594                                 msg_info.scsi.scsi_status = SCSI_STATUS_BUSY;
 595                                 msg_info.scsi.sense_len = 0;
 596                                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
 597                                     sizeof(msg_info), 0) > CTL_HA_STATUS_SUCCESS){
 598                                 }
 599                                 goto bailout;
 600                         }
 601                         ctl_zero_io(io);
 602                         // populate ctsio from msg_info
 603                         io->io_hdr.io_type = CTL_IO_SCSI;
 604                         io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
 605                         io->io_hdr.original_sc = msg_info.hdr.original_sc;
 606 #if 0
 607                         printf("pOrig %x\n", (int)msg_info.original_sc);
 608 #endif
 609                         io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
 610                                             CTL_FLAG_IO_ACTIVE;
 611                         /*
 612                          * If we're in serialization-only mode, we don't
 613                          * want to go through full done processing.  Thus
 614                          * the COPY flag.
 615                          *
 616                          * XXX KDM add another flag that is more specific.
 617                          */
 618                         if (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)
 619                                 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
 620                         io->io_hdr.nexus = msg_info.hdr.nexus;
 621 #if 0
 622                         printf("targ %d, port %d, iid %d, lun %d\n",
 623                                io->io_hdr.nexus.targ_target.id,
 624                                io->io_hdr.nexus.targ_port,
 625                                io->io_hdr.nexus.initid.id,
 626                                io->io_hdr.nexus.targ_lun);
 627 #endif
 628                         io->scsiio.tag_num = msg_info.scsi.tag_num;
 629                         io->scsiio.tag_type = msg_info.scsi.tag_type;
 630                         memcpy(io->scsiio.cdb, msg_info.scsi.cdb,
 631                                CTL_MAX_CDBLEN);
 632                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
 633                                 struct ctl_cmd_entry *entry;
 634                                 uint8_t opcode;
 635
 636                                 opcode = io->scsiio.cdb[0];
 637                                 entry = &ctl_cmd_table[opcode];
 638                                 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
 639                                 io->io_hdr.flags |=
 640                                         entry->flags & CTL_FLAG_DATA_MASK;
 641                         }
 642                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 643                                            &io->io_hdr, links);
 644                         ctl_wakeup_thread();
 645                         break;
 646
 647                 /* Performed on the Originating SC, XFER mode only */
 648                 case CTL_MSG_DATAMOVE: {
 649                         struct ctl_sg_entry *sgl;
 650                         int i, j;
 651
 652                         io = msg_info.hdr.original_sc;
 653                         if (io == NULL) {
 654                                 printf("%s: original_sc == NULL!\n", __func__);
 655                                 /* XXX KDM do something here */
 656                                 break;
 657                         }
 658                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
 659                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 660                         /*
 661                          * Keep track of this, we need to send it back over
 662                          * when the datamove is complete.
 663                          */
 664                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 665
 666                         if (msg_info.dt.sg_sequence == 0) {
 667                                 /*
 668                                  * XXX KDM we use the preallocated S/G list
 669                                  * here, but we'll need to change this to
 670                                  * dynamic allocation if we need larger S/G
 671                                  * lists.
 672                                  */
 673                                 if (msg_info.dt.kern_sg_entries >
 674                                     sizeof(io->io_hdr.remote_sglist) /
 675                                     sizeof(io->io_hdr.remote_sglist[0])) {
 676                                         printf("%s: number of S/G entries "
 677                                             "needed %u > allocated num %zd\n",
 678                                             __func__,
 679                                             msg_info.dt.kern_sg_entries,
 680                                             sizeof(io->io_hdr.remote_sglist)/
 681                                             sizeof(io->io_hdr.remote_sglist[0]));
 682
 683                                         /*
 684                                          * XXX KDM send a message back to
 685                                          * the other side to shut down the
 686                                          * DMA.  The error will come back
 687                                          * through via the normal channel.
 688                                          */
 689                                         break;
 690                                 }
 691                                 sgl = io->io_hdr.remote_sglist;
 692                                 memset(sgl, 0,
 693                                        sizeof(io->io_hdr.remote_sglist));
 694
 695                                 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
 696
 697                                 io->scsiio.kern_sg_entries =
 698                                         msg_info.dt.kern_sg_entries;
 699                                 io->scsiio.rem_sg_entries =
 700                                         msg_info.dt.kern_sg_entries;
 701                                 io->scsiio.kern_data_len =
 702                                         msg_info.dt.kern_data_len;
 703                                 io->scsiio.kern_total_len =
 704                                         msg_info.dt.kern_total_len;
 705                                 io->scsiio.kern_data_resid =
 706                                         msg_info.dt.kern_data_resid;
 707                                 io->scsiio.kern_rel_offset =
 708                                         msg_info.dt.kern_rel_offset;
 709                                 /*
 710                                  * Clear out per-DMA flags.
 711                                  */
 712                                 io->io_hdr.flags &= ~CTL_FLAG_RDMA_MASK;
 713                                 /*
 714                                  * Add per-DMA flags that are set for this
 715                                  * particular DMA request.
 716                                  */
 717                                 io->io_hdr.flags |= msg_info.dt.flags &
 718                                                     CTL_FLAG_RDMA_MASK;
 719                         } else
 720                                 sgl = (struct ctl_sg_entry *)
 721                                         io->scsiio.kern_data_ptr;
 722
 723                         for (i = msg_info.dt.sent_sg_entries, j = 0;
 724                              i < (msg_info.dt.sent_sg_entries +
 725                              msg_info.dt.cur_sg_entries); i++, j++) {
 726                                 sgl[i].addr = msg_info.dt.sg_list[j].addr;
 727                                 sgl[i].len = msg_info.dt.sg_list[j].len;
 728
 729 #if 0
 730                                 printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
 731                                        __func__,
 732                                        msg_info.dt.sg_list[j].addr,
 733                                        msg_info.dt.sg_list[j].len,
 734                                        sgl[i].addr, sgl[i].len, j, i);
 735 #endif
 736                         }
 737 #if 0
 738                         memcpy(&sgl[msg_info.dt.sent_sg_entries],
 739                                msg_info.dt.sg_list,
 740                                sizeof(*sgl) * msg_info.dt.cur_sg_entries);
 741 #endif
 742
 743                         /*
 744                          * If this is the last piece of the I/O, we've got
 745                          * the full S/G list.  Queue processing in the thread.
 746                          * Otherwise wait for the next piece.
 747                          */
 748                         if (msg_info.dt.sg_last != 0) {
 749                                 STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 750                                                    &io->io_hdr, links);
 751                                 ctl_wakeup_thread();
 752                         }
 753                         break;
 754                 }
 755                 /* Performed on the Serializing (primary) SC, XFER mode only */
 756                 case CTL_MSG_DATAMOVE_DONE: {
 757                         if (msg_info.hdr.serializing_sc == NULL) {
 758                                 printf("%s: serializing_sc == NULL!\n",
 759                                        __func__);
 760                                 /* XXX KDM now what? */
 761                                 break;
 762                         }
 763                         /*
 764                          * We grab the sense information here in case
 765                          * there was a failure, so we can return status
 766                          * back to the initiator.
 767                          */
 768                         io = msg_info.hdr.serializing_sc;
 769                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
 770                         io->io_hdr.status = msg_info.hdr.status;
 771                         io->scsiio.scsi_status = msg_info.scsi.scsi_status;
 772                         io->scsiio.sense_len = msg_info.scsi.sense_len;
 773                         io->scsiio.sense_residual =msg_info.scsi.sense_residual;
 774                         io->io_hdr.port_status = msg_info.scsi.fetd_status;
 775                         io->scsiio.residual = msg_info.scsi.residual;
 776                         memcpy(&io->scsiio.sense_data,&msg_info.scsi.sense_data,
 777                                sizeof(io->scsiio.sense_data));
 778
 779                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 780                                            &io->io_hdr, links);
 781                         ctl_wakeup_thread();
 782                         break;
 783                 }
 784
 785                 /* Preformed on Originating SC, SER_ONLY mode */
 786                 case CTL_MSG_R2R:
 787                         io = msg_info.hdr.original_sc;
 788                         if (io == NULL) {
 789                                 printf("%s: Major Bummer\n", __func__);
 790                                 mtx_unlock(&ctl_softc->ctl_lock);
 791                                 return;
 792                         } else {
 793 #if 0
 794                                 printf("pOrig %x\n",(int) ctsio);
 795 #endif
 796                         }
 797                         io->io_hdr.msg_type = CTL_MSG_R2R;
 798                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 799                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 800                                            &io->io_hdr, links);
 801                         ctl_wakeup_thread();
 802                         break;
 803
 804                 /*
 805                  * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
 806                  * mode.
 807                  * Performed on the Originating (i.e. secondary) SC in XFER
 808                  * mode
 809                  */
 810                 case CTL_MSG_FINISH_IO:
 811                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER)
 812                                 ctl_isc_handler_finish_xfer(ctl_softc,
 813                                                             &msg_info);
 814                         else
 815                                 ctl_isc_handler_finish_ser_only(ctl_softc,
 816                                                                 &msg_info);
 817                         break;
 818
 819                 /* Preformed on Originating SC */
 820                 case CTL_MSG_BAD_JUJU:
 821                         io = msg_info.hdr.original_sc;
 822                         if (io == NULL) {
 823                                 printf("%s: Bad JUJU!, original_sc is NULL!\n",
 824                                        __func__);
 825                                 break;
 826                         }
 827                         ctl_copy_sense_data(&msg_info, io);
 828                         /*
 829                          * IO should have already been cleaned up on other
 830                          * SC so clear this flag so we won't send a message
 831                          * back to finish the IO there.
 832                          */
 833                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
 834                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 835
 836                         /* io = msg_info.hdr.serializing_sc; */
 837                         io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
 838                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 839                                            &io->io_hdr, links);
 840                         ctl_wakeup_thread();
 841                         break;
 842
 843                 /* Handle resets sent from the other side */
 844                 case CTL_MSG_MANAGE_TASKS: {
 845                         struct ctl_taskio *taskio;
 846                         taskio = (struct ctl_taskio *)ctl_alloc_io(
 847                                 (void *)ctl_softc->othersc_pool);
 848                         if (taskio == NULL) {
 849                                 printf("ctl_isc_event_handler: can't allocate "
 850                                        "ctl_io!\n");
 851                                 /* Bad Juju */
 852                                 /* should I just call the proper reset func
 853                                    here??? */
 854                                 mtx_unlock(&ctl_softc->ctl_lock);
 855                                 goto bailout;
 856                         }
 857                         ctl_zero_io((union ctl_io *)taskio);
 858                         taskio->io_hdr.io_type = CTL_IO_TASK;
 859                         taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
 860                         taskio->io_hdr.nexus = msg_info.hdr.nexus;
 861                         taskio->task_action = msg_info.task.task_action;
 862                         taskio->tag_num = msg_info.task.tag_num;
 863                         taskio->tag_type = msg_info.task.tag_type;
 864 #ifdef CTL_TIME_IO
 865                         taskio->io_hdr.start_time = time_uptime;
 866                         getbintime(&taskio->io_hdr.start_bt);
 867 #if 0
 868                         cs_prof_gettime(&taskio->io_hdr.start_ticks);
 869 #endif
 870 #endif /* CTL_TIME_IO */
 871                         STAILQ_INSERT_TAIL(&ctl_softc->task_queue,
 872                                            &taskio->io_hdr, links);
 873                         ctl_softc->flags |= CTL_FLAG_TASK_PENDING;
 874                         ctl_wakeup_thread();
 875                         break;
 876                 }
 877                 /* Persistent Reserve action which needs attention */
 878                 case CTL_MSG_PERS_ACTION:
 879                         presio = (struct ctl_prio *)ctl_alloc_io(
 880                                 (void *)ctl_softc->othersc_pool);
 881                         if (presio == NULL) {
 882                                 printf("ctl_isc_event_handler: can't allocate "
 883                                        "ctl_io!\n");
 884                                 /* Bad Juju */
 885                                 /* Need to set busy and send msg back */
 886                                 mtx_unlock(&ctl_softc->ctl_lock);
 887                                 goto bailout;
 888                         }
 889                         ctl_zero_io((union ctl_io *)presio);
 890                         presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
 891                         presio->pr_msg = msg_info.pr;
 892                         STAILQ_INSERT_TAIL(&ctl_softc->isc_queue,
 893                                            &presio->io_hdr, links);
 894                         ctl_wakeup_thread();
 895                         break;
 896                 case CTL_MSG_SYNC_FE:
 897                         rcv_sync_msg = 1;
 898                         break;
 899                 case CTL_MSG_APS_LOCK: {
 900                         // It's quicker to execute this then to
 901                         // queue it.
 902                         struct ctl_lun *lun;
 903                         struct ctl_page_index *page_index;
 904                         struct copan_aps_subpage *current_sp;
 905                         uint32_t targ_lun;
 906
 907                         targ_lun = msg_info.hdr.nexus.targ_lun;
 908                         if (msg_info.hdr.nexus.lun_map_fn != NULL)
 909                                 targ_lun = msg_info.hdr.nexus.lun_map_fn(msg_info.hdr.nexus.lun_map_arg, targ_lun);
 910
 911                         lun = ctl_softc->ctl_luns[targ_lun];
 912                         page_index = &lun->mode_pages.index[index_to_aps_page];
 913                         current_sp = (struct copan_aps_subpage *)
 914                                      (page_index->page_data +
 915                                      (page_index->page_len * CTL_PAGE_CURRENT));
 916
 917                         current_sp->lock_active = msg_info.aps.lock_flag;
 918                         break;
 919                 }
 920                 default:
 921                         printf("How did I get here?\n");
 922                 }
 923                 mtx_unlock(&ctl_softc->ctl_lock);
 924         } else if (event == CTL_HA_EVT_MSG_SENT) {
 925                 if (param != CTL_HA_STATUS_SUCCESS) {
 926                         printf("Bad status from ctl_ha_msg_send status %d\n",
 927                                param);
 928                 }
 929                 return;
 930         } else if (event == CTL_HA_EVT_DISCONNECT) {
 931                 printf("CTL: Got a disconnect from Isc\n");
 932                 return;
 933         } else {
 934                 printf("ctl_isc_event_handler: Unknown event %d\n", event);
 935                 return;
 936         }
 937
 938 bailout:
 939         return;
 940 }
 941
 942 static void
 943 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
 944 {
 945         struct scsi_sense_data *sense;
 946
 947         sense = &dest->scsiio.sense_data;
 948         bcopy(&src->scsi.sense_data, sense, sizeof(*sense));
 949         dest->scsiio.scsi_status = src->scsi.scsi_status;
 950         dest->scsiio.sense_len = src->scsi.sense_len;
 951         dest->io_hdr.status = src->hdr.status;
 952 }
 953
 954 static int
 955 ctl_init(void)
 956 {
 957         struct ctl_softc *softc;
 958         struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
 959         struct ctl_frontend *fe;
 960         struct ctl_lun *lun;
 961         uint8_t sc_id =0;
 962         int i, error, retval;
 963         //int isc_retval;
 964
 965         retval = 0;
 966         ctl_pause_rtr = 0;
 967         rcv_sync_msg = 0;
 968
 969         control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
 970                                M_WAITOK | M_ZERO);
 971         softc = control_softc;
 972
 973         softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
 974                               "cam/ctl");
 975
 976         softc->dev->si_drv1 = softc;
 977
 978         /*
 979          * By default, return a "bad LUN" peripheral qualifier for unknown
 980          * LUNs.  The user can override this default using the tunable or
 981          * sysctl.  See the comment in ctl_inquiry_std() for more details.
 982          */
 983         softc->inquiry_pq_no_lun = 1;
 984         TUNABLE_INT_FETCH("kern.cam.ctl.inquiry_pq_no_lun",
 985                           &softc->inquiry_pq_no_lun);
 986         sysctl_ctx_init(&softc->sysctl_ctx);
 987         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
 988                 SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
 989                 CTLFLAG_RD, 0, "CAM Target Layer");
 990
 991         if (softc->sysctl_tree == NULL) {
 992                 printf("%s: unable to allocate sysctl tree\n", __func__);
 993                 destroy_dev(softc->dev);
 994                 free(control_softc, M_DEVBUF);
 995                 control_softc = NULL;
 996                 return (ENOMEM);
 997         }
 998
 999         SYSCTL_ADD_INT(&softc->sysctl_ctx,
1000                        SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1001                        "inquiry_pq_no_lun", CTLFLAG_RW,
1002                        &softc->inquiry_pq_no_lun, 0,
1003                        "Report no lun possible for invalid LUNs");
1004
1005         mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
1006         mtx_init(&softc->pool_lock, "CTL pool mutex", NULL, MTX_DEF);
1007         softc->open_count = 0;
1008
1009         /*
1010          * Default to actually sending a SYNCHRONIZE CACHE command down to
1011          * the drive.
1012          */
1013         softc->flags = CTL_FLAG_REAL_SYNC;
1014
1015         /*
1016          * In Copan's HA scheme, the "master" and "slave" roles are
1017          * figured out through the slot the controller is in.  Although it
1018          * is an active/active system, someone has to be in charge.
1019          */
1020 #ifdef NEEDTOPORT
1021         scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
1022 #endif
1023
1024         if (sc_id == 0) {
1025                 softc->flags |= CTL_FLAG_MASTER_SHELF;
1026                 persis_offset = 0;
1027         } else
1028                 persis_offset = CTL_MAX_INITIATORS;
1029
1030         /*
1031          * XXX KDM need to figure out where we want to get our target ID
1032          * and WWID.  Is it different on each port?
1033          */
1034         softc->target.id = 0;
1035         softc->target.wwid[0] = 0x12345678;
1036         softc->target.wwid[1] = 0x87654321;
1037         STAILQ_INIT(&softc->lun_list);
1038         STAILQ_INIT(&softc->pending_lun_queue);
1039         STAILQ_INIT(&softc->task_queue);
1040         STAILQ_INIT(&softc->incoming_queue);
1041         STAILQ_INIT(&softc->rtr_queue);
1042         STAILQ_INIT(&softc->done_queue);
1043         STAILQ_INIT(&softc->isc_queue);
1044         STAILQ_INIT(&softc->fe_list);
1045         STAILQ_INIT(&softc->be_list);
1046         STAILQ_INIT(&softc->io_pools);
1047
1048         lun = &softc->lun;
1049
1050         /*
1051          * We don't bother calling these with ctl_lock held here, because,
1052          * in theory, no one else can try to do anything while we're in our
1053          * module init routine.
1054          */
1055         if (ctl_pool_create(softc, CTL_POOL_INTERNAL, CTL_POOL_ENTRIES_INTERNAL,
1056                             &internal_pool)!= 0){
1057                 printf("ctl: can't allocate %d entry internal pool, "
1058                        "exiting\n", CTL_POOL_ENTRIES_INTERNAL);
1059                 return (ENOMEM);
1060         }
1061
1062         if (ctl_pool_create(softc, CTL_POOL_EMERGENCY,
1063                             CTL_POOL_ENTRIES_EMERGENCY, &emergency_pool) != 0) {
1064                 printf("ctl: can't allocate %d entry emergency pool, "
1065                        "exiting\n", CTL_POOL_ENTRIES_EMERGENCY);
1066                 ctl_pool_free(internal_pool);
1067                 return (ENOMEM);
1068         }
1069
1070         if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC,
1071                             &other_pool) != 0)
1072         {
1073                 printf("ctl: can't allocate %d entry other SC pool, "
1074                        "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1075                 ctl_pool_free(internal_pool);
1076                 ctl_pool_free(emergency_pool);
1077                 return (ENOMEM);
1078         }
1079
1080         softc->internal_pool = internal_pool;
1081         softc->emergency_pool = emergency_pool;
1082         softc->othersc_pool = other_pool;
1083
1084         /*
1085          * We used to allocate a processor LUN here.  The new scheme is to
1086          * just let the user allocate LUNs as he sees fit.
1087          */
1088 #if 0
1089         mtx_lock(&softc->ctl_lock);
1090         ctl_alloc_lun(softc, lun, /*be_lun*/NULL, /*target*/softc->target);
1091         mtx_unlock(&softc->ctl_lock);
1092 #endif
1093
1094         if (worker_threads > MAXCPU || worker_threads == 0) {
1095                 printf("invalid kern.cam.ctl.worker_threads value; "
1096                     "setting to 1");
1097                 worker_threads = 1;
1098         } else if (worker_threads < 0) {
1099                 if (mp_ncpus > 2) {
1100                         /*
1101                          * Using more than two worker threads actually hurts
1102                          * performance due to lock contention.
1103                          */
1104                         worker_threads = 2;
1105                 } else {
1106                         worker_threads = 1;
1107                 }
1108         }
1109
1110         for (i = 0; i < worker_threads; i++) {
1111                 error = kproc_create(ctl_work_thread, softc, &softc->work_thread, 0, 0,
1112                                 "ctl_thrd%d", i);
1113                 if (error != 0) {
1114                         printf("error creating CTL work thread!\n");
1115                         mtx_lock(&softc->ctl_lock);
1116                         ctl_free_lun(lun);
1117                         mtx_unlock(&softc->ctl_lock);
1118                         ctl_pool_free(internal_pool);
1119                         ctl_pool_free(emergency_pool);
1120                         ctl_pool_free(other_pool);
1121                         return (error);
1122                 }
1123         }
1124         if (bootverbose)
1125                 printf("ctl: CAM Target Layer loaded\n");
1126
1127         /*
1128          * Initialize the initiator and portname mappings
1129          */
1130         memset(softc->wwpn_iid, 0, sizeof(softc->wwpn_iid));
1131
1132         /*
1133          * Initialize the ioctl front end.
1134          */
1135         fe = &softc->ioctl_info.fe;
1136         sprintf(softc->ioctl_info.port_name, "CTL ioctl");
1137         fe->port_type = CTL_PORT_IOCTL;
1138         fe->num_requested_ctl_io = 100;
1139         fe->port_name = softc->ioctl_info.port_name;
1140         fe->port_online = ctl_ioctl_online;
1141         fe->port_offline = ctl_ioctl_offline;
1142         fe->onoff_arg = &softc->ioctl_info;
1143         fe->targ_enable = ctl_ioctl_targ_enable;
1144         fe->targ_disable = ctl_ioctl_targ_disable;
1145         fe->lun_enable = ctl_ioctl_lun_enable;
1146         fe->lun_disable = ctl_ioctl_lun_disable;
1147         fe->targ_lun_arg = &softc->ioctl_info;
1148         fe->fe_datamove = ctl_ioctl_datamove;
1149         fe->fe_done = ctl_ioctl_done;
1150         fe->max_targets = 15;
1151         fe->max_target_id = 15;
1152
1153         if (ctl_frontend_register(&softc->ioctl_info.fe,
1154                           (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
1155                 printf("ctl: ioctl front end registration failed, will "
1156                        "continue anyway\n");
1157         }
1158
1159 #ifdef CTL_IO_DELAY
1160         if (sizeof(struct callout) > CTL_TIMER_BYTES) {
1161                 printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
1162                        sizeof(struct callout), CTL_TIMER_BYTES);
1163                 return (EINVAL);
1164         }
1165 #endif /* CTL_IO_DELAY */
1166
1167         return (0);
1168 }
1169
1170 void
1171 ctl_shutdown(void)
1172 {
1173         struct ctl_softc *softc;
1174         struct ctl_lun *lun, *next_lun;
1175         struct ctl_io_pool *pool;
1176
1177         softc = (struct ctl_softc *)control_softc;
1178
1179         if (ctl_frontend_deregister(&softc->ioctl_info.fe) != 0)
1180                 printf("ctl: ioctl front end deregistration failed\n");
1181
1182         mtx_lock(&softc->ctl_lock);
1183
1184         /*
1185          * Free up each LUN.
1186          */
1187         for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1188                 next_lun = STAILQ_NEXT(lun, links);
1189                 ctl_free_lun(lun);
1190         }
1191
1192         mtx_unlock(&softc->ctl_lock);
1193
1194         /*
1195          * This will rip the rug out from under any FETDs or anyone else
1196          * that has a pool allocated.  Since we increment our module
1197          * refcount any time someone outside the main CTL module allocates
1198          * a pool, we shouldn't have any problems here.  The user won't be
1199          * able to unload the CTL module until client modules have
1200          * successfully unloaded.
1201          */
1202         while ((pool = STAILQ_FIRST(&softc->io_pools)) != NULL)
1203                 ctl_pool_free(pool);
1204
1205 #if 0
1206         ctl_shutdown_thread(softc->work_thread);
1207 #endif
1208
1209         mtx_destroy(&softc->pool_lock);
1210         mtx_destroy(&softc->ctl_lock);
1211
1212         destroy_dev(softc->dev);
1213
1214         sysctl_ctx_free(&softc->sysctl_ctx);
1215
1216         free(control_softc, M_DEVBUF);
1217         control_softc = NULL;
1218
1219         if (bootverbose)
1220                 printf("ctl: CAM Target Layer unloaded\n");
1221 }
1222
1223 static int
1224 ctl_module_event_handler(module_t mod, int what, void *arg)
1225 {
1226
1227         switch (what) {
1228         case MOD_LOAD:
1229                 return (ctl_init());
1230         case MOD_UNLOAD:
1231                 return (EBUSY);
1232         default:
1233                 return (EOPNOTSUPP);
1234         }
1235 }
1236
1237 /*
1238  * XXX KDM should we do some access checks here?  Bump a reference count to
1239  * prevent a CTL module from being unloaded while someone has it open?
1240  */
1241 static int
1242 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1243 {
1244         return (0);
1245 }
1246
1247 static int
1248 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1249 {
1250         return (0);
1251 }
1252
1253 int
1254 ctl_port_enable(ctl_port_type port_type)
1255 {
1256         struct ctl_softc *softc;
1257         struct ctl_frontend *fe;
1258
1259         if (ctl_is_single == 0) {
1260                 union ctl_ha_msg msg_info;
1261                 int isc_retval;
1262
1263 #if 0
1264                 printf("%s: HA mode, synchronizing frontend enable\n",
1265                         __func__);
1266 #endif
1267                 msg_info.hdr.msg_type = CTL_MSG_SYNC_FE;
1268                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1269                         sizeof(msg_info), 1 )) > CTL_HA_STATUS_SUCCESS) {
1270                         printf("Sync msg send error retval %d\n", isc_retval);
1271                 }
1272                 if (!rcv_sync_msg) {
1273                         isc_retval=ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
1274                                 sizeof(msg_info), 1);
1275                 }
1276 #if 0
1277                 printf("CTL:Frontend Enable\n");
1278         } else {
1279                 printf("%s: single mode, skipping frontend synchronization\n",
1280                         __func__);
1281 #endif
1282         }
1283
1284         softc = control_softc;
1285
1286         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1287                 if (port_type & fe->port_type)
1288                 {
1289 #if 0
1290                         printf("port %d\n", fe->targ_port);
1291 #endif
1292                         ctl_frontend_online(fe);
1293                 }
1294         }
1295
1296         return (0);
1297 }
1298
1299 int
1300 ctl_port_disable(ctl_port_type port_type)
1301 {
1302         struct ctl_softc *softc;
1303         struct ctl_frontend *fe;
1304
1305         softc = control_softc;
1306
1307         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1308                 if (port_type & fe->port_type)
1309                         ctl_frontend_offline(fe);
1310         }
1311
1312         return (0);
1313 }
1314
1315 /*
1316  * Returns 0 for success, 1 for failure.
1317  * Currently the only failure mode is if there aren't enough entries
1318  * allocated.  So, in case of a failure, look at num_entries_dropped,
1319  * reallocate and try again.
1320  */
1321 int
1322 ctl_port_list(struct ctl_port_entry *entries, int num_entries_alloced,
1323               int *num_entries_filled, int *num_entries_dropped,
1324               ctl_port_type port_type, int no_virtual)
1325 {
1326         struct ctl_softc *softc;
1327         struct ctl_frontend *fe;
1328         int entries_dropped, entries_filled;
1329         int retval;
1330         int i;
1331
1332         softc = control_softc;
1333
1334         retval = 0;
1335         entries_filled = 0;
1336         entries_dropped = 0;
1337
1338         i = 0;
1339         mtx_lock(&softc->ctl_lock);
1340         STAILQ_FOREACH(fe, &softc->fe_list, links) {
1341                 struct ctl_port_entry *entry;
1342
1343                 if ((fe->port_type & port_type) == 0)
1344                         continue;
1345
1346                 if ((no_virtual != 0)
1347                  && (fe->virtual_port != 0))
1348                         continue;
1349
1350                 if (entries_filled >= num_entries_alloced) {
1351                         entries_dropped++;
1352                         continue;
1353                 }
1354                 entry = &entries[i];
1355
1356                 entry->port_type = fe->port_type;
1357                 strlcpy(entry->port_name, fe->port_name,
1358                         sizeof(entry->port_name));
1359                 entry->physical_port = fe->physical_port;
1360                 entry->virtual_port = fe->virtual_port;
1361                 entry->wwnn = fe->wwnn;
1362                 entry->wwpn = fe->wwpn;
1363
1364                 i++;
1365                 entries_filled++;
1366         }
1367
1368         mtx_unlock(&softc->ctl_lock);
1369
1370         if (entries_dropped > 0)
1371                 retval = 1;
1372
1373         *num_entries_dropped = entries_dropped;
1374         *num_entries_filled = entries_filled;
1375
1376         return (retval);
1377 }
1378
1379 static void
1380 ctl_ioctl_online(void *arg)
1381 {
1382         struct ctl_ioctl_info *ioctl_info;
1383
1384         ioctl_info = (struct ctl_ioctl_info *)arg;
1385
1386         ioctl_info->flags |= CTL_IOCTL_FLAG_ENABLED;
1387 }
1388
1389 static void
1390 ctl_ioctl_offline(void *arg)
1391 {
1392         struct ctl_ioctl_info *ioctl_info;
1393
1394         ioctl_info = (struct ctl_ioctl_info *)arg;
1395
1396         ioctl_info->flags &= ~CTL_IOCTL_FLAG_ENABLED;
1397 }
1398
1399 /*
1400  * Remove an initiator by port number and initiator ID.
1401  * Returns 0 for success, 1 for failure.
1402  */
1403 int
1404 ctl_remove_initiator(int32_t targ_port, uint32_t iid)
1405 {
1406         struct ctl_softc *softc;
1407
1408         softc = control_softc;
1409
1410         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1411
1412         if ((targ_port < 0)
1413          || (targ_port > CTL_MAX_PORTS)) {
1414                 printf("%s: invalid port number %d\n", __func__, targ_port);
1415                 return (1);
1416         }
1417         if (iid > CTL_MAX_INIT_PER_PORT) {
1418                 printf("%s: initiator ID %u > maximun %u!\n",
1419                        __func__, iid, CTL_MAX_INIT_PER_PORT);
1420                 return (1);
1421         }
1422
1423         mtx_lock(&softc->ctl_lock);
1424
1425         softc->wwpn_iid[targ_port][iid].in_use = 0;
1426
1427         mtx_unlock(&softc->ctl_lock);
1428
1429         return (0);
1430 }
1431
1432 /*
1433  * Add an initiator to the initiator map.
1434  * Returns 0 for success, 1 for failure.
1435  */
1436 int
1437 ctl_add_initiator(uint64_t wwpn, int32_t targ_port, uint32_t iid)
1438 {
1439         struct ctl_softc *softc;
1440         int retval;
1441
1442         softc = control_softc;
1443
1444         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1445
1446         retval = 0;
1447
1448         if ((targ_port < 0)
1449          || (targ_port > CTL_MAX_PORTS)) {
1450                 printf("%s: invalid port number %d\n", __func__, targ_port);
1451                 return (1);
1452         }
1453         if (iid > CTL_MAX_INIT_PER_PORT) {
1454                 printf("%s: WWPN %#jx initiator ID %u > maximun %u!\n",
1455                        __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1456                 return (1);
1457         }
1458
1459         mtx_lock(&softc->ctl_lock);
1460
1461         if (softc->wwpn_iid[targ_port][iid].in_use != 0) {
1462                 /*
1463                  * We don't treat this as an error.
1464                  */
1465                 if (softc->wwpn_iid[targ_port][iid].wwpn == wwpn) {
1466                         printf("%s: port %d iid %u WWPN %#jx arrived again?\n",
1467                                __func__, targ_port, iid, (uintmax_t)wwpn);
1468                         goto bailout;
1469                 }
1470
1471                 /*
1472                  * This is an error, but what do we do about it?  The
1473                  * driver is telling us we have a new WWPN for this
1474                  * initiator ID, so we pretty much need to use it.
1475                  */
1476                 printf("%s: port %d iid %u WWPN %#jx arrived, WWPN %#jx is "
1477                        "still at that address\n", __func__, targ_port, iid,
1478                        (uintmax_t)wwpn,
1479                        (uintmax_t)softc->wwpn_iid[targ_port][iid].wwpn);
1480
1481                 /*
1482                  * XXX KDM clear have_ca and ua_pending on each LUN for
1483                  * this initiator.
1484                  */
1485         }
1486         softc->wwpn_iid[targ_port][iid].in_use = 1;
1487         softc->wwpn_iid[targ_port][iid].iid = iid;
1488         softc->wwpn_iid[targ_port][iid].wwpn = wwpn;
1489         softc->wwpn_iid[targ_port][iid].port = targ_port;
1490
1491 bailout:
1492
1493         mtx_unlock(&softc->ctl_lock);
1494
1495         return (retval);
1496 }
1497
1498 /*
1499  * XXX KDM should we pretend to do something in the target/lun
1500  * enable/disable functions?
1501  */
1502 static int
1503 ctl_ioctl_targ_enable(void *arg, struct ctl_id targ_id)
1504 {
1505         return (0);
1506 }
1507
1508 static int
1509 ctl_ioctl_targ_disable(void *arg, struct ctl_id targ_id)
1510 {
1511         return (0);
1512 }
1513
1514 static int
1515 ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
1516 {
1517         return (0);
1518 }
1519
1520 static int
1521 ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
1522 {
1523         return (0);
1524 }
1525
1526 /*
1527  * Data movement routine for the CTL ioctl frontend port.
1528  */
1529 static int
1530 ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
1531 {
1532         struct ctl_sg_entry *ext_sglist, *kern_sglist;
1533         struct ctl_sg_entry ext_entry, kern_entry;
1534         int ext_sglen, ext_sg_entries, kern_sg_entries;
1535         int ext_sg_start, ext_offset;
1536         int len_to_copy, len_copied;
1537         int kern_watermark, ext_watermark;
1538         int ext_sglist_malloced;
1539         int i, j;
1540
1541         ext_sglist_malloced = 0;
1542         ext_sg_start = 0;
1543         ext_offset = 0;
1544
1545         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove\n"));
1546
1547         /*
1548          * If this flag is set, fake the data transfer.
1549          */
1550         if (ctsio->io_hdr.flags & CTL_FLAG_NO_DATAMOVE) {
1551                 ctsio->ext_data_filled = ctsio->ext_data_len;
1552                 goto bailout;
1553         }
1554
1555         /*
1556          * To simplify things here, if we have a single buffer, stick it in
1557          * a S/G entry and just make it a single entry S/G list.
1558          */
1559         if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
1560                 int len_seen;
1561
1562                 ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
1563
1564                 ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
1565                                                            M_WAITOK);
1566                 ext_sglist_malloced = 1;
1567                 if (copyin(ctsio->ext_data_ptr, ext_sglist,
1568                                    ext_sglen) != 0) {
1569                         ctl_set_internal_failure(ctsio,
1570                                                  /*sks_valid*/ 0,
1571                                                  /*retry_count*/ 0);
1572                         goto bailout;
1573                 }
1574                 ext_sg_entries = ctsio->ext_sg_entries;
1575                 len_seen = 0;
1576                 for (i = 0; i < ext_sg_entries; i++) {
1577                         if ((len_seen + ext_sglist[i].len) >=
1578                              ctsio->ext_data_filled) {
1579                                 ext_sg_start = i;
1580                                 ext_offset = ctsio->ext_data_filled - len_seen;
1581                                 break;
1582                         }
1583                         len_seen += ext_sglist[i].len;
1584                 }
1585         } else {
1586                 ext_sglist = &ext_entry;
1587                 ext_sglist->addr = ctsio->ext_data_ptr;
1588                 ext_sglist->len = ctsio->ext_data_len;
1589                 ext_sg_entries = 1;
1590                 ext_sg_start = 0;
1591                 ext_offset = ctsio->ext_data_filled;
1592         }
1593
1594         if (ctsio->kern_sg_entries > 0) {
1595                 kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
1596                 kern_sg_entries = ctsio->kern_sg_entries;
1597         } else {
1598                 kern_sglist = &kern_entry;
1599                 kern_sglist->addr = ctsio->kern_data_ptr;
1600                 kern_sglist->len = ctsio->kern_data_len;
1601                 kern_sg_entries = 1;
1602         }
1603
1604
1605         kern_watermark = 0;
1606         ext_watermark = ext_offset;
1607         len_copied = 0;
1608         for (i = ext_sg_start, j = 0;
1609              i < ext_sg_entries && j < kern_sg_entries;) {
1610                 uint8_t *ext_ptr, *kern_ptr;
1611
1612                 len_to_copy = ctl_min(ext_sglist[i].len - ext_watermark,
1613                                       kern_sglist[j].len - kern_watermark);
1614
1615                 ext_ptr = (uint8_t *)ext_sglist[i].addr;
1616                 ext_ptr = ext_ptr + ext_watermark;
1617                 if (ctsio->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
1618                         /*
1619                          * XXX KDM fix this!
1620                          */
1621                         panic("need to implement bus address support");
1622 #if 0
1623                         kern_ptr = bus_to_virt(kern_sglist[j].addr);
1624 #endif
1625                 } else
1626                         kern_ptr = (uint8_t *)kern_sglist[j].addr;
1627                 kern_ptr = kern_ptr + kern_watermark;
1628
1629                 kern_watermark += len_to_copy;
1630                 ext_watermark += len_to_copy;
1631
1632                 if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
1633                      CTL_FLAG_DATA_IN) {
1634                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1635                                          "bytes to user\n", len_to_copy));
1636                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1637                                          "to %p\n", kern_ptr, ext_ptr));
1638                         if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
1639                                 ctl_set_internal_failure(ctsio,
1640                                                          /*sks_valid*/ 0,
1641                                                          /*retry_count*/ 0);
1642                                 goto bailout;
1643                         }
1644                 } else {
1645                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1646                                          "bytes from user\n", len_to_copy));
1647                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1648                                          "to %p\n", ext_ptr, kern_ptr));
1649                         if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
1650                                 ctl_set_internal_failure(ctsio,
1651                                                          /*sks_valid*/ 0,
1652                                                          /*retry_count*/0);
1653                                 goto bailout;
1654                         }
1655                 }
1656
1657                 len_copied += len_to_copy;
1658
1659                 if (ext_sglist[i].len == ext_watermark) {
1660                         i++;
1661                         ext_watermark = 0;
1662                 }
1663
1664                 if (kern_sglist[j].len == kern_watermark) {
1665                         j++;
1666                         kern_watermark = 0;
1667                 }
1668         }
1669
1670         ctsio->ext_data_filled += len_copied;
1671
1672         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_sg_entries: %d, "
1673                          "kern_sg_entries: %d\n", ext_sg_entries,
1674                          kern_sg_entries));
1675         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_data_len = %d, "
1676                          "kern_data_len = %d\n", ctsio->ext_data_len,
1677                          ctsio->kern_data_len));
1678
1679
1680         /* XXX KDM set residual?? */
1681 bailout:
1682
1683         if (ext_sglist_malloced != 0)
1684                 free(ext_sglist, M_CTL);
1685
1686         return (CTL_RETVAL_COMPLETE);
1687 }
1688
1689 /*
1690  * Serialize a command that went down the "wrong" side, and so was sent to
1691  * this controller for execution.  The logic is a little different than the
1692  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1693  * sent back to the other side, but in the success case, we execute the
1694  * command on this side (XFER mode) or tell the other side to execute it
1695  * (SER_ONLY mode).
1696  */
1697 static int
1698 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio, int have_lock)
1699 {
1700         struct ctl_softc *ctl_softc;
1701         union ctl_ha_msg msg_info;
1702         struct ctl_lun *lun;
1703         int retval = 0;
1704         uint32_t targ_lun;
1705
1706         ctl_softc = control_softc;
1707         if (have_lock == 0)
1708                 mtx_lock(&ctl_softc->ctl_lock);
1709
1710         targ_lun = ctsio->io_hdr.nexus.targ_lun;
1711         if (ctsio->io_hdr.nexus.lun_map_fn != NULL)
1712                 targ_lun = ctsio->io_hdr.nexus.lun_map_fn(ctsio->io_hdr.nexus.lun_map_arg, targ_lun);
1713         lun = ctl_softc->ctl_luns[targ_lun];
1714         if (lun==NULL)
1715         {
1716                 /*
1717                  * Why isn't LUN defined? The other side wouldn't
1718                  * send a cmd if the LUN is undefined.
1719                  */
1720                 printf("%s: Bad JUJU!, LUN is NULL!\n", __func__);
1721
1722                 /* "Logical unit not supported" */
1723                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1724                                    lun,
1725                                    /*sense_format*/SSD_TYPE_NONE,
1726                                    /*current_error*/ 1,
1727                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1728                                    /*asc*/ 0x25,
1729                                    /*ascq*/ 0x00,
1730                                    SSD_ELEM_NONE);
1731
1732                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1733                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1734                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1735                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1736                 msg_info.hdr.serializing_sc = NULL;
1737                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1738                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1739                                 sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1740                 }
1741                 if (have_lock == 0)
1742                         mtx_unlock(&ctl_softc->ctl_lock);
1743                 return(1);
1744
1745         }
1746
1747         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1748
1749         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1750                 (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1751                  ooa_links))) {
1752         case CTL_ACTION_BLOCK:
1753                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
1754                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
1755                                   blocked_links);
1756                 break;
1757         case CTL_ACTION_PASS:
1758         case CTL_ACTION_SKIP:
1759                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
1760                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
1761                         STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
1762                                            &ctsio->io_hdr, links);
1763                 } else {
1764
1765                         /* send msg back to other side */
1766                         msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1767                         msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
1768                         msg_info.hdr.msg_type = CTL_MSG_R2R;
1769 #if 0
1770                         printf("2. pOrig %x\n", (int)msg_info.hdr.original_sc);
1771 #endif
1772                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1773                             sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1774                         }
1775                 }
1776                 break;
1777         case CTL_ACTION_OVERLAP:
1778                 /* OVERLAPPED COMMANDS ATTEMPTED */
1779                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1780                                    lun,
1781                                    /*sense_format*/SSD_TYPE_NONE,
1782                                    /*current_error*/ 1,
1783                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1784                                    /*asc*/ 0x4E,
1785                                    /*ascq*/ 0x00,
1786                                    SSD_ELEM_NONE);
1787
1788                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1789                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1790                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1791                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1792                 msg_info.hdr.serializing_sc = NULL;
1793                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1794 #if 0
1795                 printf("BAD JUJU:Major Bummer Overlap\n");
1796 #endif
1797                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1798                 retval = 1;
1799                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1800                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1801                 }
1802                 break;
1803         case CTL_ACTION_OVERLAP_TAG:
1804                 /* TAGGED OVERLAPPED COMMANDS (NN = QUEUE TAG) */
1805                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1806                                    lun,
1807                                    /*sense_format*/SSD_TYPE_NONE,
1808                                    /*current_error*/ 1,
1809                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1810                                    /*asc*/ 0x4D,
1811                                    /*ascq*/ ctsio->tag_num & 0xff,
1812                                    SSD_ELEM_NONE);
1813
1814                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1815                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1816                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1817                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1818                 msg_info.hdr.serializing_sc = NULL;
1819                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1820 #if 0
1821                 printf("BAD JUJU:Major Bummer Overlap Tag\n");
1822 #endif
1823                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1824                 retval = 1;
1825                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1826                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1827                 }
1828                 break;
1829         case CTL_ACTION_ERROR:
1830         default:
1831                 /* "Internal target failure" */
1832                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1833                                    lun,
1834                                    /*sense_format*/SSD_TYPE_NONE,
1835                                    /*current_error*/ 1,
1836                                    /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
1837                                    /*asc*/ 0x44,
1838                                    /*ascq*/ 0x00,
1839                                    SSD_ELEM_NONE);
1840
1841                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1842                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1843                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1844                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1845                 msg_info.hdr.serializing_sc = NULL;
1846                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1847 #if 0
1848                 printf("BAD JUJU:Major Bummer HW Error\n");
1849 #endif
1850                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1851                 retval = 1;
1852                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1853                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1854                 }
1855                 break;
1856         }
1857         if (have_lock == 0)
1858                 mtx_unlock(&ctl_softc->ctl_lock);
1859         return (retval);
1860 }
1861
1862 static int
1863 ctl_ioctl_submit_wait(union ctl_io *io)
1864 {
1865         struct ctl_fe_ioctl_params params;
1866         ctl_fe_ioctl_state last_state;
1867         int done, retval;
1868
1869         retval = 0;
1870
1871         bzero(&params, sizeof(params));
1872
1873         mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
1874         cv_init(&params.sem, "ctlioccv");
1875         params.state = CTL_IOCTL_INPROG;
1876         last_state = params.state;
1877
1878         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = &params;
1879
1880         CTL_DEBUG_PRINT(("ctl_ioctl_submit_wait\n"));
1881
1882         /* This shouldn't happen */
1883         if ((retval = ctl_queue(io)) != CTL_RETVAL_COMPLETE)
1884                 return (retval);
1885
1886         done = 0;
1887
1888         do {
1889                 mtx_lock(&params.ioctl_mtx);
1890                 /*
1891                  * Check the state here, and don't sleep if the state has
1892                  * already changed (i.e. wakeup has already occured, but we
1893                  * weren't waiting yet).
1894                  */
1895                 if (params.state == last_state) {
1896                         /* XXX KDM cv_wait_sig instead? */
1897                         cv_wait(&params.sem, &params.ioctl_mtx);
1898                 }
1899                 last_state = params.state;
1900
1901                 switch (params.state) {
1902                 case CTL_IOCTL_INPROG:
1903                         /* Why did we wake up? */
1904                         /* XXX KDM error here? */
1905                         mtx_unlock(&params.ioctl_mtx);
1906                         break;
1907                 case CTL_IOCTL_DATAMOVE:
1908                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DATAMOVE\n"));
1909
1910                         /*
1911                          * change last_state back to INPROG to avoid
1912                          * deadlock on subsequent data moves.
1913                          */
1914                         params.state = last_state = CTL_IOCTL_INPROG;
1915
1916                         mtx_unlock(&params.ioctl_mtx);
1917                         ctl_ioctl_do_datamove(&io->scsiio);
1918                         /*
1919                          * Note that in some cases, most notably writes,
1920                          * this will queue the I/O and call us back later.
1921                          * In other cases, generally reads, this routine
1922                          * will immediately call back and wake us up,
1923                          * probably using our own context.
1924                          */
1925                         io->scsiio.be_move_done(io);
1926                         break;
1927                 case CTL_IOCTL_DONE:
1928                         mtx_unlock(&params.ioctl_mtx);
1929                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DONE\n"));
1930                         done = 1;
1931                         break;
1932                 default:
1933                         mtx_unlock(&params.ioctl_mtx);
1934                         /* XXX KDM error here? */
1935                         break;
1936                 }
1937         } while (done == 0);
1938
1939         mtx_destroy(&params.ioctl_mtx);
1940         cv_destroy(&params.sem);
1941
1942         return (CTL_RETVAL_COMPLETE);
1943 }
1944
1945 static void
1946 ctl_ioctl_datamove(union ctl_io *io)
1947 {
1948         struct ctl_fe_ioctl_params *params;
1949
1950         params = (struct ctl_fe_ioctl_params *)
1951                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1952
1953         mtx_lock(&params->ioctl_mtx);
1954         params->state = CTL_IOCTL_DATAMOVE;
1955         cv_broadcast(&params->sem);
1956         mtx_unlock(&params->ioctl_mtx);
1957 }
1958
1959 static void
1960 ctl_ioctl_done(union ctl_io *io)
1961 {
1962         struct ctl_fe_ioctl_params *params;
1963
1964         params = (struct ctl_fe_ioctl_params *)
1965                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1966
1967         mtx_lock(&params->ioctl_mtx);
1968         params->state = CTL_IOCTL_DONE;
1969         cv_broadcast(&params->sem);
1970         mtx_unlock(&params->ioctl_mtx);
1971 }
1972
1973 static void
1974 ctl_ioctl_hard_startstop_callback(void *arg, struct cfi_metatask *metatask)
1975 {
1976         struct ctl_fe_ioctl_startstop_info *sd_info;
1977
1978         sd_info = (struct ctl_fe_ioctl_startstop_info *)arg;
1979
1980         sd_info->hs_info.status = metatask->status;
1981         sd_info->hs_info.total_luns = metatask->taskinfo.startstop.total_luns;
1982         sd_info->hs_info.luns_complete =
1983                 metatask->taskinfo.startstop.luns_complete;
1984         sd_info->hs_info.luns_failed = metatask->taskinfo.startstop.luns_failed;
1985
1986         cv_broadcast(&sd_info->sem);
1987 }
1988
1989 static void
1990 ctl_ioctl_bbrread_callback(void *arg, struct cfi_metatask *metatask)
1991 {
1992         struct ctl_fe_ioctl_bbrread_info *fe_bbr_info;
1993
1994         fe_bbr_info = (struct ctl_fe_ioctl_bbrread_info *)arg;
1995
1996         mtx_lock(fe_bbr_info->lock);
1997         fe_bbr_info->bbr_info->status = metatask->status;
1998         fe_bbr_info->bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
1999         fe_bbr_info->wakeup_done = 1;
2000         mtx_unlock(fe_bbr_info->lock);
2001
2002         cv_broadcast(&fe_bbr_info->sem);
2003 }
2004
2005 /*
2006  * Returns 0 for success, errno for failure.
2007  */
2008 static int
2009 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
2010                    struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
2011 {
2012         union ctl_io *io;
2013         int retval;
2014
2015         retval = 0;
2016
2017         mtx_assert(&control_softc->ctl_lock, MA_OWNED);
2018
2019         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
2020              (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2021              ooa_links)) {
2022                 struct ctl_ooa_entry *entry;
2023
2024                 /*
2025                  * If we've got more than we can fit, just count the
2026                  * remaining entries.
2027                  */
2028                 if (*cur_fill_num >= ooa_hdr->alloc_num)
2029                         continue;
2030
2031                 entry = &kern_entries[*cur_fill_num];
2032
2033                 entry->tag_num = io->scsiio.tag_num;
2034                 entry->lun_num = lun->lun;
2035 #ifdef CTL_TIME_IO
2036                 entry->start_bt = io->io_hdr.start_bt;
2037 #endif
2038                 bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
2039                 entry->cdb_len = io->scsiio.cdb_len;
2040                 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
2041                         entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
2042
2043                 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
2044                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
2045
2046                 if (io->io_hdr.flags & CTL_FLAG_ABORT)
2047                         entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
2048
2049                 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
2050                         entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
2051
2052                 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
2053                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
2054         }
2055
2056         return (retval);
2057 }
2058
2059 static void *
2060 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2061                  size_t error_str_len)
2062 {
2063         void *kptr;
2064
2065         kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2066
2067         if (copyin(user_addr, kptr, len) != 0) {
2068                 snprintf(error_str, error_str_len, "Error copying %d bytes "
2069                          "from user address %p to kernel address %p", len,
2070                          user_addr, kptr);
2071                 free(kptr, M_CTL);
2072                 return (NULL);
2073         }
2074
2075         return (kptr);
2076 }
2077
2078 static void
2079 ctl_free_args(int num_be_args, struct ctl_be_arg *be_args)
2080 {
2081         int i;
2082
2083         if (be_args == NULL)
2084                 return;
2085
2086         for (i = 0; i < num_be_args; i++) {
2087                 free(be_args[i].kname, M_CTL);
2088                 free(be_args[i].kvalue, M_CTL);
2089         }
2090
2091         free(be_args, M_CTL);
2092 }
2093
2094 static struct ctl_be_arg *
2095 ctl_copyin_args(int num_be_args, struct ctl_be_arg *be_args,
2096                 char *error_str, size_t error_str_len)
2097 {
2098         struct ctl_be_arg *args;
2099         int i;
2100
2101         args = ctl_copyin_alloc(be_args, num_be_args * sizeof(*be_args),
2102                                 error_str, error_str_len);
2103
2104         if (args == NULL)
2105                 goto bailout;
2106
2107         for (i = 0; i < num_be_args; i++) {
2108                 args[i].kname = NULL;
2109                 args[i].kvalue = NULL;
2110         }
2111
2112         for (i = 0; i < num_be_args; i++) {
2113                 uint8_t *tmpptr;
2114
2115                 args[i].kname = ctl_copyin_alloc(args[i].name,
2116                         args[i].namelen, error_str, error_str_len);
2117                 if (args[i].kname == NULL)
2118                         goto bailout;
2119
2120                 if (args[i].kname[args[i].namelen - 1] != '\0') {
2121                         snprintf(error_str, error_str_len, "Argument %d "
2122                                  "name is not NUL-terminated", i);
2123                         goto bailout;
2124                 }
2125
2126                 args[i].kvalue = NULL;
2127
2128                 tmpptr = ctl_copyin_alloc(args[i].value,
2129                         args[i].vallen, error_str, error_str_len);
2130                 if (tmpptr == NULL)
2131                         goto bailout;
2132
2133                 args[i].kvalue = tmpptr;
2134
2135                 if ((args[i].flags & CTL_BEARG_ASCII)
2136                  && (tmpptr[args[i].vallen - 1] != '\0')) {
2137                         snprintf(error_str, error_str_len, "Argument %d "
2138                                  "value is not NUL-terminated", i);
2139                         goto bailout;
2140                 }
2141         }
2142
2143         return (args);
2144 bailout:
2145
2146         ctl_free_args(num_be_args, args);
2147
2148         return (NULL);
2149 }
2150
2151 /*
2152  * Escape characters that are illegal or not recommended in XML.
2153  */
2154 int
2155 ctl_sbuf_printf_esc(struct sbuf *sb, char *str)
2156 {
2157         int retval;
2158
2159         retval = 0;
2160
2161         for (; *str; str++) {
2162                 switch (*str) {
2163                 case '&':
2164                         retval = sbuf_printf(sb, "&amp;");
2165                         break;
2166                 case '>':
2167                         retval = sbuf_printf(sb, "&gt;");
2168                         break;
2169                 case '<':
2170                         retval = sbuf_printf(sb, "&lt;");
2171                         break;
2172                 default:
2173                         retval = sbuf_putc(sb, *str);
2174                         break;
2175                 }
2176
2177                 if (retval != 0)
2178                         break;
2179
2180         }
2181
2182         return (retval);
2183 }
2184
2185 static int
2186 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2187           struct thread *td)
2188 {
2189         struct ctl_softc *softc;
2190         int retval;
2191
2192         softc = control_softc;
2193
2194         retval = 0;
2195
2196         switch (cmd) {
2197         case CTL_IO: {
2198                 union ctl_io *io;
2199                 void *pool_tmp;
2200
2201                 /*
2202                  * If we haven't been "enabled", don't allow any SCSI I/O
2203                  * to this FETD.
2204                  */
2205                 if ((softc->ioctl_info.flags & CTL_IOCTL_FLAG_ENABLED) == 0) {
2206                         retval = -EPERM;
2207                         break;
2208                 }
2209
2210                 io = ctl_alloc_io(softc->ioctl_info.fe.ctl_pool_ref);
2211                 if (io == NULL) {
2212                         printf("ctl_ioctl: can't allocate ctl_io!\n");
2213                         retval = -ENOSPC;
2214                         break;
2215                 }
2216
2217                 /*
2218                  * Need to save the pool reference so it doesn't get
2219                  * spammed by the user's ctl_io.
2220                  */
2221                 pool_tmp = io->io_hdr.pool;
2222
2223                 memcpy(io, (void *)addr, sizeof(*io));
2224
2225                 io->io_hdr.pool = pool_tmp;
2226                 /*
2227                  * No status yet, so make sure the status is set properly.
2228                  */
2229                 io->io_hdr.status = CTL_STATUS_NONE;
2230
2231                 /*
2232                  * The user sets the initiator ID, target and LUN IDs.
2233                  */
2234                 io->io_hdr.nexus.targ_port = softc->ioctl_info.fe.targ_port;
2235                 io->io_hdr.flags |= CTL_FLAG_USER_REQ;
2236                 if ((io->io_hdr.io_type == CTL_IO_SCSI)
2237                  && (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
2238                         io->scsiio.tag_num = softc->ioctl_info.cur_tag_num++;
2239
2240                 retval = ctl_ioctl_submit_wait(io);
2241
2242                 if (retval != 0) {
2243                         ctl_free_io(io);
2244                         break;
2245                 }
2246
2247                 memcpy((void *)addr, io, sizeof(*io));
2248
2249                 /* return this to our pool */
2250                 ctl_free_io(io);
2251
2252                 break;
2253         }
2254         case CTL_ENABLE_PORT:
2255         case CTL_DISABLE_PORT:
2256         case CTL_SET_PORT_WWNS: {
2257                 struct ctl_frontend *fe;
2258                 struct ctl_port_entry *entry;
2259
2260                 entry = (struct ctl_port_entry *)addr;
2261
2262                 mtx_lock(&softc->ctl_lock);
2263                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2264                         int action, done;
2265
2266                         action = 0;
2267                         done = 0;
2268
2269                         if ((entry->port_type == CTL_PORT_NONE)
2270                          && (entry->targ_port == fe->targ_port)) {
2271                                 /*
2272                                  * If the user only wants to enable or
2273                                  * disable or set WWNs on a specific port,
2274                                  * do the operation and we're done.
2275                                  */
2276                                 action = 1;
2277                                 done = 1;
2278                         } else if (entry->port_type & fe->port_type) {
2279                                 /*
2280                                  * Compare the user's type mask with the
2281                                  * particular frontend type to see if we
2282                                  * have a match.
2283                                  */
2284                                 action = 1;
2285                                 done = 0;
2286
2287                                 /*
2288                                  * Make sure the user isn't trying to set
2289                                  * WWNs on multiple ports at the same time.
2290                                  */
2291                                 if (cmd == CTL_SET_PORT_WWNS) {
2292                                         printf("%s: Can't set WWNs on "
2293                                                "multiple ports\n", __func__);
2294                                         retval = EINVAL;
2295                                         break;
2296                                 }
2297                         }
2298                         if (action != 0) {
2299                                 /*
2300                                  * XXX KDM we have to drop the lock here,
2301                                  * because the online/offline operations
2302                                  * can potentially block.  We need to
2303                                  * reference count the frontends so they
2304                                  * can't go away,
2305                                  */
2306                                 mtx_unlock(&softc->ctl_lock);
2307
2308                                 if (cmd == CTL_ENABLE_PORT) {
2309                                         struct ctl_lun *lun;
2310
2311                                         STAILQ_FOREACH(lun, &softc->lun_list,
2312                                                        links) {
2313                                                 fe->lun_enable(fe->targ_lun_arg,
2314                                                     lun->target,
2315                                                     lun->lun);
2316                                         }
2317
2318                                         ctl_frontend_online(fe);
2319                                 } else if (cmd == CTL_DISABLE_PORT) {
2320                                         struct ctl_lun *lun;
2321
2322                                         ctl_frontend_offline(fe);
2323
2324                                         STAILQ_FOREACH(lun, &softc->lun_list,
2325                                                        links) {
2326                                                 fe->lun_disable(
2327                                                     fe->targ_lun_arg,
2328                                                     lun->target,
2329                                                     lun->lun);
2330                                         }
2331                                 }
2332
2333                                 mtx_lock(&softc->ctl_lock);
2334
2335                                 if (cmd == CTL_SET_PORT_WWNS)
2336                                         ctl_frontend_set_wwns(fe,
2337                                             (entry->flags & CTL_PORT_WWNN_VALID) ?
2338                                             1 : 0, entry->wwnn,
2339                                             (entry->flags & CTL_PORT_WWPN_VALID) ?
2340                                             1 : 0, entry->wwpn);
2341                         }
2342                         if (done != 0)
2343                                 break;
2344                 }
2345                 mtx_unlock(&softc->ctl_lock);
2346                 break;
2347         }
2348         case CTL_GET_PORT_LIST: {
2349                 struct ctl_frontend *fe;
2350                 struct ctl_port_list *list;
2351                 int i;
2352
2353                 list = (struct ctl_port_list *)addr;
2354
2355                 if (list->alloc_len != (list->alloc_num *
2356                     sizeof(struct ctl_port_entry))) {
2357                         printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2358                                "alloc_num %u * sizeof(struct ctl_port_entry) "
2359                                "%zu\n", __func__, list->alloc_len,
2360                                list->alloc_num, sizeof(struct ctl_port_entry));
2361                         retval = EINVAL;
2362                         break;
2363                 }
2364                 list->fill_len = 0;
2365                 list->fill_num = 0;
2366                 list->dropped_num = 0;
2367                 i = 0;
2368                 mtx_lock(&softc->ctl_lock);
2369                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2370                         struct ctl_port_entry entry, *list_entry;
2371
2372                         if (list->fill_num >= list->alloc_num) {
2373                                 list->dropped_num++;
2374                                 continue;
2375                         }
2376
2377                         entry.port_type = fe->port_type;
2378                         strlcpy(entry.port_name, fe->port_name,
2379                                 sizeof(entry.port_name));
2380                         entry.targ_port = fe->targ_port;
2381                         entry.physical_port = fe->physical_port;
2382                         entry.virtual_port = fe->virtual_port;
2383                         entry.wwnn = fe->wwnn;
2384                         entry.wwpn = fe->wwpn;
2385                         if (fe->status & CTL_PORT_STATUS_ONLINE)
2386                                 entry.online = 1;
2387                         else
2388                                 entry.online = 0;
2389
2390                         list_entry = &list->entries[i];
2391
2392                         retval = copyout(&entry, list_entry, sizeof(entry));
2393                         if (retval != 0) {
2394                                 printf("%s: CTL_GET_PORT_LIST: copyout "
2395                                        "returned %d\n", __func__, retval);
2396                                 break;
2397                         }
2398                         i++;
2399                         list->fill_num++;
2400                         list->fill_len += sizeof(entry);
2401                 }
2402                 mtx_unlock(&softc->ctl_lock);
2403
2404                 /*
2405                  * If this is non-zero, we had a copyout fault, so there's
2406                  * probably no point in attempting to set the status inside
2407                  * the structure.
2408                  */
2409                 if (retval != 0)
2410                         break;
2411
2412                 if (list->dropped_num > 0)
2413                         list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2414                 else
2415                         list->status = CTL_PORT_LIST_OK;
2416                 break;
2417         }
2418         case CTL_DUMP_OOA: {
2419                 struct ctl_lun *lun;
2420                 union ctl_io *io;
2421                 char printbuf[128];
2422                 struct sbuf sb;
2423
2424                 mtx_lock(&softc->ctl_lock);
2425                 printf("Dumping OOA queues:\n");
2426                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2427                         for (io = (union ctl_io *)TAILQ_FIRST(
2428                              &lun->ooa_queue); io != NULL;
2429                              io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2430                              ooa_links)) {
2431                                 sbuf_new(&sb, printbuf, sizeof(printbuf),
2432                                          SBUF_FIXEDLEN);
2433                                 sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2434                                             (intmax_t)lun->lun,
2435                                             io->scsiio.tag_num,
2436                                             (io->io_hdr.flags &
2437                                             CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2438                                             (io->io_hdr.flags &
2439                                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2440                                             (io->io_hdr.flags &
2441                                             CTL_FLAG_ABORT) ? " ABORT" : "",
2442                                             (io->io_hdr.flags &
2443                                         CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2444                                 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2445                                 sbuf_finish(&sb);
2446                                 printf("%s\n", sbuf_data(&sb));
2447                         }
2448                 }
2449                 printf("OOA queues dump done\n");
2450                 mtx_unlock(&softc->ctl_lock);
2451                 break;
2452         }
2453         case CTL_GET_OOA: {
2454                 struct ctl_lun *lun;
2455                 struct ctl_ooa *ooa_hdr;
2456                 struct ctl_ooa_entry *entries;
2457                 uint32_t cur_fill_num;
2458
2459                 ooa_hdr = (struct ctl_ooa *)addr;
2460
2461                 if ((ooa_hdr->alloc_len == 0)
2462                  || (ooa_hdr->alloc_num == 0)) {
2463                         printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2464                                "must be non-zero\n", __func__,
2465                                ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2466                         retval = EINVAL;
2467                         break;
2468                 }
2469
2470                 if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2471                     sizeof(struct ctl_ooa_entry))) {
2472                         printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2473                                "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2474                                __func__, ooa_hdr->alloc_len,
2475                                ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2476                         retval = EINVAL;
2477                         break;
2478                 }
2479
2480                 entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2481                 if (entries == NULL) {
2482                         printf("%s: could not allocate %d bytes for OOA "
2483                                "dump\n", __func__, ooa_hdr->alloc_len);
2484                         retval = ENOMEM;
2485                         break;
2486                 }
2487
2488                 mtx_lock(&softc->ctl_lock);
2489                 if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2490                  && ((ooa_hdr->lun_num > CTL_MAX_LUNS)
2491                   || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2492                         mtx_unlock(&softc->ctl_lock);
2493                         free(entries, M_CTL);
2494                         printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2495                                __func__, (uintmax_t)ooa_hdr->lun_num);
2496                         retval = EINVAL;
2497                         break;
2498                 }
2499
2500                 cur_fill_num = 0;
2501
2502                 if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2503                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
2504                                 retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2505                                         ooa_hdr, entries);
2506                                 if (retval != 0)
2507                                         break;
2508                         }
2509                         if (retval != 0) {
2510                                 mtx_unlock(&softc->ctl_lock);
2511                                 free(entries, M_CTL);
2512                                 break;
2513                         }
2514                 } else {
2515                         lun = softc->ctl_luns[ooa_hdr->lun_num];
2516
2517                         retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
2518                                                     entries);
2519                 }
2520                 mtx_unlock(&softc->ctl_lock);
2521
2522                 ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2523                 ooa_hdr->fill_len = ooa_hdr->fill_num *
2524                         sizeof(struct ctl_ooa_entry);
2525                 retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2526                 if (retval != 0) {
2527                         printf("%s: error copying out %d bytes for OOA dump\n",
2528                                __func__, ooa_hdr->fill_len);
2529                 }
2530
2531                 getbintime(&ooa_hdr->cur_bt);
2532
2533                 if (cur_fill_num > ooa_hdr->alloc_num) {
2534                         ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2535                         ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2536                 } else {
2537                         ooa_hdr->dropped_num = 0;
2538                         ooa_hdr->status = CTL_OOA_OK;
2539                 }
2540
2541                 free(entries, M_CTL);
2542                 break;
2543         }
2544         case CTL_CHECK_OOA: {
2545                 union ctl_io *io;
2546                 struct ctl_lun *lun;
2547                 struct ctl_ooa_info *ooa_info;
2548
2549
2550                 ooa_info = (struct ctl_ooa_info *)addr;
2551
2552                 if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2553                         ooa_info->status = CTL_OOA_INVALID_LUN;
2554                         break;
2555                 }
2556                 mtx_lock(&softc->ctl_lock);
2557                 lun = softc->ctl_luns[ooa_info->lun_id];
2558                 if (lun == NULL) {
2559                         mtx_unlock(&softc->ctl_lock);
2560                         ooa_info->status = CTL_OOA_INVALID_LUN;
2561                         break;
2562                 }
2563
2564                 ooa_info->num_entries = 0;
2565                 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2566                      io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2567                      &io->io_hdr, ooa_links)) {
2568                         ooa_info->num_entries++;
2569                 }
2570
2571                 mtx_unlock(&softc->ctl_lock);
2572                 ooa_info->status = CTL_OOA_SUCCESS;
2573
2574                 break;
2575         }
2576         case CTL_HARD_START:
2577         case CTL_HARD_STOP: {
2578                 struct ctl_fe_ioctl_startstop_info ss_info;
2579                 struct cfi_metatask *metatask;
2580                 struct mtx hs_mtx;
2581
2582                 mtx_init(&hs_mtx, "HS Mutex", NULL, MTX_DEF);
2583
2584                 cv_init(&ss_info.sem, "hard start/stop cv" );
2585
2586                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2587                 if (metatask == NULL) {
2588                         retval = ENOMEM;
2589                         mtx_destroy(&hs_mtx);
2590                         break;
2591                 }
2592
2593                 if (cmd == CTL_HARD_START)
2594                         metatask->tasktype = CFI_TASK_STARTUP;
2595                 else
2596                         metatask->tasktype = CFI_TASK_SHUTDOWN;
2597
2598                 metatask->callback = ctl_ioctl_hard_startstop_callback;
2599                 metatask->callback_arg = &ss_info;
2600
2601                 cfi_action(metatask);
2602
2603                 /* Wait for the callback */
2604                 mtx_lock(&hs_mtx);
2605                 cv_wait_sig(&ss_info.sem, &hs_mtx);
2606                 mtx_unlock(&hs_mtx);
2607
2608                 /*
2609                  * All information has been copied from the metatask by the
2610                  * time cv_broadcast() is called, so we free the metatask here.
2611                  */
2612                 cfi_free_metatask(metatask);
2613
2614                 memcpy((void *)addr, &ss_info.hs_info, sizeof(ss_info.hs_info));
2615
2616                 mtx_destroy(&hs_mtx);
2617                 break;
2618         }
2619         case CTL_BBRREAD: {
2620                 struct ctl_bbrread_info *bbr_info;
2621                 struct ctl_fe_ioctl_bbrread_info fe_bbr_info;
2622                 struct mtx bbr_mtx;
2623                 struct cfi_metatask *metatask;
2624
2625                 bbr_info = (struct ctl_bbrread_info *)addr;
2626
2627                 bzero(&fe_bbr_info, sizeof(fe_bbr_info));
2628
2629                 bzero(&bbr_mtx, sizeof(bbr_mtx));
2630                 mtx_init(&bbr_mtx, "BBR Mutex", NULL, MTX_DEF);
2631
2632                 fe_bbr_info.bbr_info = bbr_info;
2633                 fe_bbr_info.lock = &bbr_mtx;
2634
2635                 cv_init(&fe_bbr_info.sem, "BBR read cv");
2636                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2637
2638                 if (metatask == NULL) {
2639                         mtx_destroy(&bbr_mtx);
2640                         cv_destroy(&fe_bbr_info.sem);
2641                         retval = ENOMEM;
2642                         break;
2643                 }
2644                 metatask->tasktype = CFI_TASK_BBRREAD;
2645                 metatask->callback = ctl_ioctl_bbrread_callback;
2646                 metatask->callback_arg = &fe_bbr_info;
2647                 metatask->taskinfo.bbrread.lun_num = bbr_info->lun_num;
2648                 metatask->taskinfo.bbrread.lba = bbr_info->lba;
2649                 metatask->taskinfo.bbrread.len = bbr_info->len;
2650
2651                 cfi_action(metatask);
2652
2653                 mtx_lock(&bbr_mtx);
2654                 while (fe_bbr_info.wakeup_done == 0)
2655                         cv_wait_sig(&fe_bbr_info.sem, &bbr_mtx);
2656                 mtx_unlock(&bbr_mtx);
2657
2658                 bbr_info->status = metatask->status;
2659                 bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2660                 bbr_info->scsi_status = metatask->taskinfo.bbrread.scsi_status;
2661                 memcpy(&bbr_info->sense_data,
2662                        &metatask->taskinfo.bbrread.sense_data,
2663                        ctl_min(sizeof(bbr_info->sense_data),
2664                                sizeof(metatask->taskinfo.bbrread.sense_data)));
2665
2666                 cfi_free_metatask(metatask);
2667
2668                 mtx_destroy(&bbr_mtx);
2669                 cv_destroy(&fe_bbr_info.sem);
2670
2671                 break;
2672         }
2673         case CTL_DELAY_IO: {
2674                 struct ctl_io_delay_info *delay_info;
2675 #ifdef CTL_IO_DELAY
2676                 struct ctl_lun *lun;
2677 #endif /* CTL_IO_DELAY */
2678
2679                 delay_info = (struct ctl_io_delay_info *)addr;
2680
2681 #ifdef CTL_IO_DELAY
2682                 mtx_lock(&softc->ctl_lock);
2683
2684                 if ((delay_info->lun_id > CTL_MAX_LUNS)
2685                  || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2686                         delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2687                 } else {
2688                         lun = softc->ctl_luns[delay_info->lun_id];
2689
2690                         delay_info->status = CTL_DELAY_STATUS_OK;
2691
2692                         switch (delay_info->delay_type) {
2693                         case CTL_DELAY_TYPE_CONT:
2694                                 break;
2695                         case CTL_DELAY_TYPE_ONESHOT:
2696                                 break;
2697                         default:
2698                                 delay_info->status =
2699                                         CTL_DELAY_STATUS_INVALID_TYPE;
2700                                 break;
2701                         }
2702
2703                         switch (delay_info->delay_loc) {
2704                         case CTL_DELAY_LOC_DATAMOVE:
2705                                 lun->delay_info.datamove_type =
2706                                         delay_info->delay_type;
2707                                 lun->delay_info.datamove_delay =
2708                                         delay_info->delay_secs;
2709                                 break;
2710                         case CTL_DELAY_LOC_DONE:
2711                                 lun->delay_info.done_type =
2712                                         delay_info->delay_type;
2713                                 lun->delay_info.done_delay =
2714                                         delay_info->delay_secs;
2715                                 break;
2716                         default:
2717                                 delay_info->status =
2718                                         CTL_DELAY_STATUS_INVALID_LOC;
2719                                 break;
2720                         }
2721                 }
2722
2723                 mtx_unlock(&softc->ctl_lock);
2724 #else
2725                 delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2726 #endif /* CTL_IO_DELAY */
2727                 break;
2728         }
2729         case CTL_REALSYNC_SET: {
2730                 int *syncstate;
2731
2732                 syncstate = (int *)addr;
2733
2734                 mtx_lock(&softc->ctl_lock);
2735                 switch (*syncstate) {
2736                 case 0:
2737                         softc->flags &= ~CTL_FLAG_REAL_SYNC;
2738                         break;
2739                 case 1:
2740                         softc->flags |= CTL_FLAG_REAL_SYNC;
2741                         break;
2742                 default:
2743                         retval = -EINVAL;
2744                         break;
2745                 }
2746                 mtx_unlock(&softc->ctl_lock);
2747                 break;
2748         }
2749         case CTL_REALSYNC_GET: {
2750                 int *syncstate;
2751
2752                 syncstate = (int*)addr;
2753
2754                 mtx_lock(&softc->ctl_lock);
2755                 if (softc->flags & CTL_FLAG_REAL_SYNC)
2756                         *syncstate = 1;
2757                 else
2758                         *syncstate = 0;
2759                 mtx_unlock(&softc->ctl_lock);
2760
2761                 break;
2762         }
2763         case CTL_SETSYNC:
2764         case CTL_GETSYNC: {
2765                 struct ctl_sync_info *sync_info;
2766                 struct ctl_lun *lun;
2767
2768                 sync_info = (struct ctl_sync_info *)addr;
2769
2770                 mtx_lock(&softc->ctl_lock);
2771                 lun = softc->ctl_luns[sync_info->lun_id];
2772                 if (lun == NULL) {
2773                         mtx_unlock(&softc->ctl_lock);
2774                         sync_info->status = CTL_GS_SYNC_NO_LUN;
2775                 }
2776                 /*
2777                  * Get or set the sync interval.  We're not bounds checking
2778                  * in the set case, hopefully the user won't do something
2779                  * silly.
2780                  */
2781                 if (cmd == CTL_GETSYNC)
2782                         sync_info->sync_interval = lun->sync_interval;
2783                 else
2784                         lun->sync_interval = sync_info->sync_interval;
2785
2786                 mtx_unlock(&softc->ctl_lock);
2787
2788                 sync_info->status = CTL_GS_SYNC_OK;
2789
2790                 break;
2791         }
2792         case CTL_GETSTATS: {
2793                 struct ctl_stats *stats;
2794                 struct ctl_lun *lun;
2795                 int i;
2796
2797                 stats = (struct ctl_stats *)addr;
2798
2799                 if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2800                      stats->alloc_len) {
2801                         stats->status = CTL_SS_NEED_MORE_SPACE;
2802                         stats->num_luns = softc->num_luns;
2803                         break;
2804                 }
2805                 /*
2806                  * XXX KDM no locking here.  If the LUN list changes,
2807                  * things can blow up.
2808                  */
2809                 for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2810                      i++, lun = STAILQ_NEXT(lun, links)) {
2811                         retval = copyout(&lun->stats, &stats->lun_stats[i],
2812                                          sizeof(lun->stats));
2813                         if (retval != 0)
2814                                 break;
2815                 }
2816                 stats->num_luns = softc->num_luns;
2817                 stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2818                                  softc->num_luns;
2819                 stats->status = CTL_SS_OK;
2820 #ifdef CTL_TIME_IO
2821                 stats->flags = CTL_STATS_FLAG_TIME_VALID;
2822 #else
2823                 stats->flags = CTL_STATS_FLAG_NONE;
2824 #endif
2825                 getnanouptime(&stats->timestamp);
2826                 break;
2827         }
2828         case CTL_ERROR_INJECT: {
2829                 struct ctl_error_desc *err_desc, *new_err_desc;
2830                 struct ctl_lun *lun;
2831
2832                 err_desc = (struct ctl_error_desc *)addr;
2833
2834                 new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2835                                       M_WAITOK | M_ZERO);
2836                 bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2837
2838                 mtx_lock(&softc->ctl_lock);
2839                 lun = softc->ctl_luns[err_desc->lun_id];
2840                 if (lun == NULL) {
2841                         mtx_unlock(&softc->ctl_lock);
2842                         printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2843                                __func__, (uintmax_t)err_desc->lun_id);
2844                         retval = EINVAL;
2845                         break;
2846                 }
2847
2848                 /*
2849                  * We could do some checking here to verify the validity
2850                  * of the request, but given the complexity of error
2851                  * injection requests, the checking logic would be fairly
2852                  * complex.
2853                  *
2854                  * For now, if the request is invalid, it just won't get
2855                  * executed and might get deleted.
2856                  */
2857                 STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2858
2859                 /*
2860                  * XXX KDM check to make sure the serial number is unique,
2861                  * in case we somehow manage to wrap.  That shouldn't
2862                  * happen for a very long time, but it's the right thing to
2863                  * do.
2864                  */
2865                 new_err_desc->serial = lun->error_serial;
2866                 err_desc->serial = lun->error_serial;
2867                 lun->error_serial++;
2868
2869                 mtx_unlock(&softc->ctl_lock);
2870                 break;
2871         }
2872         case CTL_ERROR_INJECT_DELETE: {
2873                 struct ctl_error_desc *delete_desc, *desc, *desc2;
2874                 struct ctl_lun *lun;
2875                 int delete_done;
2876
2877                 delete_desc = (struct ctl_error_desc *)addr;
2878                 delete_done = 0;
2879
2880                 mtx_lock(&softc->ctl_lock);
2881                 lun = softc->ctl_luns[delete_desc->lun_id];
2882                 if (lun == NULL) {
2883                         mtx_unlock(&softc->ctl_lock);
2884                         printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
2885                                __func__, (uintmax_t)delete_desc->lun_id);
2886                         retval = EINVAL;
2887                         break;
2888                 }
2889                 STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
2890                         if (desc->serial != delete_desc->serial)
2891                                 continue;
2892
2893                         STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
2894                                       links);
2895                         free(desc, M_CTL);
2896                         delete_done = 1;
2897                 }
2898                 mtx_unlock(&softc->ctl_lock);
2899                 if (delete_done == 0) {
2900                         printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
2901                                "error serial %ju on LUN %u\n", __func__,
2902                                delete_desc->serial, delete_desc->lun_id);
2903                         retval = EINVAL;
2904                         break;
2905                 }
2906                 break;
2907         }
2908         case CTL_DUMP_STRUCTS: {
2909                 int i, j, k;
2910                 struct ctl_frontend *fe;
2911
2912                 printf("CTL IID to WWPN map start:\n");
2913                 for (i = 0; i < CTL_MAX_PORTS; i++) {
2914                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
2915                                 if (softc->wwpn_iid[i][j].in_use == 0)
2916                                         continue;
2917
2918                                 printf("port %d iid %u WWPN %#jx\n",
2919                                        softc->wwpn_iid[i][j].port,
2920                                        softc->wwpn_iid[i][j].iid,
2921                                        (uintmax_t)softc->wwpn_iid[i][j].wwpn);
2922                         }
2923                 }
2924                 printf("CTL IID to WWPN map end\n");
2925                 printf("CTL Persistent Reservation information start:\n");
2926                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2927                         struct ctl_lun *lun;
2928
2929                         lun = softc->ctl_luns[i];
2930
2931                         if ((lun == NULL)
2932                          || ((lun->flags & CTL_LUN_DISABLED) != 0))
2933                                 continue;
2934
2935                         for (j = 0; j < (CTL_MAX_PORTS * 2); j++) {
2936                                 for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
2937                                         if (lun->per_res[j+k].registered == 0)
2938                                                 continue;
2939                                         printf("LUN %d port %d iid %d key "
2940                                                "%#jx\n", i, j, k,
2941                                                (uintmax_t)scsi_8btou64(
2942                                                lun->per_res[j+k].res_key.key));
2943                                 }
2944                         }
2945                 }
2946                 printf("CTL Persistent Reservation information end\n");
2947                 printf("CTL Frontends:\n");
2948                 /*
2949                  * XXX KDM calling this without a lock.  We'd likely want
2950                  * to drop the lock before calling the frontend's dump
2951                  * routine anyway.
2952                  */
2953                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2954                         printf("Frontend %s Type %u pport %d vport %d WWNN "
2955                                "%#jx WWPN %#jx\n", fe->port_name, fe->port_type,
2956                                fe->physical_port, fe->virtual_port,
2957                                (uintmax_t)fe->wwnn, (uintmax_t)fe->wwpn);
2958
2959                         /*
2960                          * Frontends are not required to support the dump
2961                          * routine.
2962                          */
2963                         if (fe->fe_dump == NULL)
2964                                 continue;
2965
2966                         fe->fe_dump();
2967                 }
2968                 printf("CTL Frontend information end\n");
2969                 break;
2970         }
2971         case CTL_LUN_REQ: {
2972                 struct ctl_lun_req *lun_req;
2973                 struct ctl_backend_driver *backend;
2974
2975                 lun_req = (struct ctl_lun_req *)addr;
2976
2977                 backend = ctl_backend_find(lun_req->backend);
2978                 if (backend == NULL) {
2979                         lun_req->status = CTL_LUN_ERROR;
2980                         snprintf(lun_req->error_str,
2981                                  sizeof(lun_req->error_str),
2982                                  "Backend \"%s\" not found.",
2983                                  lun_req->backend);
2984                         break;
2985                 }
2986                 if (lun_req->num_be_args > 0) {
2987                         lun_req->kern_be_args = ctl_copyin_args(
2988                                 lun_req->num_be_args,
2989                                 lun_req->be_args,
2990                                 lun_req->error_str,
2991                                 sizeof(lun_req->error_str));
2992                         if (lun_req->kern_be_args == NULL) {
2993                                 lun_req->status = CTL_LUN_ERROR;
2994                                 break;
2995                         }
2996                 }
2997
2998                 retval = backend->ioctl(dev, cmd, addr, flag, td);
2999
3000                 if (lun_req->num_be_args > 0) {
3001                         ctl_free_args(lun_req->num_be_args,
3002                                       lun_req->kern_be_args);
3003                 }
3004                 break;
3005         }
3006         case CTL_LUN_LIST: {
3007                 struct sbuf *sb;
3008                 struct ctl_lun *lun;
3009                 struct ctl_lun_list *list;
3010                 struct ctl_be_lun_option *opt;
3011
3012                 list = (struct ctl_lun_list *)addr;
3013
3014                 /*
3015                  * Allocate a fixed length sbuf here, based on the length
3016                  * of the user's buffer.  We could allocate an auto-extending
3017                  * buffer, and then tell the user how much larger our
3018                  * amount of data is than his buffer, but that presents
3019                  * some problems:
3020                  *
3021                  * 1.  The sbuf(9) routines use a blocking malloc, and so
3022                  *     we can't hold a lock while calling them with an
3023                  *     auto-extending buffer.
3024                  *
3025                  * 2.  There is not currently a LUN reference counting
3026                  *     mechanism, outside of outstanding transactions on
3027                  *     the LUN's OOA queue.  So a LUN could go away on us
3028                  *     while we're getting the LUN number, backend-specific
3029                  *     information, etc.  Thus, given the way things
3030                  *     currently work, we need to hold the CTL lock while
3031                  *     grabbing LUN information.
3032                  *
3033                  * So, from the user's standpoint, the best thing to do is
3034                  * allocate what he thinks is a reasonable buffer length,
3035                  * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
3036                  * double the buffer length and try again.  (And repeat
3037                  * that until he succeeds.)
3038                  */
3039                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3040                 if (sb == NULL) {
3041                         list->status = CTL_LUN_LIST_ERROR;
3042                         snprintf(list->error_str, sizeof(list->error_str),
3043                                  "Unable to allocate %d bytes for LUN list",
3044                                  list->alloc_len);
3045                         break;
3046                 }
3047
3048                 sbuf_printf(sb, "<ctllunlist>\n");
3049
3050                 mtx_lock(&softc->ctl_lock);
3051
3052                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
3053                         retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3054                                              (uintmax_t)lun->lun);
3055
3056                         /*
3057                          * Bail out as soon as we see that we've overfilled
3058                          * the buffer.
3059                          */
3060                         if (retval != 0)
3061                                 break;
3062
3063                         retval = sbuf_printf(sb, "<backend_type>%s"
3064                                              "</backend_type>\n",
3065                                              (lun->backend == NULL) ?  "none" :
3066                                              lun->backend->name);
3067
3068                         if (retval != 0)
3069                                 break;
3070
3071                         retval = sbuf_printf(sb, "<lun_type>%d</lun_type>\n",
3072                                              lun->be_lun->lun_type);
3073
3074                         if (retval != 0)
3075                                 break;
3076
3077                         if (lun->backend == NULL) {
3078                                 retval = sbuf_printf(sb, "</lun>\n");
3079                                 if (retval != 0)
3080                                         break;
3081                                 continue;
3082                         }
3083
3084                         retval = sbuf_printf(sb, "<size>%ju</size>\n",
3085                                              (lun->be_lun->maxlba > 0) ?
3086                                              lun->be_lun->maxlba + 1 : 0);
3087
3088                         if (retval != 0)
3089                                 break;
3090
3091                         retval = sbuf_printf(sb, "<blocksize>%u</blocksize>\n",
3092                                              lun->be_lun->blocksize);
3093
3094                         if (retval != 0)
3095                                 break;
3096
3097                         retval = sbuf_printf(sb, "<serial_number>");
3098
3099                         if (retval != 0)
3100                                 break;
3101
3102                         retval = ctl_sbuf_printf_esc(sb,
3103                                                      lun->be_lun->serial_num);
3104
3105                         if (retval != 0)
3106                                 break;
3107
3108                         retval = sbuf_printf(sb, "</serial_number>\n");
3109
3110                         if (retval != 0)
3111                                 break;
3112
3113                         retval = sbuf_printf(sb, "<device_id>");
3114
3115                         if (retval != 0)
3116                                 break;
3117
3118                         retval = ctl_sbuf_printf_esc(sb,lun->be_lun->device_id);
3119
3120                         if (retval != 0)
3121                                 break;
3122
3123                         retval = sbuf_printf(sb, "</device_id>\n");
3124
3125                         if (retval != 0)
3126                                 break;
3127
3128                         if (lun->backend->lun_info != NULL) {
3129                                 retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3130                                 if (retval != 0)
3131                                         break;
3132                         }
3133                         STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3134                                 retval = sbuf_printf(sb, "<%s>%s</%s>", opt->name, opt->value, opt->name);
3135                                 if (retval != 0)
3136                                         break;
3137                         }
3138
3139                         retval = sbuf_printf(sb, "</lun>\n");
3140
3141                         if (retval != 0)
3142                                 break;
3143                 }
3144                 mtx_unlock(&softc->ctl_lock);
3145
3146                 if ((retval != 0)
3147                  || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3148                         retval = 0;
3149                         sbuf_delete(sb);
3150                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3151                         snprintf(list->error_str, sizeof(list->error_str),
3152                                  "Out of space, %d bytes is too small",
3153                                  list->alloc_len);
3154                         break;
3155                 }
3156
3157                 sbuf_finish(sb);
3158
3159                 retval = copyout(sbuf_data(sb), list->lun_xml,
3160                                  sbuf_len(sb) + 1);
3161
3162                 list->fill_len = sbuf_len(sb) + 1;
3163                 list->status = CTL_LUN_LIST_OK;
3164                 sbuf_delete(sb);
3165                 break;
3166         }
3167         case CTL_ISCSI: {
3168                 struct ctl_iscsi *ci;
3169                 struct ctl_frontend *fe;
3170
3171                 ci = (struct ctl_iscsi *)addr;
3172
3173                 mtx_lock(&softc->ctl_lock);
3174                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
3175                         if (strcmp(fe->port_name, "iscsi") == 0)
3176                                 break;
3177                 }
3178                 mtx_unlock(&softc->ctl_lock);
3179
3180                 if (fe == NULL) {
3181                         ci->status = CTL_ISCSI_ERROR;
3182                         snprintf(ci->error_str, sizeof(ci->error_str), "Backend \"iscsi\" not found.");
3183                         break;
3184                 }
3185
3186                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3187                 break;
3188         }
3189         default: {
3190                 /* XXX KDM should we fix this? */
3191 #if 0
3192                 struct ctl_backend_driver *backend;
3193                 unsigned int type;
3194                 int found;
3195
3196                 found = 0;
3197
3198                 /*
3199                  * We encode the backend type as the ioctl type for backend
3200                  * ioctls.  So parse it out here, and then search for a
3201                  * backend of this type.
3202                  */
3203                 type = _IOC_TYPE(cmd);
3204
3205                 STAILQ_FOREACH(backend, &softc->be_list, links) {
3206                         if (backend->type == type) {
3207                                 found = 1;
3208                                 break;
3209                         }
3210                 }
3211                 if (found == 0) {
3212                         printf("ctl: unknown ioctl command %#lx or backend "
3213                                "%d\n", cmd, type);
3214                         retval = -EINVAL;
3215                         break;
3216                 }
3217                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3218 #endif
3219                 retval = ENOTTY;
3220                 break;
3221         }
3222         }
3223         return (retval);
3224 }
3225
3226 uint32_t
3227 ctl_get_initindex(struct ctl_nexus *nexus)
3228 {
3229         if (nexus->targ_port < CTL_MAX_PORTS)
3230                 return (nexus->initid.id +
3231                         (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3232         else
3233                 return (nexus->initid.id +
3234                        ((nexus->targ_port - CTL_MAX_PORTS) *
3235                         CTL_MAX_INIT_PER_PORT));
3236 }
3237
3238 uint32_t
3239 ctl_get_resindex(struct ctl_nexus *nexus)
3240 {
3241         return (nexus->initid.id + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3242 }
3243
3244 uint32_t
3245 ctl_port_idx(int port_num)
3246 {
3247         if (port_num < CTL_MAX_PORTS)
3248                 return(port_num);
3249         else
3250                 return(port_num - CTL_MAX_PORTS);
3251 }
3252
3253 /*
3254  * Note:  This only works for bitmask sizes that are at least 32 bits, and
3255  * that are a power of 2.
3256  */
3257 int
3258 ctl_ffz(uint32_t *mask, uint32_t size)
3259 {
3260         uint32_t num_chunks, num_pieces;
3261         int i, j;
3262
3263         num_chunks = (size >> 5);
3264         if (num_chunks == 0)
3265                 num_chunks++;
3266         num_pieces = ctl_min((sizeof(uint32_t) * 8), size);
3267
3268         for (i = 0; i < num_chunks; i++) {
3269                 for (j = 0; j < num_pieces; j++) {
3270                         if ((mask[i] & (1 << j)) == 0)
3271                                 return ((i << 5) + j);
3272                 }
3273         }
3274
3275         return (-1);
3276 }
3277
3278 int
3279 ctl_set_mask(uint32_t *mask, uint32_t bit)
3280 {
3281         uint32_t chunk, piece;
3282
3283         chunk = bit >> 5;
3284         piece = bit % (sizeof(uint32_t) * 8);
3285
3286         if ((mask[chunk] & (1 << piece)) != 0)
3287                 return (-1);
3288         else
3289                 mask[chunk] |= (1 << piece);
3290
3291         return (0);
3292 }
3293
3294 int
3295 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3296 {
3297         uint32_t chunk, piece;
3298
3299         chunk = bit >> 5;
3300         piece = bit % (sizeof(uint32_t) * 8);
3301
3302         if ((mask[chunk] & (1 << piece)) == 0)
3303                 return (-1);
3304         else
3305                 mask[chunk] &= ~(1 << piece);
3306
3307         return (0);
3308 }
3309
3310 int
3311 ctl_is_set(uint32_t *mask, uint32_t bit)
3312 {
3313         uint32_t chunk, piece;
3314
3315         chunk = bit >> 5;
3316         piece = bit % (sizeof(uint32_t) * 8);
3317
3318         if ((mask[chunk] & (1 << piece)) == 0)
3319                 return (0);
3320         else
3321                 return (1);
3322 }
3323
3324 #ifdef unused
3325 /*
3326  * The bus, target and lun are optional, they can be filled in later.
3327  * can_wait is used to determine whether we can wait on the malloc or not.
3328  */
3329 union ctl_io*
3330 ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port, uint32_t targ_target,
3331               uint32_t targ_lun, int can_wait)
3332 {
3333         union ctl_io *io;
3334
3335         if (can_wait)
3336                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_WAITOK);
3337         else
3338                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3339
3340         if (io != NULL) {
3341                 io->io_hdr.io_type = io_type;
3342                 io->io_hdr.targ_port = targ_port;
3343                 /*
3344                  * XXX KDM this needs to change/go away.  We need to move
3345                  * to a preallocated pool of ctl_scsiio structures.
3346                  */
3347                 io->io_hdr.nexus.targ_target.id = targ_target;
3348                 io->io_hdr.nexus.targ_lun = targ_lun;
3349         }
3350
3351         return (io);
3352 }
3353
3354 void
3355 ctl_kfree_io(union ctl_io *io)
3356 {
3357         free(io, M_CTL);
3358 }
3359 #endif /* unused */
3360
3361 /*
3362  * ctl_softc, pool_type, total_ctl_io are passed in.
3363  * npool is passed out.
3364  */
3365 int
3366 ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type,
3367                 uint32_t total_ctl_io, struct ctl_io_pool **npool)
3368 {
3369         uint32_t i;
3370         union ctl_io *cur_io, *next_io;
3371         struct ctl_io_pool *pool;
3372         int retval;
3373
3374         retval = 0;
3375
3376         pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3377                                             M_NOWAIT | M_ZERO);
3378         if (pool == NULL) {
3379                 retval = -ENOMEM;
3380                 goto bailout;
3381         }
3382
3383         pool->type = pool_type;
3384         pool->ctl_softc = ctl_softc;
3385
3386         mtx_lock(&ctl_softc->pool_lock);
3387         pool->id = ctl_softc->cur_pool_id++;
3388         mtx_unlock(&ctl_softc->pool_lock);
3389
3390         pool->flags = CTL_POOL_FLAG_NONE;
3391         pool->refcount = 1;             /* Reference for validity. */
3392         STAILQ_INIT(&pool->free_queue);
3393
3394         /*
3395          * XXX KDM other options here:
3396          * - allocate a page at a time
3397          * - allocate one big chunk of memory.
3398          * Page allocation might work well, but would take a little more
3399          * tracking.
3400          */
3401         for (i = 0; i < total_ctl_io; i++) {
3402                 cur_io = (union ctl_io *)malloc(sizeof(*cur_io), M_CTL,
3403                                                 M_NOWAIT);
3404                 if (cur_io == NULL) {
3405                         retval = ENOMEM;
3406                         break;
3407                 }
3408                 cur_io->io_hdr.pool = pool;
3409                 STAILQ_INSERT_TAIL(&pool->free_queue, &cur_io->io_hdr, links);
3410                 pool->total_ctl_io++;
3411                 pool->free_ctl_io++;
3412         }
3413
3414         if (retval != 0) {
3415                 for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3416                      cur_io != NULL; cur_io = next_io) {
3417                         next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr,
3418                                                               links);
3419                         STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr,
3420                                       ctl_io_hdr, links);
3421                         free(cur_io, M_CTL);
3422                 }
3423
3424                 free(pool, M_CTL);
3425                 goto bailout;
3426         }
3427         mtx_lock(&ctl_softc->pool_lock);
3428         ctl_softc->num_pools++;
3429         STAILQ_INSERT_TAIL(&ctl_softc->io_pools, pool, links);
3430         /*
3431          * Increment our usage count if this is an external consumer, so we
3432          * can't get unloaded until the external consumer (most likely a
3433          * FETD) unloads and frees his pool.
3434          *
3435          * XXX KDM will this increment the caller's module use count, or
3436          * mine?
3437          */
3438 #if 0
3439         if ((pool_type != CTL_POOL_EMERGENCY)
3440          && (pool_type != CTL_POOL_INTERNAL)
3441          && (pool_type != CTL_POOL_IOCTL)
3442          && (pool_type != CTL_POOL_4OTHERSC))
3443                 MOD_INC_USE_COUNT;
3444 #endif
3445
3446         mtx_unlock(&ctl_softc->pool_lock);
3447
3448         *npool = pool;
3449
3450 bailout:
3451
3452         return (retval);
3453 }
3454
3455 static int
3456 ctl_pool_acquire(struct ctl_io_pool *pool)
3457 {
3458
3459         mtx_assert(&pool->ctl_softc->pool_lock, MA_OWNED);
3460
3461         if (pool->flags & CTL_POOL_FLAG_INVALID)
3462                 return (-EINVAL);
3463
3464         pool->refcount++;
3465
3466         return (0);
3467 }
3468
3469 static void
3470 ctl_pool_release(struct ctl_io_pool *pool)
3471 {
3472         struct ctl_softc *ctl_softc = pool->ctl_softc;
3473         union ctl_io *io;
3474
3475         mtx_assert(&ctl_softc->pool_lock, MA_OWNED);
3476
3477         if (--pool->refcount != 0)
3478                 return;
3479
3480         while ((io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue)) != NULL) {
3481                 STAILQ_REMOVE(&pool->free_queue, &io->io_hdr, ctl_io_hdr,
3482                               links);
3483                 free(io, M_CTL);
3484         }
3485
3486         STAILQ_REMOVE(&ctl_softc->io_pools, pool, ctl_io_pool, links);
3487         ctl_softc->num_pools--;
3488
3489         /*
3490          * XXX KDM will this decrement the caller's usage count or mine?
3491          */
3492 #if 0
3493         if ((pool->type != CTL_POOL_EMERGENCY)
3494          && (pool->type != CTL_POOL_INTERNAL)
3495          && (pool->type != CTL_POOL_IOCTL))
3496                 MOD_DEC_USE_COUNT;
3497 #endif
3498
3499         free(pool, M_CTL);
3500 }
3501
3502 void
3503 ctl_pool_free(struct ctl_io_pool *pool)
3504 {
3505         struct ctl_softc *ctl_softc;
3506
3507         if (pool == NULL)
3508                 return;
3509
3510         ctl_softc = pool->ctl_softc;
3511         mtx_lock(&ctl_softc->pool_lock);
3512         pool->flags |= CTL_POOL_FLAG_INVALID;
3513         ctl_pool_release(pool);
3514         mtx_unlock(&ctl_softc->pool_lock);
3515 }
3516
3517 /*
3518  * This routine does not block (except for spinlocks of course).
3519  * It tries to allocate a ctl_io union from the caller's pool as quickly as
3520  * possible.
3521  */
3522 union ctl_io *
3523 ctl_alloc_io(void *pool_ref)
3524 {
3525         union ctl_io *io;
3526         struct ctl_softc *ctl_softc;
3527         struct ctl_io_pool *pool, *npool;
3528         struct ctl_io_pool *emergency_pool;
3529
3530         pool = (struct ctl_io_pool *)pool_ref;
3531
3532         if (pool == NULL) {
3533                 printf("%s: pool is NULL\n", __func__);
3534                 return (NULL);
3535         }
3536
3537         emergency_pool = NULL;
3538
3539         ctl_softc = pool->ctl_softc;
3540
3541         mtx_lock(&ctl_softc->pool_lock);
3542         /*
3543          * First, try to get the io structure from the user's pool.
3544          */
3545         if (ctl_pool_acquire(pool) == 0) {
3546                 io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3547                 if (io != NULL) {
3548                         STAILQ_REMOVE_HEAD(&pool->free_queue, links);
3549                         pool->total_allocated++;
3550                         pool->free_ctl_io--;
3551                         mtx_unlock(&ctl_softc->pool_lock);
3552                         return (io);
3553                 } else
3554                         ctl_pool_release(pool);
3555         }
3556         /*
3557          * If he doesn't have any io structures left, search for an
3558          * emergency pool and grab one from there.
3559          */
3560         STAILQ_FOREACH(npool, &ctl_softc->io_pools, links) {
3561                 if (npool->type != CTL_POOL_EMERGENCY)
3562                         continue;
3563
3564                 if (ctl_pool_acquire(npool) != 0)
3565                         continue;
3566
3567                 emergency_pool = npool;
3568
3569                 io = (union ctl_io *)STAILQ_FIRST(&npool->free_queue);
3570                 if (io != NULL) {
3571                         STAILQ_REMOVE_HEAD(&npool->free_queue, links);
3572                         npool->total_allocated++;
3573                         npool->free_ctl_io--;
3574                         mtx_unlock(&ctl_softc->pool_lock);
3575                         return (io);
3576                 } else
3577                         ctl_pool_release(npool);
3578         }
3579
3580         /* Drop the spinlock before we malloc */
3581         mtx_unlock(&ctl_softc->pool_lock);
3582
3583         /*
3584          * The emergency pool (if it exists) didn't have one, so try an
3585          * atomic (i.e. nonblocking) malloc and see if we get lucky.
3586          */
3587         io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3588         if (io != NULL) {
3589                 /*
3590                  * If the emergency pool exists but is empty, add this
3591                  * ctl_io to its list when it gets freed.
3592                  */
3593                 if (emergency_pool != NULL) {
3594                         mtx_lock(&ctl_softc->pool_lock);
3595                         if (ctl_pool_acquire(emergency_pool) == 0) {
3596                                 io->io_hdr.pool = emergency_pool;
3597                                 emergency_pool->total_ctl_io++;
3598                                 /*
3599                                  * Need to bump this, otherwise
3600                                  * total_allocated and total_freed won't
3601                                  * match when we no longer have anything
3602                                  * outstanding.
3603                                  */
3604                                 emergency_pool->total_allocated++;
3605                         }
3606                         mtx_unlock(&ctl_softc->pool_lock);
3607                 } else
3608                         io->io_hdr.pool = NULL;
3609         }
3610
3611         return (io);
3612 }
3613
3614 void
3615 ctl_free_io(union ctl_io *io)
3616 {
3617         if (io == NULL)
3618                 return;
3619
3620         /*
3621          * If this ctl_io has a pool, return it to that pool.
3622          */
3623         if (io->io_hdr.pool != NULL) {
3624                 struct ctl_io_pool *pool;
3625 #if 0
3626                 struct ctl_softc *ctl_softc;
3627                 union ctl_io *tmp_io;
3628                 unsigned long xflags;
3629                 int i;
3630
3631                 ctl_softc = control_softc;
3632 #endif
3633
3634                 pool = (struct ctl_io_pool *)io->io_hdr.pool;
3635
3636                 mtx_lock(&pool->ctl_softc->pool_lock);
3637 #if 0
3638                 save_flags(xflags);
3639
3640                 for (i = 0, tmp_io = (union ctl_io *)STAILQ_FIRST(
3641                      &ctl_softc->task_queue); tmp_io != NULL; i++,
3642                      tmp_io = (union ctl_io *)STAILQ_NEXT(&tmp_io->io_hdr,
3643                      links)) {
3644                         if (tmp_io == io) {
3645                                 printf("%s: %p is still on the task queue!\n",
3646                                        __func__, tmp_io);
3647                                 printf("%s: (%d): type %d "
3648                                        "msg %d cdb %x iptl: "
3649                                        "%d:%d:%d:%d tag 0x%04x "
3650                                        "flg %#lx\n",
3651                                         __func__, i,
3652                                         tmp_io->io_hdr.io_type,
3653                                         tmp_io->io_hdr.msg_type,
3654                                         tmp_io->scsiio.cdb[0],
3655                                         tmp_io->io_hdr.nexus.initid.id,
3656                                         tmp_io->io_hdr.nexus.targ_port,
3657                                         tmp_io->io_hdr.nexus.targ_target.id,
3658                                         tmp_io->io_hdr.nexus.targ_lun,
3659                                         (tmp_io->io_hdr.io_type ==
3660                                         CTL_IO_TASK) ?
3661                                         tmp_io->taskio.tag_num :
3662                                         tmp_io->scsiio.tag_num,
3663                                         xflags);
3664                                 panic("I/O still on the task queue!");
3665                         }
3666                 }
3667 #endif
3668                 io->io_hdr.io_type = 0xff;
3669                 STAILQ_INSERT_TAIL(&pool->free_queue, &io->io_hdr, links);
3670                 pool->total_freed++;
3671                 pool->free_ctl_io++;
3672                 ctl_pool_release(pool);
3673                 mtx_unlock(&pool->ctl_softc->pool_lock);
3674         } else {
3675                 /*
3676                  * Otherwise, just free it.  We probably malloced it and
3677                  * the emergency pool wasn't available.
3678                  */
3679                 free(io, M_CTL);
3680         }
3681
3682 }
3683
3684 void
3685 ctl_zero_io(union ctl_io *io)
3686 {
3687         void *pool_ref;
3688
3689         if (io == NULL)
3690                 return;
3691
3692         /*
3693          * May need to preserve linked list pointers at some point too.
3694          */
3695         pool_ref = io->io_hdr.pool;
3696
3697         memset(io, 0, sizeof(*io));
3698
3699         io->io_hdr.pool = pool_ref;
3700 }
3701
3702 /*
3703  * This routine is currently used for internal copies of ctl_ios that need
3704  * to persist for some reason after we've already returned status to the
3705  * FETD.  (Thus the flag set.)
3706  *
3707  * XXX XXX
3708  * Note that this makes a blind copy of all fields in the ctl_io, except
3709  * for the pool reference.  This includes any memory that has been
3710  * allocated!  That memory will no longer be valid after done has been
3711  * called, so this would be VERY DANGEROUS for command that actually does
3712  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3713  * start and stop commands, which don't transfer any data, so this is not a
3714  * problem.  If it is used for anything else, the caller would also need to
3715  * allocate data buffer space and this routine would need to be modified to
3716  * copy the data buffer(s) as well.
3717  */
3718 void
3719 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3720 {
3721         void *pool_ref;
3722
3723         if ((src == NULL)
3724          || (dest == NULL))
3725                 return;
3726
3727         /*
3728          * May need to preserve linked list pointers at some point too.
3729          */
3730         pool_ref = dest->io_hdr.pool;
3731
3732         memcpy(dest, src, ctl_min(sizeof(*src), sizeof(*dest)));
3733
3734         dest->io_hdr.pool = pool_ref;
3735         /*
3736          * We need to know that this is an internal copy, and doesn't need
3737          * to get passed back to the FETD that allocated it.
3738          */
3739         dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3740 }
3741
3742 #ifdef NEEDTOPORT
3743 static void
3744 ctl_update_power_subpage(struct copan_power_subpage *page)
3745 {
3746         int num_luns, num_partitions, config_type;
3747         struct ctl_softc *softc;
3748         cs_BOOL_t aor_present, shelf_50pct_power;
3749         cs_raidset_personality_t rs_type;
3750         int max_active_luns;
3751
3752         softc = control_softc;
3753
3754         /* subtract out the processor LUN */
3755         num_luns = softc->num_luns - 1;
3756         /*
3757          * Default to 7 LUNs active, which was the only number we allowed
3758          * in the past.
3759          */
3760         max_active_luns = 7;
3761
3762         num_partitions = config_GetRsPartitionInfo();
3763         config_type = config_GetConfigType();
3764         shelf_50pct_power = config_GetShelfPowerMode();
3765         aor_present = config_IsAorRsPresent();
3766
3767         rs_type = ddb_GetRsRaidType(1);
3768         if ((rs_type != CS_RAIDSET_PERSONALITY_RAID5)
3769          && (rs_type != CS_RAIDSET_PERSONALITY_RAID1)) {
3770                 EPRINT(0, "Unsupported RS type %d!", rs_type);
3771         }
3772
3773
3774         page->total_luns = num_luns;
3775
3776         switch (config_type) {
3777         case 40:
3778                 /*
3779                  * In a 40 drive configuration, it doesn't matter what DC
3780                  * cards we have, whether we have AOR enabled or not,
3781                  * partitioning or not, or what type of RAIDset we have.
3782                  * In that scenario, we can power up every LUN we present
3783                  * to the user.
3784                  */
3785                 max_active_luns = num_luns;
3786
3787                 break;
3788         case 64:
3789                 if (shelf_50pct_power == CS_FALSE) {
3790                         /* 25% power */
3791                         if (aor_present == CS_TRUE) {
3792                                 if (rs_type ==
3793                                      CS_RAIDSET_PERSONALITY_RAID5) {
3794                                         max_active_luns = 7;
3795                                 } else if (rs_type ==
3796                                          CS_RAIDSET_PERSONALITY_RAID1){
3797                                         max_active_luns = 14;
3798                                 } else {
3799                                         /* XXX KDM now what?? */
3800                                 }
3801                         } else {
3802                                 if (rs_type ==
3803                                      CS_RAIDSET_PERSONALITY_RAID5) {
3804                                         max_active_luns = 8;
3805                                 } else if (rs_type ==
3806                                          CS_RAIDSET_PERSONALITY_RAID1){
3807                                         max_active_luns = 16;
3808                                 } else {
3809                                         /* XXX KDM now what?? */
3810                                 }
3811                         }
3812                 } else {
3813                         /* 50% power */
3814                         /*
3815                          * With 50% power in a 64 drive configuration, we
3816                          * can power all LUNs we present.
3817                          */
3818                         max_active_luns = num_luns;
3819                 }
3820                 break;
3821         case 112:
3822                 if (shelf_50pct_power == CS_FALSE) {
3823                         /* 25% power */
3824                         if (aor_present == CS_TRUE) {
3825                                 if (rs_type ==
3826                                      CS_RAIDSET_PERSONALITY_RAID5) {
3827                                         max_active_luns = 7;
3828                                 } else if (rs_type ==
3829                                          CS_RAIDSET_PERSONALITY_RAID1){
3830                                         max_active_luns = 14;
3831                                 } else {
3832                                         /* XXX KDM now what?? */
3833                                 }
3834                         } else {
3835                                 if (rs_type ==
3836                                      CS_RAIDSET_PERSONALITY_RAID5) {
3837                                         max_active_luns = 8;
3838                                 } else if (rs_type ==
3839                                          CS_RAIDSET_PERSONALITY_RAID1){
3840                                         max_active_luns = 16;
3841                                 } else {
3842                                         /* XXX KDM now what?? */
3843                                 }
3844                         }
3845                 } else {
3846                         /* 50% power */
3847                         if (aor_present == CS_TRUE) {
3848                                 if (rs_type ==
3849                                      CS_RAIDSET_PERSONALITY_RAID5) {
3850                                         max_active_luns = 14;
3851                                 } else if (rs_type ==
3852                                          CS_RAIDSET_PERSONALITY_RAID1){
3853                                         /*
3854                                          * We're assuming here that disk
3855                                          * caching is enabled, and so we're
3856                                          * able to power up half of each
3857                                          * LUN, and cache all writes.
3858                                          */
3859                                         max_active_luns = num_luns;
3860                                 } else {
3861                                         /* XXX KDM now what?? */
3862                                 }
3863                         } else {
3864                                 if (rs_type ==
3865                                      CS_RAIDSET_PERSONALITY_RAID5) {
3866                                         max_active_luns = 15;
3867                                 } else if (rs_type ==
3868                                          CS_RAIDSET_PERSONALITY_RAID1){
3869                                         max_active_luns = 30;
3870                                 } else {
3871                                         /* XXX KDM now what?? */
3872                                 }
3873                         }
3874                 }
3875                 break;
3876         default:
3877                 /*
3878                  * In this case, we have an unknown configuration, so we
3879                  * just use the default from above.
3880                  */
3881                 break;
3882         }
3883
3884         page->max_active_luns = max_active_luns;
3885 #if 0
3886         printk("%s: total_luns = %d, max_active_luns = %d\n", __func__,
3887                page->total_luns, page->max_active_luns);
3888 #endif
3889 }
3890 #endif /* NEEDTOPORT */
3891
3892 /*
3893  * This routine could be used in the future to load default and/or saved
3894  * mode page parameters for a particuar lun.
3895  */
3896 static int
3897 ctl_init_page_index(struct ctl_lun *lun)
3898 {
3899         int i;
3900         struct ctl_page_index *page_index;
3901         struct ctl_softc *softc;
3902
3903         memcpy(&lun->mode_pages.index, page_index_template,
3904                sizeof(page_index_template));
3905
3906         softc = lun->ctl_softc;
3907
3908         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
3909
3910                 page_index = &lun->mode_pages.index[i];
3911                 /*
3912                  * If this is a disk-only mode page, there's no point in
3913                  * setting it up.  For some pages, we have to have some
3914                  * basic information about the disk in order to calculate the
3915                  * mode page data.
3916                  */
3917                 if ((lun->be_lun->lun_type != T_DIRECT)
3918                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
3919                         continue;
3920
3921                 switch (page_index->page_code & SMPH_PC_MASK) {
3922                 case SMS_FORMAT_DEVICE_PAGE: {
3923                         struct scsi_format_page *format_page;
3924
3925                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3926                                 panic("subpage is incorrect!");
3927
3928                         /*
3929                          * Sectors per track are set above.  Bytes per
3930                          * sector need to be set here on a per-LUN basis.
3931                          */
3932                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
3933                                &format_page_default,
3934                                sizeof(format_page_default));
3935                         memcpy(&lun->mode_pages.format_page[
3936                                CTL_PAGE_CHANGEABLE], &format_page_changeable,
3937                                sizeof(format_page_changeable));
3938                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
3939                                &format_page_default,
3940                                sizeof(format_page_default));
3941                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
3942                                &format_page_default,
3943                                sizeof(format_page_default));
3944
3945                         format_page = &lun->mode_pages.format_page[
3946                                 CTL_PAGE_CURRENT];
3947                         scsi_ulto2b(lun->be_lun->blocksize,
3948                                     format_page->bytes_per_sector);
3949
3950                         format_page = &lun->mode_pages.format_page[
3951                                 CTL_PAGE_DEFAULT];
3952                         scsi_ulto2b(lun->be_lun->blocksize,
3953                                     format_page->bytes_per_sector);
3954
3955                         format_page = &lun->mode_pages.format_page[
3956                                 CTL_PAGE_SAVED];
3957                         scsi_ulto2b(lun->be_lun->blocksize,
3958                                     format_page->bytes_per_sector);
3959
3960                         page_index->page_data =
3961                                 (uint8_t *)lun->mode_pages.format_page;
3962                         break;
3963                 }
3964                 case SMS_RIGID_DISK_PAGE: {
3965                         struct scsi_rigid_disk_page *rigid_disk_page;
3966                         uint32_t sectors_per_cylinder;
3967                         uint64_t cylinders;
3968 #ifndef __XSCALE__
3969                         int shift;
3970 #endif /* !__XSCALE__ */
3971
3972                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3973                                 panic("invalid subpage value %d",
3974                                       page_index->subpage);
3975
3976                         /*
3977                          * Rotation rate and sectors per track are set
3978                          * above.  We calculate the cylinders here based on
3979                          * capacity.  Due to the number of heads and
3980                          * sectors per track we're using, smaller arrays
3981                          * may turn out to have 0 cylinders.  Linux and
3982                          * FreeBSD don't pay attention to these mode pages
3983                          * to figure out capacity, but Solaris does.  It
3984                          * seems to deal with 0 cylinders just fine, and
3985                          * works out a fake geometry based on the capacity.
3986                          */
3987                         memcpy(&lun->mode_pages.rigid_disk_page[
3988                                CTL_PAGE_CURRENT], &rigid_disk_page_default,
3989                                sizeof(rigid_disk_page_default));
3990                         memcpy(&lun->mode_pages.rigid_disk_page[
3991                                CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
3992                                sizeof(rigid_disk_page_changeable));
3993                         memcpy(&lun->mode_pages.rigid_disk_page[
3994                                CTL_PAGE_DEFAULT], &rigid_disk_page_default,
3995                                sizeof(rigid_disk_page_default));
3996                         memcpy(&lun->mode_pages.rigid_disk_page[
3997                                CTL_PAGE_SAVED], &rigid_disk_page_default,
3998                                sizeof(rigid_disk_page_default));
3999
4000                         sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
4001                                 CTL_DEFAULT_HEADS;
4002
4003                         /*
4004                          * The divide method here will be more accurate,
4005                          * probably, but results in floating point being
4006                          * used in the kernel on i386 (__udivdi3()).  On the
4007                          * XScale, though, __udivdi3() is implemented in
4008                          * software.
4009                          *
4010                          * The shift method for cylinder calculation is
4011                          * accurate if sectors_per_cylinder is a power of
4012                          * 2.  Otherwise it might be slightly off -- you
4013                          * might have a bit of a truncation problem.
4014                          */
4015 #ifdef  __XSCALE__
4016                         cylinders = (lun->be_lun->maxlba + 1) /
4017                                 sectors_per_cylinder;
4018 #else
4019                         for (shift = 31; shift > 0; shift--) {
4020                                 if (sectors_per_cylinder & (1 << shift))
4021                                         break;
4022                         }
4023                         cylinders = (lun->be_lun->maxlba + 1) >> shift;
4024 #endif
4025
4026                         /*
4027                          * We've basically got 3 bytes, or 24 bits for the
4028                          * cylinder size in the mode page.  If we're over,
4029                          * just round down to 2^24.
4030                          */
4031                         if (cylinders > 0xffffff)
4032                                 cylinders = 0xffffff;
4033
4034                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4035                                 CTL_PAGE_CURRENT];
4036                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4037
4038                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4039                                 CTL_PAGE_DEFAULT];
4040                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4041
4042                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4043                                 CTL_PAGE_SAVED];
4044                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4045
4046                         page_index->page_data =
4047                                 (uint8_t *)lun->mode_pages.rigid_disk_page;
4048                         break;
4049                 }
4050                 case SMS_CACHING_PAGE: {
4051
4052                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4053                                 panic("invalid subpage value %d",
4054                                       page_index->subpage);
4055                         /*
4056                          * Defaults should be okay here, no calculations
4057                          * needed.
4058                          */
4059                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
4060                                &caching_page_default,
4061                                sizeof(caching_page_default));
4062                         memcpy(&lun->mode_pages.caching_page[
4063                                CTL_PAGE_CHANGEABLE], &caching_page_changeable,
4064                                sizeof(caching_page_changeable));
4065                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
4066                                &caching_page_default,
4067                                sizeof(caching_page_default));
4068                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4069                                &caching_page_default,
4070                                sizeof(caching_page_default));
4071                         page_index->page_data =
4072                                 (uint8_t *)lun->mode_pages.caching_page;
4073                         break;
4074                 }
4075                 case SMS_CONTROL_MODE_PAGE: {
4076
4077                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4078                                 panic("invalid subpage value %d",
4079                                       page_index->subpage);
4080
4081                         /*
4082                          * Defaults should be okay here, no calculations
4083                          * needed.
4084                          */
4085                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
4086                                &control_page_default,
4087                                sizeof(control_page_default));
4088                         memcpy(&lun->mode_pages.control_page[
4089                                CTL_PAGE_CHANGEABLE], &control_page_changeable,
4090                                sizeof(control_page_changeable));
4091                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
4092                                &control_page_default,
4093                                sizeof(control_page_default));
4094                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
4095                                &control_page_default,
4096                                sizeof(control_page_default));
4097                         page_index->page_data =
4098                                 (uint8_t *)lun->mode_pages.control_page;
4099                         break;
4100
4101                 }
4102                 case SMS_VENDOR_SPECIFIC_PAGE:{
4103                         switch (page_index->subpage) {
4104                         case PWR_SUBPAGE_CODE: {
4105                                 struct copan_power_subpage *current_page,
4106                                                            *saved_page;
4107
4108                                 memcpy(&lun->mode_pages.power_subpage[
4109                                        CTL_PAGE_CURRENT],
4110                                        &power_page_default,
4111                                        sizeof(power_page_default));
4112                                 memcpy(&lun->mode_pages.power_subpage[
4113                                        CTL_PAGE_CHANGEABLE],
4114                                        &power_page_changeable,
4115                                        sizeof(power_page_changeable));
4116                                 memcpy(&lun->mode_pages.power_subpage[
4117                                        CTL_PAGE_DEFAULT],
4118                                        &power_page_default,
4119                                        sizeof(power_page_default));
4120                                 memcpy(&lun->mode_pages.power_subpage[
4121                                        CTL_PAGE_SAVED],
4122                                        &power_page_default,
4123                                        sizeof(power_page_default));
4124                                 page_index->page_data =
4125                                     (uint8_t *)lun->mode_pages.power_subpage;
4126
4127                                 current_page = (struct copan_power_subpage *)
4128                                         (page_index->page_data +
4129                                          (page_index->page_len *
4130                                           CTL_PAGE_CURRENT));
4131                                 saved_page = (struct copan_power_subpage *)
4132                                         (page_index->page_data +
4133                                          (page_index->page_len *
4134                                           CTL_PAGE_SAVED));
4135                                 break;
4136                         }
4137                         case APS_SUBPAGE_CODE: {
4138                                 struct copan_aps_subpage *current_page,
4139                                                          *saved_page;
4140
4141                                 // This gets set multiple times but
4142                                 // it should always be the same. It's
4143                                 // only done during init so who cares.
4144                                 index_to_aps_page = i;
4145
4146                                 memcpy(&lun->mode_pages.aps_subpage[
4147                                        CTL_PAGE_CURRENT],
4148                                        &aps_page_default,
4149                                        sizeof(aps_page_default));
4150                                 memcpy(&lun->mode_pages.aps_subpage[
4151                                        CTL_PAGE_CHANGEABLE],
4152                                        &aps_page_changeable,
4153                                        sizeof(aps_page_changeable));
4154                                 memcpy(&lun->mode_pages.aps_subpage[
4155                                        CTL_PAGE_DEFAULT],
4156                                        &aps_page_default,
4157                                        sizeof(aps_page_default));
4158                                 memcpy(&lun->mode_pages.aps_subpage[
4159                                        CTL_PAGE_SAVED],
4160                                        &aps_page_default,
4161                                        sizeof(aps_page_default));
4162                                 page_index->page_data =
4163                                         (uint8_t *)lun->mode_pages.aps_subpage;
4164
4165                                 current_page = (struct copan_aps_subpage *)
4166                                         (page_index->page_data +
4167                                          (page_index->page_len *
4168                                           CTL_PAGE_CURRENT));
4169                                 saved_page = (struct copan_aps_subpage *)
4170                                         (page_index->page_data +
4171                                          (page_index->page_len *
4172                                           CTL_PAGE_SAVED));
4173                                 break;
4174                         }
4175                         case DBGCNF_SUBPAGE_CODE: {
4176                                 struct copan_debugconf_subpage *current_page,
4177                                                                *saved_page;
4178
4179                                 memcpy(&lun->mode_pages.debugconf_subpage[
4180                                        CTL_PAGE_CURRENT],
4181                                        &debugconf_page_default,
4182                                        sizeof(debugconf_page_default));
4183                                 memcpy(&lun->mode_pages.debugconf_subpage[
4184                                        CTL_PAGE_CHANGEABLE],
4185                                        &debugconf_page_changeable,
4186                                        sizeof(debugconf_page_changeable));
4187                                 memcpy(&lun->mode_pages.debugconf_subpage[
4188                                        CTL_PAGE_DEFAULT],
4189                                        &debugconf_page_default,
4190                                        sizeof(debugconf_page_default));
4191                                 memcpy(&lun->mode_pages.debugconf_subpage[
4192                                        CTL_PAGE_SAVED],
4193                                        &debugconf_page_default,
4194                                        sizeof(debugconf_page_default));
4195                                 page_index->page_data =
4196                                         (uint8_t *)lun->mode_pages.debugconf_subpage;
4197
4198                                 current_page = (struct copan_debugconf_subpage *)
4199                                         (page_index->page_data +
4200                                          (page_index->page_len *
4201                                           CTL_PAGE_CURRENT));
4202                                 saved_page = (struct copan_debugconf_subpage *)
4203                                         (page_index->page_data +
4204                                          (page_index->page_len *
4205                                           CTL_PAGE_SAVED));
4206                                 break;
4207                         }
4208                         default:
4209                                 panic("invalid subpage value %d",
4210                                       page_index->subpage);
4211                                 break;
4212                         }
4213                         break;
4214                 }
4215                 default:
4216                         panic("invalid page value %d",
4217                               page_index->page_code & SMPH_PC_MASK);
4218                         break;
4219         }
4220         }
4221
4222         return (CTL_RETVAL_COMPLETE);
4223 }
4224
4225 /*
4226  * LUN allocation.
4227  *
4228  * Requirements:
4229  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4230  *   wants us to allocate the LUN and he can block.
4231  * - ctl_softc is always set
4232  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4233  *
4234  * Returns 0 for success, non-zero (errno) for failure.
4235  */
4236 static int
4237 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4238               struct ctl_be_lun *const be_lun, struct ctl_id target_id)
4239 {
4240         struct ctl_lun *nlun, *lun;
4241         struct ctl_frontend *fe;
4242         int lun_number, i, lun_malloced;
4243
4244         if (be_lun == NULL)
4245                 return (EINVAL);
4246
4247         /*
4248          * We currently only support Direct Access or Processor LUN types.
4249          */
4250         switch (be_lun->lun_type) {
4251         case T_DIRECT:
4252                 break;
4253         case T_PROCESSOR:
4254                 break;
4255         case T_SEQUENTIAL:
4256         case T_CHANGER:
4257         default:
4258                 be_lun->lun_config_status(be_lun->be_lun,
4259                                           CTL_LUN_CONFIG_FAILURE);
4260                 break;
4261         }
4262         if (ctl_lun == NULL) {
4263                 lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4264                 lun_malloced = 1;
4265         } else {
4266                 lun_malloced = 0;
4267                 lun = ctl_lun;
4268         }
4269
4270         memset(lun, 0, sizeof(*lun));
4271         if (lun_malloced)
4272                 lun->flags = CTL_LUN_MALLOCED;
4273
4274         mtx_lock(&ctl_softc->ctl_lock);
4275         /*
4276          * See if the caller requested a particular LUN number.  If so, see
4277          * if it is available.  Otherwise, allocate the first available LUN.
4278          */
4279         if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4280                 if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4281                  || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4282                         mtx_unlock(&ctl_softc->ctl_lock);
4283                         if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4284                                 printf("ctl: requested LUN ID %d is higher "
4285                                        "than CTL_MAX_LUNS - 1 (%d)\n",
4286                                        be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4287                         } else {
4288                                 /*
4289                                  * XXX KDM return an error, or just assign
4290                                  * another LUN ID in this case??
4291                                  */
4292                                 printf("ctl: requested LUN ID %d is already "
4293                                        "in use\n", be_lun->req_lun_id);
4294                         }
4295                         if (lun->flags & CTL_LUN_MALLOCED)
4296                                 free(lun, M_CTL);
4297                         be_lun->lun_config_status(be_lun->be_lun,
4298                                                   CTL_LUN_CONFIG_FAILURE);
4299                         return (ENOSPC);
4300                 }
4301                 lun_number = be_lun->req_lun_id;
4302         } else {
4303                 lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, CTL_MAX_LUNS);
4304                 if (lun_number == -1) {
4305                         mtx_unlock(&ctl_softc->ctl_lock);
4306                         printf("ctl: can't allocate LUN on target %ju, out of "
4307                                "LUNs\n", (uintmax_t)target_id.id);
4308                         if (lun->flags & CTL_LUN_MALLOCED)
4309                                 free(lun, M_CTL);
4310                         be_lun->lun_config_status(be_lun->be_lun,
4311                                                   CTL_LUN_CONFIG_FAILURE);
4312                         return (ENOSPC);
4313                 }
4314         }
4315         ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4316
4317         lun->target = target_id;
4318         lun->lun = lun_number;
4319         lun->be_lun = be_lun;
4320         /*
4321          * The processor LUN is always enabled.  Disk LUNs come on line
4322          * disabled, and must be enabled by the backend.
4323          */
4324         lun->flags |= CTL_LUN_DISABLED;
4325         lun->backend = be_lun->be;
4326         be_lun->ctl_lun = lun;
4327         be_lun->lun_id = lun_number;
4328         atomic_add_int(&be_lun->be->num_luns, 1);
4329         if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4330                 lun->flags |= CTL_LUN_STOPPED;
4331
4332         if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4333                 lun->flags |= CTL_LUN_INOPERABLE;
4334
4335         if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4336                 lun->flags |= CTL_LUN_PRIMARY_SC;
4337
4338         lun->ctl_softc = ctl_softc;
4339         TAILQ_INIT(&lun->ooa_queue);
4340         TAILQ_INIT(&lun->blocked_queue);
4341         STAILQ_INIT(&lun->error_list);
4342
4343         /*
4344          * Initialize the mode page index.
4345          */
4346         ctl_init_page_index(lun);
4347
4348         /*
4349          * Set the poweron UA for all initiators on this LUN only.
4350          */
4351         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4352                 lun->pending_sense[i].ua_pending = CTL_UA_POWERON;
4353
4354         /*
4355          * Now, before we insert this lun on the lun list, set the lun
4356          * inventory changed UA for all other luns.
4357          */
4358         STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4359                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4360                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4361                 }
4362         }
4363
4364         STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4365
4366         ctl_softc->ctl_luns[lun_number] = lun;
4367
4368         ctl_softc->num_luns++;
4369
4370         /* Setup statistics gathering */
4371         lun->stats.device_type = be_lun->lun_type;
4372         lun->stats.lun_number = lun_number;
4373         if (lun->stats.device_type == T_DIRECT)
4374                 lun->stats.blocksize = be_lun->blocksize;
4375         else
4376                 lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4377         for (i = 0;i < CTL_MAX_PORTS;i++)
4378                 lun->stats.ports[i].targ_port = i;
4379
4380         mtx_unlock(&ctl_softc->ctl_lock);
4381
4382         lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4383
4384         /*
4385          * Run through each registered FETD and bring it online if it isn't
4386          * already.  Enable the target ID if it hasn't been enabled, and
4387          * enable this particular LUN.
4388          */
4389         STAILQ_FOREACH(fe, &ctl_softc->fe_list, links) {
4390                 int retval;
4391
4392                 /*
4393                  * XXX KDM this only works for ONE TARGET ID.  We'll need
4394                  * to do things differently if we go to a multiple target
4395                  * ID scheme.
4396                  */
4397                 if ((fe->status & CTL_PORT_STATUS_TARG_ONLINE) == 0) {
4398
4399                         retval = fe->targ_enable(fe->targ_lun_arg, target_id);
4400                         if (retval != 0) {
4401                                 printf("ctl_alloc_lun: FETD %s port %d "
4402                                        "returned error %d for targ_enable on "
4403                                        "target %ju\n", fe->port_name,
4404                                        fe->targ_port, retval,
4405                                        (uintmax_t)target_id.id);
4406                         } else
4407                                 fe->status |= CTL_PORT_STATUS_TARG_ONLINE;
4408                 }
4409
4410                 retval = fe->lun_enable(fe->targ_lun_arg, target_id,lun_number);
4411                 if (retval != 0) {
4412                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4413                                "%d for lun_enable on target %ju lun %d\n",
4414                                fe->port_name, fe->targ_port, retval,
4415                                (uintmax_t)target_id.id, lun_number);
4416                 } else
4417                         fe->status |= CTL_PORT_STATUS_LUN_ONLINE;
4418         }
4419         return (0);
4420 }
4421
4422 /*
4423  * Delete a LUN.
4424  * Assumptions:
4425  * - LUN has already been marked invalid and any pending I/O has been taken
4426  *   care of.
4427  */
4428 static int
4429 ctl_free_lun(struct ctl_lun *lun)
4430 {
4431         struct ctl_softc *softc;
4432 #if 0
4433         struct ctl_frontend *fe;
4434 #endif
4435         struct ctl_lun *nlun;
4436         union ctl_io *io, *next_io;
4437         int i;
4438
4439         softc = lun->ctl_softc;
4440
4441         mtx_assert(&softc->ctl_lock, MA_OWNED);
4442
4443         STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4444
4445         ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4446
4447         softc->ctl_luns[lun->lun] = NULL;
4448
4449         if (TAILQ_FIRST(&lun->ooa_queue) != NULL) {
4450                 printf("ctl_free_lun: aieee!! freeing a LUN with "
4451                        "outstanding I/O!!\n");
4452         }
4453
4454         /*
4455          * If we have anything pending on the RtR queue, remove it.
4456          */
4457         for (io = (union ctl_io *)STAILQ_FIRST(&softc->rtr_queue); io != NULL;
4458              io = next_io) {
4459                 uint32_t targ_lun;
4460
4461                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
4462                 targ_lun = io->io_hdr.nexus.targ_lun;
4463                 if (io->io_hdr.nexus.lun_map_fn != NULL)
4464                         targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
4465                 if ((io->io_hdr.nexus.targ_target.id == lun->target.id)
4466                  && (targ_lun == lun->lun))
4467                         STAILQ_REMOVE(&softc->rtr_queue, &io->io_hdr,
4468                                       ctl_io_hdr, links);
4469         }
4470
4471         /*
4472          * Then remove everything from the blocked queue.
4473          */
4474         for (io = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue); io != NULL;
4475              io = next_io) {
4476                 next_io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,blocked_links);
4477                 TAILQ_REMOVE(&lun->blocked_queue, &io->io_hdr, blocked_links);
4478                 io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
4479         }
4480
4481         /*
4482          * Now clear out the OOA queue, and free all the I/O.
4483          * XXX KDM should we notify the FETD here?  We probably need to
4484          * quiesce the LUN before deleting it.
4485          */
4486         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); io != NULL;
4487              io = next_io) {
4488                 next_io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr, ooa_links);
4489                 TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
4490                 ctl_free_io(io);
4491         }
4492
4493         softc->num_luns--;
4494
4495         /*
4496          * XXX KDM this scheme only works for a single target/multiple LUN
4497          * setup.  It needs to be revamped for a multiple target scheme.
4498          *
4499          * XXX KDM this results in fe->lun_disable() getting called twice,
4500          * once when ctl_disable_lun() is called, and a second time here.
4501          * We really need to re-think the LUN disable semantics.  There
4502          * should probably be several steps/levels to LUN removal:
4503          *  - disable
4504          *  - invalidate
4505          *  - free
4506          *
4507          * Right now we only have a disable method when communicating to
4508          * the front end ports, at least for individual LUNs.
4509          */
4510 #if 0
4511         STAILQ_FOREACH(fe, &softc->fe_list, links) {
4512                 int retval;
4513
4514                 retval = fe->lun_disable(fe->targ_lun_arg, lun->target,
4515                                          lun->lun);
4516                 if (retval != 0) {
4517                         printf("ctl_free_lun: FETD %s port %d returned error "
4518                                "%d for lun_disable on target %ju lun %jd\n",
4519                                fe->port_name, fe->targ_port, retval,
4520                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4521                 }
4522
4523                 if (STAILQ_FIRST(&softc->lun_list) == NULL) {
4524                         fe->status &= ~CTL_PORT_STATUS_LUN_ONLINE;
4525
4526                         retval = fe->targ_disable(fe->targ_lun_arg,lun->target);
4527                         if (retval != 0) {
4528                                 printf("ctl_free_lun: FETD %s port %d "
4529                                        "returned error %d for targ_disable on "
4530                                        "target %ju\n", fe->port_name,
4531                                        fe->targ_port, retval,
4532                                        (uintmax_t)lun->target.id);
4533                         } else
4534                                 fe->status &= ~CTL_PORT_STATUS_TARG_ONLINE;
4535
4536                         if ((fe->status & CTL_PORT_STATUS_TARG_ONLINE) != 0)
4537                                 continue;
4538
4539 #if 0
4540                         fe->port_offline(fe->onoff_arg);
4541                         fe->status &= ~CTL_PORT_STATUS_ONLINE;
4542 #endif
4543                 }
4544         }
4545 #endif
4546
4547         /*
4548          * Tell the backend to free resources, if this LUN has a backend.
4549          */
4550         atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4551         lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4552
4553         if (lun->flags & CTL_LUN_MALLOCED)
4554                 free(lun, M_CTL);
4555
4556         STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4557                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4558                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4559                 }
4560         }
4561
4562         return (0);
4563 }
4564
4565 static void
4566 ctl_create_lun(struct ctl_be_lun *be_lun)
4567 {
4568         struct ctl_softc *ctl_softc;
4569
4570         ctl_softc = control_softc;
4571
4572         /*
4573          * ctl_alloc_lun() should handle all potential failure cases.
4574          */
4575         ctl_alloc_lun(ctl_softc, NULL, be_lun, ctl_softc->target);
4576 }
4577
4578 int
4579 ctl_add_lun(struct ctl_be_lun *be_lun)
4580 {
4581         struct ctl_softc *ctl_softc;
4582
4583         ctl_softc = control_softc;
4584
4585         mtx_lock(&ctl_softc->ctl_lock);
4586         STAILQ_INSERT_TAIL(&ctl_softc->pending_lun_queue, be_lun, links);
4587         mtx_unlock(&ctl_softc->ctl_lock);
4588
4589         ctl_wakeup_thread();
4590
4591         return (0);
4592 }
4593
4594 int
4595 ctl_enable_lun(struct ctl_be_lun *be_lun)
4596 {
4597         struct ctl_softc *ctl_softc;
4598         struct ctl_frontend *fe, *nfe;
4599         struct ctl_lun *lun;
4600         int retval;
4601
4602         ctl_softc = control_softc;
4603
4604         lun = (struct ctl_lun *)be_lun->ctl_lun;
4605
4606         mtx_lock(&ctl_softc->ctl_lock);
4607         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4608                 /*
4609                  * eh?  Why did we get called if the LUN is already
4610                  * enabled?
4611                  */
4612                 mtx_unlock(&ctl_softc->ctl_lock);
4613                 return (0);
4614         }
4615         lun->flags &= ~CTL_LUN_DISABLED;
4616
4617         for (fe = STAILQ_FIRST(&ctl_softc->fe_list); fe != NULL; fe = nfe) {
4618                 nfe = STAILQ_NEXT(fe, links);
4619
4620                 /*
4621                  * Drop the lock while we call the FETD's enable routine.
4622                  * This can lead to a callback into CTL (at least in the
4623                  * case of the internal initiator frontend.
4624                  */
4625                 mtx_unlock(&ctl_softc->ctl_lock);
4626                 retval = fe->lun_enable(fe->targ_lun_arg, lun->target,lun->lun);
4627                 mtx_lock(&ctl_softc->ctl_lock);
4628                 if (retval != 0) {
4629                         printf("%s: FETD %s port %d returned error "
4630                                "%d for lun_enable on target %ju lun %jd\n",
4631                                __func__, fe->port_name, fe->targ_port, retval,
4632                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4633                 }
4634 #if 0
4635                  else {
4636             /* NOTE:  TODO:  why does lun enable affect port status? */
4637                         fe->status |= CTL_PORT_STATUS_LUN_ONLINE;
4638                 }
4639 #endif
4640         }
4641
4642         mtx_unlock(&ctl_softc->ctl_lock);
4643
4644         return (0);
4645 }
4646
4647 int
4648 ctl_disable_lun(struct ctl_be_lun *be_lun)
4649 {
4650         struct ctl_softc *ctl_softc;
4651         struct ctl_frontend *fe;
4652         struct ctl_lun *lun;
4653         int retval;
4654
4655         ctl_softc = control_softc;
4656
4657         lun = (struct ctl_lun *)be_lun->ctl_lun;
4658
4659         mtx_lock(&ctl_softc->ctl_lock);
4660
4661         if (lun->flags & CTL_LUN_DISABLED) {
4662                 mtx_unlock(&ctl_softc->ctl_lock);
4663                 return (0);
4664         }
4665         lun->flags |= CTL_LUN_DISABLED;
4666
4667         STAILQ_FOREACH(fe, &ctl_softc->fe_list, links) {
4668                 mtx_unlock(&ctl_softc->ctl_lock);
4669                 /*
4670                  * Drop the lock before we call the frontend's disable
4671                  * routine, to avoid lock order reversals.
4672                  *
4673                  * XXX KDM what happens if the frontend list changes while
4674                  * we're traversing it?  It's unlikely, but should be handled.
4675                  */
4676                 retval = fe->lun_disable(fe->targ_lun_arg, lun->target,
4677                                          lun->lun);
4678                 mtx_lock(&ctl_softc->ctl_lock);
4679                 if (retval != 0) {
4680                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4681                                "%d for lun_disable on target %ju lun %jd\n",
4682                                fe->port_name, fe->targ_port, retval,
4683                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4684                 }
4685         }
4686
4687         mtx_unlock(&ctl_softc->ctl_lock);
4688
4689         return (0);
4690 }
4691
4692 int
4693 ctl_start_lun(struct ctl_be_lun *be_lun)
4694 {
4695         struct ctl_softc *ctl_softc;
4696         struct ctl_lun *lun;
4697
4698         ctl_softc = control_softc;
4699
4700         lun = (struct ctl_lun *)be_lun->ctl_lun;
4701
4702         mtx_lock(&ctl_softc->ctl_lock);
4703         lun->flags &= ~CTL_LUN_STOPPED;
4704         mtx_unlock(&ctl_softc->ctl_lock);
4705
4706         return (0);
4707 }
4708
4709 int
4710 ctl_stop_lun(struct ctl_be_lun *be_lun)
4711 {
4712         struct ctl_softc *ctl_softc;
4713         struct ctl_lun *lun;
4714
4715         ctl_softc = control_softc;
4716
4717         lun = (struct ctl_lun *)be_lun->ctl_lun;
4718
4719         mtx_lock(&ctl_softc->ctl_lock);
4720         lun->flags |= CTL_LUN_STOPPED;
4721         mtx_unlock(&ctl_softc->ctl_lock);
4722
4723         return (0);
4724 }
4725
4726 int
4727 ctl_lun_offline(struct ctl_be_lun *be_lun)
4728 {
4729         struct ctl_softc *ctl_softc;
4730         struct ctl_lun *lun;
4731
4732         ctl_softc = control_softc;
4733
4734         lun = (struct ctl_lun *)be_lun->ctl_lun;
4735
4736         mtx_lock(&ctl_softc->ctl_lock);
4737         lun->flags |= CTL_LUN_OFFLINE;
4738         mtx_unlock(&ctl_softc->ctl_lock);
4739
4740         return (0);
4741 }
4742
4743 int
4744 ctl_lun_online(struct ctl_be_lun *be_lun)
4745 {
4746         struct ctl_softc *ctl_softc;
4747         struct ctl_lun *lun;
4748
4749         ctl_softc = control_softc;
4750
4751         lun = (struct ctl_lun *)be_lun->ctl_lun;
4752
4753         mtx_lock(&ctl_softc->ctl_lock);
4754         lun->flags &= ~CTL_LUN_OFFLINE;
4755         mtx_unlock(&ctl_softc->ctl_lock);
4756
4757         return (0);
4758 }
4759
4760 int
4761 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4762 {
4763         struct ctl_softc *ctl_softc;
4764         struct ctl_lun *lun;
4765
4766         ctl_softc = control_softc;
4767
4768         lun = (struct ctl_lun *)be_lun->ctl_lun;
4769
4770         mtx_lock(&ctl_softc->ctl_lock);
4771
4772         /*
4773          * The LUN needs to be disabled before it can be marked invalid.
4774          */
4775         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4776                 mtx_unlock(&ctl_softc->ctl_lock);
4777                 return (-1);
4778         }
4779         /*
4780          * Mark the LUN invalid.
4781          */
4782         lun->flags |= CTL_LUN_INVALID;
4783
4784         /*
4785          * If there is nothing in the OOA queue, go ahead and free the LUN.
4786          * If we have something in the OOA queue, we'll free it when the
4787          * last I/O completes.
4788          */
4789         if (TAILQ_FIRST(&lun->ooa_queue) == NULL)
4790                 ctl_free_lun(lun);
4791         mtx_unlock(&ctl_softc->ctl_lock);
4792
4793         return (0);
4794 }
4795
4796 int
4797 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
4798 {
4799         struct ctl_softc *ctl_softc;
4800         struct ctl_lun *lun;
4801
4802         ctl_softc = control_softc;
4803         lun = (struct ctl_lun *)be_lun->ctl_lun;
4804
4805         mtx_lock(&ctl_softc->ctl_lock);
4806         lun->flags |= CTL_LUN_INOPERABLE;
4807         mtx_unlock(&ctl_softc->ctl_lock);
4808
4809         return (0);
4810 }
4811
4812 int
4813 ctl_lun_operable(struct ctl_be_lun *be_lun)
4814 {
4815         struct ctl_softc *ctl_softc;
4816         struct ctl_lun *lun;
4817
4818         ctl_softc = control_softc;
4819         lun = (struct ctl_lun *)be_lun->ctl_lun;
4820
4821         mtx_lock(&ctl_softc->ctl_lock);
4822         lun->flags &= ~CTL_LUN_INOPERABLE;
4823         mtx_unlock(&ctl_softc->ctl_lock);
4824
4825         return (0);
4826 }
4827
4828 int
4829 ctl_lun_power_lock(struct ctl_be_lun *be_lun, struct ctl_nexus *nexus,
4830                    int lock)
4831 {
4832         struct ctl_softc *softc;
4833         struct ctl_lun *lun;
4834         struct copan_aps_subpage *current_sp;
4835         struct ctl_page_index *page_index;
4836         int i;
4837
4838         softc = control_softc;
4839
4840         mtx_lock(&softc->ctl_lock);
4841
4842         lun = (struct ctl_lun *)be_lun->ctl_lun;
4843
4844         page_index = NULL;
4845         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4846                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
4847                      APS_PAGE_CODE)
4848                         continue;
4849
4850                 if (lun->mode_pages.index[i].subpage != APS_SUBPAGE_CODE)
4851                         continue;
4852                 page_index = &lun->mode_pages.index[i];
4853         }
4854
4855         if (page_index == NULL) {
4856                 mtx_unlock(&softc->ctl_lock);
4857                 printf("%s: APS subpage not found for lun %ju!\n", __func__,
4858                        (uintmax_t)lun->lun);
4859                 return (1);
4860         }
4861 #if 0
4862         if ((softc->aps_locked_lun != 0)
4863          && (softc->aps_locked_lun != lun->lun)) {
4864                 printf("%s: attempt to lock LUN %llu when %llu is already "
4865                        "locked\n");
4866                 mtx_unlock(&softc->ctl_lock);
4867                 return (1);
4868         }
4869 #endif
4870
4871         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
4872                 (page_index->page_len * CTL_PAGE_CURRENT));
4873
4874         if (lock != 0) {
4875                 current_sp->lock_active = APS_LOCK_ACTIVE;
4876                 softc->aps_locked_lun = lun->lun;
4877         } else {
4878                 current_sp->lock_active = 0;
4879                 softc->aps_locked_lun = 0;
4880         }
4881
4882
4883         /*
4884          * If we're in HA mode, try to send the lock message to the other
4885          * side.
4886          */
4887         if (ctl_is_single == 0) {
4888                 int isc_retval;
4889                 union ctl_ha_msg lock_msg;
4890
4891                 lock_msg.hdr.nexus = *nexus;
4892                 lock_msg.hdr.msg_type = CTL_MSG_APS_LOCK;
4893                 if (lock != 0)
4894                         lock_msg.aps.lock_flag = 1;
4895                 else
4896                         lock_msg.aps.lock_flag = 0;
4897                 isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &lock_msg,
4898                                          sizeof(lock_msg), 0);
4899                 if (isc_retval > CTL_HA_STATUS_SUCCESS) {
4900                         printf("%s: APS (lock=%d) error returned from "
4901                                "ctl_ha_msg_send: %d\n", __func__, lock, isc_retval);
4902                         mtx_unlock(&softc->ctl_lock);
4903                         return (1);
4904                 }
4905         }
4906
4907         mtx_unlock(&softc->ctl_lock);
4908
4909         return (0);
4910 }
4911
4912 void
4913 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
4914 {
4915         struct ctl_lun *lun;
4916         struct ctl_softc *softc;
4917         int i;
4918
4919         softc = control_softc;
4920
4921         mtx_lock(&softc->ctl_lock);
4922
4923         lun = (struct ctl_lun *)be_lun->ctl_lun;
4924
4925         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4926                 lun->pending_sense[i].ua_pending |= CTL_UA_CAPACITY_CHANGED;
4927
4928         mtx_unlock(&softc->ctl_lock);
4929 }
4930
4931 /*
4932  * Backend "memory move is complete" callback for requests that never
4933  * make it down to say RAIDCore's configuration code.
4934  */
4935 int
4936 ctl_config_move_done(union ctl_io *io)
4937 {
4938         int retval;
4939
4940         retval = CTL_RETVAL_COMPLETE;
4941
4942
4943         CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
4944         /*
4945          * XXX KDM this shouldn't happen, but what if it does?
4946          */
4947         if (io->io_hdr.io_type != CTL_IO_SCSI)
4948                 panic("I/O type isn't CTL_IO_SCSI!");
4949
4950         if ((io->io_hdr.port_status == 0)
4951          && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
4952          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
4953                 io->io_hdr.status = CTL_SUCCESS;
4954         else if ((io->io_hdr.port_status != 0)
4955               && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
4956               && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)){
4957                 /*
4958                  * For hardware error sense keys, the sense key
4959                  * specific value is defined to be a retry count,
4960                  * but we use it to pass back an internal FETD
4961                  * error code.  XXX KDM  Hopefully the FETD is only
4962                  * using 16 bits for an error code, since that's
4963                  * all the space we have in the sks field.
4964                  */
4965                 ctl_set_internal_failure(&io->scsiio,
4966                                          /*sks_valid*/ 1,
4967                                          /*retry_count*/
4968                                          io->io_hdr.port_status);
4969                 free(io->scsiio.kern_data_ptr, M_CTL);
4970                 ctl_done(io);
4971                 goto bailout;
4972         }
4973
4974         if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
4975          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
4976          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
4977                 /*
4978                  * XXX KDM just assuming a single pointer here, and not a
4979                  * S/G list.  If we start using S/G lists for config data,
4980                  * we'll need to know how to clean them up here as well.
4981                  */
4982                 free(io->scsiio.kern_data_ptr, M_CTL);
4983                 /* Hopefully the user has already set the status... */
4984                 ctl_done(io);
4985         } else {
4986                 /*
4987                  * XXX KDM now we need to continue data movement.  Some
4988                  * options:
4989                  * - call ctl_scsiio() again?  We don't do this for data
4990                  *   writes, because for those at least we know ahead of
4991                  *   time where the write will go and how long it is.  For
4992                  *   config writes, though, that information is largely
4993                  *   contained within the write itself, thus we need to
4994                  *   parse out the data again.
4995                  *
4996                  * - Call some other function once the data is in?
4997                  */
4998
4999                 /*
5000                  * XXX KDM call ctl_scsiio() again for now, and check flag
5001                  * bits to see whether we're allocated or not.
5002                  */
5003                 retval = ctl_scsiio(&io->scsiio);
5004         }
5005 bailout:
5006         return (retval);
5007 }
5008
5009 /*
5010  * This gets called by a backend driver when it is done with a
5011  * configuration write.
5012  */
5013 void
5014 ctl_config_write_done(union ctl_io *io)
5015 {
5016         /*
5017          * If the IO_CONT flag is set, we need to call the supplied
5018          * function to continue processing the I/O, instead of completing
5019          * the I/O just yet.
5020          *
5021          * If there is an error, though, we don't want to keep processing.
5022          * Instead, just send status back to the initiator.
5023          */
5024         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT)
5025          && (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)
5026           || ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))) {
5027                 io->scsiio.io_cont(io);
5028                 return;
5029         }
5030         /*
5031          * Since a configuration write can be done for commands that actually
5032          * have data allocated, like write buffer, and commands that have
5033          * no data, like start/stop unit, we need to check here.
5034          */
5035         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
5036                 free(io->scsiio.kern_data_ptr, M_CTL);
5037         ctl_done(io);
5038 }
5039
5040 /*
5041  * SCSI release command.
5042  */
5043 int
5044 ctl_scsi_release(struct ctl_scsiio *ctsio)
5045 {
5046         int length, longid, thirdparty_id, resv_id;
5047         struct ctl_softc *ctl_softc;
5048         struct ctl_lun *lun;
5049
5050         length = 0;
5051         resv_id = 0;
5052
5053         CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
5054
5055         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5056         ctl_softc = control_softc;
5057
5058         switch (ctsio->cdb[0]) {
5059         case RELEASE: {
5060                 struct scsi_release *cdb;
5061
5062                 cdb = (struct scsi_release *)ctsio->cdb;
5063                 if ((cdb->byte2 & 0x1f) != 0) {
5064                         ctl_set_invalid_field(ctsio,
5065                                               /*sks_valid*/ 1,
5066                                               /*command*/ 1,
5067                                               /*field*/ 1,
5068                                               /*bit_valid*/ 0,
5069                                               /*bit*/ 0);
5070                         ctl_done((union ctl_io *)ctsio);
5071                         return (CTL_RETVAL_COMPLETE);
5072                 }
5073                 break;
5074         }
5075         case RELEASE_10: {
5076                 struct scsi_release_10 *cdb;
5077
5078                 cdb = (struct scsi_release_10 *)ctsio->cdb;
5079
5080                 if ((cdb->byte2 & SR10_EXTENT) != 0) {
5081                         ctl_set_invalid_field(ctsio,
5082                                               /*sks_valid*/ 1,
5083                                               /*command*/ 1,
5084                                               /*field*/ 1,
5085                                               /*bit_valid*/ 1,
5086                                               /*bit*/ 0);
5087                         ctl_done((union ctl_io *)ctsio);
5088                         return (CTL_RETVAL_COMPLETE);
5089
5090                 }
5091
5092                 if ((cdb->byte2 & SR10_3RDPTY) != 0) {
5093                         ctl_set_invalid_field(ctsio,
5094                                               /*sks_valid*/ 1,
5095                                               /*command*/ 1,
5096                                               /*field*/ 1,
5097                                               /*bit_valid*/ 1,
5098                                               /*bit*/ 4);
5099                         ctl_done((union ctl_io *)ctsio);
5100                         return (CTL_RETVAL_COMPLETE);
5101                 }
5102
5103                 if (cdb->byte2 & SR10_LONGID)
5104                         longid = 1;
5105                 else
5106                         thirdparty_id = cdb->thirdparty_id;
5107
5108                 resv_id = cdb->resv_id;
5109                 length = scsi_2btoul(cdb->length);
5110                 break;
5111         }
5112         }
5113
5114
5115         /*
5116          * XXX KDM right now, we only support LUN reservation.  We don't
5117          * support 3rd party reservations, or extent reservations, which
5118          * might actually need the parameter list.  If we've gotten this
5119          * far, we've got a LUN reservation.  Anything else got kicked out
5120          * above.  So, according to SPC, ignore the length.
5121          */
5122         length = 0;
5123
5124         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5125          && (length > 0)) {
5126                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5127                 ctsio->kern_data_len = length;
5128                 ctsio->kern_total_len = length;
5129                 ctsio->kern_data_resid = 0;
5130                 ctsio->kern_rel_offset = 0;
5131                 ctsio->kern_sg_entries = 0;
5132                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5133                 ctsio->be_move_done = ctl_config_move_done;
5134                 ctl_datamove((union ctl_io *)ctsio);
5135
5136                 return (CTL_RETVAL_COMPLETE);
5137         }
5138
5139         if (length > 0)
5140                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5141
5142         mtx_lock(&ctl_softc->ctl_lock);
5143
5144         /*
5145          * According to SPC, it is not an error for an intiator to attempt
5146          * to release a reservation on a LUN that isn't reserved, or that
5147          * is reserved by another initiator.  The reservation can only be
5148          * released, though, by the initiator who made it or by one of
5149          * several reset type events.
5150          */
5151         if (lun->flags & CTL_LUN_RESERVED) {
5152                 if ((ctsio->io_hdr.nexus.initid.id == lun->rsv_nexus.initid.id)
5153                  && (ctsio->io_hdr.nexus.targ_port == lun->rsv_nexus.targ_port)
5154                  && (ctsio->io_hdr.nexus.targ_target.id ==
5155                      lun->rsv_nexus.targ_target.id)) {
5156                         lun->flags &= ~CTL_LUN_RESERVED;
5157                 }
5158         }
5159
5160         ctsio->scsi_status = SCSI_STATUS_OK;
5161         ctsio->io_hdr.status = CTL_SUCCESS;
5162
5163         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5164                 free(ctsio->kern_data_ptr, M_CTL);
5165                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5166         }
5167
5168         mtx_unlock(&ctl_softc->ctl_lock);
5169
5170         ctl_done((union ctl_io *)ctsio);
5171         return (CTL_RETVAL_COMPLETE);
5172 }
5173
5174 int
5175 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5176 {
5177         int extent, thirdparty, longid;
5178         int resv_id, length;
5179         uint64_t thirdparty_id;
5180         struct ctl_softc *ctl_softc;
5181         struct ctl_lun *lun;
5182
5183         extent = 0;
5184         thirdparty = 0;
5185         longid = 0;
5186         resv_id = 0;
5187         length = 0;
5188         thirdparty_id = 0;
5189
5190         CTL_DEBUG_PRINT(("ctl_reserve\n"));
5191
5192         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5193         ctl_softc = control_softc;
5194
5195         switch (ctsio->cdb[0]) {
5196         case RESERVE: {
5197                 struct scsi_reserve *cdb;
5198
5199                 cdb = (struct scsi_reserve *)ctsio->cdb;
5200                 if ((cdb->byte2 & 0x1f) != 0) {
5201                         ctl_set_invalid_field(ctsio,
5202                                               /*sks_valid*/ 1,
5203                                               /*command*/ 1,
5204                                               /*field*/ 1,
5205                                               /*bit_valid*/ 0,
5206                                               /*bit*/ 0);
5207                         ctl_done((union ctl_io *)ctsio);
5208                         return (CTL_RETVAL_COMPLETE);
5209                 }
5210                 resv_id = cdb->resv_id;
5211                 length = scsi_2btoul(cdb->length);
5212                 break;
5213         }
5214         case RESERVE_10: {
5215                 struct scsi_reserve_10 *cdb;
5216
5217                 cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5218
5219                 if ((cdb->byte2 & SR10_EXTENT) != 0) {
5220                         ctl_set_invalid_field(ctsio,
5221                                               /*sks_valid*/ 1,
5222                                               /*command*/ 1,
5223                                               /*field*/ 1,
5224                                               /*bit_valid*/ 1,
5225                                               /*bit*/ 0);
5226                         ctl_done((union ctl_io *)ctsio);
5227                         return (CTL_RETVAL_COMPLETE);
5228                 }
5229                 if ((cdb->byte2 & SR10_3RDPTY) != 0) {
5230                         ctl_set_invalid_field(ctsio,
5231                                               /*sks_valid*/ 1,
5232                                               /*command*/ 1,
5233                                               /*field*/ 1,
5234                                               /*bit_valid*/ 1,
5235                                               /*bit*/ 4);
5236                         ctl_done((union ctl_io *)ctsio);
5237                         return (CTL_RETVAL_COMPLETE);
5238                 }
5239                 if (cdb->byte2 & SR10_LONGID)
5240                         longid = 1;
5241                 else
5242                         thirdparty_id = cdb->thirdparty_id;
5243
5244                 resv_id = cdb->resv_id;
5245                 length = scsi_2btoul(cdb->length);
5246                 break;
5247         }
5248         }
5249
5250         /*
5251          * XXX KDM right now, we only support LUN reservation.  We don't
5252          * support 3rd party reservations, or extent reservations, which
5253          * might actually need the parameter list.  If we've gotten this
5254          * far, we've got a LUN reservation.  Anything else got kicked out
5255          * above.  So, according to SPC, ignore the length.
5256          */
5257         length = 0;
5258
5259         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5260          && (length > 0)) {
5261                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5262                 ctsio->kern_data_len = length;
5263                 ctsio->kern_total_len = length;
5264                 ctsio->kern_data_resid = 0;
5265                 ctsio->kern_rel_offset = 0;
5266                 ctsio->kern_sg_entries = 0;
5267                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5268                 ctsio->be_move_done = ctl_config_move_done;
5269                 ctl_datamove((union ctl_io *)ctsio);
5270
5271                 return (CTL_RETVAL_COMPLETE);
5272         }
5273
5274         if (length > 0)
5275                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5276
5277         mtx_lock(&ctl_softc->ctl_lock);
5278         if (lun->flags & CTL_LUN_RESERVED) {
5279                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
5280                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
5281                  || (ctsio->io_hdr.nexus.targ_target.id !=
5282                      lun->rsv_nexus.targ_target.id)) {
5283                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
5284                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
5285                         goto bailout;
5286                 }
5287         }
5288
5289         lun->flags |= CTL_LUN_RESERVED;
5290         lun->rsv_nexus = ctsio->io_hdr.nexus;
5291
5292         ctsio->scsi_status = SCSI_STATUS_OK;
5293         ctsio->io_hdr.status = CTL_SUCCESS;
5294
5295 bailout:
5296         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5297                 free(ctsio->kern_data_ptr, M_CTL);
5298                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5299         }
5300
5301         mtx_unlock(&ctl_softc->ctl_lock);
5302
5303         ctl_done((union ctl_io *)ctsio);
5304         return (CTL_RETVAL_COMPLETE);
5305 }
5306
5307 int
5308 ctl_start_stop(struct ctl_scsiio *ctsio)
5309 {
5310         struct scsi_start_stop_unit *cdb;
5311         struct ctl_lun *lun;
5312         struct ctl_softc *ctl_softc;
5313         int retval;
5314
5315         CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5316
5317         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5318         ctl_softc = control_softc;
5319         retval = 0;
5320
5321         cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5322
5323         /*
5324          * XXX KDM
5325          * We don't support the immediate bit on a stop unit.  In order to
5326          * do that, we would need to code up a way to know that a stop is
5327          * pending, and hold off any new commands until it completes, one
5328          * way or another.  Then we could accept or reject those commands
5329          * depending on its status.  We would almost need to do the reverse
5330          * of what we do below for an immediate start -- return the copy of
5331          * the ctl_io to the FETD with status to send to the host (and to
5332          * free the copy!) and then free the original I/O once the stop
5333          * actually completes.  That way, the OOA queue mechanism can work
5334          * to block commands that shouldn't proceed.  Another alternative
5335          * would be to put the copy in the queue in place of the original,
5336          * and return the original back to the caller.  That could be
5337          * slightly safer..
5338          */
5339         if ((cdb->byte2 & SSS_IMMED)
5340          && ((cdb->how & SSS_START) == 0)) {
5341                 ctl_set_invalid_field(ctsio,
5342                                       /*sks_valid*/ 1,
5343                                       /*command*/ 1,
5344                                       /*field*/ 1,
5345                                       /*bit_valid*/ 1,
5346                                       /*bit*/ 0);
5347                 ctl_done((union ctl_io *)ctsio);
5348                 return (CTL_RETVAL_COMPLETE);
5349         }
5350
5351         /*
5352          * We don't support the power conditions field.  We need to check
5353          * this prior to checking the load/eject and start/stop bits.
5354          */
5355         if ((cdb->how & SSS_PC_MASK) != SSS_PC_START_VALID) {
5356                 ctl_set_invalid_field(ctsio,
5357                                       /*sks_valid*/ 1,
5358                                       /*command*/ 1,
5359                                       /*field*/ 4,
5360                                       /*bit_valid*/ 1,
5361                                       /*bit*/ 4);
5362                 ctl_done((union ctl_io *)ctsio);
5363                 return (CTL_RETVAL_COMPLETE);
5364         }
5365
5366         /*
5367          * Media isn't removable, so we can't load or eject it.
5368          */
5369         if ((cdb->how & SSS_LOEJ) != 0) {
5370                 ctl_set_invalid_field(ctsio,
5371                                       /*sks_valid*/ 1,
5372                                       /*command*/ 1,
5373                                       /*field*/ 4,
5374                                       /*bit_valid*/ 1,
5375                                       /*bit*/ 1);
5376                 ctl_done((union ctl_io *)ctsio);
5377                 return (CTL_RETVAL_COMPLETE);
5378         }
5379
5380         if ((lun->flags & CTL_LUN_PR_RESERVED)
5381          && ((cdb->how & SSS_START)==0)) {
5382                 uint32_t residx;
5383
5384                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5385                 if (!lun->per_res[residx].registered
5386                  || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5387
5388                         ctl_set_reservation_conflict(ctsio);
5389                         ctl_done((union ctl_io *)ctsio);
5390                         return (CTL_RETVAL_COMPLETE);
5391                 }
5392         }
5393
5394         /*
5395          * If there is no backend on this device, we can't start or stop
5396          * it.  In theory we shouldn't get any start/stop commands in the
5397          * first place at this level if the LUN doesn't have a backend.
5398          * That should get stopped by the command decode code.
5399          */
5400         if (lun->backend == NULL) {
5401                 ctl_set_invalid_opcode(ctsio);
5402                 ctl_done((union ctl_io *)ctsio);
5403                 return (CTL_RETVAL_COMPLETE);
5404         }
5405
5406         /*
5407          * XXX KDM Copan-specific offline behavior.
5408          * Figure out a reasonable way to port this?
5409          */
5410 #ifdef NEEDTOPORT
5411         mtx_lock(&ctl_softc->ctl_lock);
5412
5413         if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5414          && (lun->flags & CTL_LUN_OFFLINE)) {
5415                 /*
5416                  * If the LUN is offline, and the on/offline bit isn't set,
5417                  * reject the start or stop.  Otherwise, let it through.
5418                  */
5419                 mtx_unlock(&ctl_softc->ctl_lock);
5420                 ctl_set_lun_not_ready(ctsio);
5421                 ctl_done((union ctl_io *)ctsio);
5422         } else {
5423                 mtx_unlock(&ctl_softc->ctl_lock);
5424 #endif /* NEEDTOPORT */
5425                 /*
5426                  * This could be a start or a stop when we're online,
5427                  * or a stop/offline or start/online.  A start or stop when
5428                  * we're offline is covered in the case above.
5429                  */
5430                 /*
5431                  * In the non-immediate case, we send the request to
5432                  * the backend and return status to the user when
5433                  * it is done.
5434                  *
5435                  * In the immediate case, we allocate a new ctl_io
5436                  * to hold a copy of the request, and send that to
5437                  * the backend.  We then set good status on the
5438                  * user's request and return it immediately.
5439                  */
5440                 if (cdb->byte2 & SSS_IMMED) {
5441                         union ctl_io *new_io;
5442
5443                         new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5444                         if (new_io == NULL) {
5445                                 ctl_set_busy(ctsio);
5446                                 ctl_done((union ctl_io *)ctsio);
5447                         } else {
5448                                 ctl_copy_io((union ctl_io *)ctsio,
5449                                             new_io);
5450                                 retval = lun->backend->config_write(new_io);
5451                                 ctl_set_success(ctsio);
5452                                 ctl_done((union ctl_io *)ctsio);
5453                         }
5454                 } else {
5455                         retval = lun->backend->config_write(
5456                                 (union ctl_io *)ctsio);
5457                 }
5458 #ifdef NEEDTOPORT
5459         }
5460 #endif
5461         return (retval);
5462 }
5463
5464 /*
5465  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5466  * we don't really do anything with the LBA and length fields if the user
5467  * passes them in.  Instead we'll just flush out the cache for the entire
5468  * LUN.
5469  */
5470 int
5471 ctl_sync_cache(struct ctl_scsiio *ctsio)
5472 {
5473         struct ctl_lun *lun;
5474         struct ctl_softc *ctl_softc;
5475         uint64_t starting_lba;
5476         uint32_t block_count;
5477         int reladr, immed;
5478         int retval;
5479
5480         CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5481
5482         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5483         ctl_softc = control_softc;
5484         retval = 0;
5485         reladr = 0;
5486         immed = 0;
5487
5488         switch (ctsio->cdb[0]) {
5489         case SYNCHRONIZE_CACHE: {
5490                 struct scsi_sync_cache *cdb;
5491                 cdb = (struct scsi_sync_cache *)ctsio->cdb;
5492
5493                 if (cdb->byte2 & SSC_RELADR)
5494                         reladr = 1;
5495
5496                 if (cdb->byte2 & SSC_IMMED)
5497                         immed = 1;
5498
5499                 starting_lba = scsi_4btoul(cdb->begin_lba);
5500                 block_count = scsi_2btoul(cdb->lb_count);
5501                 break;
5502         }
5503         case SYNCHRONIZE_CACHE_16: {
5504                 struct scsi_sync_cache_16 *cdb;
5505                 cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5506
5507                 if (cdb->byte2 & SSC_RELADR)
5508                         reladr = 1;
5509
5510                 if (cdb->byte2 & SSC_IMMED)
5511                         immed = 1;
5512
5513                 starting_lba = scsi_8btou64(cdb->begin_lba);
5514                 block_count = scsi_4btoul(cdb->lb_count);
5515                 break;
5516         }
5517         default:
5518                 ctl_set_invalid_opcode(ctsio);
5519                 ctl_done((union ctl_io *)ctsio);
5520                 goto bailout;
5521                 break; /* NOTREACHED */
5522         }
5523
5524         if (immed) {
5525                 /*
5526                  * We don't support the immediate bit.  Since it's in the
5527                  * same place for the 10 and 16 byte SYNCHRONIZE CACHE
5528                  * commands, we can just return the same error in either
5529                  * case.
5530                  */
5531                 ctl_set_invalid_field(ctsio,
5532                                       /*sks_valid*/ 1,
5533                                       /*command*/ 1,
5534                                       /*field*/ 1,
5535                                       /*bit_valid*/ 1,
5536                                       /*bit*/ 1);
5537                 ctl_done((union ctl_io *)ctsio);
5538                 goto bailout;
5539         }
5540
5541         if (reladr) {
5542                 /*
5543                  * We don't support the reladr bit either.  It can only be
5544                  * used with linked commands, and we don't support linked
5545                  * commands.  Since the bit is in the same place for the
5546                  * 10 and 16 byte SYNCHRONIZE CACHE * commands, we can
5547                  * just return the same error in either case.
5548                  */
5549                 ctl_set_invalid_field(ctsio,
5550                                       /*sks_valid*/ 1,
5551                                       /*command*/ 1,
5552                                       /*field*/ 1,
5553                                       /*bit_valid*/ 1,
5554                                       /*bit*/ 0);
5555                 ctl_done((union ctl_io *)ctsio);
5556                 goto bailout;
5557         }
5558
5559         /*
5560          * We check the LBA and length, but don't do anything with them.
5561          * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5562          * get flushed.  This check will just help satisfy anyone who wants
5563          * to see an error for an out of range LBA.
5564          */
5565         if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5566                 ctl_set_lba_out_of_range(ctsio);
5567                 ctl_done((union ctl_io *)ctsio);
5568                 goto bailout;
5569         }
5570
5571         /*
5572          * If this LUN has no backend, we can't flush the cache anyway.
5573          */
5574         if (lun->backend == NULL) {
5575                 ctl_set_invalid_opcode(ctsio);
5576                 ctl_done((union ctl_io *)ctsio);
5577                 goto bailout;
5578         }
5579
5580         /*
5581          * Check to see whether we're configured to send the SYNCHRONIZE
5582          * CACHE command directly to the back end.
5583          */
5584         mtx_lock(&ctl_softc->ctl_lock);
5585         if ((ctl_softc->flags & CTL_FLAG_REAL_SYNC)
5586          && (++(lun->sync_count) >= lun->sync_interval)) {
5587                 lun->sync_count = 0;
5588                 mtx_unlock(&ctl_softc->ctl_lock);
5589                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5590         } else {
5591                 mtx_unlock(&ctl_softc->ctl_lock);
5592                 ctl_set_success(ctsio);
5593                 ctl_done((union ctl_io *)ctsio);
5594         }
5595
5596 bailout:
5597
5598         return (retval);
5599 }
5600
5601 int
5602 ctl_format(struct ctl_scsiio *ctsio)
5603 {
5604         struct scsi_format *cdb;
5605         struct ctl_lun *lun;
5606         struct ctl_softc *ctl_softc;
5607         int length, defect_list_len;
5608
5609         CTL_DEBUG_PRINT(("ctl_format\n"));
5610
5611         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5612         ctl_softc = control_softc;
5613
5614         cdb = (struct scsi_format *)ctsio->cdb;
5615
5616         length = 0;
5617         if (cdb->byte2 & SF_FMTDATA) {
5618                 if (cdb->byte2 & SF_LONGLIST)
5619                         length = sizeof(struct scsi_format_header_long);
5620                 else
5621                         length = sizeof(struct scsi_format_header_short);
5622         }
5623
5624         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5625          && (length > 0)) {
5626                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5627                 ctsio->kern_data_len = length;
5628                 ctsio->kern_total_len = length;
5629                 ctsio->kern_data_resid = 0;
5630                 ctsio->kern_rel_offset = 0;
5631                 ctsio->kern_sg_entries = 0;
5632                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5633                 ctsio->be_move_done = ctl_config_move_done;
5634                 ctl_datamove((union ctl_io *)ctsio);
5635
5636                 return (CTL_RETVAL_COMPLETE);
5637         }
5638
5639         defect_list_len = 0;
5640
5641         if (cdb->byte2 & SF_FMTDATA) {
5642                 if (cdb->byte2 & SF_LONGLIST) {
5643                         struct scsi_format_header_long *header;
5644
5645                         header = (struct scsi_format_header_long *)
5646                                 ctsio->kern_data_ptr;
5647
5648                         defect_list_len = scsi_4btoul(header->defect_list_len);
5649                         if (defect_list_len != 0) {
5650                                 ctl_set_invalid_field(ctsio,
5651                                                       /*sks_valid*/ 1,
5652                                                       /*command*/ 0,
5653                                                       /*field*/ 2,
5654                                                       /*bit_valid*/ 0,
5655                                                       /*bit*/ 0);
5656                                 goto bailout;
5657                         }
5658                 } else {
5659                         struct scsi_format_header_short *header;
5660
5661                         header = (struct scsi_format_header_short *)
5662                                 ctsio->kern_data_ptr;
5663
5664                         defect_list_len = scsi_2btoul(header->defect_list_len);
5665                         if (defect_list_len != 0) {
5666                                 ctl_set_invalid_field(ctsio,
5667                                                       /*sks_valid*/ 1,
5668                                                       /*command*/ 0,
5669                                                       /*field*/ 2,
5670                                                       /*bit_valid*/ 0,
5671                                                       /*bit*/ 0);
5672                                 goto bailout;
5673                         }
5674                 }
5675         }
5676
5677         /*
5678          * The format command will clear out the "Medium format corrupted"
5679          * status if set by the configuration code.  That status is really
5680          * just a way to notify the host that we have lost the media, and
5681          * get them to issue a command that will basically make them think
5682          * they're blowing away the media.
5683          */
5684         mtx_lock(&ctl_softc->ctl_lock);
5685         lun->flags &= ~CTL_LUN_INOPERABLE;
5686         mtx_unlock(&ctl_softc->ctl_lock);
5687
5688         ctsio->scsi_status = SCSI_STATUS_OK;
5689         ctsio->io_hdr.status = CTL_SUCCESS;
5690 bailout:
5691
5692         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5693                 free(ctsio->kern_data_ptr, M_CTL);
5694                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5695         }
5696
5697         ctl_done((union ctl_io *)ctsio);
5698         return (CTL_RETVAL_COMPLETE);
5699 }
5700
5701 int
5702 ctl_write_buffer(struct ctl_scsiio *ctsio)
5703 {
5704         struct scsi_write_buffer *cdb;
5705         struct copan_page_header *header;
5706         struct ctl_lun *lun;
5707         struct ctl_softc *ctl_softc;
5708         int buffer_offset, len;
5709         int retval;
5710
5711         header = NULL;
5712
5713         retval = CTL_RETVAL_COMPLETE;
5714
5715         CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5716
5717         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5718         ctl_softc = control_softc;
5719         cdb = (struct scsi_write_buffer *)ctsio->cdb;
5720
5721         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5722                 ctl_set_invalid_field(ctsio,
5723                                       /*sks_valid*/ 1,
5724                                       /*command*/ 1,
5725                                       /*field*/ 1,
5726                                       /*bit_valid*/ 1,
5727                                       /*bit*/ 4);
5728                 ctl_done((union ctl_io *)ctsio);
5729                 return (CTL_RETVAL_COMPLETE);
5730         }
5731         if (cdb->buffer_id != 0) {
5732                 ctl_set_invalid_field(ctsio,
5733                                       /*sks_valid*/ 1,
5734                                       /*command*/ 1,
5735                                       /*field*/ 2,
5736                                       /*bit_valid*/ 0,
5737                                       /*bit*/ 0);
5738                 ctl_done((union ctl_io *)ctsio);
5739                 return (CTL_RETVAL_COMPLETE);
5740         }
5741
5742         len = scsi_3btoul(cdb->length);
5743         buffer_offset = scsi_3btoul(cdb->offset);
5744
5745         if (len > sizeof(lun->write_buffer)) {
5746                 ctl_set_invalid_field(ctsio,
5747                                       /*sks_valid*/ 1,
5748                                       /*command*/ 1,
5749                                       /*field*/ 6,
5750                                       /*bit_valid*/ 0,
5751                                       /*bit*/ 0);
5752                 ctl_done((union ctl_io *)ctsio);
5753                 return (CTL_RETVAL_COMPLETE);
5754         }
5755
5756         if (buffer_offset != 0) {
5757                 ctl_set_invalid_field(ctsio,
5758                                       /*sks_valid*/ 1,
5759                                       /*command*/ 1,
5760                                       /*field*/ 3,
5761                                       /*bit_valid*/ 0,
5762                                       /*bit*/ 0);
5763                 ctl_done((union ctl_io *)ctsio);
5764                 return (CTL_RETVAL_COMPLETE);
5765         }
5766
5767         /*
5768          * If we've got a kernel request that hasn't been malloced yet,
5769          * malloc it and tell the caller the data buffer is here.
5770          */
5771         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5772                 ctsio->kern_data_ptr = lun->write_buffer;
5773                 ctsio->kern_data_len = len;
5774                 ctsio->kern_total_len = len;
5775                 ctsio->kern_data_resid = 0;
5776                 ctsio->kern_rel_offset = 0;
5777                 ctsio->kern_sg_entries = 0;
5778                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5779                 ctsio->be_move_done = ctl_config_move_done;
5780                 ctl_datamove((union ctl_io *)ctsio);
5781
5782                 return (CTL_RETVAL_COMPLETE);
5783         }
5784
5785         ctl_done((union ctl_io *)ctsio);
5786
5787         return (CTL_RETVAL_COMPLETE);
5788 }
5789
5790 /*
5791  * Note that this function currently doesn't actually do anything inside
5792  * CTL to enforce things if the DQue bit is turned on.
5793  *
5794  * Also note that this function can't be used in the default case, because
5795  * the DQue bit isn't set in the changeable mask for the control mode page
5796  * anyway.  This is just here as an example for how to implement a page
5797  * handler, and a placeholder in case we want to allow the user to turn
5798  * tagged queueing on and off.
5799  *
5800  * The D_SENSE bit handling is functional, however, and will turn
5801  * descriptor sense on and off for a given LUN.
5802  */
5803 int
5804 ctl_control_page_handler(struct ctl_scsiio *ctsio,
5805                          struct ctl_page_index *page_index, uint8_t *page_ptr)
5806 {
5807         struct scsi_control_page *current_cp, *saved_cp, *user_cp;
5808         struct ctl_lun *lun;
5809         struct ctl_softc *softc;
5810         int set_ua;
5811         uint32_t initidx;
5812
5813         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5814         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5815         set_ua = 0;
5816
5817         user_cp = (struct scsi_control_page *)page_ptr;
5818         current_cp = (struct scsi_control_page *)
5819                 (page_index->page_data + (page_index->page_len *
5820                 CTL_PAGE_CURRENT));
5821         saved_cp = (struct scsi_control_page *)
5822                 (page_index->page_data + (page_index->page_len *
5823                 CTL_PAGE_SAVED));
5824
5825         softc = control_softc;
5826
5827         mtx_lock(&softc->ctl_lock);
5828         if (((current_cp->rlec & SCP_DSENSE) == 0)
5829          && ((user_cp->rlec & SCP_DSENSE) != 0)) {
5830                 /*
5831                  * Descriptor sense is currently turned off and the user
5832                  * wants to turn it on.
5833                  */
5834                 current_cp->rlec |= SCP_DSENSE;
5835                 saved_cp->rlec |= SCP_DSENSE;
5836                 lun->flags |= CTL_LUN_SENSE_DESC;
5837                 set_ua = 1;
5838         } else if (((current_cp->rlec & SCP_DSENSE) != 0)
5839                 && ((user_cp->rlec & SCP_DSENSE) == 0)) {
5840                 /*
5841                  * Descriptor sense is currently turned on, and the user
5842                  * wants to turn it off.
5843                  */
5844                 current_cp->rlec &= ~SCP_DSENSE;
5845                 saved_cp->rlec &= ~SCP_DSENSE;
5846                 lun->flags &= ~CTL_LUN_SENSE_DESC;
5847                 set_ua = 1;
5848         }
5849         if (current_cp->queue_flags & SCP_QUEUE_DQUE) {
5850                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
5851 #ifdef NEEDTOPORT
5852                         csevent_log(CSC_CTL | CSC_SHELF_SW |
5853                                     CTL_UNTAG_TO_UNTAG,
5854                                     csevent_LogType_Trace,
5855                                     csevent_Severity_Information,
5856                                     csevent_AlertLevel_Green,
5857                                     csevent_FRU_Firmware,
5858                                     csevent_FRU_Unknown,
5859                                     "Received untagged to untagged transition");
5860 #endif /* NEEDTOPORT */
5861                 } else {
5862 #ifdef NEEDTOPORT
5863                         csevent_log(CSC_CTL | CSC_SHELF_SW |
5864                                     CTL_UNTAG_TO_TAG,
5865                                     csevent_LogType_ConfigChange,
5866                                     csevent_Severity_Information,
5867                                     csevent_AlertLevel_Green,
5868                                     csevent_FRU_Firmware,
5869                                     csevent_FRU_Unknown,
5870                                     "Received untagged to tagged "
5871                                     "queueing transition");
5872 #endif /* NEEDTOPORT */
5873
5874                         current_cp->queue_flags &= ~SCP_QUEUE_DQUE;
5875                         saved_cp->queue_flags &= ~SCP_QUEUE_DQUE;
5876                         set_ua = 1;
5877                 }
5878         } else {
5879                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
5880 #ifdef NEEDTOPORT
5881                         csevent_log(CSC_CTL | CSC_SHELF_SW |
5882                                     CTL_TAG_TO_UNTAG,
5883                                     csevent_LogType_ConfigChange,
5884                                     csevent_Severity_Warning,
5885                                     csevent_AlertLevel_Yellow,
5886                                     csevent_FRU_Firmware,
5887                                     csevent_FRU_Unknown,
5888                                     "Received tagged queueing to untagged "
5889                                     "transition");
5890 #endif /* NEEDTOPORT */
5891
5892                         current_cp->queue_flags |= SCP_QUEUE_DQUE;
5893                         saved_cp->queue_flags |= SCP_QUEUE_DQUE;
5894                         set_ua = 1;
5895                 } else {
5896 #ifdef NEEDTOPORT
5897                         csevent_log(CSC_CTL | CSC_SHELF_SW |
5898                                     CTL_TAG_TO_TAG,
5899                                     csevent_LogType_Trace,
5900                                     csevent_Severity_Information,
5901                                     csevent_AlertLevel_Green,
5902                                     csevent_FRU_Firmware,
5903                                     csevent_FRU_Unknown,
5904                                     "Received tagged queueing to tagged "
5905                                     "queueing transition");
5906 #endif /* NEEDTOPORT */
5907                 }
5908         }
5909         if (set_ua != 0) {
5910                 int i;
5911                 /*
5912                  * Let other initiators know that the mode
5913                  * parameters for this LUN have changed.
5914                  */
5915                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
5916                         if (i == initidx)
5917                                 continue;
5918
5919                         lun->pending_sense[i].ua_pending |=
5920                                 CTL_UA_MODE_CHANGE;
5921                 }
5922         }
5923         mtx_unlock(&softc->ctl_lock);
5924
5925         return (0);
5926 }
5927
5928 int
5929 ctl_power_sp_handler(struct ctl_scsiio *ctsio,
5930                      struct ctl_page_index *page_index, uint8_t *page_ptr)
5931 {
5932         return (0);
5933 }
5934
5935 int
5936 ctl_power_sp_sense_handler(struct ctl_scsiio *ctsio,
5937                            struct ctl_page_index *page_index, int pc)
5938 {
5939         struct copan_power_subpage *page;
5940
5941         page = (struct copan_power_subpage *)page_index->page_data +
5942                 (page_index->page_len * pc);
5943
5944         switch (pc) {
5945         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
5946                 /*
5947                  * We don't update the changable bits for this page.
5948                  */
5949                 break;
5950         case SMS_PAGE_CTRL_CURRENT >> 6:
5951         case SMS_PAGE_CTRL_DEFAULT >> 6:
5952         case SMS_PAGE_CTRL_SAVED >> 6:
5953 #ifdef NEEDTOPORT
5954                 ctl_update_power_subpage(page);
5955 #endif
5956                 break;
5957         default:
5958 #ifdef NEEDTOPORT
5959                 EPRINT(0, "Invalid PC %d!!", pc);
5960 #endif
5961                 break;
5962         }
5963         return (0);
5964 }
5965
5966
5967 int
5968 ctl_aps_sp_handler(struct ctl_scsiio *ctsio,
5969                    struct ctl_page_index *page_index, uint8_t *page_ptr)
5970 {
5971         struct copan_aps_subpage *user_sp;
5972         struct copan_aps_subpage *current_sp;
5973         union ctl_modepage_info *modepage_info;
5974         struct ctl_softc *softc;
5975         struct ctl_lun *lun;
5976         int retval;
5977
5978         retval = CTL_RETVAL_COMPLETE;
5979         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
5980                      (page_index->page_len * CTL_PAGE_CURRENT));
5981         softc = control_softc;
5982         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5983
5984         user_sp = (struct copan_aps_subpage *)page_ptr;
5985
5986         modepage_info = (union ctl_modepage_info *)
5987                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
5988
5989         modepage_info->header.page_code = page_index->page_code & SMPH_PC_MASK;
5990         modepage_info->header.subpage = page_index->subpage;
5991         modepage_info->aps.lock_active = user_sp->lock_active;
5992
5993         mtx_lock(&softc->ctl_lock);
5994
5995         /*
5996          * If there is a request to lock the LUN and another LUN is locked
5997          * this is an error. If the requested LUN is already locked ignore
5998          * the request. If no LUN is locked attempt to lock it.
5999          * if there is a request to unlock the LUN and the LUN is currently
6000          * locked attempt to unlock it. Otherwise ignore the request. i.e.
6001          * if another LUN is locked or no LUN is locked.
6002          */
6003         if (user_sp->lock_active & APS_LOCK_ACTIVE) {
6004                 if (softc->aps_locked_lun == lun->lun) {
6005                         /*
6006                          * This LUN is already locked, so we're done.
6007                          */
6008                         retval = CTL_RETVAL_COMPLETE;
6009                 } else if (softc->aps_locked_lun == 0) {
6010                         /*
6011                          * No one has the lock, pass the request to the
6012                          * backend.
6013                          */
6014                         retval = lun->backend->config_write(
6015                                 (union ctl_io *)ctsio);
6016                 } else {
6017                         /*
6018                          * Someone else has the lock, throw out the request.
6019                          */
6020                         ctl_set_already_locked(ctsio);
6021                         free(ctsio->kern_data_ptr, M_CTL);
6022                         ctl_done((union ctl_io *)ctsio);
6023
6024                         /*
6025                          * Set the return value so that ctl_do_mode_select()
6026                          * won't try to complete the command.  We already
6027                          * completed it here.
6028                          */
6029                         retval = CTL_RETVAL_ERROR;
6030                 }
6031         } else if (softc->aps_locked_lun == lun->lun) {
6032                 /*
6033                  * This LUN is locked, so pass the unlock request to the
6034                  * backend.
6035                  */
6036                 retval = lun->backend->config_write((union ctl_io *)ctsio);
6037         }
6038         mtx_unlock(&softc->ctl_lock);
6039
6040         return (retval);
6041 }
6042
6043 int
6044 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6045                                 struct ctl_page_index *page_index,
6046                                 uint8_t *page_ptr)
6047 {
6048         uint8_t *c;
6049         int i;
6050
6051         c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6052         ctl_time_io_secs =
6053                 (c[0] << 8) |
6054                 (c[1] << 0) |
6055                 0;
6056         CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6057         printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6058         printf("page data:");
6059         for (i=0; i<8; i++)
6060                 printf(" %.2x",page_ptr[i]);
6061         printf("\n");
6062         return (0);
6063 }
6064
6065 int
6066 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6067                                struct ctl_page_index *page_index,
6068                                int pc)
6069 {
6070         struct copan_debugconf_subpage *page;
6071
6072         page = (struct copan_debugconf_subpage *)page_index->page_data +
6073                 (page_index->page_len * pc);
6074
6075         switch (pc) {
6076         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6077         case SMS_PAGE_CTRL_DEFAULT >> 6:
6078         case SMS_PAGE_CTRL_SAVED >> 6:
6079                 /*
6080                  * We don't update the changable or default bits for this page.
6081                  */
6082                 break;
6083         case SMS_PAGE_CTRL_CURRENT >> 6:
6084                 page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6085                 page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6086                 break;
6087         default:
6088 #ifdef NEEDTOPORT
6089                 EPRINT(0, "Invalid PC %d!!", pc);
6090 #endif /* NEEDTOPORT */
6091                 break;
6092         }
6093         return (0);
6094 }
6095
6096
6097 static int
6098 ctl_do_mode_select(union ctl_io *io)
6099 {
6100         struct scsi_mode_page_header *page_header;
6101         struct ctl_page_index *page_index;
6102         struct ctl_scsiio *ctsio;
6103         int control_dev, page_len;
6104         int page_len_offset, page_len_size;
6105         union ctl_modepage_info *modepage_info;
6106         struct ctl_lun *lun;
6107         int *len_left, *len_used;
6108         int retval, i;
6109
6110         ctsio = &io->scsiio;
6111         page_index = NULL;
6112         page_len = 0;
6113         retval = CTL_RETVAL_COMPLETE;
6114
6115         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6116
6117         if (lun->be_lun->lun_type != T_DIRECT)
6118                 control_dev = 1;
6119         else
6120                 control_dev = 0;
6121
6122         modepage_info = (union ctl_modepage_info *)
6123                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6124         len_left = &modepage_info->header.len_left;
6125         len_used = &modepage_info->header.len_used;
6126
6127 do_next_page:
6128
6129         page_header = (struct scsi_mode_page_header *)
6130                 (ctsio->kern_data_ptr + *len_used);
6131
6132         if (*len_left == 0) {
6133                 free(ctsio->kern_data_ptr, M_CTL);
6134                 ctl_set_success(ctsio);
6135                 ctl_done((union ctl_io *)ctsio);
6136                 return (CTL_RETVAL_COMPLETE);
6137         } else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6138
6139                 free(ctsio->kern_data_ptr, M_CTL);
6140                 ctl_set_param_len_error(ctsio);
6141                 ctl_done((union ctl_io *)ctsio);
6142                 return (CTL_RETVAL_COMPLETE);
6143
6144         } else if ((page_header->page_code & SMPH_SPF)
6145                 && (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6146
6147                 free(ctsio->kern_data_ptr, M_CTL);
6148                 ctl_set_param_len_error(ctsio);
6149                 ctl_done((union ctl_io *)ctsio);
6150                 return (CTL_RETVAL_COMPLETE);
6151         }
6152
6153
6154         /*
6155          * XXX KDM should we do something with the block descriptor?
6156          */
6157         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6158
6159                 if ((control_dev != 0)
6160                  && (lun->mode_pages.index[i].page_flags &
6161                      CTL_PAGE_FLAG_DISK_ONLY))
6162                         continue;
6163
6164                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6165                     (page_header->page_code & SMPH_PC_MASK))
6166                         continue;
6167
6168                 /*
6169                  * If neither page has a subpage code, then we've got a
6170                  * match.
6171                  */
6172                 if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6173                  && ((page_header->page_code & SMPH_SPF) == 0)) {
6174                         page_index = &lun->mode_pages.index[i];
6175                         page_len = page_header->page_length;
6176                         break;
6177                 }
6178
6179                 /*
6180                  * If both pages have subpages, then the subpage numbers
6181                  * have to match.
6182                  */
6183                 if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6184                   && (page_header->page_code & SMPH_SPF)) {
6185                         struct scsi_mode_page_header_sp *sph;
6186
6187                         sph = (struct scsi_mode_page_header_sp *)page_header;
6188
6189                         if (lun->mode_pages.index[i].subpage ==
6190                             sph->subpage) {
6191                                 page_index = &lun->mode_pages.index[i];
6192                                 page_len = scsi_2btoul(sph->page_length);
6193                                 break;
6194                         }
6195                 }
6196         }
6197
6198         /*
6199          * If we couldn't find the page, or if we don't have a mode select
6200          * handler for it, send back an error to the user.
6201          */
6202         if ((page_index == NULL)
6203          || (page_index->select_handler == NULL)) {
6204                 ctl_set_invalid_field(ctsio,
6205                                       /*sks_valid*/ 1,
6206                                       /*command*/ 0,
6207                                       /*field*/ *len_used,
6208                                       /*bit_valid*/ 0,
6209                                       /*bit*/ 0);
6210                 free(ctsio->kern_data_ptr, M_CTL);
6211                 ctl_done((union ctl_io *)ctsio);
6212                 return (CTL_RETVAL_COMPLETE);
6213         }
6214
6215         if (page_index->page_code & SMPH_SPF) {
6216                 page_len_offset = 2;
6217                 page_len_size = 2;
6218         } else {
6219                 page_len_size = 1;
6220                 page_len_offset = 1;
6221         }
6222
6223         /*
6224          * If the length the initiator gives us isn't the one we specify in
6225          * the mode page header, or if they didn't specify enough data in
6226          * the CDB to avoid truncating this page, kick out the request.
6227          */
6228         if ((page_len != (page_index->page_len - page_len_offset -
6229                           page_len_size))
6230          || (*len_left < page_index->page_len)) {
6231
6232
6233                 ctl_set_invalid_field(ctsio,
6234                                       /*sks_valid*/ 1,
6235                                       /*command*/ 0,
6236                                       /*field*/ *len_used + page_len_offset,
6237                                       /*bit_valid*/ 0,
6238                                       /*bit*/ 0);
6239                 free(ctsio->kern_data_ptr, M_CTL);
6240                 ctl_done((union ctl_io *)ctsio);
6241                 return (CTL_RETVAL_COMPLETE);
6242         }
6243
6244         /*
6245          * Run through the mode page, checking to make sure that the bits
6246          * the user changed are actually legal for him to change.
6247          */
6248         for (i = 0; i < page_index->page_len; i++) {
6249                 uint8_t *user_byte, *change_mask, *current_byte;
6250                 int bad_bit;
6251                 int j;
6252
6253                 user_byte = (uint8_t *)page_header + i;
6254                 change_mask = page_index->page_data +
6255                               (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6256                 current_byte = page_index->page_data +
6257                                (page_index->page_len * CTL_PAGE_CURRENT) + i;
6258
6259                 /*
6260                  * Check to see whether the user set any bits in this byte
6261                  * that he is not allowed to set.
6262                  */
6263                 if ((*user_byte & ~(*change_mask)) ==
6264                     (*current_byte & ~(*change_mask)))
6265                         continue;
6266
6267                 /*
6268                  * Go through bit by bit to determine which one is illegal.
6269                  */
6270                 bad_bit = 0;
6271                 for (j = 7; j >= 0; j--) {
6272                         if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6273                             (((1 << i) & ~(*change_mask)) & *current_byte)) {
6274                                 bad_bit = i;
6275                                 break;
6276                         }
6277                 }
6278                 ctl_set_invalid_field(ctsio,
6279                                       /*sks_valid*/ 1,
6280                                       /*command*/ 0,
6281                                       /*field*/ *len_used + i,
6282                                       /*bit_valid*/ 1,
6283                                       /*bit*/ bad_bit);
6284                 free(ctsio->kern_data_ptr, M_CTL);
6285                 ctl_done((union ctl_io *)ctsio);
6286                 return (CTL_RETVAL_COMPLETE);
6287         }
6288
6289         /*
6290          * Decrement these before we call the page handler, since we may
6291          * end up getting called back one way or another before the handler
6292          * returns to this context.
6293          */
6294         *len_left -= page_index->page_len;
6295         *len_used += page_index->page_len;
6296
6297         retval = page_index->select_handler(ctsio, page_index,
6298                                             (uint8_t *)page_header);
6299
6300         /*
6301          * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6302          * wait until this queued command completes to finish processing
6303          * the mode page.  If it returns anything other than
6304          * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6305          * already set the sense information, freed the data pointer, and
6306          * completed the io for us.
6307          */
6308         if (retval != CTL_RETVAL_COMPLETE)
6309                 goto bailout_no_done;
6310
6311         /*
6312          * If the initiator sent us more than one page, parse the next one.
6313          */
6314         if (*len_left > 0)
6315                 goto do_next_page;
6316
6317         ctl_set_success(ctsio);
6318         free(ctsio->kern_data_ptr, M_CTL);
6319         ctl_done((union ctl_io *)ctsio);
6320
6321 bailout_no_done:
6322
6323         return (CTL_RETVAL_COMPLETE);
6324
6325 }
6326
6327 int
6328 ctl_mode_select(struct ctl_scsiio *ctsio)
6329 {
6330         int param_len, pf, sp;
6331         int header_size, bd_len;
6332         int len_left, len_used;
6333         struct ctl_page_index *page_index;
6334         struct ctl_lun *lun;
6335         int control_dev, page_len;
6336         union ctl_modepage_info *modepage_info;
6337         int retval;
6338
6339         pf = 0;
6340         sp = 0;
6341         page_len = 0;
6342         len_used = 0;
6343         len_left = 0;
6344         retval = 0;
6345         bd_len = 0;
6346         page_index = NULL;
6347
6348         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6349
6350         if (lun->be_lun->lun_type != T_DIRECT)
6351                 control_dev = 1;
6352         else
6353                 control_dev = 0;
6354
6355         switch (ctsio->cdb[0]) {
6356         case MODE_SELECT_6: {
6357                 struct scsi_mode_select_6 *cdb;
6358
6359                 cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6360
6361                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6362                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6363
6364                 param_len = cdb->length;
6365                 header_size = sizeof(struct scsi_mode_header_6);
6366                 break;
6367         }
6368         case MODE_SELECT_10: {
6369                 struct scsi_mode_select_10 *cdb;
6370
6371                 cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6372
6373                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6374                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6375
6376                 param_len = scsi_2btoul(cdb->length);
6377                 header_size = sizeof(struct scsi_mode_header_10);
6378                 break;
6379         }
6380         default:
6381                 ctl_set_invalid_opcode(ctsio);
6382                 ctl_done((union ctl_io *)ctsio);
6383                 return (CTL_RETVAL_COMPLETE);
6384                 break; /* NOTREACHED */
6385         }
6386
6387         /*
6388          * From SPC-3:
6389          * "A parameter list length of zero indicates that the Data-Out Buffer
6390          * shall be empty. This condition shall not be considered as an error."
6391          */
6392         if (param_len == 0) {
6393                 ctl_set_success(ctsio);
6394                 ctl_done((union ctl_io *)ctsio);
6395                 return (CTL_RETVAL_COMPLETE);
6396         }
6397
6398         /*
6399          * Since we'll hit this the first time through, prior to
6400          * allocation, we don't need to free a data buffer here.
6401          */
6402         if (param_len < header_size) {
6403                 ctl_set_param_len_error(ctsio);
6404                 ctl_done((union ctl_io *)ctsio);
6405                 return (CTL_RETVAL_COMPLETE);
6406         }
6407
6408         /*
6409          * Allocate the data buffer and grab the user's data.  In theory,
6410          * we shouldn't have to sanity check the parameter list length here
6411          * because the maximum size is 64K.  We should be able to malloc
6412          * that much without too many problems.
6413          */
6414         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6415                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6416                 ctsio->kern_data_len = param_len;
6417                 ctsio->kern_total_len = param_len;
6418                 ctsio->kern_data_resid = 0;
6419                 ctsio->kern_rel_offset = 0;
6420                 ctsio->kern_sg_entries = 0;
6421                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6422                 ctsio->be_move_done = ctl_config_move_done;
6423                 ctl_datamove((union ctl_io *)ctsio);
6424
6425                 return (CTL_RETVAL_COMPLETE);
6426         }
6427
6428         switch (ctsio->cdb[0]) {
6429         case MODE_SELECT_6: {
6430                 struct scsi_mode_header_6 *mh6;
6431
6432                 mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6433                 bd_len = mh6->blk_desc_len;
6434                 break;
6435         }
6436         case MODE_SELECT_10: {
6437                 struct scsi_mode_header_10 *mh10;
6438
6439                 mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6440                 bd_len = scsi_2btoul(mh10->blk_desc_len);
6441                 break;
6442         }
6443         default:
6444                 panic("Invalid CDB type %#x", ctsio->cdb[0]);
6445                 break;
6446         }
6447
6448         if (param_len < (header_size + bd_len)) {
6449                 free(ctsio->kern_data_ptr, M_CTL);
6450                 ctl_set_param_len_error(ctsio);
6451                 ctl_done((union ctl_io *)ctsio);
6452                 return (CTL_RETVAL_COMPLETE);
6453         }
6454
6455         /*
6456          * Set the IO_CONT flag, so that if this I/O gets passed to
6457          * ctl_config_write_done(), it'll get passed back to
6458          * ctl_do_mode_select() for further processing, or completion if
6459          * we're all done.
6460          */
6461         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6462         ctsio->io_cont = ctl_do_mode_select;
6463
6464         modepage_info = (union ctl_modepage_info *)
6465                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6466
6467         memset(modepage_info, 0, sizeof(*modepage_info));
6468
6469         len_left = param_len - header_size - bd_len;
6470         len_used = header_size + bd_len;
6471
6472         modepage_info->header.len_left = len_left;
6473         modepage_info->header.len_used = len_used;
6474
6475         return (ctl_do_mode_select((union ctl_io *)ctsio));
6476 }
6477
6478 int
6479 ctl_mode_sense(struct ctl_scsiio *ctsio)
6480 {
6481         struct ctl_lun *lun;
6482         int pc, page_code, dbd, llba, subpage;
6483         int alloc_len, page_len, header_len, total_len;
6484         struct scsi_mode_block_descr *block_desc;
6485         struct ctl_page_index *page_index;
6486         int control_dev;
6487
6488         dbd = 0;
6489         llba = 0;
6490         block_desc = NULL;
6491         page_index = NULL;
6492
6493         CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6494
6495         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6496
6497         if (lun->be_lun->lun_type != T_DIRECT)
6498                 control_dev = 1;
6499         else
6500                 control_dev = 0;
6501
6502         switch (ctsio->cdb[0]) {
6503         case MODE_SENSE_6: {
6504                 struct scsi_mode_sense_6 *cdb;
6505
6506                 cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6507
6508                 header_len = sizeof(struct scsi_mode_hdr_6);
6509                 if (cdb->byte2 & SMS_DBD)
6510                         dbd = 1;
6511                 else
6512                         header_len += sizeof(struct scsi_mode_block_descr);
6513
6514                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6515                 page_code = cdb->page & SMS_PAGE_CODE;
6516                 subpage = cdb->subpage;
6517                 alloc_len = cdb->length;
6518                 break;
6519         }
6520         case MODE_SENSE_10: {
6521                 struct scsi_mode_sense_10 *cdb;
6522
6523                 cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6524
6525                 header_len = sizeof(struct scsi_mode_hdr_10);
6526
6527                 if (cdb->byte2 & SMS_DBD)
6528                         dbd = 1;
6529                 else
6530                         header_len += sizeof(struct scsi_mode_block_descr);
6531                 if (cdb->byte2 & SMS10_LLBAA)
6532                         llba = 1;
6533                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6534                 page_code = cdb->page & SMS_PAGE_CODE;
6535                 subpage = cdb->subpage;
6536                 alloc_len = scsi_2btoul(cdb->length);
6537                 break;
6538         }
6539         default:
6540                 ctl_set_invalid_opcode(ctsio);
6541                 ctl_done((union ctl_io *)ctsio);
6542                 return (CTL_RETVAL_COMPLETE);
6543                 break; /* NOTREACHED */
6544         }
6545
6546         /*
6547          * We have to make a first pass through to calculate the size of
6548          * the pages that match the user's query.  Then we allocate enough
6549          * memory to hold it, and actually copy the data into the buffer.
6550          */
6551         switch (page_code) {
6552         case SMS_ALL_PAGES_PAGE: {
6553                 int i;
6554
6555                 page_len = 0;
6556
6557                 /*
6558                  * At the moment, values other than 0 and 0xff here are
6559                  * reserved according to SPC-3.
6560                  */
6561                 if ((subpage != SMS_SUBPAGE_PAGE_0)
6562                  && (subpage != SMS_SUBPAGE_ALL)) {
6563                         ctl_set_invalid_field(ctsio,
6564                                               /*sks_valid*/ 1,
6565                                               /*command*/ 1,
6566                                               /*field*/ 3,
6567                                               /*bit_valid*/ 0,
6568                                               /*bit*/ 0);
6569                         ctl_done((union ctl_io *)ctsio);
6570                         return (CTL_RETVAL_COMPLETE);
6571                 }
6572
6573                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6574                         if ((control_dev != 0)
6575                          && (lun->mode_pages.index[i].page_flags &
6576                              CTL_PAGE_FLAG_DISK_ONLY))
6577                                 continue;
6578
6579                         /*
6580                          * We don't use this subpage if the user didn't
6581                          * request all subpages.
6582                          */
6583                         if ((lun->mode_pages.index[i].subpage != 0)
6584                          && (subpage == SMS_SUBPAGE_PAGE_0))
6585                                 continue;
6586
6587 #if 0
6588                         printf("found page %#x len %d\n",
6589                                lun->mode_pages.index[i].page_code &
6590                                SMPH_PC_MASK,
6591                                lun->mode_pages.index[i].page_len);
6592 #endif
6593                         page_len += lun->mode_pages.index[i].page_len;
6594                 }
6595                 break;
6596         }
6597         default: {
6598                 int i;
6599
6600                 page_len = 0;
6601
6602                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6603                         /* Look for the right page code */
6604                         if ((lun->mode_pages.index[i].page_code &
6605                              SMPH_PC_MASK) != page_code)
6606                                 continue;
6607
6608                         /* Look for the right subpage or the subpage wildcard*/
6609                         if ((lun->mode_pages.index[i].subpage != subpage)
6610                          && (subpage != SMS_SUBPAGE_ALL))
6611                                 continue;
6612
6613                         /* Make sure the page is supported for this dev type */
6614                         if ((control_dev != 0)
6615                          && (lun->mode_pages.index[i].page_flags &
6616                              CTL_PAGE_FLAG_DISK_ONLY))
6617                                 continue;
6618
6619 #if 0
6620                         printf("found page %#x len %d\n",
6621                                lun->mode_pages.index[i].page_code &
6622                                SMPH_PC_MASK,
6623                                lun->mode_pages.index[i].page_len);
6624 #endif
6625
6626                         page_len += lun->mode_pages.index[i].page_len;
6627                 }
6628
6629                 if (page_len == 0) {
6630                         ctl_set_invalid_field(ctsio,
6631                                               /*sks_valid*/ 1,
6632                                               /*command*/ 1,
6633                                               /*field*/ 2,
6634                                               /*bit_valid*/ 1,
6635                                               /*bit*/ 5);
6636                         ctl_done((union ctl_io *)ctsio);
6637                         return (CTL_RETVAL_COMPLETE);
6638                 }
6639                 break;
6640         }
6641         }
6642
6643         total_len = header_len + page_len;
6644 #if 0
6645         printf("header_len = %d, page_len = %d, total_len = %d\n",
6646                header_len, page_len, total_len);
6647 #endif
6648
6649         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6650         ctsio->kern_sg_entries = 0;
6651         ctsio->kern_data_resid = 0;
6652         ctsio->kern_rel_offset = 0;
6653         if (total_len < alloc_len) {
6654                 ctsio->residual = alloc_len - total_len;
6655                 ctsio->kern_data_len = total_len;
6656                 ctsio->kern_total_len = total_len;
6657         } else {
6658                 ctsio->residual = 0;
6659                 ctsio->kern_data_len = alloc_len;
6660                 ctsio->kern_total_len = alloc_len;
6661         }
6662
6663         switch (ctsio->cdb[0]) {
6664         case MODE_SENSE_6: {
6665                 struct scsi_mode_hdr_6 *header;
6666
6667                 header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6668
6669                 header->datalen = ctl_min(total_len - 1, 254);
6670
6671                 if (dbd)
6672                         header->block_descr_len = 0;
6673                 else
6674                         header->block_descr_len =
6675                                 sizeof(struct scsi_mode_block_descr);
6676                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6677                 break;
6678         }
6679         case MODE_SENSE_10: {
6680                 struct scsi_mode_hdr_10 *header;
6681                 int datalen;
6682
6683                 header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6684
6685                 datalen = ctl_min(total_len - 2, 65533);
6686                 scsi_ulto2b(datalen, header->datalen);
6687                 if (dbd)
6688                         scsi_ulto2b(0, header->block_descr_len);
6689                 else
6690                         scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
6691                                     header->block_descr_len);
6692                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6693                 break;
6694         }
6695         default:
6696                 panic("invalid CDB type %#x", ctsio->cdb[0]);
6697                 break; /* NOTREACHED */
6698         }
6699
6700         /*
6701          * If we've got a disk, use its blocksize in the block
6702          * descriptor.  Otherwise, just set it to 0.
6703          */
6704         if (dbd == 0) {
6705                 if (control_dev != 0)
6706                         scsi_ulto3b(lun->be_lun->blocksize,
6707                                     block_desc->block_len);
6708                 else
6709                         scsi_ulto3b(0, block_desc->block_len);
6710         }
6711
6712         switch (page_code) {
6713         case SMS_ALL_PAGES_PAGE: {
6714                 int i, data_used;
6715
6716                 data_used = header_len;
6717                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6718                         struct ctl_page_index *page_index;
6719
6720                         page_index = &lun->mode_pages.index[i];
6721
6722                         if ((control_dev != 0)
6723                          && (page_index->page_flags &
6724                             CTL_PAGE_FLAG_DISK_ONLY))
6725                                 continue;
6726
6727                         /*
6728                          * We don't use this subpage if the user didn't
6729                          * request all subpages.  We already checked (above)
6730                          * to make sure the user only specified a subpage
6731                          * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
6732                          */
6733                         if ((page_index->subpage != 0)
6734                          && (subpage == SMS_SUBPAGE_PAGE_0))
6735                                 continue;
6736
6737                         /*
6738                          * Call the handler, if it exists, to update the
6739                          * page to the latest values.
6740                          */
6741                         if (page_index->sense_handler != NULL)
6742                                 page_index->sense_handler(ctsio, page_index,pc);
6743
6744                         memcpy(ctsio->kern_data_ptr + data_used,
6745                                page_index->page_data +
6746                                (page_index->page_len * pc),
6747                                page_index->page_len);
6748                         data_used += page_index->page_len;
6749                 }
6750                 break;
6751         }
6752         default: {
6753                 int i, data_used;
6754
6755                 data_used = header_len;
6756
6757                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6758                         struct ctl_page_index *page_index;
6759
6760                         page_index = &lun->mode_pages.index[i];
6761
6762                         /* Look for the right page code */
6763                         if ((page_index->page_code & SMPH_PC_MASK) != page_code)
6764                                 continue;
6765
6766                         /* Look for the right subpage or the subpage wildcard*/
6767                         if ((page_index->subpage != subpage)
6768                          && (subpage != SMS_SUBPAGE_ALL))
6769                                 continue;
6770
6771                         /* Make sure the page is supported for this dev type */
6772                         if ((control_dev != 0)
6773                          && (page_index->page_flags &
6774                              CTL_PAGE_FLAG_DISK_ONLY))
6775                                 continue;
6776
6777                         /*
6778                          * Call the handler, if it exists, to update the
6779                          * page to the latest values.
6780                          */
6781                         if (page_index->sense_handler != NULL)
6782                                 page_index->sense_handler(ctsio, page_index,pc);
6783
6784                         memcpy(ctsio->kern_data_ptr + data_used,
6785                                page_index->page_data +
6786                                (page_index->page_len * pc),
6787                                page_index->page_len);
6788                         data_used += page_index->page_len;
6789                 }
6790                 break;
6791         }
6792         }
6793
6794         ctsio->scsi_status = SCSI_STATUS_OK;
6795
6796         ctsio->be_move_done = ctl_config_move_done;
6797         ctl_datamove((union ctl_io *)ctsio);
6798
6799         return (CTL_RETVAL_COMPLETE);
6800 }
6801
6802 int
6803 ctl_read_capacity(struct ctl_scsiio *ctsio)
6804 {
6805         struct scsi_read_capacity *cdb;
6806         struct scsi_read_capacity_data *data;
6807         struct ctl_lun *lun;
6808         uint32_t lba;
6809
6810         CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
6811
6812         cdb = (struct scsi_read_capacity *)ctsio->cdb;
6813
6814         lba = scsi_4btoul(cdb->addr);
6815         if (((cdb->pmi & SRC_PMI) == 0)
6816          && (lba != 0)) {
6817                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6818                                       /*sks_valid*/ 1,
6819                                       /*command*/ 1,
6820                                       /*field*/ 2,
6821                                       /*bit_valid*/ 0,
6822                                       /*bit*/ 0);
6823                 ctl_done((union ctl_io *)ctsio);
6824                 return (CTL_RETVAL_COMPLETE);
6825         }
6826
6827         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6828
6829         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
6830         data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
6831         ctsio->residual = 0;
6832         ctsio->kern_data_len = sizeof(*data);
6833         ctsio->kern_total_len = sizeof(*data);
6834         ctsio->kern_data_resid = 0;
6835         ctsio->kern_rel_offset = 0;
6836         ctsio->kern_sg_entries = 0;
6837
6838         /*
6839          * If the maximum LBA is greater than 0xfffffffe, the user must
6840          * issue a SERVICE ACTION IN (16) command, with the read capacity
6841          * serivce action set.
6842          */
6843         if (lun->be_lun->maxlba > 0xfffffffe)
6844                 scsi_ulto4b(0xffffffff, data->addr);
6845         else
6846                 scsi_ulto4b(lun->be_lun->maxlba, data->addr);
6847
6848         /*
6849          * XXX KDM this may not be 512 bytes...
6850          */
6851         scsi_ulto4b(lun->be_lun->blocksize, data->length);
6852
6853         ctsio->scsi_status = SCSI_STATUS_OK;
6854
6855         ctsio->be_move_done = ctl_config_move_done;
6856         ctl_datamove((union ctl_io *)ctsio);
6857
6858         return (CTL_RETVAL_COMPLETE);
6859 }
6860
6861 static int
6862 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
6863 {
6864         struct scsi_read_capacity_16 *cdb;
6865         struct scsi_read_capacity_data_long *data;
6866         struct ctl_lun *lun;
6867         uint64_t lba;
6868         uint32_t alloc_len;
6869
6870         CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
6871
6872         cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
6873
6874         alloc_len = scsi_4btoul(cdb->alloc_len);
6875         lba = scsi_8btou64(cdb->addr);
6876
6877         if ((cdb->reladr & SRC16_PMI)
6878          && (lba != 0)) {
6879                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6880                                       /*sks_valid*/ 1,
6881                                       /*command*/ 1,
6882                                       /*field*/ 2,
6883                                       /*bit_valid*/ 0,
6884                                       /*bit*/ 0);
6885                 ctl_done((union ctl_io *)ctsio);
6886                 return (CTL_RETVAL_COMPLETE);
6887         }
6888
6889         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6890
6891         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
6892         data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
6893
6894         if (sizeof(*data) < alloc_len) {
6895                 ctsio->residual = alloc_len - sizeof(*data);
6896                 ctsio->kern_data_len = sizeof(*data);
6897                 ctsio->kern_total_len = sizeof(*data);
6898         } else {
6899                 ctsio->residual = 0;
6900                 ctsio->kern_data_len = alloc_len;
6901                 ctsio->kern_total_len = alloc_len;
6902         }
6903         ctsio->kern_data_resid = 0;
6904         ctsio->kern_rel_offset = 0;
6905         ctsio->kern_sg_entries = 0;
6906
6907         scsi_u64to8b(lun->be_lun->maxlba, data->addr);
6908         /* XXX KDM this may not be 512 bytes... */
6909         scsi_ulto4b(lun->be_lun->blocksize, data->length);
6910         data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
6911         scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
6912
6913         ctsio->scsi_status = SCSI_STATUS_OK;
6914
6915         ctsio->be_move_done = ctl_config_move_done;
6916         ctl_datamove((union ctl_io *)ctsio);
6917
6918         return (CTL_RETVAL_COMPLETE);
6919 }
6920
6921 int
6922 ctl_service_action_in(struct ctl_scsiio *ctsio)
6923 {
6924         struct scsi_service_action_in *cdb;
6925         int retval;
6926
6927         CTL_DEBUG_PRINT(("ctl_service_action_in\n"));
6928
6929         cdb = (struct scsi_service_action_in *)ctsio->cdb;
6930
6931         retval = CTL_RETVAL_COMPLETE;
6932
6933         switch (cdb->service_action) {
6934         case SRC16_SERVICE_ACTION:
6935                 retval = ctl_read_capacity_16(ctsio);
6936                 break;
6937         default:
6938                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6939                                       /*sks_valid*/ 1,
6940                                       /*command*/ 1,
6941                                       /*field*/ 1,
6942                                       /*bit_valid*/ 1,
6943                                       /*bit*/ 4);
6944                 ctl_done((union ctl_io *)ctsio);
6945                 break;
6946         }
6947
6948         return (retval);
6949 }
6950
6951 int
6952 ctl_maintenance_in(struct ctl_scsiio *ctsio)
6953 {
6954         struct scsi_maintenance_in *cdb;
6955         int retval;
6956         int alloc_len, total_len = 0;
6957         int num_target_port_groups, single;
6958         struct ctl_lun *lun;
6959         struct ctl_softc *softc;
6960         struct scsi_target_group_data *rtg_ptr;
6961         struct scsi_target_port_group_descriptor *tpg_desc_ptr1, *tpg_desc_ptr2;
6962         struct scsi_target_port_descriptor  *tp_desc_ptr1_1, *tp_desc_ptr1_2,
6963                                             *tp_desc_ptr2_1, *tp_desc_ptr2_2;
6964
6965         CTL_DEBUG_PRINT(("ctl_maintenance_in\n"));
6966
6967         cdb = (struct scsi_maintenance_in *)ctsio->cdb;
6968         softc = control_softc;
6969         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6970
6971         retval = CTL_RETVAL_COMPLETE;
6972
6973         if ((cdb->byte2 & SERVICE_ACTION_MASK) != SA_RPRT_TRGT_GRP) {
6974                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6975                                       /*sks_valid*/ 1,
6976                                       /*command*/ 1,
6977                                       /*field*/ 1,
6978                                       /*bit_valid*/ 1,
6979                                       /*bit*/ 4);
6980                 ctl_done((union ctl_io *)ctsio);
6981                 return(retval);
6982         }
6983
6984         mtx_lock(&softc->ctl_lock);
6985         single = ctl_is_single;
6986         mtx_unlock(&softc->ctl_lock);
6987
6988         if (single)
6989                 num_target_port_groups = NUM_TARGET_PORT_GROUPS - 1;
6990         else
6991                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
6992
6993         total_len = sizeof(struct scsi_target_group_data) +
6994                 sizeof(struct scsi_target_port_group_descriptor) *
6995                 num_target_port_groups +
6996                 sizeof(struct scsi_target_port_descriptor) *
6997                 NUM_PORTS_PER_GRP * num_target_port_groups;
6998
6999         alloc_len = scsi_4btoul(cdb->length);
7000
7001         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7002
7003         ctsio->kern_sg_entries = 0;
7004
7005         if (total_len < alloc_len) {
7006                 ctsio->residual = alloc_len - total_len;
7007                 ctsio->kern_data_len = total_len;
7008                 ctsio->kern_total_len = total_len;
7009         } else {
7010                 ctsio->residual = 0;
7011                 ctsio->kern_data_len = alloc_len;
7012                 ctsio->kern_total_len = alloc_len;
7013         }
7014         ctsio->kern_data_resid = 0;
7015         ctsio->kern_rel_offset = 0;
7016
7017         rtg_ptr = (struct scsi_target_group_data *)ctsio->kern_data_ptr;
7018
7019         tpg_desc_ptr1 = &rtg_ptr->groups[0];
7020         tp_desc_ptr1_1 = &tpg_desc_ptr1->descriptors[0];
7021         tp_desc_ptr1_2 = (struct scsi_target_port_descriptor *)
7022                 &tp_desc_ptr1_1->desc_list[0];
7023
7024         if (single == 0) {
7025                 tpg_desc_ptr2 = (struct scsi_target_port_group_descriptor *)
7026                         &tp_desc_ptr1_2->desc_list[0];
7027                 tp_desc_ptr2_1 = &tpg_desc_ptr2->descriptors[0];
7028                 tp_desc_ptr2_2 = (struct scsi_target_port_descriptor *)
7029                         &tp_desc_ptr2_1->desc_list[0];
7030         } else {
7031                 tpg_desc_ptr2 = NULL;
7032                 tp_desc_ptr2_1 = NULL;
7033                 tp_desc_ptr2_2 = NULL;
7034         }
7035
7036         scsi_ulto4b(total_len - 4, rtg_ptr->length);
7037         if (single == 0) {
7038                 if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7039                         if (lun->flags & CTL_LUN_PRIMARY_SC) {
7040                                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7041                                 tpg_desc_ptr2->pref_state =
7042                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7043                         } else {
7044                                 tpg_desc_ptr1->pref_state =
7045                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7046                                 tpg_desc_ptr2->pref_state = TPG_PRIMARY;
7047                         }
7048                 } else {
7049                         if (lun->flags & CTL_LUN_PRIMARY_SC) {
7050                                 tpg_desc_ptr1->pref_state =
7051                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7052                                 tpg_desc_ptr2->pref_state = TPG_PRIMARY;
7053                         } else {
7054                                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7055                                 tpg_desc_ptr2->pref_state =
7056                                         TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7057                         }
7058                 }
7059         } else {
7060                 tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7061         }
7062         tpg_desc_ptr1->support = 0;
7063         tpg_desc_ptr1->target_port_group[1] = 1;
7064         tpg_desc_ptr1->status = TPG_IMPLICIT;
7065         tpg_desc_ptr1->target_port_count= NUM_PORTS_PER_GRP;
7066
7067         if (single == 0) {
7068                 tpg_desc_ptr2->support = 0;
7069                 tpg_desc_ptr2->target_port_group[1] = 2;
7070                 tpg_desc_ptr2->status = TPG_IMPLICIT;
7071                 tpg_desc_ptr2->target_port_count = NUM_PORTS_PER_GRP;
7072
7073                 tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7074                 tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7075
7076                 tp_desc_ptr2_1->relative_target_port_identifier[1] = 9;
7077                 tp_desc_ptr2_2->relative_target_port_identifier[1] = 10;
7078         } else {
7079                 if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7080                         tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7081                         tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7082                 } else {
7083                         tp_desc_ptr1_1->relative_target_port_identifier[1] = 9;
7084                         tp_desc_ptr1_2->relative_target_port_identifier[1] = 10;
7085                 }
7086         }
7087
7088         ctsio->be_move_done = ctl_config_move_done;
7089
7090         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7091                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7092                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7093                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7094                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7095
7096         ctl_datamove((union ctl_io *)ctsio);
7097         return(retval);
7098 }
7099
7100 int
7101 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7102 {
7103         struct scsi_per_res_in *cdb;
7104         int alloc_len, total_len = 0;
7105         /* struct scsi_per_res_in_rsrv in_data; */
7106         struct ctl_lun *lun;
7107         struct ctl_softc *softc;
7108
7109         CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7110
7111         softc = control_softc;
7112
7113         cdb = (struct scsi_per_res_in *)ctsio->cdb;
7114
7115         alloc_len = scsi_2btoul(cdb->length);
7116
7117         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7118
7119 retry:
7120         mtx_lock(&softc->ctl_lock);
7121         switch (cdb->action) {
7122         case SPRI_RK: /* read keys */
7123                 total_len = sizeof(struct scsi_per_res_in_keys) +
7124                         lun->pr_key_count *
7125                         sizeof(struct scsi_per_res_key);
7126                 break;
7127         case SPRI_RR: /* read reservation */
7128                 if (lun->flags & CTL_LUN_PR_RESERVED)
7129                         total_len = sizeof(struct scsi_per_res_in_rsrv);
7130                 else
7131                         total_len = sizeof(struct scsi_per_res_in_header);
7132                 break;
7133         case SPRI_RC: /* report capabilities */
7134                 total_len = sizeof(struct scsi_per_res_cap);
7135                 break;
7136         case SPRI_RS: /* read full status */
7137         default:
7138                 mtx_unlock(&softc->ctl_lock);
7139                 ctl_set_invalid_field(ctsio,
7140                                       /*sks_valid*/ 1,
7141                                       /*command*/ 1,
7142                                       /*field*/ 1,
7143                                       /*bit_valid*/ 1,
7144                                       /*bit*/ 0);
7145                 ctl_done((union ctl_io *)ctsio);
7146                 return (CTL_RETVAL_COMPLETE);
7147                 break; /* NOTREACHED */
7148         }
7149         mtx_unlock(&softc->ctl_lock);
7150
7151         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7152
7153         if (total_len < alloc_len) {
7154                 ctsio->residual = alloc_len - total_len;
7155                 ctsio->kern_data_len = total_len;
7156                 ctsio->kern_total_len = total_len;
7157         } else {
7158                 ctsio->residual = 0;
7159                 ctsio->kern_data_len = alloc_len;
7160                 ctsio->kern_total_len = alloc_len;
7161         }
7162
7163         ctsio->kern_data_resid = 0;
7164         ctsio->kern_rel_offset = 0;
7165         ctsio->kern_sg_entries = 0;
7166
7167         mtx_lock(&softc->ctl_lock);
7168         switch (cdb->action) {
7169         case SPRI_RK: { // read keys
7170         struct scsi_per_res_in_keys *res_keys;
7171                 int i, key_count;
7172
7173                 res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7174
7175                 /*
7176                  * We had to drop the lock to allocate our buffer, which
7177                  * leaves time for someone to come in with another
7178                  * persistent reservation.  (That is unlikely, though,
7179                  * since this should be the only persistent reservation
7180                  * command active right now.)
7181                  */
7182                 if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7183                     (lun->pr_key_count *
7184                      sizeof(struct scsi_per_res_key)))){
7185                         mtx_unlock(&softc->ctl_lock);
7186                         free(ctsio->kern_data_ptr, M_CTL);
7187                         printf("%s: reservation length changed, retrying\n",
7188                                __func__);
7189                         goto retry;
7190                 }
7191
7192                 scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7193
7194                 scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7195                              lun->pr_key_count, res_keys->header.length);
7196
7197                 for (i = 0, key_count = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7198                         if (!lun->per_res[i].registered)
7199                                 continue;
7200
7201                         /*
7202                          * We used lun->pr_key_count to calculate the
7203                          * size to allocate.  If it turns out the number of
7204                          * initiators with the registered flag set is
7205                          * larger than that (i.e. they haven't been kept in
7206                          * sync), we've got a problem.
7207                          */
7208                         if (key_count >= lun->pr_key_count) {
7209 #ifdef NEEDTOPORT
7210                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
7211                                             CTL_PR_ERROR,
7212                                             csevent_LogType_Fault,
7213                                             csevent_AlertLevel_Yellow,
7214                                             csevent_FRU_ShelfController,
7215                                             csevent_FRU_Firmware,
7216                                         csevent_FRU_Unknown,
7217                                             "registered keys %d >= key "
7218                                             "count %d", key_count,
7219                                             lun->pr_key_count);
7220 #endif
7221                                 key_count++;
7222                                 continue;
7223                         }
7224                         memcpy(res_keys->keys[key_count].key,
7225                                lun->per_res[i].res_key.key,
7226                                ctl_min(sizeof(res_keys->keys[key_count].key),
7227                                sizeof(lun->per_res[i].res_key)));
7228                         key_count++;
7229                 }
7230                 break;
7231         }
7232         case SPRI_RR: { // read reservation
7233                 struct scsi_per_res_in_rsrv *res;
7234                 int tmp_len, header_only;
7235
7236                 res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7237
7238                 scsi_ulto4b(lun->PRGeneration, res->header.generation);
7239
7240                 if (lun->flags & CTL_LUN_PR_RESERVED)
7241                 {
7242                         tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7243                         scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7244                                     res->header.length);
7245                         header_only = 0;
7246                 } else {
7247                         tmp_len = sizeof(struct scsi_per_res_in_header);
7248                         scsi_ulto4b(0, res->header.length);
7249                         header_only = 1;
7250                 }
7251
7252                 /*
7253                  * We had to drop the lock to allocate our buffer, which
7254                  * leaves time for someone to come in with another
7255                  * persistent reservation.  (That is unlikely, though,
7256                  * since this should be the only persistent reservation
7257                  * command active right now.)
7258                  */
7259                 if (tmp_len != total_len) {
7260                         mtx_unlock(&softc->ctl_lock);
7261                         free(ctsio->kern_data_ptr, M_CTL);
7262                         printf("%s: reservation status changed, retrying\n",
7263                                __func__);
7264                         goto retry;
7265                 }
7266
7267                 /*
7268                  * No reservation held, so we're done.
7269                  */
7270                 if (header_only != 0)
7271                         break;
7272
7273                 /*
7274                  * If the registration is an All Registrants type, the key
7275                  * is 0, since it doesn't really matter.
7276                  */
7277                 if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7278                         memcpy(res->data.reservation,
7279                                &lun->per_res[lun->pr_res_idx].res_key,
7280                                sizeof(struct scsi_per_res_key));
7281                 }
7282                 res->data.scopetype = lun->res_type;
7283                 break;
7284         }
7285         case SPRI_RC:     //report capabilities
7286         {
7287                 struct scsi_per_res_cap *res_cap;
7288                 uint16_t type_mask;
7289
7290                 res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7291                 scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7292                 res_cap->flags2 |= SPRI_TMV;
7293                 type_mask = SPRI_TM_WR_EX_AR |
7294                             SPRI_TM_EX_AC_RO |
7295                             SPRI_TM_WR_EX_RO |
7296                             SPRI_TM_EX_AC |
7297                             SPRI_TM_WR_EX |
7298                             SPRI_TM_EX_AC_AR;
7299                 scsi_ulto2b(type_mask, res_cap->type_mask);
7300                 break;
7301         }
7302         case SPRI_RS: //read full status
7303         default:
7304                 /*
7305                  * This is a bug, because we just checked for this above,
7306                  * and should have returned an error.
7307                  */
7308                 panic("Invalid PR type %x", cdb->action);
7309                 break; /* NOTREACHED */
7310         }
7311         mtx_unlock(&softc->ctl_lock);
7312
7313         ctsio->be_move_done = ctl_config_move_done;
7314
7315         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7316                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7317                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7318                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7319                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7320
7321         ctl_datamove((union ctl_io *)ctsio);
7322
7323         return (CTL_RETVAL_COMPLETE);
7324 }
7325
7326 /*
7327  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
7328  * it should return.
7329  */
7330 static int
7331 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
7332                 uint64_t sa_res_key, uint8_t type, uint32_t residx,
7333                 struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
7334                 struct scsi_per_res_out_parms* param)
7335 {
7336         union ctl_ha_msg persis_io;
7337         int retval, i;
7338         int isc_retval;
7339
7340         retval = 0;
7341
7342         if (sa_res_key == 0) {
7343                 mtx_lock(&softc->ctl_lock);
7344                 if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7345                         /* validate scope and type */
7346                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7347                              SPR_LU_SCOPE) {
7348                                 mtx_unlock(&softc->ctl_lock);
7349                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7350                                                       /*sks_valid*/ 1,
7351                                                       /*command*/ 1,
7352                                                       /*field*/ 2,
7353                                                       /*bit_valid*/ 1,
7354                                                       /*bit*/ 4);
7355                                 ctl_done((union ctl_io *)ctsio);
7356                                 return (1);
7357                         }
7358
7359                         if (type>8 || type==2 || type==4 || type==0) {
7360                                 mtx_unlock(&softc->ctl_lock);
7361                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7362                                                       /*sks_valid*/ 1,
7363                                                       /*command*/ 1,
7364                                                       /*field*/ 2,
7365                                                       /*bit_valid*/ 1,
7366                                                       /*bit*/ 0);
7367                                 ctl_done((union ctl_io *)ctsio);
7368                                 return (1);
7369                         }
7370
7371                         /* temporarily unregister this nexus */
7372                         lun->per_res[residx].registered = 0;
7373
7374                         /*
7375                          * Unregister everybody else and build UA for
7376                          * them
7377                          */
7378                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7379                                 if (lun->per_res[i].registered == 0)
7380                                         continue;
7381
7382                                 if (!persis_offset
7383                                  && i <CTL_MAX_INITIATORS)
7384                                         lun->pending_sense[i].ua_pending |=
7385                                                 CTL_UA_REG_PREEMPT;
7386                                 else if (persis_offset
7387                                       && i >= persis_offset)
7388                                         lun->pending_sense[i-persis_offset
7389                                                 ].ua_pending |=
7390                                                 CTL_UA_REG_PREEMPT;
7391                                 lun->per_res[i].registered = 0;
7392                                 memset(&lun->per_res[i].res_key, 0,
7393                                        sizeof(struct scsi_per_res_key));
7394                         }
7395                         lun->per_res[residx].registered = 1;
7396                         lun->pr_key_count = 1;
7397                         lun->res_type = type;
7398                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7399                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7400                                 lun->pr_res_idx = residx;
7401
7402                         mtx_unlock(&softc->ctl_lock);
7403                         /* send msg to other side */
7404                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7405                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7406                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7407                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7408                         persis_io.pr.pr_info.res_type = type;
7409                         memcpy(persis_io.pr.pr_info.sa_res_key,
7410                                param->serv_act_res_key,
7411                                sizeof(param->serv_act_res_key));
7412                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7413                              &persis_io, sizeof(persis_io), 0)) >
7414                              CTL_HA_STATUS_SUCCESS) {
7415                                 printf("CTL:Persis Out error returned "
7416                                        "from ctl_ha_msg_send %d\n",
7417                                        isc_retval);
7418                         }
7419                 } else {
7420                         /* not all registrants */
7421                         mtx_unlock(&softc->ctl_lock);
7422                         free(ctsio->kern_data_ptr, M_CTL);
7423                         ctl_set_invalid_field(ctsio,
7424                                               /*sks_valid*/ 1,
7425                                               /*command*/ 0,
7426                                               /*field*/ 8,
7427                                               /*bit_valid*/ 0,
7428                                               /*bit*/ 0);
7429                         ctl_done((union ctl_io *)ctsio);
7430                         return (1);
7431                 }
7432         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7433                 || !(lun->flags & CTL_LUN_PR_RESERVED)) {
7434                 int found = 0;
7435
7436                 mtx_lock(&softc->ctl_lock);
7437                 if (res_key == sa_res_key) {
7438                         /* special case */
7439                         /*
7440                          * The spec implies this is not good but doesn't
7441                          * say what to do. There are two choices either
7442                          * generate a res conflict or check condition
7443                          * with illegal field in parameter data. Since
7444                          * that is what is done when the sa_res_key is
7445                          * zero I'll take that approach since this has
7446                          * to do with the sa_res_key.
7447                          */
7448                         mtx_unlock(&softc->ctl_lock);
7449                         free(ctsio->kern_data_ptr, M_CTL);
7450                         ctl_set_invalid_field(ctsio,
7451                                               /*sks_valid*/ 1,
7452                                               /*command*/ 0,
7453                                               /*field*/ 8,
7454                                               /*bit_valid*/ 0,
7455                                               /*bit*/ 0);
7456                         ctl_done((union ctl_io *)ctsio);
7457                         return (1);
7458                 }
7459
7460                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7461                         if (lun->per_res[i].registered
7462                          && memcmp(param->serv_act_res_key,
7463                             lun->per_res[i].res_key.key,
7464                             sizeof(struct scsi_per_res_key)) != 0)
7465                                 continue;
7466
7467                         found = 1;
7468                         lun->per_res[i].registered = 0;
7469                         memset(&lun->per_res[i].res_key, 0,
7470                                sizeof(struct scsi_per_res_key));
7471                         lun->pr_key_count--;
7472
7473                         if (!persis_offset
7474                          && i < CTL_MAX_INITIATORS)
7475                                 lun->pending_sense[i].ua_pending |=
7476                                         CTL_UA_REG_PREEMPT;
7477                         else if (persis_offset
7478                               && i >= persis_offset)
7479                                 lun->pending_sense[i-persis_offset].ua_pending|=
7480                                         CTL_UA_REG_PREEMPT;
7481                 }
7482                 mtx_unlock(&softc->ctl_lock);
7483                 if (!found) {
7484                         free(ctsio->kern_data_ptr, M_CTL);
7485                         ctl_set_reservation_conflict(ctsio);
7486                         ctl_done((union ctl_io *)ctsio);
7487                         return (CTL_RETVAL_COMPLETE);
7488                 }
7489                 /* send msg to other side */
7490                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7491                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7492                 persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7493                 persis_io.pr.pr_info.residx = lun->pr_res_idx;
7494                 persis_io.pr.pr_info.res_type = type;
7495                 memcpy(persis_io.pr.pr_info.sa_res_key,
7496                        param->serv_act_res_key,
7497                        sizeof(param->serv_act_res_key));
7498                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7499                      &persis_io, sizeof(persis_io), 0)) >
7500                      CTL_HA_STATUS_SUCCESS) {
7501                         printf("CTL:Persis Out error returned from "
7502                                "ctl_ha_msg_send %d\n", isc_retval);
7503                 }
7504         } else {
7505                 /* Reserved but not all registrants */
7506                 /* sa_res_key is res holder */
7507                 if (memcmp(param->serv_act_res_key,
7508                    lun->per_res[lun->pr_res_idx].res_key.key,
7509                    sizeof(struct scsi_per_res_key)) == 0) {
7510                         /* validate scope and type */
7511                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7512                              SPR_LU_SCOPE) {
7513                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7514                                                       /*sks_valid*/ 1,
7515                                                       /*command*/ 1,
7516                                                       /*field*/ 2,
7517                                                       /*bit_valid*/ 1,
7518                                                       /*bit*/ 4);
7519                                 ctl_done((union ctl_io *)ctsio);
7520                                 return (1);
7521                         }
7522
7523                         if (type>8 || type==2 || type==4 || type==0) {
7524                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7525                                                       /*sks_valid*/ 1,
7526                                                       /*command*/ 1,
7527                                                       /*field*/ 2,
7528                                                       /*bit_valid*/ 1,
7529                                                       /*bit*/ 0);
7530                                 ctl_done((union ctl_io *)ctsio);
7531                                 return (1);
7532                         }
7533
7534                         /*
7535                          * Do the following:
7536                          * if sa_res_key != res_key remove all
7537                          * registrants w/sa_res_key and generate UA
7538                          * for these registrants(Registrations
7539                          * Preempted) if it wasn't an exclusive
7540                          * reservation generate UA(Reservations
7541                          * Preempted) for all other registered nexuses
7542                          * if the type has changed. Establish the new
7543                          * reservation and holder. If res_key and
7544                          * sa_res_key are the same do the above
7545                          * except don't unregister the res holder.
7546                          */
7547
7548                         /*
7549                          * Temporarily unregister so it won't get
7550                          * removed or UA generated
7551                          */
7552                         lun->per_res[residx].registered = 0;
7553                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7554                                 if (lun->per_res[i].registered == 0)
7555                                         continue;
7556
7557                                 if (memcmp(param->serv_act_res_key,
7558                                     lun->per_res[i].res_key.key,
7559                                     sizeof(struct scsi_per_res_key)) == 0) {
7560                                         lun->per_res[i].registered = 0;
7561                                         memset(&lun->per_res[i].res_key,
7562                                                0,
7563                                                sizeof(struct scsi_per_res_key));
7564                                         lun->pr_key_count--;
7565
7566                                         if (!persis_offset
7567                                          && i < CTL_MAX_INITIATORS)
7568                                                 lun->pending_sense[i
7569                                                         ].ua_pending |=
7570                                                         CTL_UA_REG_PREEMPT;
7571                                         else if (persis_offset
7572                                               && i >= persis_offset)
7573                                                 lun->pending_sense[
7574                                                   i-persis_offset].ua_pending |=
7575                                                   CTL_UA_REG_PREEMPT;
7576                                 } else if (type != lun->res_type
7577                                         && (lun->res_type == SPR_TYPE_WR_EX_RO
7578                                          || lun->res_type ==SPR_TYPE_EX_AC_RO)){
7579                                                 if (!persis_offset
7580                                                  && i < CTL_MAX_INITIATORS)
7581                                                         lun->pending_sense[i
7582                                                         ].ua_pending |=
7583                                                         CTL_UA_RES_RELEASE;
7584                                                 else if (persis_offset
7585                                                       && i >= persis_offset)
7586                                                         lun->pending_sense[
7587                                                         i-persis_offset
7588                                                         ].ua_pending |=
7589                                                         CTL_UA_RES_RELEASE;
7590                                 }
7591                         }
7592                         lun->per_res[residx].registered = 1;
7593                         lun->res_type = type;
7594                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7595                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7596                                 lun->pr_res_idx = residx;
7597                         else
7598                                 lun->pr_res_idx =
7599                                         CTL_PR_ALL_REGISTRANTS;
7600
7601                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7602                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7603                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7604                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7605                         persis_io.pr.pr_info.res_type = type;
7606                         memcpy(persis_io.pr.pr_info.sa_res_key,
7607                                param->serv_act_res_key,
7608                                sizeof(param->serv_act_res_key));
7609                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7610                              &persis_io, sizeof(persis_io), 0)) >
7611                              CTL_HA_STATUS_SUCCESS) {
7612                                 printf("CTL:Persis Out error returned "
7613                                        "from ctl_ha_msg_send %d\n",
7614                                        isc_retval);
7615                         }
7616                 } else {
7617                         /*
7618                          * sa_res_key is not the res holder just
7619                          * remove registrants
7620                          */
7621                         int found=0;
7622                         mtx_lock(&softc->ctl_lock);
7623
7624                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7625                                 if (memcmp(param->serv_act_res_key,
7626                                     lun->per_res[i].res_key.key,
7627                                     sizeof(struct scsi_per_res_key)) != 0)
7628                                         continue;
7629
7630                                 found = 1;
7631                                 lun->per_res[i].registered = 0;
7632                                 memset(&lun->per_res[i].res_key, 0,
7633                                        sizeof(struct scsi_per_res_key));
7634                                 lun->pr_key_count--;
7635
7636                                 if (!persis_offset
7637                                  && i < CTL_MAX_INITIATORS)
7638                                         lun->pending_sense[i].ua_pending |=
7639                                                 CTL_UA_REG_PREEMPT;
7640                                 else if (persis_offset
7641                                       && i >= persis_offset)
7642                                         lun->pending_sense[
7643                                                 i-persis_offset].ua_pending |=
7644                                                 CTL_UA_REG_PREEMPT;
7645                         }
7646
7647                         if (!found) {
7648                                 mtx_unlock(&softc->ctl_lock);
7649                                 free(ctsio->kern_data_ptr, M_CTL);
7650                                 ctl_set_reservation_conflict(ctsio);
7651                                 ctl_done((union ctl_io *)ctsio);
7652                                 return (1);
7653                         }
7654                         mtx_unlock(&softc->ctl_lock);
7655                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7656                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7657                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7658                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7659                         persis_io.pr.pr_info.res_type = type;
7660                         memcpy(persis_io.pr.pr_info.sa_res_key,
7661                                param->serv_act_res_key,
7662                                sizeof(param->serv_act_res_key));
7663                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7664                              &persis_io, sizeof(persis_io), 0)) >
7665                              CTL_HA_STATUS_SUCCESS) {
7666                                 printf("CTL:Persis Out error returned "
7667                                        "from ctl_ha_msg_send %d\n",
7668                                 isc_retval);
7669                         }
7670                 }
7671         }
7672
7673         lun->PRGeneration++;
7674
7675         return (retval);
7676 }
7677
7678 static void
7679 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
7680 {
7681         int i;
7682
7683         if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7684          || lun->pr_res_idx == CTL_PR_NO_RESERVATION
7685          || memcmp(&lun->per_res[lun->pr_res_idx].res_key,
7686                    msg->pr.pr_info.sa_res_key,
7687                    sizeof(struct scsi_per_res_key)) != 0) {
7688                 uint64_t sa_res_key;
7689                 sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
7690
7691                 if (sa_res_key == 0) {
7692                         /* temporarily unregister this nexus */
7693                         lun->per_res[msg->pr.pr_info.residx].registered = 0;
7694
7695                         /*
7696                          * Unregister everybody else and build UA for
7697                          * them
7698                          */
7699                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7700                                 if (lun->per_res[i].registered == 0)
7701                                         continue;
7702
7703                                 if (!persis_offset
7704                                  && i < CTL_MAX_INITIATORS)
7705                                         lun->pending_sense[i].ua_pending |=
7706                                                 CTL_UA_REG_PREEMPT;
7707                                 else if (persis_offset && i >= persis_offset)
7708                                         lun->pending_sense[i -
7709                                                 persis_offset].ua_pending |=
7710                                                 CTL_UA_REG_PREEMPT;
7711                                 lun->per_res[i].registered = 0;
7712                                 memset(&lun->per_res[i].res_key, 0,
7713                                        sizeof(struct scsi_per_res_key));
7714                         }
7715
7716                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
7717                         lun->pr_key_count = 1;
7718                         lun->res_type = msg->pr.pr_info.res_type;
7719                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7720                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7721                                 lun->pr_res_idx = msg->pr.pr_info.residx;
7722                 } else {
7723                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7724                                 if (memcmp(msg->pr.pr_info.sa_res_key,
7725                                    lun->per_res[i].res_key.key,
7726                                    sizeof(struct scsi_per_res_key)) != 0)
7727                                         continue;
7728
7729                                 lun->per_res[i].registered = 0;
7730                                 memset(&lun->per_res[i].res_key, 0,
7731                                        sizeof(struct scsi_per_res_key));
7732                                 lun->pr_key_count--;
7733
7734                                 if (!persis_offset
7735                                  && i < persis_offset)
7736                                         lun->pending_sense[i].ua_pending |=
7737                                                 CTL_UA_REG_PREEMPT;
7738                                 else if (persis_offset
7739                                       && i >= persis_offset)
7740                                         lun->pending_sense[i -
7741                                                 persis_offset].ua_pending |=
7742                                                 CTL_UA_REG_PREEMPT;
7743                         }
7744                 }
7745         } else {
7746                 /*
7747                  * Temporarily unregister so it won't get removed
7748                  * or UA generated
7749                  */
7750                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
7751                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7752                         if (lun->per_res[i].registered == 0)
7753                                 continue;
7754
7755                         if (memcmp(msg->pr.pr_info.sa_res_key,
7756                            lun->per_res[i].res_key.key,
7757                            sizeof(struct scsi_per_res_key)) == 0) {
7758                                 lun->per_res[i].registered = 0;
7759                                 memset(&lun->per_res[i].res_key, 0,
7760                                        sizeof(struct scsi_per_res_key));
7761                                 lun->pr_key_count--;
7762                                 if (!persis_offset
7763                                  && i < CTL_MAX_INITIATORS)
7764                                         lun->pending_sense[i].ua_pending |=
7765                                                 CTL_UA_REG_PREEMPT;
7766                                 else if (persis_offset
7767                                       && i >= persis_offset)
7768                                         lun->pending_sense[i -
7769                                                 persis_offset].ua_pending |=
7770                                                 CTL_UA_REG_PREEMPT;
7771                         } else if (msg->pr.pr_info.res_type != lun->res_type
7772                                 && (lun->res_type == SPR_TYPE_WR_EX_RO
7773                                  || lun->res_type == SPR_TYPE_EX_AC_RO)) {
7774                                         if (!persis_offset
7775                                          && i < persis_offset)
7776                                                 lun->pending_sense[i
7777                                                         ].ua_pending |=
7778                                                         CTL_UA_RES_RELEASE;
7779                                         else if (persis_offset
7780                                               && i >= persis_offset)
7781                                         lun->pending_sense[i -
7782                                                 persis_offset].ua_pending |=
7783                                                 CTL_UA_RES_RELEASE;
7784                         }
7785                 }
7786                 lun->per_res[msg->pr.pr_info.residx].registered = 1;
7787                 lun->res_type = msg->pr.pr_info.res_type;
7788                 if (lun->res_type != SPR_TYPE_WR_EX_AR
7789                  && lun->res_type != SPR_TYPE_EX_AC_AR)
7790                         lun->pr_res_idx = msg->pr.pr_info.residx;
7791                 else
7792                         lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
7793         }
7794         lun->PRGeneration++;
7795
7796 }
7797
7798
7799 int
7800 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
7801 {
7802         int retval;
7803         int isc_retval;
7804         u_int32_t param_len;
7805         struct scsi_per_res_out *cdb;
7806         struct ctl_lun *lun;
7807         struct scsi_per_res_out_parms* param;
7808         struct ctl_softc *softc;
7809         uint32_t residx;
7810         uint64_t res_key, sa_res_key;
7811         uint8_t type;
7812         union ctl_ha_msg persis_io;
7813         int    i;
7814
7815         CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
7816
7817         retval = CTL_RETVAL_COMPLETE;
7818
7819         softc = control_softc;
7820
7821         cdb = (struct scsi_per_res_out *)ctsio->cdb;
7822         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7823
7824         /*
7825          * We only support whole-LUN scope.  The scope & type are ignored for
7826          * register, register and ignore existing key and clear.
7827          * We sometimes ignore scope and type on preempts too!!
7828          * Verify reservation type here as well.
7829          */
7830         type = cdb->scope_type & SPR_TYPE_MASK;
7831         if ((cdb->action == SPRO_RESERVE)
7832          || (cdb->action == SPRO_RELEASE)) {
7833                 if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
7834                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7835                                               /*sks_valid*/ 1,
7836                                               /*command*/ 1,
7837                                               /*field*/ 2,
7838                                               /*bit_valid*/ 1,
7839                                               /*bit*/ 4);
7840                         ctl_done((union ctl_io *)ctsio);
7841                         return (CTL_RETVAL_COMPLETE);
7842                 }
7843
7844                 if (type>8 || type==2 || type==4 || type==0) {
7845                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7846                                               /*sks_valid*/ 1,
7847                                               /*command*/ 1,
7848                                               /*field*/ 2,
7849                                               /*bit_valid*/ 1,
7850                                               /*bit*/ 0);
7851                         ctl_done((union ctl_io *)ctsio);
7852                         return (CTL_RETVAL_COMPLETE);
7853                 }
7854         }
7855
7856         switch (cdb->action & SPRO_ACTION_MASK) {
7857         case SPRO_REGISTER:
7858         case SPRO_RESERVE:
7859         case SPRO_RELEASE:
7860         case SPRO_CLEAR:
7861         case SPRO_PREEMPT:
7862         case SPRO_REG_IGNO:
7863                 break;
7864         case SPRO_REG_MOVE:
7865         case SPRO_PRE_ABO:
7866         default:
7867                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7868                                       /*sks_valid*/ 1,
7869                                       /*command*/ 1,
7870                                       /*field*/ 1,
7871                                       /*bit_valid*/ 1,
7872                                       /*bit*/ 0);
7873                 ctl_done((union ctl_io *)ctsio);
7874                 return (CTL_RETVAL_COMPLETE);
7875                 break; /* NOTREACHED */
7876         }
7877
7878         param_len = scsi_4btoul(cdb->length);
7879
7880         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
7881                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
7882                 ctsio->kern_data_len = param_len;
7883                 ctsio->kern_total_len = param_len;
7884                 ctsio->kern_data_resid = 0;
7885                 ctsio->kern_rel_offset = 0;
7886                 ctsio->kern_sg_entries = 0;
7887                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7888                 ctsio->be_move_done = ctl_config_move_done;
7889                 ctl_datamove((union ctl_io *)ctsio);
7890
7891                 return (CTL_RETVAL_COMPLETE);
7892         }
7893
7894         param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
7895
7896         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
7897         res_key = scsi_8btou64(param->res_key.key);
7898         sa_res_key = scsi_8btou64(param->serv_act_res_key);
7899
7900         /*
7901          * Validate the reservation key here except for SPRO_REG_IGNO
7902          * This must be done for all other service actions
7903          */
7904         if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
7905                 mtx_lock(&softc->ctl_lock);
7906                 if (lun->per_res[residx].registered) {
7907                     if (memcmp(param->res_key.key,
7908                                lun->per_res[residx].res_key.key,
7909                                ctl_min(sizeof(param->res_key),
7910                                sizeof(lun->per_res[residx].res_key))) != 0) {
7911                                 /*
7912                                  * The current key passed in doesn't match
7913                                  * the one the initiator previously
7914                                  * registered.
7915                                  */
7916                                 mtx_unlock(&softc->ctl_lock);
7917                                 free(ctsio->kern_data_ptr, M_CTL);
7918                                 ctl_set_reservation_conflict(ctsio);
7919                                 ctl_done((union ctl_io *)ctsio);
7920                                 return (CTL_RETVAL_COMPLETE);
7921                         }
7922                 } else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
7923                         /*
7924                          * We are not registered
7925                          */
7926                         mtx_unlock(&softc->ctl_lock);
7927                         free(ctsio->kern_data_ptr, M_CTL);
7928                         ctl_set_reservation_conflict(ctsio);
7929                         ctl_done((union ctl_io *)ctsio);
7930                         return (CTL_RETVAL_COMPLETE);
7931                 } else if (res_key != 0) {
7932                         /*
7933                          * We are not registered and trying to register but
7934                          * the register key isn't zero.
7935                          */
7936                         mtx_unlock(&softc->ctl_lock);
7937                         free(ctsio->kern_data_ptr, M_CTL);
7938                         ctl_set_reservation_conflict(ctsio);
7939                         ctl_done((union ctl_io *)ctsio);
7940                         return (CTL_RETVAL_COMPLETE);
7941                 }
7942                 mtx_unlock(&softc->ctl_lock);
7943         }
7944
7945         switch (cdb->action & SPRO_ACTION_MASK) {
7946         case SPRO_REGISTER:
7947         case SPRO_REG_IGNO: {
7948
7949 #if 0
7950                 printf("Registration received\n");
7951 #endif
7952
7953                 /*
7954                  * We don't support any of these options, as we report in
7955                  * the read capabilities request (see
7956                  * ctl_persistent_reserve_in(), above).
7957                  */
7958                 if ((param->flags & SPR_SPEC_I_PT)
7959                  || (param->flags & SPR_ALL_TG_PT)
7960                  || (param->flags & SPR_APTPL)) {
7961                         int bit_ptr;
7962
7963                         if (param->flags & SPR_APTPL)
7964                                 bit_ptr = 0;
7965                         else if (param->flags & SPR_ALL_TG_PT)
7966                                 bit_ptr = 2;
7967                         else /* SPR_SPEC_I_PT */
7968                                 bit_ptr = 3;
7969
7970                         free(ctsio->kern_data_ptr, M_CTL);
7971                         ctl_set_invalid_field(ctsio,
7972                                               /*sks_valid*/ 1,
7973                                               /*command*/ 0,
7974                                               /*field*/ 20,
7975                                               /*bit_valid*/ 1,
7976                                               /*bit*/ bit_ptr);
7977                         ctl_done((union ctl_io *)ctsio);
7978                         return (CTL_RETVAL_COMPLETE);
7979                 }
7980
7981                 mtx_lock(&softc->ctl_lock);
7982
7983                 /*
7984                  * The initiator wants to clear the
7985                  * key/unregister.
7986                  */
7987                 if (sa_res_key == 0) {
7988                         if ((res_key == 0
7989                           && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
7990                          || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
7991                           && !lun->per_res[residx].registered)) {
7992                                 mtx_unlock(&softc->ctl_lock);
7993                                 goto done;
7994                         }
7995
7996                         lun->per_res[residx].registered = 0;
7997                         memset(&lun->per_res[residx].res_key,
7998                                0, sizeof(lun->per_res[residx].res_key));
7999                         lun->pr_key_count--;
8000
8001                         if (residx == lun->pr_res_idx) {
8002                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8003                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8004
8005                                 if ((lun->res_type == SPR_TYPE_WR_EX_RO
8006                                   || lun->res_type == SPR_TYPE_EX_AC_RO)
8007                                  && lun->pr_key_count) {
8008                                         /*
8009                                          * If the reservation is a registrants
8010                                          * only type we need to generate a UA
8011                                          * for other registered inits.  The
8012                                          * sense code should be RESERVATIONS
8013                                          * RELEASED
8014                                          */
8015
8016                                         for (i = 0; i < CTL_MAX_INITIATORS;i++){
8017                                                 if (lun->per_res[
8018                                                     i+persis_offset].registered
8019                                                     == 0)
8020                                                         continue;
8021                                                 lun->pending_sense[i
8022                                                         ].ua_pending |=
8023                                                         CTL_UA_RES_RELEASE;
8024                                         }
8025                                 }
8026                                 lun->res_type = 0;
8027                         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8028                                 if (lun->pr_key_count==0) {
8029                                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8030                                         lun->res_type = 0;
8031                                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8032                                 }
8033                         }
8034                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8035                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8036                         persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8037                         persis_io.pr.pr_info.residx = residx;
8038                         if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8039                              &persis_io, sizeof(persis_io), 0 )) >
8040                              CTL_HA_STATUS_SUCCESS) {
8041                                 printf("CTL:Persis Out error returned from "
8042                                        "ctl_ha_msg_send %d\n", isc_retval);
8043                         }
8044                         mtx_unlock(&softc->ctl_lock);
8045                 } else /* sa_res_key != 0 */ {
8046
8047                         /*
8048                          * If we aren't registered currently then increment
8049                          * the key count and set the registered flag.
8050                          */
8051                         if (!lun->per_res[residx].registered) {
8052                                 lun->pr_key_count++;
8053                                 lun->per_res[residx].registered = 1;
8054                         }
8055
8056                         memcpy(&lun->per_res[residx].res_key,
8057                                param->serv_act_res_key,
8058                                ctl_min(sizeof(param->serv_act_res_key),
8059                                sizeof(lun->per_res[residx].res_key)));
8060
8061                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8062                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8063                         persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8064                         persis_io.pr.pr_info.residx = residx;
8065                         memcpy(persis_io.pr.pr_info.sa_res_key,
8066                                param->serv_act_res_key,
8067                                sizeof(param->serv_act_res_key));
8068                         mtx_unlock(&softc->ctl_lock);
8069                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8070                              &persis_io, sizeof(persis_io), 0)) >
8071                              CTL_HA_STATUS_SUCCESS) {
8072                                 printf("CTL:Persis Out error returned from "
8073                                        "ctl_ha_msg_send %d\n", isc_retval);
8074                         }
8075                 }
8076                 lun->PRGeneration++;
8077
8078                 break;
8079         }
8080         case SPRO_RESERVE:
8081 #if 0
8082                 printf("Reserve executed type %d\n", type);
8083 #endif
8084                 mtx_lock(&softc->ctl_lock);
8085                 if (lun->flags & CTL_LUN_PR_RESERVED) {
8086                         /*
8087                          * if this isn't the reservation holder and it's
8088                          * not a "all registrants" type or if the type is
8089                          * different then we have a conflict
8090                          */
8091                         if ((lun->pr_res_idx != residx
8092                           && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8093                          || lun->res_type != type) {
8094                                 mtx_unlock(&softc->ctl_lock);
8095                                 free(ctsio->kern_data_ptr, M_CTL);
8096                                 ctl_set_reservation_conflict(ctsio);
8097                                 ctl_done((union ctl_io *)ctsio);
8098                                 return (CTL_RETVAL_COMPLETE);
8099                         }
8100                         mtx_unlock(&softc->ctl_lock);
8101                 } else /* create a reservation */ {
8102                         /*
8103                          * If it's not an "all registrants" type record
8104                          * reservation holder
8105                          */
8106                         if (type != SPR_TYPE_WR_EX_AR
8107                          && type != SPR_TYPE_EX_AC_AR)
8108                                 lun->pr_res_idx = residx; /* Res holder */
8109                         else
8110                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8111
8112                         lun->flags |= CTL_LUN_PR_RESERVED;
8113                         lun->res_type = type;
8114
8115                         mtx_unlock(&softc->ctl_lock);
8116
8117                         /* send msg to other side */
8118                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8119                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8120                         persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8121                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8122                         persis_io.pr.pr_info.res_type = type;
8123                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8124                              &persis_io, sizeof(persis_io), 0)) >
8125                              CTL_HA_STATUS_SUCCESS) {
8126                                 printf("CTL:Persis Out error returned from "
8127                                        "ctl_ha_msg_send %d\n", isc_retval);
8128                         }
8129                 }
8130                 break;
8131
8132         case SPRO_RELEASE:
8133                 mtx_lock(&softc->ctl_lock);
8134                 if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8135                         /* No reservation exists return good status */
8136                         mtx_unlock(&softc->ctl_lock);
8137                         goto done;
8138                 }
8139                 /*
8140                  * Is this nexus a reservation holder?
8141                  */
8142                 if (lun->pr_res_idx != residx
8143                  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8144                         /*
8145                          * not a res holder return good status but
8146                          * do nothing
8147                          */
8148                         mtx_unlock(&softc->ctl_lock);
8149                         goto done;
8150                 }
8151
8152                 if (lun->res_type != type) {
8153                         mtx_unlock(&softc->ctl_lock);
8154                         free(ctsio->kern_data_ptr, M_CTL);
8155                         ctl_set_illegal_pr_release(ctsio);
8156                         ctl_done((union ctl_io *)ctsio);
8157                         return (CTL_RETVAL_COMPLETE);
8158                 }
8159
8160                 /* okay to release */
8161                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8162                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8163                 lun->res_type = 0;
8164
8165                 /*
8166                  * if this isn't an exclusive access
8167                  * res generate UA for all other
8168                  * registrants.
8169                  */
8170                 if (type != SPR_TYPE_EX_AC
8171                  && type != SPR_TYPE_WR_EX) {
8172                         /*
8173                          * temporarily unregister so we don't generate UA
8174                          */
8175                         lun->per_res[residx].registered = 0;
8176
8177                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8178                                 if (lun->per_res[i+persis_offset].registered
8179                                     == 0)
8180                                         continue;
8181                                 lun->pending_sense[i].ua_pending |=
8182                                         CTL_UA_RES_RELEASE;
8183                         }
8184
8185                         lun->per_res[residx].registered = 1;
8186                 }
8187                 mtx_unlock(&softc->ctl_lock);
8188                 /* Send msg to other side */
8189                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8190                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8191                 persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8192                 if ((isc_retval=ctl_ha_msg_send( CTL_HA_CHAN_CTL, &persis_io,
8193                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8194                         printf("CTL:Persis Out error returned from "
8195                                "ctl_ha_msg_send %d\n", isc_retval);
8196                 }
8197                 break;
8198
8199         case SPRO_CLEAR:
8200                 /* send msg to other side */
8201
8202                 mtx_lock(&softc->ctl_lock);
8203                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8204                 lun->res_type = 0;
8205                 lun->pr_key_count = 0;
8206                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8207
8208
8209                 memset(&lun->per_res[residx].res_key,
8210                        0, sizeof(lun->per_res[residx].res_key));
8211                 lun->per_res[residx].registered = 0;
8212
8213                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++)
8214                         if (lun->per_res[i].registered) {
8215                                 if (!persis_offset && i < CTL_MAX_INITIATORS)
8216                                         lun->pending_sense[i].ua_pending |=
8217                                                 CTL_UA_RES_PREEMPT;
8218                                 else if (persis_offset && i >= persis_offset)
8219                                         lun->pending_sense[i-persis_offset
8220                                             ].ua_pending |= CTL_UA_RES_PREEMPT;
8221
8222                                 memset(&lun->per_res[i].res_key,
8223                                        0, sizeof(struct scsi_per_res_key));
8224                                 lun->per_res[i].registered = 0;
8225                         }
8226                 lun->PRGeneration++;
8227                 mtx_unlock(&softc->ctl_lock);
8228                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8229                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8230                 persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8231                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8232                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8233                         printf("CTL:Persis Out error returned from "
8234                                "ctl_ha_msg_send %d\n", isc_retval);
8235                 }
8236                 break;
8237
8238         case SPRO_PREEMPT: {
8239                 int nretval;
8240
8241                 nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8242                                           residx, ctsio, cdb, param);
8243                 if (nretval != 0)
8244                         return (CTL_RETVAL_COMPLETE);
8245                 break;
8246         }
8247         case SPRO_REG_MOVE:
8248         case SPRO_PRE_ABO:
8249         default:
8250                 free(ctsio->kern_data_ptr, M_CTL);
8251                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8252                                       /*sks_valid*/ 1,
8253                                       /*command*/ 1,
8254                                       /*field*/ 1,
8255                                       /*bit_valid*/ 1,
8256                                       /*bit*/ 0);
8257                 ctl_done((union ctl_io *)ctsio);
8258                 return (CTL_RETVAL_COMPLETE);
8259                 break; /* NOTREACHED */
8260         }
8261
8262 done:
8263         free(ctsio->kern_data_ptr, M_CTL);
8264         ctl_set_success(ctsio);
8265         ctl_done((union ctl_io *)ctsio);
8266
8267         return (retval);
8268 }
8269
8270 /*
8271  * This routine is for handling a message from the other SC pertaining to
8272  * persistent reserve out. All the error checking will have been done
8273  * so only perorming the action need be done here to keep the two
8274  * in sync.
8275  */
8276 static void
8277 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8278 {
8279         struct ctl_lun *lun;
8280         struct ctl_softc *softc;
8281         int i;
8282         uint32_t targ_lun;
8283
8284         softc = control_softc;
8285
8286         mtx_lock(&softc->ctl_lock);
8287
8288         targ_lun = msg->hdr.nexus.targ_lun;
8289         if (msg->hdr.nexus.lun_map_fn != NULL)
8290                 targ_lun = msg->hdr.nexus.lun_map_fn(msg->hdr.nexus.lun_map_arg, targ_lun);
8291         lun = softc->ctl_luns[targ_lun];
8292         switch(msg->pr.pr_info.action) {
8293         case CTL_PR_REG_KEY:
8294                 if (!lun->per_res[msg->pr.pr_info.residx].registered) {
8295                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
8296                         lun->pr_key_count++;
8297                 }
8298                 lun->PRGeneration++;
8299                 memcpy(&lun->per_res[msg->pr.pr_info.residx].res_key,
8300                        msg->pr.pr_info.sa_res_key,
8301                        sizeof(struct scsi_per_res_key));
8302                 break;
8303
8304         case CTL_PR_UNREG_KEY:
8305                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
8306                 memset(&lun->per_res[msg->pr.pr_info.residx].res_key,
8307                        0, sizeof(struct scsi_per_res_key));
8308                 lun->pr_key_count--;
8309
8310                 /* XXX Need to see if the reservation has been released */
8311                 /* if so do we need to generate UA? */
8312                 if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8313                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8314                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8315
8316                         if ((lun->res_type == SPR_TYPE_WR_EX_RO
8317                           || lun->res_type == SPR_TYPE_EX_AC_RO)
8318                          && lun->pr_key_count) {
8319                                 /*
8320                                  * If the reservation is a registrants
8321                                  * only type we need to generate a UA
8322                                  * for other registered inits.  The
8323                                  * sense code should be RESERVATIONS
8324                                  * RELEASED
8325                                  */
8326
8327                                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8328                                         if (lun->per_res[i+
8329                                             persis_offset].registered == 0)
8330                                                 continue;
8331
8332                                         lun->pending_sense[i
8333                                                 ].ua_pending |=
8334                                                 CTL_UA_RES_RELEASE;
8335                                 }
8336                         }
8337                         lun->res_type = 0;
8338                 } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8339                         if (lun->pr_key_count==0) {
8340                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8341                                 lun->res_type = 0;
8342                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8343                         }
8344                 }
8345                 lun->PRGeneration++;
8346                 break;
8347
8348         case CTL_PR_RESERVE:
8349                 lun->flags |= CTL_LUN_PR_RESERVED;
8350                 lun->res_type = msg->pr.pr_info.res_type;
8351                 lun->pr_res_idx = msg->pr.pr_info.residx;
8352
8353                 break;
8354
8355         case CTL_PR_RELEASE:
8356                 /*
8357                  * if this isn't an exclusive access res generate UA for all
8358                  * other registrants.
8359                  */
8360                 if (lun->res_type != SPR_TYPE_EX_AC
8361                  && lun->res_type != SPR_TYPE_WR_EX) {
8362                         for (i = 0; i < CTL_MAX_INITIATORS; i++)
8363                                 if (lun->per_res[i+persis_offset].registered)
8364                                         lun->pending_sense[i].ua_pending |=
8365                                                 CTL_UA_RES_RELEASE;
8366                 }
8367
8368                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8369                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8370                 lun->res_type = 0;
8371                 break;
8372
8373         case CTL_PR_PREEMPT:
8374                 ctl_pro_preempt_other(lun, msg);
8375                 break;
8376         case CTL_PR_CLEAR:
8377                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8378                 lun->res_type = 0;
8379                 lun->pr_key_count = 0;
8380                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8381
8382                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8383                         if (lun->per_res[i].registered == 0)
8384                                 continue;
8385                         if (!persis_offset
8386                          && i < CTL_MAX_INITIATORS)
8387                                 lun->pending_sense[i].ua_pending |=
8388                                         CTL_UA_RES_PREEMPT;
8389                         else if (persis_offset
8390                               && i >= persis_offset)
8391                                 lun->pending_sense[i-persis_offset].ua_pending|=
8392                                         CTL_UA_RES_PREEMPT;
8393                         memset(&lun->per_res[i].res_key, 0,
8394                                sizeof(struct scsi_per_res_key));
8395                         lun->per_res[i].registered = 0;
8396                 }
8397                 lun->PRGeneration++;
8398                 break;
8399         }
8400
8401         mtx_unlock(&softc->ctl_lock);
8402 }
8403
8404 int
8405 ctl_read_write(struct ctl_scsiio *ctsio)
8406 {
8407         struct ctl_lun *lun;
8408         struct ctl_lba_len lbalen;
8409         uint64_t lba;
8410         uint32_t num_blocks;
8411         int reladdr, fua, dpo, ebp;
8412         int retval;
8413         int isread;
8414
8415         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8416
8417         CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8418
8419         reladdr = 0;
8420         fua = 0;
8421         dpo = 0;
8422         ebp = 0;
8423
8424         retval = CTL_RETVAL_COMPLETE;
8425
8426         isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8427               || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8428         if (lun->flags & CTL_LUN_PR_RESERVED && isread) {
8429                 uint32_t residx;
8430
8431                 /*
8432                  * XXX KDM need a lock here.
8433                  */
8434                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8435                 if ((lun->res_type == SPR_TYPE_EX_AC
8436                   && residx != lun->pr_res_idx)
8437                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
8438                    || lun->res_type == SPR_TYPE_EX_AC_AR)
8439                   && !lun->per_res[residx].registered)) {
8440                         ctl_set_reservation_conflict(ctsio);
8441                         ctl_done((union ctl_io *)ctsio);
8442                         return (CTL_RETVAL_COMPLETE);
8443                 }
8444         }
8445
8446         switch (ctsio->cdb[0]) {
8447         case READ_6:
8448         case WRITE_6: {
8449                 struct scsi_rw_6 *cdb;
8450
8451                 cdb = (struct scsi_rw_6 *)ctsio->cdb;
8452
8453                 lba = scsi_3btoul(cdb->addr);
8454                 /* only 5 bits are valid in the most significant address byte */
8455                 lba &= 0x1fffff;
8456                 num_blocks = cdb->length;
8457                 /*
8458                  * This is correct according to SBC-2.
8459                  */
8460                 if (num_blocks == 0)
8461                         num_blocks = 256;
8462                 break;
8463         }
8464         case READ_10:
8465         case WRITE_10: {
8466                 struct scsi_rw_10 *cdb;
8467
8468                 cdb = (struct scsi_rw_10 *)ctsio->cdb;
8469
8470                 if (cdb->byte2 & SRW10_RELADDR)
8471                         reladdr = 1;
8472                 if (cdb->byte2 & SRW10_FUA)
8473                         fua = 1;
8474                 if (cdb->byte2 & SRW10_DPO)
8475                         dpo = 1;
8476
8477                 if ((cdb->opcode == WRITE_10)
8478                  && (cdb->byte2 & SRW10_EBP))
8479                         ebp = 1;
8480
8481                 lba = scsi_4btoul(cdb->addr);
8482                 num_blocks = scsi_2btoul(cdb->length);
8483                 break;
8484         }
8485         case WRITE_VERIFY_10: {
8486                 struct scsi_write_verify_10 *cdb;
8487
8488                 cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
8489
8490                 /*
8491                  * XXX KDM we should do actual write verify support at some
8492                  * point.  This is obviously fake, we're just translating
8493                  * things to a write.  So we don't even bother checking the
8494                  * BYTCHK field, since we don't do any verification.  If
8495                  * the user asks for it, we'll just pretend we did it.
8496                  */
8497                 if (cdb->byte2 & SWV_DPO)
8498                         dpo = 1;
8499
8500                 lba = scsi_4btoul(cdb->addr);
8501                 num_blocks = scsi_2btoul(cdb->length);
8502                 break;
8503         }
8504         case READ_12:
8505         case WRITE_12: {
8506                 struct scsi_rw_12 *cdb;
8507
8508                 cdb = (struct scsi_rw_12 *)ctsio->cdb;
8509
8510                 if (cdb->byte2 & SRW12_RELADDR)
8511                         reladdr = 1;
8512                 if (cdb->byte2 & SRW12_FUA)
8513                         fua = 1;
8514                 if (cdb->byte2 & SRW12_DPO)
8515                         dpo = 1;
8516                 lba = scsi_4btoul(cdb->addr);
8517                 num_blocks = scsi_4btoul(cdb->length);
8518                 break;
8519         }
8520         case WRITE_VERIFY_12: {
8521                 struct scsi_write_verify_12 *cdb;
8522
8523                 cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
8524
8525                 if (cdb->byte2 & SWV_DPO)
8526                         dpo = 1;
8527
8528                 lba = scsi_4btoul(cdb->addr);
8529                 num_blocks = scsi_4btoul(cdb->length);
8530
8531                 break;
8532         }
8533         case READ_16:
8534         case WRITE_16: {
8535                 struct scsi_rw_16 *cdb;
8536
8537                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
8538
8539                 if (cdb->byte2 & SRW12_RELADDR)
8540                         reladdr = 1;
8541                 if (cdb->byte2 & SRW12_FUA)
8542                         fua = 1;
8543                 if (cdb->byte2 & SRW12_DPO)
8544                         dpo = 1;
8545
8546                 lba = scsi_8btou64(cdb->addr);
8547                 num_blocks = scsi_4btoul(cdb->length);
8548                 break;
8549         }
8550         case WRITE_VERIFY_16: {
8551                 struct scsi_write_verify_16 *cdb;
8552
8553                 cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
8554
8555                 if (cdb->byte2 & SWV_DPO)
8556                         dpo = 1;
8557
8558                 lba = scsi_8btou64(cdb->addr);
8559                 num_blocks = scsi_4btoul(cdb->length);
8560                 break;
8561         }
8562         default:
8563                 /*
8564                  * We got a command we don't support.  This shouldn't
8565                  * happen, commands should be filtered out above us.
8566                  */
8567                 ctl_set_invalid_opcode(ctsio);
8568                 ctl_done((union ctl_io *)ctsio);
8569
8570                 return (CTL_RETVAL_COMPLETE);
8571                 break; /* NOTREACHED */
8572         }
8573
8574         /*
8575          * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
8576          * interesting for us, but if RAIDCore is in write-back mode,
8577          * getting it to do write-through for a particular transaction may
8578          * not be possible.
8579          */
8580         /*
8581          * We don't support relative addressing.  That also requires
8582          * supporting linked commands, which we don't do.
8583          */
8584         if (reladdr != 0) {
8585                 ctl_set_invalid_field(ctsio,
8586                                       /*sks_valid*/ 1,
8587                                       /*command*/ 1,
8588                                       /*field*/ 1,
8589                                       /*bit_valid*/ 1,
8590                                       /*bit*/ 0);
8591                 ctl_done((union ctl_io *)ctsio);
8592                 return (CTL_RETVAL_COMPLETE);
8593         }
8594
8595         /*
8596          * The first check is to make sure we're in bounds, the second
8597          * check is to catch wrap-around problems.  If the lba + num blocks
8598          * is less than the lba, then we've wrapped around and the block
8599          * range is invalid anyway.
8600          */
8601         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
8602          || ((lba + num_blocks) < lba)) {
8603                 ctl_set_lba_out_of_range(ctsio);
8604                 ctl_done((union ctl_io *)ctsio);
8605                 return (CTL_RETVAL_COMPLETE);
8606         }
8607
8608         /*
8609          * According to SBC-3, a transfer length of 0 is not an error.
8610          * Note that this cannot happen with WRITE(6) or READ(6), since 0
8611          * translates to 256 blocks for those commands.
8612          */
8613         if (num_blocks == 0) {
8614                 ctl_set_success(ctsio);
8615                 ctl_done((union ctl_io *)ctsio);
8616                 return (CTL_RETVAL_COMPLETE);
8617         }
8618
8619         lbalen.lba = lba;
8620         lbalen.len = num_blocks;
8621         memcpy(ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes, &lbalen,
8622                sizeof(lbalen));
8623
8624         CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
8625
8626         retval = lun->backend->data_submit((union ctl_io *)ctsio);
8627
8628         return (retval);
8629 }
8630
8631 int
8632 ctl_report_luns(struct ctl_scsiio *ctsio)
8633 {
8634         struct scsi_report_luns *cdb;
8635         struct scsi_report_luns_data *lun_data;
8636         struct ctl_lun *lun, *request_lun;
8637         int num_luns, retval;
8638         uint32_t alloc_len, lun_datalen;
8639         int num_filled, well_known;
8640         uint32_t initidx, targ_lun_id, lun_id;
8641
8642         retval = CTL_RETVAL_COMPLETE;
8643         well_known = 0;
8644
8645         cdb = (struct scsi_report_luns *)ctsio->cdb;
8646
8647         CTL_DEBUG_PRINT(("ctl_report_luns\n"));
8648
8649         mtx_lock(&control_softc->ctl_lock);
8650         num_luns = control_softc->num_luns;
8651         mtx_unlock(&control_softc->ctl_lock);
8652
8653         switch (cdb->select_report) {
8654         case RPL_REPORT_DEFAULT:
8655         case RPL_REPORT_ALL:
8656                 break;
8657         case RPL_REPORT_WELLKNOWN:
8658                 well_known = 1;
8659                 num_luns = 0;
8660                 break;
8661         default:
8662                 ctl_set_invalid_field(ctsio,
8663                                       /*sks_valid*/ 1,
8664                                       /*command*/ 1,
8665                                       /*field*/ 2,
8666                                       /*bit_valid*/ 0,
8667                                       /*bit*/ 0);
8668                 ctl_done((union ctl_io *)ctsio);
8669                 return (retval);
8670                 break; /* NOTREACHED */
8671         }
8672
8673         alloc_len = scsi_4btoul(cdb->length);
8674         /*
8675          * The initiator has to allocate at least 16 bytes for this request,
8676          * so he can at least get the header and the first LUN.  Otherwise
8677          * we reject the request (per SPC-3 rev 14, section 6.21).
8678          */
8679         if (alloc_len < (sizeof(struct scsi_report_luns_data) +
8680             sizeof(struct scsi_report_luns_lundata))) {
8681                 ctl_set_invalid_field(ctsio,
8682                                       /*sks_valid*/ 1,
8683                                       /*command*/ 1,
8684                                       /*field*/ 6,
8685                                       /*bit_valid*/ 0,
8686                                       /*bit*/ 0);
8687                 ctl_done((union ctl_io *)ctsio);
8688                 return (retval);
8689         }
8690
8691         request_lun = (struct ctl_lun *)
8692                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8693
8694         lun_datalen = sizeof(*lun_data) +
8695                 (num_luns * sizeof(struct scsi_report_luns_lundata));
8696
8697         ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
8698         lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
8699         ctsio->kern_sg_entries = 0;
8700
8701         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
8702
8703         mtx_lock(&control_softc->ctl_lock);
8704         for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
8705                 lun_id = targ_lun_id;
8706                 if (ctsio->io_hdr.nexus.lun_map_fn != NULL)
8707                         lun_id = ctsio->io_hdr.nexus.lun_map_fn(ctsio->io_hdr.nexus.lun_map_arg, lun_id);
8708                 if (lun_id >= CTL_MAX_LUNS)
8709                         continue;
8710                 lun = control_softc->ctl_luns[lun_id];
8711                 if (lun == NULL)
8712                         continue;
8713
8714                 if (targ_lun_id <= 0xff) {
8715                         /*
8716                          * Peripheral addressing method, bus number 0.
8717                          */
8718                         lun_data->luns[num_filled].lundata[0] =
8719                                 RPL_LUNDATA_ATYP_PERIPH;
8720                         lun_data->luns[num_filled].lundata[1] = targ_lun_id;
8721                         num_filled++;
8722                 } else if (targ_lun_id <= 0x3fff) {
8723                         /*
8724                          * Flat addressing method.
8725                          */
8726                         lun_data->luns[num_filled].lundata[0] =
8727                                 RPL_LUNDATA_ATYP_FLAT |
8728                                 (targ_lun_id & RPL_LUNDATA_FLAT_LUN_MASK);
8729 #ifdef OLDCTLHEADERS
8730                                 (SRLD_ADDR_FLAT << SRLD_ADDR_SHIFT) |
8731                                 (targ_lun_id & SRLD_BUS_LUN_MASK);
8732 #endif
8733                         lun_data->luns[num_filled].lundata[1] =
8734 #ifdef OLDCTLHEADERS
8735                                 targ_lun_id >> SRLD_BUS_LUN_BITS;
8736 #endif
8737                                 targ_lun_id >> RPL_LUNDATA_FLAT_LUN_BITS;
8738                         num_filled++;
8739                 } else {
8740                         printf("ctl_report_luns: bogus LUN number %jd, "
8741                                "skipping\n", (intmax_t)targ_lun_id);
8742                 }
8743                 /*
8744                  * According to SPC-3, rev 14 section 6.21:
8745                  *
8746                  * "The execution of a REPORT LUNS command to any valid and
8747                  * installed logical unit shall clear the REPORTED LUNS DATA
8748                  * HAS CHANGED unit attention condition for all logical
8749                  * units of that target with respect to the requesting
8750                  * initiator. A valid and installed logical unit is one
8751                  * having a PERIPHERAL QUALIFIER of 000b in the standard
8752                  * INQUIRY data (see 6.4.2)."
8753                  *
8754                  * If request_lun is NULL, the LUN this report luns command
8755                  * was issued to is either disabled or doesn't exist. In that
8756                  * case, we shouldn't clear any pending lun change unit
8757                  * attention.
8758                  */
8759                 if (request_lun != NULL)
8760                         lun->pending_sense[initidx].ua_pending &=
8761                                 ~CTL_UA_LUN_CHANGE;
8762         }
8763         mtx_unlock(&control_softc->ctl_lock);
8764
8765         /*
8766          * It's quite possible that we've returned fewer LUNs than we allocated
8767          * space for.  Trim it.
8768          */
8769         lun_datalen = sizeof(*lun_data) +
8770                 (num_filled * sizeof(struct scsi_report_luns_lundata));
8771
8772         if (lun_datalen < alloc_len) {
8773                 ctsio->residual = alloc_len - lun_datalen;
8774                 ctsio->kern_data_len = lun_datalen;
8775                 ctsio->kern_total_len = lun_datalen;
8776         } else {
8777                 ctsio->residual = 0;
8778                 ctsio->kern_data_len = alloc_len;
8779                 ctsio->kern_total_len = alloc_len;
8780         }
8781         ctsio->kern_data_resid = 0;
8782         ctsio->kern_rel_offset = 0;
8783         ctsio->kern_sg_entries = 0;
8784
8785         /*
8786          * We set this to the actual data length, regardless of how much
8787          * space we actually have to return results.  If the user looks at
8788          * this value, he'll know whether or not he allocated enough space
8789          * and reissue the command if necessary.  We don't support well
8790          * known logical units, so if the user asks for that, return none.
8791          */
8792         scsi_ulto4b(lun_datalen - 8, lun_data->length);
8793
8794         /*
8795          * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
8796          * this request.
8797          */
8798         ctsio->scsi_status = SCSI_STATUS_OK;
8799
8800         ctsio->be_move_done = ctl_config_move_done;
8801         ctl_datamove((union ctl_io *)ctsio);
8802
8803         return (retval);
8804 }
8805
8806 int
8807 ctl_request_sense(struct ctl_scsiio *ctsio)
8808 {
8809         struct scsi_request_sense *cdb;
8810         struct scsi_sense_data *sense_ptr;
8811         struct ctl_lun *lun;
8812         uint32_t initidx;
8813         int have_error;
8814         scsi_sense_data_type sense_format;
8815
8816         cdb = (struct scsi_request_sense *)ctsio->cdb;
8817
8818         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8819
8820         CTL_DEBUG_PRINT(("ctl_request_sense\n"));
8821
8822         /*
8823          * Determine which sense format the user wants.
8824          */
8825         if (cdb->byte2 & SRS_DESC)
8826                 sense_format = SSD_TYPE_DESC;
8827         else
8828                 sense_format = SSD_TYPE_FIXED;
8829
8830         ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
8831         sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
8832         ctsio->kern_sg_entries = 0;
8833
8834         /*
8835          * struct scsi_sense_data, which is currently set to 256 bytes, is
8836          * larger than the largest allowed value for the length field in the
8837          * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
8838          */
8839         ctsio->residual = 0;
8840         ctsio->kern_data_len = cdb->length;
8841         ctsio->kern_total_len = cdb->length;
8842
8843         ctsio->kern_data_resid = 0;
8844         ctsio->kern_rel_offset = 0;
8845         ctsio->kern_sg_entries = 0;
8846
8847         /*
8848          * If we don't have a LUN, we don't have any pending sense.
8849          */
8850         if (lun == NULL)
8851                 goto no_sense;
8852
8853         have_error = 0;
8854         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
8855         /*
8856          * Check for pending sense, and then for pending unit attentions.
8857          * Pending sense gets returned first, then pending unit attentions.
8858          */
8859         mtx_lock(&lun->ctl_softc->ctl_lock);
8860         if (ctl_is_set(lun->have_ca, initidx)) {
8861                 scsi_sense_data_type stored_format;
8862
8863                 /*
8864                  * Check to see which sense format was used for the stored
8865                  * sense data.
8866                  */
8867                 stored_format = scsi_sense_type(
8868                     &lun->pending_sense[initidx].sense);
8869
8870                 /*
8871                  * If the user requested a different sense format than the
8872                  * one we stored, then we need to convert it to the other
8873                  * format.  If we're going from descriptor to fixed format
8874                  * sense data, we may lose things in translation, depending
8875                  * on what options were used.
8876                  *
8877                  * If the stored format is SSD_TYPE_NONE (i.e. invalid),
8878                  * for some reason we'll just copy it out as-is.
8879                  */
8880                 if ((stored_format == SSD_TYPE_FIXED)
8881                  && (sense_format == SSD_TYPE_DESC))
8882                         ctl_sense_to_desc((struct scsi_sense_data_fixed *)
8883                             &lun->pending_sense[initidx].sense,
8884                             (struct scsi_sense_data_desc *)sense_ptr);
8885                 else if ((stored_format == SSD_TYPE_DESC)
8886                       && (sense_format == SSD_TYPE_FIXED))
8887                         ctl_sense_to_fixed((struct scsi_sense_data_desc *)
8888                             &lun->pending_sense[initidx].sense,
8889                             (struct scsi_sense_data_fixed *)sense_ptr);
8890                 else
8891                         memcpy(sense_ptr, &lun->pending_sense[initidx].sense,
8892                                ctl_min(sizeof(*sense_ptr),
8893                                sizeof(lun->pending_sense[initidx].sense)));
8894
8895                 ctl_clear_mask(lun->have_ca, initidx);
8896                 have_error = 1;
8897         } else if (lun->pending_sense[initidx].ua_pending != CTL_UA_NONE) {
8898                 ctl_ua_type ua_type;
8899
8900                 ua_type = ctl_build_ua(lun->pending_sense[initidx].ua_pending,
8901                                        sense_ptr, sense_format);
8902                 if (ua_type != CTL_UA_NONE) {
8903                         have_error = 1;
8904                         /* We're reporting this UA, so clear it */
8905                         lun->pending_sense[initidx].ua_pending &= ~ua_type;
8906                 }
8907         }
8908         mtx_unlock(&lun->ctl_softc->ctl_lock);
8909
8910         /*
8911          * We already have a pending error, return it.
8912          */
8913         if (have_error != 0) {
8914                 /*
8915                  * We report the SCSI status as OK, since the status of the
8916                  * request sense command itself is OK.
8917                  */
8918                 ctsio->scsi_status = SCSI_STATUS_OK;
8919
8920                 /*
8921                  * We report 0 for the sense length, because we aren't doing
8922                  * autosense in this case.  We're reporting sense as
8923                  * parameter data.
8924                  */
8925                 ctsio->sense_len = 0;
8926
8927                 ctsio->be_move_done = ctl_config_move_done;
8928                 ctl_datamove((union ctl_io *)ctsio);
8929
8930                 return (CTL_RETVAL_COMPLETE);
8931         }
8932
8933 no_sense:
8934
8935         /*
8936          * No sense information to report, so we report that everything is
8937          * okay.
8938          */
8939         ctl_set_sense_data(sense_ptr,
8940                            lun,
8941                            sense_format,
8942                            /*current_error*/ 1,
8943                            /*sense_key*/ SSD_KEY_NO_SENSE,
8944                            /*asc*/ 0x00,
8945                            /*ascq*/ 0x00,
8946                            SSD_ELEM_NONE);
8947
8948         ctsio->scsi_status = SCSI_STATUS_OK;
8949
8950         /*
8951          * We report 0 for the sense length, because we aren't doing
8952          * autosense in this case.  We're reporting sense as parameter data.
8953          */
8954         ctsio->sense_len = 0;
8955         ctsio->be_move_done = ctl_config_move_done;
8956         ctl_datamove((union ctl_io *)ctsio);
8957
8958         return (CTL_RETVAL_COMPLETE);
8959 }
8960
8961 int
8962 ctl_tur(struct ctl_scsiio *ctsio)
8963 {
8964         struct ctl_lun *lun;
8965
8966         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8967
8968         CTL_DEBUG_PRINT(("ctl_tur\n"));
8969
8970         if (lun == NULL)
8971                 return (-EINVAL);
8972
8973         ctsio->scsi_status = SCSI_STATUS_OK;
8974         ctsio->io_hdr.status = CTL_SUCCESS;
8975
8976         ctl_done((union ctl_io *)ctsio);
8977
8978         return (CTL_RETVAL_COMPLETE);
8979 }
8980
8981 #ifdef notyet
8982 static int
8983 ctl_cmddt_inquiry(struct ctl_scsiio *ctsio)
8984 {
8985
8986 }
8987 #endif
8988
8989 static int
8990 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
8991 {
8992         struct scsi_vpd_supported_pages *pages;
8993         int sup_page_size;
8994         struct ctl_lun *lun;
8995
8996         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8997
8998         sup_page_size = sizeof(struct scsi_vpd_supported_pages) +
8999                 SCSI_EVPD_NUM_SUPPORTED_PAGES;
9000         ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9001         pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9002         ctsio->kern_sg_entries = 0;
9003
9004         if (sup_page_size < alloc_len) {
9005                 ctsio->residual = alloc_len - sup_page_size;
9006                 ctsio->kern_data_len = sup_page_size;
9007                 ctsio->kern_total_len = sup_page_size;
9008         } else {
9009                 ctsio->residual = 0;
9010                 ctsio->kern_data_len = alloc_len;
9011                 ctsio->kern_total_len = alloc_len;
9012         }
9013         ctsio->kern_data_resid = 0;
9014         ctsio->kern_rel_offset = 0;
9015         ctsio->kern_sg_entries = 0;
9016
9017         /*
9018          * The control device is always connected.  The disk device, on the
9019          * other hand, may not be online all the time.  Need to change this
9020          * to figure out whether the disk device is actually online or not.
9021          */
9022         if (lun != NULL)
9023                 pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9024                                 lun->be_lun->lun_type;
9025         else
9026                 pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9027
9028         pages->length = SCSI_EVPD_NUM_SUPPORTED_PAGES;
9029         /* Supported VPD pages */
9030         pages->page_list[0] = SVPD_SUPPORTED_PAGES;
9031         /* Serial Number */
9032         pages->page_list[1] = SVPD_UNIT_SERIAL_NUMBER;
9033         /* Device Identification */
9034         pages->page_list[2] = SVPD_DEVICE_ID;
9035
9036         ctsio->scsi_status = SCSI_STATUS_OK;
9037
9038         ctsio->be_move_done = ctl_config_move_done;
9039         ctl_datamove((union ctl_io *)ctsio);
9040
9041         return (CTL_RETVAL_COMPLETE);
9042 }
9043
9044 static int
9045 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9046 {
9047         struct scsi_vpd_unit_serial_number *sn_ptr;
9048         struct ctl_lun *lun;
9049 #ifndef CTL_USE_BACKEND_SN
9050         char tmpstr[32];
9051 #endif
9052
9053         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9054
9055         ctsio->kern_data_ptr = malloc(sizeof(*sn_ptr), M_CTL, M_WAITOK | M_ZERO);
9056         sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9057         ctsio->kern_sg_entries = 0;
9058
9059         if (sizeof(*sn_ptr) < alloc_len) {
9060                 ctsio->residual = alloc_len - sizeof(*sn_ptr);
9061                 ctsio->kern_data_len = sizeof(*sn_ptr);
9062                 ctsio->kern_total_len = sizeof(*sn_ptr);
9063         } else {
9064                 ctsio->residual = 0;
9065                 ctsio->kern_data_len = alloc_len;
9066                 ctsio->kern_total_len = alloc_len;
9067         }
9068         ctsio->kern_data_resid = 0;
9069         ctsio->kern_rel_offset = 0;
9070         ctsio->kern_sg_entries = 0;
9071
9072         /*
9073          * The control device is always connected.  The disk device, on the
9074          * other hand, may not be online all the time.  Need to change this
9075          * to figure out whether the disk device is actually online or not.
9076          */
9077         if (lun != NULL)
9078                 sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9079                                   lun->be_lun->lun_type;
9080         else
9081                 sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9082
9083         sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9084         sn_ptr->length = ctl_min(sizeof(*sn_ptr) - 4, CTL_SN_LEN);
9085 #ifdef CTL_USE_BACKEND_SN
9086         /*
9087          * If we don't have a LUN, we just leave the serial number as
9088          * all spaces.
9089          */
9090         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9091         if (lun != NULL) {
9092                 strncpy((char *)sn_ptr->serial_num,
9093                         (char *)lun->be_lun->serial_num, CTL_SN_LEN);
9094         }
9095 #else
9096         /*
9097          * Note that we're using a non-unique serial number here,
9098          */
9099         snprintf(tmpstr, sizeof(tmpstr), "MYSERIALNUMIS000");
9100         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9101         strncpy(sn_ptr->serial_num, tmpstr, ctl_min(CTL_SN_LEN,
9102                 ctl_min(sizeof(tmpstr), sizeof(*sn_ptr) - 4)));
9103 #endif
9104         ctsio->scsi_status = SCSI_STATUS_OK;
9105
9106         ctsio->be_move_done = ctl_config_move_done;
9107         ctl_datamove((union ctl_io *)ctsio);
9108
9109         return (CTL_RETVAL_COMPLETE);
9110 }
9111
9112
9113 static int
9114 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9115 {
9116         struct scsi_vpd_device_id *devid_ptr;
9117         struct scsi_vpd_id_descriptor *desc, *desc1;
9118         struct scsi_vpd_id_descriptor *desc2, *desc3; /* for types 4h and 5h */
9119         struct scsi_vpd_id_t10 *t10id;
9120         struct ctl_softc *ctl_softc;
9121         struct ctl_lun *lun;
9122         struct ctl_frontend *fe;
9123 #ifndef CTL_USE_BACKEND_SN
9124         char tmpstr[32];
9125 #endif /* CTL_USE_BACKEND_SN */
9126         int devid_len;
9127
9128         ctl_softc = control_softc;
9129
9130         mtx_lock(&ctl_softc->ctl_lock);
9131         fe = ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)];
9132         mtx_unlock(&ctl_softc->ctl_lock);
9133
9134         if (fe->devid != NULL)
9135                 return ((fe->devid)(ctsio, alloc_len));
9136
9137         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9138
9139         devid_len = sizeof(struct scsi_vpd_device_id) +
9140                 sizeof(struct scsi_vpd_id_descriptor) +
9141                 sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN +
9142                 sizeof(struct scsi_vpd_id_descriptor) + CTL_WWPN_LEN +
9143                 sizeof(struct scsi_vpd_id_descriptor) +
9144                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9145                 sizeof(struct scsi_vpd_id_descriptor) +
9146                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9147
9148         ctsio->kern_data_ptr = malloc(devid_len, M_CTL, M_WAITOK | M_ZERO);
9149         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9150         ctsio->kern_sg_entries = 0;
9151
9152         if (devid_len < alloc_len) {
9153                 ctsio->residual = alloc_len - devid_len;
9154                 ctsio->kern_data_len = devid_len;
9155                 ctsio->kern_total_len = devid_len;
9156         } else {
9157                 ctsio->residual = 0;
9158                 ctsio->kern_data_len = alloc_len;
9159                 ctsio->kern_total_len = alloc_len;
9160         }
9161         ctsio->kern_data_resid = 0;
9162         ctsio->kern_rel_offset = 0;
9163         ctsio->kern_sg_entries = 0;
9164
9165         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
9166         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
9167         desc1 = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9168                 sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN);
9169         desc2 = (struct scsi_vpd_id_descriptor *)(&desc1->identifier[0] +
9170                   CTL_WWPN_LEN);
9171         desc3 = (struct scsi_vpd_id_descriptor *)(&desc2->identifier[0] +
9172                  sizeof(struct scsi_vpd_id_rel_trgt_port_id));
9173
9174         /*
9175          * The control device is always connected.  The disk device, on the
9176          * other hand, may not be online all the time.
9177          */
9178         if (lun != NULL)
9179                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9180                                      lun->be_lun->lun_type;
9181         else
9182                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9183
9184         devid_ptr->page_code = SVPD_DEVICE_ID;
9185
9186         scsi_ulto2b(devid_len - 4, devid_ptr->length);
9187
9188         mtx_lock(&ctl_softc->ctl_lock);
9189
9190         /*
9191          * For Fibre channel,
9192          */
9193         if (fe->port_type == CTL_PORT_FC)
9194         {
9195                 desc->proto_codeset = (SCSI_PROTO_FC << 4) |
9196                                       SVPD_ID_CODESET_ASCII;
9197                 desc1->proto_codeset = (SCSI_PROTO_FC << 4) |
9198                               SVPD_ID_CODESET_BINARY;
9199         }
9200         else
9201         {
9202                 desc->proto_codeset = (SCSI_PROTO_SPI << 4) |
9203                                       SVPD_ID_CODESET_ASCII;
9204                 desc1->proto_codeset = (SCSI_PROTO_SPI << 4) |
9205                               SVPD_ID_CODESET_BINARY;
9206         }
9207         desc2->proto_codeset = desc3->proto_codeset = desc1->proto_codeset;
9208         mtx_unlock(&ctl_softc->ctl_lock);
9209
9210         /*
9211          * We're using a LUN association here.  i.e., this device ID is a
9212          * per-LUN identifier.
9213          */
9214         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
9215         desc->length = sizeof(*t10id) + CTL_DEVID_LEN;
9216         strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
9217
9218         /*
9219          * desc1 is for the WWPN which is a port asscociation.
9220          */
9221         desc1->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT | SVPD_ID_TYPE_NAA;
9222         desc1->length = CTL_WWPN_LEN;
9223         /* XXX Call Reggie's get_WWNN func here then add port # to the end */
9224         /* For testing just create the WWPN */
9225 #if 0
9226         ddb_GetWWNN((char *)desc1->identifier);
9227
9228         /* NOTE: if the port is 0 or 8 we don't want to subtract 1 */
9229         /* This is so Copancontrol will return something sane */
9230         if (ctsio->io_hdr.nexus.targ_port!=0 &&
9231             ctsio->io_hdr.nexus.targ_port!=8)
9232                 desc1->identifier[7] += ctsio->io_hdr.nexus.targ_port-1;
9233         else
9234                 desc1->identifier[7] += ctsio->io_hdr.nexus.targ_port;
9235 #endif
9236
9237         be64enc(desc1->identifier, fe->wwpn);
9238
9239         /*
9240          * desc2 is for the Relative Target Port(type 4h) identifier
9241          */
9242         desc2->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT
9243                          | SVPD_ID_TYPE_RELTARG;
9244         desc2->length = 4;
9245 //#if 0
9246         /* NOTE: if the port is 0 or 8 we don't want to subtract 1 */
9247         /* This is so Copancontrol will return something sane */
9248         if (ctsio->io_hdr.nexus.targ_port!=0 &&
9249             ctsio->io_hdr.nexus.targ_port!=8)
9250                 desc2->identifier[3] = ctsio->io_hdr.nexus.targ_port - 1;
9251         else
9252                 desc2->identifier[3] = ctsio->io_hdr.nexus.targ_port;
9253 //#endif
9254
9255         /*
9256          * desc3 is for the Target Port Group(type 5h) identifier
9257          */
9258         desc3->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT
9259                          | SVPD_ID_TYPE_TPORTGRP;
9260         desc3->length = 4;
9261         if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS || ctl_is_single)
9262                 desc3->identifier[3] = 1;
9263         else
9264                 desc3->identifier[3] = 2;
9265
9266 #ifdef CTL_USE_BACKEND_SN
9267         /*
9268          * If we've actually got a backend, copy the device id from the
9269          * per-LUN data.  Otherwise, set it to all spaces.
9270          */
9271         if (lun != NULL) {
9272                 /*
9273                  * Copy the backend's LUN ID.
9274                  */
9275                 strncpy((char *)t10id->vendor_spec_id,
9276                         (char *)lun->be_lun->device_id, CTL_DEVID_LEN);
9277         } else {
9278                 /*
9279                  * No backend, set this to spaces.
9280                  */
9281                 memset(t10id->vendor_spec_id, 0x20, CTL_DEVID_LEN);
9282         }
9283 #else
9284         snprintf(tmpstr, sizeof(tmpstr), "MYDEVICEIDIS%4d",
9285                  (lun != NULL) ?  (int)lun->lun : 0);
9286         strncpy(t10id->vendor_spec_id, tmpstr, ctl_min(CTL_DEVID_LEN,
9287                 sizeof(tmpstr)));
9288 #endif
9289
9290         ctsio->scsi_status = SCSI_STATUS_OK;
9291
9292         ctsio->be_move_done = ctl_config_move_done;
9293         ctl_datamove((union ctl_io *)ctsio);
9294
9295         return (CTL_RETVAL_COMPLETE);
9296 }
9297
9298 static int
9299 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
9300 {
9301         struct scsi_inquiry *cdb;
9302         int alloc_len, retval;
9303
9304         cdb = (struct scsi_inquiry *)ctsio->cdb;
9305
9306         retval = CTL_RETVAL_COMPLETE;
9307
9308         alloc_len = scsi_2btoul(cdb->length);
9309
9310         switch (cdb->page_code) {
9311         case SVPD_SUPPORTED_PAGES:
9312                 retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
9313                 break;
9314         case SVPD_UNIT_SERIAL_NUMBER:
9315                 retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
9316                 break;
9317         case SVPD_DEVICE_ID:
9318                 retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
9319                 break;
9320         default:
9321                 ctl_set_invalid_field(ctsio,
9322                                       /*sks_valid*/ 1,
9323                                       /*command*/ 1,
9324                                       /*field*/ 2,
9325                                       /*bit_valid*/ 0,
9326                                       /*bit*/ 0);
9327                 ctl_done((union ctl_io *)ctsio);
9328                 retval = CTL_RETVAL_COMPLETE;
9329                 break;
9330         }
9331
9332         return (retval);
9333 }
9334
9335 static int
9336 ctl_inquiry_std(struct ctl_scsiio *ctsio)
9337 {
9338         struct scsi_inquiry_data *inq_ptr;
9339         struct scsi_inquiry *cdb;
9340         struct ctl_softc *ctl_softc;
9341         struct ctl_lun *lun;
9342         uint32_t alloc_len;
9343         int is_fc;
9344
9345         ctl_softc = control_softc;
9346
9347         /*
9348          * Figure out whether we're talking to a Fibre Channel port or not.
9349          * We treat the ioctl front end, and any SCSI adapters, as packetized
9350          * SCSI front ends.
9351          */
9352         mtx_lock(&ctl_softc->ctl_lock);
9353         if (ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)]->port_type !=
9354             CTL_PORT_FC)
9355                 is_fc = 0;
9356         else
9357                 is_fc = 1;
9358         mtx_unlock(&ctl_softc->ctl_lock);
9359
9360         lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9361         cdb = (struct scsi_inquiry *)ctsio->cdb;
9362         alloc_len = scsi_2btoul(cdb->length);
9363
9364         /*
9365          * We malloc the full inquiry data size here and fill it
9366          * in.  If the user only asks for less, we'll give him
9367          * that much.
9368          */
9369         ctsio->kern_data_ptr = malloc(sizeof(*inq_ptr), M_CTL, M_WAITOK | M_ZERO);
9370         inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
9371         ctsio->kern_sg_entries = 0;
9372         ctsio->kern_data_resid = 0;
9373         ctsio->kern_rel_offset = 0;
9374
9375         if (sizeof(*inq_ptr) < alloc_len) {
9376                 ctsio->residual = alloc_len - sizeof(*inq_ptr);
9377                 ctsio->kern_data_len = sizeof(*inq_ptr);
9378                 ctsio->kern_total_len = sizeof(*inq_ptr);
9379         } else {
9380                 ctsio->residual = 0;
9381                 ctsio->kern_data_len = alloc_len;
9382                 ctsio->kern_total_len = alloc_len;
9383         }
9384
9385         /*
9386          * If we have a LUN configured, report it as connected.  Otherwise,
9387          * report that it is offline or no device is supported, depending
9388          * on the value of inquiry_pq_no_lun.
9389          *
9390          * According to the spec (SPC-4 r34), the peripheral qualifier
9391          * SID_QUAL_LU_OFFLINE (001b) is used in the following scenario:
9392          *
9393          * "A peripheral device having the specified peripheral device type
9394          * is not connected to this logical unit. However, the device
9395          * server is capable of supporting the specified peripheral device
9396          * type on this logical unit."
9397          *
9398          * According to the same spec, the peripheral qualifier
9399          * SID_QUAL_BAD_LU (011b) is used in this scenario:
9400          *
9401          * "The device server is not capable of supporting a peripheral
9402          * device on this logical unit. For this peripheral qualifier the
9403          * peripheral device type shall be set to 1Fh. All other peripheral
9404          * device type values are reserved for this peripheral qualifier."
9405          *
9406          * Given the text, it would seem that we probably want to report that
9407          * the LUN is offline here.  There is no LUN connected, but we can
9408          * support a LUN at the given LUN number.
9409          *
9410          * In the real world, though, it sounds like things are a little
9411          * different:
9412          *
9413          * - Linux, when presented with a LUN with the offline peripheral
9414          *   qualifier, will create an sg driver instance for it.  So when
9415          *   you attach it to CTL, you wind up with a ton of sg driver
9416          *   instances.  (One for every LUN that Linux bothered to probe.)
9417          *   Linux does this despite the fact that it issues a REPORT LUNs
9418          *   to LUN 0 to get the inventory of supported LUNs.
9419          *
9420          * - There is other anecdotal evidence (from Emulex folks) about
9421          *   arrays that use the offline peripheral qualifier for LUNs that
9422          *   are on the "passive" path in an active/passive array.
9423          *
9424          * So the solution is provide a hopefully reasonable default
9425          * (return bad/no LUN) and allow the user to change the behavior
9426          * with a tunable/sysctl variable.
9427          */
9428         if (lun != NULL)
9429                 inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9430                                   lun->be_lun->lun_type;
9431         else if (ctl_softc->inquiry_pq_no_lun == 0)
9432                 inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9433         else
9434                 inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
9435
9436         /* RMB in byte 2 is 0 */
9437         inq_ptr->version = SCSI_REV_SPC3;
9438
9439         /*
9440          * According to SAM-3, even if a device only supports a single
9441          * level of LUN addressing, it should still set the HISUP bit:
9442          *
9443          * 4.9.1 Logical unit numbers overview
9444          *
9445          * All logical unit number formats described in this standard are
9446          * hierarchical in structure even when only a single level in that
9447          * hierarchy is used. The HISUP bit shall be set to one in the
9448          * standard INQUIRY data (see SPC-2) when any logical unit number
9449          * format described in this standard is used.  Non-hierarchical
9450          * formats are outside the scope of this standard.
9451          *
9452          * Therefore we set the HiSup bit here.
9453          *
9454          * The reponse format is 2, per SPC-3.
9455          */
9456         inq_ptr->response_format = SID_HiSup | 2;
9457
9458         inq_ptr->additional_length = sizeof(*inq_ptr) - 4;
9459         CTL_DEBUG_PRINT(("additional_length = %d\n",
9460                          inq_ptr->additional_length));
9461
9462         inq_ptr->spc3_flags = SPC3_SID_TPGS_IMPLICIT;
9463         /* 16 bit addressing */
9464         if (is_fc == 0)
9465                 inq_ptr->spc2_flags = SPC2_SID_ADDR16;
9466         /* XXX set the SID_MultiP bit here if we're actually going to
9467            respond on multiple ports */
9468         inq_ptr->spc2_flags |= SPC2_SID_MultiP;
9469
9470         /* 16 bit data bus, synchronous transfers */
9471         /* XXX these flags don't apply for FC */
9472         if (is_fc == 0)
9473                 inq_ptr->flags = SID_WBus16 | SID_Sync;
9474         /*
9475          * XXX KDM do we want to support tagged queueing on the control
9476          * device at all?
9477          */
9478         if ((lun == NULL)
9479          || (lun->be_lun->lun_type != T_PROCESSOR))
9480                 inq_ptr->flags |= SID_CmdQue;
9481         /*
9482          * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
9483          * We have 8 bytes for the vendor name, and 16 bytes for the device
9484          * name and 4 bytes for the revision.
9485          */
9486         strncpy(inq_ptr->vendor, CTL_VENDOR, sizeof(inq_ptr->vendor));
9487         if (lun == NULL) {
9488                 strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
9489         } else {
9490                 switch (lun->be_lun->lun_type) {
9491                 case T_DIRECT:
9492                         strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
9493                         break;
9494                 case T_PROCESSOR:
9495                         strcpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT);
9496                         break;
9497                 default:
9498                         strcpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT);
9499                         break;
9500                 }
9501         }
9502
9503         /*
9504          * XXX make this a macro somewhere so it automatically gets
9505          * incremented when we make changes.
9506          */
9507         strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
9508
9509         /*
9510          * For parallel SCSI, we support double transition and single
9511          * transition clocking.  We also support QAS (Quick Arbitration
9512          * and Selection) and Information Unit transfers on both the
9513          * control and array devices.
9514          */
9515         if (is_fc == 0)
9516                 inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
9517                                     SID_SPI_IUS;
9518
9519         /* SAM-3 */
9520         scsi_ulto2b(0x0060, inq_ptr->version1);
9521         /* SPC-3 (no version claimed) XXX should we claim a version? */
9522         scsi_ulto2b(0x0300, inq_ptr->version2);
9523         if (is_fc) {
9524                 /* FCP-2 ANSI INCITS.350:2003 */
9525                 scsi_ulto2b(0x0917, inq_ptr->version3);
9526         } else {
9527                 /* SPI-4 ANSI INCITS.362:200x */
9528                 scsi_ulto2b(0x0B56, inq_ptr->version3);
9529         }
9530
9531         if (lun == NULL) {
9532                 /* SBC-2 (no version claimed) XXX should we claim a version? */
9533                 scsi_ulto2b(0x0320, inq_ptr->version4);
9534         } else {
9535                 switch (lun->be_lun->lun_type) {
9536                 case T_DIRECT:
9537                         /*
9538                          * SBC-2 (no version claimed) XXX should we claim a
9539                          * version?
9540                          */
9541                         scsi_ulto2b(0x0320, inq_ptr->version4);
9542                         break;
9543                 case T_PROCESSOR:
9544                 default:
9545                         break;
9546                 }
9547         }
9548
9549         ctsio->scsi_status = SCSI_STATUS_OK;
9550         if (ctsio->kern_data_len > 0) {
9551                 ctsio->be_move_done = ctl_config_move_done;
9552                 ctl_datamove((union ctl_io *)ctsio);
9553         } else {
9554                 ctsio->io_hdr.status = CTL_SUCCESS;
9555                 ctl_done((union ctl_io *)ctsio);
9556         }
9557
9558         return (CTL_RETVAL_COMPLETE);
9559 }
9560
9561 int
9562 ctl_inquiry(struct ctl_scsiio *ctsio)
9563 {
9564         struct scsi_inquiry *cdb;
9565         int retval;
9566
9567         cdb = (struct scsi_inquiry *)ctsio->cdb;
9568
9569         retval = 0;
9570
9571         CTL_DEBUG_PRINT(("ctl_inquiry\n"));
9572
9573         /*
9574          * Right now, we don't support the CmdDt inquiry information.
9575          * This would be nice to support in the future.  When we do
9576          * support it, we should change this test so that it checks to make
9577          * sure SI_EVPD and SI_CMDDT aren't both set at the same time.
9578          */
9579 #ifdef notyet
9580         if (((cdb->byte2 & SI_EVPD)
9581          && (cdb->byte2 & SI_CMDDT)))
9582 #endif
9583         if (cdb->byte2 & SI_CMDDT) {
9584                 /*
9585                  * Point to the SI_CMDDT bit.  We might change this
9586                  * when we support SI_CMDDT, but since both bits would be
9587                  * "wrong", this should probably just stay as-is then.
9588                  */
9589                 ctl_set_invalid_field(ctsio,
9590                                       /*sks_valid*/ 1,
9591                                       /*command*/ 1,
9592                                       /*field*/ 1,
9593                                       /*bit_valid*/ 1,
9594                                       /*bit*/ 1);
9595                 ctl_done((union ctl_io *)ctsio);
9596                 return (CTL_RETVAL_COMPLETE);
9597         }
9598         if (cdb->byte2 & SI_EVPD)
9599                 retval = ctl_inquiry_evpd(ctsio);
9600 #ifdef notyet
9601         else if (cdb->byte2 & SI_CMDDT)
9602                 retval = ctl_inquiry_cmddt(ctsio);
9603 #endif
9604         else
9605                 retval = ctl_inquiry_std(ctsio);
9606
9607         return (retval);
9608 }
9609
9610 /*
9611  * For known CDB types, parse the LBA and length.
9612  */
9613 static int
9614 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len)
9615 {
9616         if (io->io_hdr.io_type != CTL_IO_SCSI)
9617                 return (1);
9618
9619         switch (io->scsiio.cdb[0]) {
9620         case READ_6:
9621         case WRITE_6: {
9622                 struct scsi_rw_6 *cdb;
9623
9624                 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
9625
9626                 *lba = scsi_3btoul(cdb->addr);
9627                 /* only 5 bits are valid in the most significant address byte */
9628                 *lba &= 0x1fffff;
9629                 *len = cdb->length;
9630                 break;
9631         }
9632         case READ_10:
9633         case WRITE_10: {
9634                 struct scsi_rw_10 *cdb;
9635
9636                 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
9637
9638                 *lba = scsi_4btoul(cdb->addr);
9639                 *len = scsi_2btoul(cdb->length);
9640                 break;
9641         }
9642         case WRITE_VERIFY_10: {
9643                 struct scsi_write_verify_10 *cdb;
9644
9645                 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
9646
9647                 *lba = scsi_4btoul(cdb->addr);
9648                 *len = scsi_2btoul(cdb->length);
9649                 break;
9650         }
9651         case READ_12:
9652         case WRITE_12: {
9653                 struct scsi_rw_12 *cdb;
9654
9655                 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
9656
9657                 *lba = scsi_4btoul(cdb->addr);
9658                 *len = scsi_4btoul(cdb->length);
9659                 break;
9660         }
9661         case WRITE_VERIFY_12: {
9662                 struct scsi_write_verify_12 *cdb;
9663
9664                 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
9665
9666                 *lba = scsi_4btoul(cdb->addr);
9667                 *len = scsi_4btoul(cdb->length);
9668                 break;
9669         }
9670         case READ_16:
9671         case WRITE_16: {
9672                 struct scsi_rw_16 *cdb;
9673
9674                 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
9675
9676                 *lba = scsi_8btou64(cdb->addr);
9677                 *len = scsi_4btoul(cdb->length);
9678                 break;
9679         }
9680         case WRITE_VERIFY_16: {
9681                 struct scsi_write_verify_16 *cdb;
9682
9683                 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
9684
9685
9686                 *lba = scsi_8btou64(cdb->addr);
9687                 *len = scsi_4btoul(cdb->length);
9688                 break;
9689         }
9690         default:
9691                 return (1);
9692                 break; /* NOTREACHED */
9693         }
9694
9695         return (0);
9696 }
9697
9698 static ctl_action
9699 ctl_extent_check_lba(uint64_t lba1, uint32_t len1, uint64_t lba2, uint32_t len2)
9700 {
9701         uint64_t endlba1, endlba2;
9702
9703         endlba1 = lba1 + len1 - 1;
9704         endlba2 = lba2 + len2 - 1;
9705
9706         if ((endlba1 < lba2)
9707          || (endlba2 < lba1))
9708                 return (CTL_ACTION_PASS);
9709         else
9710                 return (CTL_ACTION_BLOCK);
9711 }
9712
9713 static ctl_action
9714 ctl_extent_check(union ctl_io *io1, union ctl_io *io2)
9715 {
9716         uint64_t lba1, lba2;
9717         uint32_t len1, len2;
9718         int retval;
9719
9720         retval = ctl_get_lba_len(io1, &lba1, &len1);
9721         if (retval != 0)
9722                 return (CTL_ACTION_ERROR);
9723
9724         retval = ctl_get_lba_len(io2, &lba2, &len2);
9725         if (retval != 0)
9726                 return (CTL_ACTION_ERROR);
9727
9728         return (ctl_extent_check_lba(lba1, len1, lba2, len2));
9729 }
9730
9731 static ctl_action
9732 ctl_check_for_blockage(union ctl_io *pending_io, union ctl_io *ooa_io)
9733 {
9734         struct ctl_cmd_entry *pending_entry, *ooa_entry;
9735         ctl_serialize_action *serialize_row;
9736
9737         /*
9738          * The initiator attempted multiple untagged commands at the same
9739          * time.  Can't do that.
9740          */
9741         if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
9742          && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
9743          && ((pending_io->io_hdr.nexus.targ_port ==
9744               ooa_io->io_hdr.nexus.targ_port)
9745           && (pending_io->io_hdr.nexus.initid.id ==
9746               ooa_io->io_hdr.nexus.initid.id))
9747          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
9748                 return (CTL_ACTION_OVERLAP);
9749
9750         /*
9751          * The initiator attempted to send multiple tagged commands with
9752          * the same ID.  (It's fine if different initiators have the same
9753          * tag ID.)
9754          *
9755          * Even if all of those conditions are true, we don't kill the I/O
9756          * if the command ahead of us has been aborted.  We won't end up
9757          * sending it to the FETD, and it's perfectly legal to resend a
9758          * command with the same tag number as long as the previous
9759          * instance of this tag number has been aborted somehow.
9760          */
9761         if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
9762          && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
9763          && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
9764          && ((pending_io->io_hdr.nexus.targ_port ==
9765               ooa_io->io_hdr.nexus.targ_port)
9766           && (pending_io->io_hdr.nexus.initid.id ==
9767               ooa_io->io_hdr.nexus.initid.id))
9768          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
9769                 return (CTL_ACTION_OVERLAP_TAG);
9770
9771         /*
9772          * If we get a head of queue tag, SAM-3 says that we should
9773          * immediately execute it.
9774          *
9775          * What happens if this command would normally block for some other
9776          * reason?  e.g. a request sense with a head of queue tag
9777          * immediately after a write.  Normally that would block, but this
9778          * will result in its getting executed immediately...
9779          *
9780          * We currently return "pass" instead of "skip", so we'll end up
9781          * going through the rest of the queue to check for overlapped tags.
9782          *
9783          * XXX KDM check for other types of blockage first??
9784          */
9785         if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
9786                 return (CTL_ACTION_PASS);
9787
9788         /*
9789          * Ordered tags have to block until all items ahead of them
9790          * have completed.  If we get called with an ordered tag, we always
9791          * block, if something else is ahead of us in the queue.
9792          */
9793         if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
9794                 return (CTL_ACTION_BLOCK);
9795
9796         /*
9797          * Simple tags get blocked until all head of queue and ordered tags
9798          * ahead of them have completed.  I'm lumping untagged commands in
9799          * with simple tags here.  XXX KDM is that the right thing to do?
9800          */
9801         if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
9802           || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
9803          && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
9804           || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
9805                 return (CTL_ACTION_BLOCK);
9806
9807         pending_entry = &ctl_cmd_table[pending_io->scsiio.cdb[0]];
9808         ooa_entry = &ctl_cmd_table[ooa_io->scsiio.cdb[0]];
9809
9810         serialize_row = ctl_serialize_table[ooa_entry->seridx];
9811
9812         switch (serialize_row[pending_entry->seridx]) {
9813         case CTL_SER_BLOCK:
9814                 return (CTL_ACTION_BLOCK);
9815                 break; /* NOTREACHED */
9816         case CTL_SER_EXTENT:
9817                 return (ctl_extent_check(pending_io, ooa_io));
9818                 break; /* NOTREACHED */
9819         case CTL_SER_PASS:
9820                 return (CTL_ACTION_PASS);
9821                 break; /* NOTREACHED */
9822         case CTL_SER_SKIP:
9823                 return (CTL_ACTION_SKIP);
9824                 break;
9825         default:
9826                 panic("invalid serialization value %d",
9827                       serialize_row[pending_entry->seridx]);
9828                 break; /* NOTREACHED */
9829         }
9830
9831         return (CTL_ACTION_ERROR);
9832 }
9833
9834 /*
9835  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
9836  * Assumptions:
9837  * - pending_io is generally either incoming, or on the blocked queue
9838  * - starting I/O is the I/O we want to start the check with.
9839  */
9840 static ctl_action
9841 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
9842               union ctl_io *starting_io)
9843 {
9844         union ctl_io *ooa_io;
9845         ctl_action action;
9846
9847         mtx_assert(&control_softc->ctl_lock, MA_OWNED);
9848
9849         /*
9850          * Run back along the OOA queue, starting with the current
9851          * blocked I/O and going through every I/O before it on the
9852          * queue.  If starting_io is NULL, we'll just end up returning
9853          * CTL_ACTION_PASS.
9854          */
9855         for (ooa_io = starting_io; ooa_io != NULL;
9856              ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
9857              ooa_links)){
9858
9859                 /*
9860                  * This routine just checks to see whether
9861                  * cur_blocked is blocked by ooa_io, which is ahead
9862                  * of it in the queue.  It doesn't queue/dequeue
9863                  * cur_blocked.
9864                  */
9865                 action = ctl_check_for_blockage(pending_io, ooa_io);
9866                 switch (action) {
9867                 case CTL_ACTION_BLOCK:
9868                 case CTL_ACTION_OVERLAP:
9869                 case CTL_ACTION_OVERLAP_TAG:
9870                 case CTL_ACTION_SKIP:
9871                 case CTL_ACTION_ERROR:
9872                         return (action);
9873                         break; /* NOTREACHED */
9874                 case CTL_ACTION_PASS:
9875                         break;
9876                 default:
9877                         panic("invalid action %d", action);
9878                         break;  /* NOTREACHED */
9879                 }
9880         }
9881
9882         return (CTL_ACTION_PASS);
9883 }
9884
9885 /*
9886  * Assumptions:
9887  * - An I/O has just completed, and has been removed from the per-LUN OOA
9888  *   queue, so some items on the blocked queue may now be unblocked.
9889  */
9890 static int
9891 ctl_check_blocked(struct ctl_lun *lun)
9892 {
9893         union ctl_io *cur_blocked, *next_blocked;
9894
9895         mtx_assert(&control_softc->ctl_lock, MA_OWNED);
9896
9897         /*
9898          * Run forward from the head of the blocked queue, checking each
9899          * entry against the I/Os prior to it on the OOA queue to see if
9900          * there is still any blockage.
9901          *
9902          * We cannot use the TAILQ_FOREACH() macro, because it can't deal
9903          * with our removing a variable on it while it is traversing the
9904          * list.
9905          */
9906         for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
9907              cur_blocked != NULL; cur_blocked = next_blocked) {
9908                 union ctl_io *prev_ooa;
9909                 ctl_action action;
9910
9911                 next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
9912                                                           blocked_links);
9913
9914                 prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
9915                                                       ctl_ooaq, ooa_links);
9916
9917                 /*
9918                  * If cur_blocked happens to be the first item in the OOA
9919                  * queue now, prev_ooa will be NULL, and the action
9920                  * returned will just be CTL_ACTION_PASS.
9921                  */
9922                 action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
9923
9924                 switch (action) {
9925                 case CTL_ACTION_BLOCK:
9926                         /* Nothing to do here, still blocked */
9927                         break;
9928                 case CTL_ACTION_OVERLAP:
9929                 case CTL_ACTION_OVERLAP_TAG:
9930                         /*
9931                          * This shouldn't happen!  In theory we've already
9932                          * checked this command for overlap...
9933                          */
9934                         break;
9935                 case CTL_ACTION_PASS:
9936                 case CTL_ACTION_SKIP: {
9937                         struct ctl_softc *softc;
9938                         struct ctl_cmd_entry *entry;
9939                         uint32_t initidx;
9940                         uint8_t opcode;
9941                         int isc_retval;
9942
9943                         /*
9944                          * The skip case shouldn't happen, this transaction
9945                          * should have never made it onto the blocked queue.
9946                          */
9947                         /*
9948                          * This I/O is no longer blocked, we can remove it
9949                          * from the blocked queue.  Since this is a TAILQ
9950                          * (doubly linked list), we can do O(1) removals
9951                          * from any place on the list.
9952                          */
9953                         TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
9954                                      blocked_links);
9955                         cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
9956
9957                         if (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC){
9958                                 /*
9959                                  * Need to send IO back to original side to
9960                                  * run
9961                                  */
9962                                 union ctl_ha_msg msg_info;
9963
9964                                 msg_info.hdr.original_sc =
9965                                         cur_blocked->io_hdr.original_sc;
9966                                 msg_info.hdr.serializing_sc = cur_blocked;
9967                                 msg_info.hdr.msg_type = CTL_MSG_R2R;
9968                                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
9969                                      &msg_info, sizeof(msg_info), 0)) >
9970                                      CTL_HA_STATUS_SUCCESS) {
9971                                         printf("CTL:Check Blocked error from "
9972                                                "ctl_ha_msg_send %d\n",
9973                                                isc_retval);
9974                                 }
9975                                 break;
9976                         }
9977                         opcode = cur_blocked->scsiio.cdb[0];
9978                         entry = &ctl_cmd_table[opcode];
9979                         softc = control_softc;
9980
9981                         initidx = ctl_get_initindex(&cur_blocked->io_hdr.nexus);
9982
9983                         /*
9984                          * Check this I/O for LUN state changes that may
9985                          * have happened while this command was blocked.
9986                          * The LUN state may have been changed by a command
9987                          * ahead of us in the queue, so we need to re-check
9988                          * for any states that can be caused by SCSI
9989                          * commands.
9990                          */
9991                         if (ctl_scsiio_lun_check(softc, lun, entry,
9992                                                  &cur_blocked->scsiio) == 0) {
9993                                 cur_blocked->io_hdr.flags |=
9994                                                       CTL_FLAG_IS_WAS_ON_RTR;
9995                                 STAILQ_INSERT_TAIL(&lun->ctl_softc->rtr_queue,
9996                                                    &cur_blocked->io_hdr, links);
9997                                 /*
9998                                  * In the non CTL_DONE_THREAD case, we need
9999                                  * to wake up the work thread here.  When
10000                                  * we're processing completed requests from
10001                                  * the work thread context, we'll pop back
10002                                  * around and end up pulling things off the
10003                                  * RtR queue.  When we aren't processing
10004                                  * things from the work thread context,
10005                                  * though, we won't ever check the RtR queue.
10006                                  * So we need to wake up the thread to clear
10007                                  * things off the queue.  Otherwise this
10008                                  * transaction will just sit on the RtR queue
10009                                  * until a new I/O comes in.  (Which may or
10010                                  * may not happen...)
10011                                  */
10012 #ifndef CTL_DONE_THREAD
10013                                 ctl_wakeup_thread();
10014 #endif
10015                         } else
10016                                 ctl_done_lock(cur_blocked, /*have_lock*/ 1);
10017                         break;
10018                 }
10019                 default:
10020                         /*
10021                          * This probably shouldn't happen -- we shouldn't
10022                          * get CTL_ACTION_ERROR, or anything else.
10023                          */
10024                         break;
10025                 }
10026         }
10027
10028         return (CTL_RETVAL_COMPLETE);
10029 }
10030
10031 /*
10032  * This routine (with one exception) checks LUN flags that can be set by
10033  * commands ahead of us in the OOA queue.  These flags have to be checked
10034  * when a command initially comes in, and when we pull a command off the
10035  * blocked queue and are preparing to execute it.  The reason we have to
10036  * check these flags for commands on the blocked queue is that the LUN
10037  * state may have been changed by a command ahead of us while we're on the
10038  * blocked queue.
10039  *
10040  * Ordering is somewhat important with these checks, so please pay
10041  * careful attention to the placement of any new checks.
10042  */
10043 static int
10044 ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
10045                      struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
10046 {
10047         int retval;
10048
10049         retval = 0;
10050
10051         /*
10052          * If this shelf is a secondary shelf controller, we have to reject
10053          * any media access commands.
10054          */
10055 #if 0
10056         /* No longer needed for HA */
10057         if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
10058          && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
10059                 ctl_set_lun_standby(ctsio);
10060                 retval = 1;
10061                 goto bailout;
10062         }
10063 #endif
10064
10065         /*
10066          * Check for a reservation conflict.  If this command isn't allowed
10067          * even on reserved LUNs, and if this initiator isn't the one who
10068          * reserved us, reject the command with a reservation conflict.
10069          */
10070         if ((lun->flags & CTL_LUN_RESERVED)
10071          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
10072                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
10073                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
10074                  || (ctsio->io_hdr.nexus.targ_target.id !=
10075                      lun->rsv_nexus.targ_target.id)) {
10076                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10077                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
10078                         retval = 1;
10079                         goto bailout;
10080                 }
10081         }
10082
10083         if ( (lun->flags & CTL_LUN_PR_RESERVED)
10084          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV) == 0)) {
10085                 uint32_t residx;
10086
10087                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
10088                 /*
10089                  * if we aren't registered or it's a res holder type
10090                  * reservation and this isn't the res holder then set a
10091                  * conflict.
10092                  * NOTE: Commands which might be allowed on write exclusive
10093                  * type reservations are checked in the particular command
10094                  * for a conflict. Read and SSU are the only ones.
10095                  */
10096                 if (!lun->per_res[residx].registered
10097                  || (residx != lun->pr_res_idx && lun->res_type < 4)) {
10098                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10099                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
10100                         retval = 1;
10101                         goto bailout;
10102                 }
10103
10104         }
10105
10106         if ((lun->flags & CTL_LUN_OFFLINE)
10107          && ((entry->flags & CTL_CMD_FLAG_OK_ON_OFFLINE) == 0)) {
10108                 ctl_set_lun_not_ready(ctsio);
10109                 retval = 1;
10110                 goto bailout;
10111         }
10112
10113         /*
10114          * If the LUN is stopped, see if this particular command is allowed
10115          * for a stopped lun.  Otherwise, reject it with 0x04,0x02.
10116          */
10117         if ((lun->flags & CTL_LUN_STOPPED)
10118          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
10119                 /* "Logical unit not ready, initializing cmd. required" */
10120                 ctl_set_lun_stopped(ctsio);
10121                 retval = 1;
10122                 goto bailout;
10123         }
10124
10125         if ((lun->flags & CTL_LUN_INOPERABLE)
10126          && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
10127                 /* "Medium format corrupted" */
10128                 ctl_set_medium_format_corrupted(ctsio);
10129                 retval = 1;
10130                 goto bailout;
10131         }
10132
10133 bailout:
10134         return (retval);
10135
10136 }
10137
10138 static void
10139 ctl_failover_io(union ctl_io *io, int have_lock)
10140 {
10141         ctl_set_busy(&io->scsiio);
10142         ctl_done_lock(io, have_lock);
10143 }
10144
10145 static void
10146 ctl_failover(void)
10147 {
10148         struct ctl_lun *lun;
10149         struct ctl_softc *ctl_softc;
10150         union ctl_io *next_io, *pending_io;
10151         union ctl_io *io;
10152         int lun_idx;
10153         int i;
10154
10155         ctl_softc = control_softc;
10156
10157         mtx_lock(&ctl_softc->ctl_lock);
10158         /*
10159          * Remove any cmds from the other SC from the rtr queue.  These
10160          * will obviously only be for LUNs for which we're the primary.
10161          * We can't send status or get/send data for these commands.
10162          * Since they haven't been executed yet, we can just remove them.
10163          * We'll either abort them or delete them below, depending on
10164          * which HA mode we're in.
10165          */
10166         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->rtr_queue);
10167              io != NULL; io = next_io) {
10168                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
10169                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10170                         STAILQ_REMOVE(&ctl_softc->rtr_queue, &io->io_hdr,
10171                                       ctl_io_hdr, links);
10172         }
10173
10174         for (lun_idx=0; lun_idx < ctl_softc->num_luns; lun_idx++) {
10175                 lun = ctl_softc->ctl_luns[lun_idx];
10176                 if (lun==NULL)
10177                         continue;
10178
10179                 /*
10180                  * Processor LUNs are primary on both sides.
10181                  * XXX will this always be true?
10182                  */
10183                 if (lun->be_lun->lun_type == T_PROCESSOR)
10184                         continue;
10185
10186                 if ((lun->flags & CTL_LUN_PRIMARY_SC)
10187                  && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10188                         printf("FAILOVER: primary lun %d\n", lun_idx);
10189                         /*
10190                          * Remove all commands from the other SC. First from the
10191                          * blocked queue then from the ooa queue. Once we have
10192                          * removed them. Call ctl_check_blocked to see if there
10193                          * is anything that can run.
10194                          */
10195                         for (io = (union ctl_io *)TAILQ_FIRST(
10196                              &lun->blocked_queue); io != NULL; io = next_io) {
10197
10198                                 next_io = (union ctl_io *)TAILQ_NEXT(
10199                                     &io->io_hdr, blocked_links);
10200
10201                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10202                                         TAILQ_REMOVE(&lun->blocked_queue,
10203                                                      &io->io_hdr,blocked_links);
10204                                         io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10205                                         TAILQ_REMOVE(&lun->ooa_queue,
10206                                                      &io->io_hdr, ooa_links);
10207
10208                                         ctl_free_io(io);
10209                                 }
10210                         }
10211
10212                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10213                              io != NULL; io = next_io) {
10214
10215                                 next_io = (union ctl_io *)TAILQ_NEXT(
10216                                     &io->io_hdr, ooa_links);
10217
10218                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10219
10220                                         TAILQ_REMOVE(&lun->ooa_queue,
10221                                                 &io->io_hdr,
10222                                                 ooa_links);
10223
10224                                         ctl_free_io(io);
10225                                 }
10226                         }
10227                         ctl_check_blocked(lun);
10228                 } else if ((lun->flags & CTL_LUN_PRIMARY_SC)
10229                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10230
10231                         printf("FAILOVER: primary lun %d\n", lun_idx);
10232                         /*
10233                          * Abort all commands from the other SC.  We can't
10234                          * send status back for them now.  These should get
10235                          * cleaned up when they are completed or come out
10236                          * for a datamove operation.
10237                          */
10238                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10239                              io != NULL; io = next_io) {
10240                                 next_io = (union ctl_io *)TAILQ_NEXT(
10241                                         &io->io_hdr, ooa_links);
10242
10243                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10244                                         io->io_hdr.flags |= CTL_FLAG_ABORT;
10245                         }
10246                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10247                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10248
10249                         printf("FAILOVER: secondary lun %d\n", lun_idx);
10250
10251                         lun->flags |= CTL_LUN_PRIMARY_SC;
10252
10253                         /*
10254                          * We send all I/O that was sent to this controller
10255                          * and redirected to the other side back with
10256                          * busy status, and have the initiator retry it.
10257                          * Figuring out how much data has been transferred,
10258                          * etc. and picking up where we left off would be
10259                          * very tricky.
10260                          *
10261                          * XXX KDM need to remove I/O from the blocked
10262                          * queue as well!
10263                          */
10264                         for (pending_io = (union ctl_io *)TAILQ_FIRST(
10265                              &lun->ooa_queue); pending_io != NULL;
10266                              pending_io = next_io) {
10267
10268                                 next_io =  (union ctl_io *)TAILQ_NEXT(
10269                                         &pending_io->io_hdr, ooa_links);
10270
10271                                 pending_io->io_hdr.flags &=
10272                                         ~CTL_FLAG_SENT_2OTHER_SC;
10273
10274                                 if (pending_io->io_hdr.flags &
10275                                     CTL_FLAG_IO_ACTIVE) {
10276                                         pending_io->io_hdr.flags |=
10277                                                 CTL_FLAG_FAILOVER;
10278                                 } else {
10279                                         ctl_set_busy(&pending_io->scsiio);
10280                                         ctl_done_lock(pending_io,
10281                                                       /*have_lock*/1);
10282                                 }
10283                         }
10284
10285                         /*
10286                          * Build Unit Attention
10287                          */
10288                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10289                                 lun->pending_sense[i].ua_pending |=
10290                                                      CTL_UA_ASYM_ACC_CHANGE;
10291                         }
10292                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10293                         && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10294                         printf("FAILOVER: secondary lun %d\n", lun_idx);
10295                         /*
10296                          * if the first io on the OOA is not on the RtR queue
10297                          * add it.
10298                          */
10299                         lun->flags |= CTL_LUN_PRIMARY_SC;
10300
10301                         pending_io = (union ctl_io *)TAILQ_FIRST(
10302                             &lun->ooa_queue);
10303                         if (pending_io==NULL) {
10304                                 printf("Nothing on OOA queue\n");
10305                                 continue;
10306                         }
10307
10308                         pending_io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
10309                         if ((pending_io->io_hdr.flags &
10310                              CTL_FLAG_IS_WAS_ON_RTR) == 0) {
10311                                 pending_io->io_hdr.flags |=
10312                                     CTL_FLAG_IS_WAS_ON_RTR;
10313                                 STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
10314                                                    &pending_io->io_hdr, links);
10315                         }
10316 #if 0
10317                         else
10318                         {
10319                                 printf("Tag 0x%04x is running\n",
10320                                       pending_io->scsiio.tag_num);
10321                         }
10322 #endif
10323
10324                         next_io = (union ctl_io *)TAILQ_NEXT(
10325                             &pending_io->io_hdr, ooa_links);
10326                         for (pending_io=next_io; pending_io != NULL;
10327                              pending_io = next_io) {
10328                                 pending_io->io_hdr.flags &=
10329                                     ~CTL_FLAG_SENT_2OTHER_SC;
10330                                 next_io = (union ctl_io *)TAILQ_NEXT(
10331                                         &pending_io->io_hdr, ooa_links);
10332                                 if (pending_io->io_hdr.flags &
10333                                     CTL_FLAG_IS_WAS_ON_RTR) {
10334 #if 0
10335                                         printf("Tag 0x%04x is running\n",
10336                                                 pending_io->scsiio.tag_num);
10337 #endif
10338                                         continue;
10339                                 }
10340
10341                                 switch (ctl_check_ooa(lun, pending_io,
10342                                     (union ctl_io *)TAILQ_PREV(
10343                                     &pending_io->io_hdr, ctl_ooaq,
10344                                     ooa_links))) {
10345
10346                                 case CTL_ACTION_BLOCK:
10347                                         TAILQ_INSERT_TAIL(&lun->blocked_queue,
10348                                                           &pending_io->io_hdr,
10349                                                           blocked_links);
10350                                         pending_io->io_hdr.flags |=
10351                                             CTL_FLAG_BLOCKED;
10352                                         break;
10353                                 case CTL_ACTION_PASS:
10354                                 case CTL_ACTION_SKIP:
10355                                         pending_io->io_hdr.flags |=
10356                                             CTL_FLAG_IS_WAS_ON_RTR;
10357                                         STAILQ_INSERT_TAIL(
10358                                             &ctl_softc->rtr_queue,
10359                                             &pending_io->io_hdr, links);
10360                                         break;
10361                                 case CTL_ACTION_OVERLAP:
10362                                         ctl_set_overlapped_cmd(
10363                                             (struct ctl_scsiio *)pending_io);
10364                                         ctl_done_lock(pending_io,
10365                                                       /*have_lock*/ 1);
10366                                         break;
10367                                 case CTL_ACTION_OVERLAP_TAG:
10368                                         ctl_set_overlapped_tag(
10369                                             (struct ctl_scsiio *)pending_io,
10370                                             pending_io->scsiio.tag_num & 0xff);
10371                                         ctl_done_lock(pending_io,
10372                                                       /*have_lock*/ 1);
10373                                         break;
10374                                 case CTL_ACTION_ERROR:
10375                                 default:
10376                                         ctl_set_internal_failure(
10377                                                 (struct ctl_scsiio *)pending_io,
10378                                                 0,  // sks_valid
10379                                                 0); //retry count
10380                                         ctl_done_lock(pending_io,
10381                                                       /*have_lock*/ 1);
10382                                         break;
10383                                 }
10384                         }
10385
10386                         /*
10387                          * Build Unit Attention
10388                          */
10389                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10390                                 lun->pending_sense[i].ua_pending |=
10391                                                      CTL_UA_ASYM_ACC_CHANGE;
10392                         }
10393                 } else {
10394                         panic("Unhandled HA mode failover, LUN flags = %#x, "
10395                               "ha_mode = #%x", lun->flags, ctl_softc->ha_mode);
10396                 }
10397         }
10398         ctl_pause_rtr = 0;
10399         mtx_unlock(&ctl_softc->ctl_lock);
10400 }
10401
10402 static int
10403 ctl_scsiio_precheck(struct ctl_softc *ctl_softc, struct ctl_scsiio *ctsio)
10404 {
10405         struct ctl_lun *lun;
10406         struct ctl_cmd_entry *entry;
10407         uint8_t opcode;
10408         uint32_t initidx, targ_lun;
10409         int retval;
10410
10411         retval = 0;
10412
10413         lun = NULL;
10414
10415         opcode = ctsio->cdb[0];
10416
10417         mtx_lock(&ctl_softc->ctl_lock);
10418
10419         targ_lun = ctsio->io_hdr.nexus.targ_lun;
10420         if (ctsio->io_hdr.nexus.lun_map_fn != NULL)
10421                 targ_lun = ctsio->io_hdr.nexus.lun_map_fn(ctsio->io_hdr.nexus.lun_map_arg, targ_lun);
10422         if ((targ_lun < CTL_MAX_LUNS)
10423          && (ctl_softc->ctl_luns[targ_lun] != NULL)) {
10424                 lun = ctl_softc->ctl_luns[targ_lun];
10425                 /*
10426                  * If the LUN is invalid, pretend that it doesn't exist.
10427                  * It will go away as soon as all pending I/O has been
10428                  * completed.
10429                  */
10430                 if (lun->flags & CTL_LUN_DISABLED) {
10431                         lun = NULL;
10432                 } else {
10433                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
10434                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
10435                                 lun->be_lun;
10436                         if (lun->be_lun->lun_type == T_PROCESSOR) {
10437                                 ctsio->io_hdr.flags |= CTL_FLAG_CONTROL_DEV;
10438                         }
10439                 }
10440         } else {
10441                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
10442                 ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
10443         }
10444
10445         entry = &ctl_cmd_table[opcode];
10446
10447         ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
10448         ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
10449
10450         /*
10451          * Check to see whether we can send this command to LUNs that don't
10452          * exist.  This should pretty much only be the case for inquiry
10453          * and request sense.  Further checks, below, really require having
10454          * a LUN, so we can't really check the command anymore.  Just put
10455          * it on the rtr queue.
10456          */
10457         if (lun == NULL) {
10458                 if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10459                         goto queue_rtr;
10460
10461                 ctl_set_unsupported_lun(ctsio);
10462                 mtx_unlock(&ctl_softc->ctl_lock);
10463                 ctl_done((union ctl_io *)ctsio);
10464                 CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
10465                 goto bailout;
10466         } else {
10467                 /*
10468                  * Every I/O goes into the OOA queue for a particular LUN, and
10469                  * stays there until completion.
10470                  */
10471                 TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
10472
10473                 /*
10474                  * Make sure we support this particular command on this LUN.
10475                  * e.g., we don't support writes to the control LUN.
10476                  */
10477                 switch (lun->be_lun->lun_type) {
10478                 case T_PROCESSOR:
10479                         if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0)
10480                          && ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10481                               == 0)) {
10482                                 ctl_set_invalid_opcode(ctsio);
10483                                 mtx_unlock(&ctl_softc->ctl_lock);
10484                                 ctl_done((union ctl_io *)ctsio);
10485                                 goto bailout;
10486                         }
10487                         break;
10488                 case T_DIRECT:
10489                         if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0)
10490                          && ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
10491                               == 0)){
10492                                 ctl_set_invalid_opcode(ctsio);
10493                                 mtx_unlock(&ctl_softc->ctl_lock);
10494                                 ctl_done((union ctl_io *)ctsio);
10495                                 goto bailout;
10496                         }
10497                         break;
10498                 default:
10499                         printf("Unsupported CTL LUN type %d\n",
10500                                lun->be_lun->lun_type);
10501                         panic("Unsupported CTL LUN type %d\n",
10502                               lun->be_lun->lun_type);
10503                         break; /* NOTREACHED */
10504                 }
10505         }
10506
10507         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
10508
10509         /*
10510          * If we've got a request sense, it'll clear the contingent
10511          * allegiance condition.  Otherwise, if we have a CA condition for
10512          * this initiator, clear it, because it sent down a command other
10513          * than request sense.
10514          */
10515         if ((opcode != REQUEST_SENSE)
10516          && (ctl_is_set(lun->have_ca, initidx)))
10517                 ctl_clear_mask(lun->have_ca, initidx);
10518
10519         /*
10520          * If the command has this flag set, it handles its own unit
10521          * attention reporting, we shouldn't do anything.  Otherwise we
10522          * check for any pending unit attentions, and send them back to the
10523          * initiator.  We only do this when a command initially comes in,
10524          * not when we pull it off the blocked queue.
10525          *
10526          * According to SAM-3, section 5.3.2, the order that things get
10527          * presented back to the host is basically unit attentions caused
10528          * by some sort of reset event, busy status, reservation conflicts
10529          * or task set full, and finally any other status.
10530          *
10531          * One issue here is that some of the unit attentions we report
10532          * don't fall into the "reset" category (e.g. "reported luns data
10533          * has changed").  So reporting it here, before the reservation
10534          * check, may be technically wrong.  I guess the only thing to do
10535          * would be to check for and report the reset events here, and then
10536          * check for the other unit attention types after we check for a
10537          * reservation conflict.
10538          *
10539          * XXX KDM need to fix this
10540          */
10541         if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
10542                 ctl_ua_type ua_type;
10543
10544                 ua_type = lun->pending_sense[initidx].ua_pending;
10545                 if (ua_type != CTL_UA_NONE) {
10546                         scsi_sense_data_type sense_format;
10547
10548                         if (lun != NULL)
10549                                 sense_format = (lun->flags &
10550                                     CTL_LUN_SENSE_DESC) ? SSD_TYPE_DESC :
10551                                     SSD_TYPE_FIXED;
10552                         else
10553                                 sense_format = SSD_TYPE_FIXED;
10554
10555                         ua_type = ctl_build_ua(ua_type, &ctsio->sense_data,
10556                                                sense_format);
10557                         if (ua_type != CTL_UA_NONE) {
10558                                 ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
10559                                 ctsio->io_hdr.status = CTL_SCSI_ERROR |
10560                                                        CTL_AUTOSENSE;
10561                                 ctsio->sense_len = SSD_FULL_SIZE;
10562                                 lun->pending_sense[initidx].ua_pending &=
10563                                         ~ua_type;
10564                                 mtx_unlock(&ctl_softc->ctl_lock);
10565                                 ctl_done((union ctl_io *)ctsio);
10566                                 goto bailout;
10567                         }
10568                 }
10569         }
10570
10571
10572         if (ctl_scsiio_lun_check(ctl_softc, lun, entry, ctsio) != 0) {
10573                 mtx_unlock(&ctl_softc->ctl_lock);
10574                 ctl_done((union ctl_io *)ctsio);
10575                 goto bailout;
10576         }
10577
10578         /*
10579          * XXX CHD this is where we want to send IO to other side if
10580          * this LUN is secondary on this SC. We will need to make a copy
10581          * of the IO and flag the IO on this side as SENT_2OTHER and the flag
10582          * the copy we send as FROM_OTHER.
10583          * We also need to stuff the address of the original IO so we can
10584          * find it easily. Something similar will need be done on the other
10585          * side so when we are done we can find the copy.
10586          */
10587         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
10588                 union ctl_ha_msg msg_info;
10589                 int isc_retval;
10590
10591                 ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
10592
10593                 msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
10594                 msg_info.hdr.original_sc = (union ctl_io *)ctsio;
10595 #if 0
10596                 printf("1. ctsio %p\n", ctsio);
10597 #endif
10598                 msg_info.hdr.serializing_sc = NULL;
10599                 msg_info.hdr.nexus = ctsio->io_hdr.nexus;
10600                 msg_info.scsi.tag_num = ctsio->tag_num;
10601                 msg_info.scsi.tag_type = ctsio->tag_type;
10602                 memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
10603
10604                 ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
10605
10606                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10607                     (void *)&msg_info, sizeof(msg_info), 0)) >
10608                     CTL_HA_STATUS_SUCCESS) {
10609                         printf("CTL:precheck, ctl_ha_msg_send returned %d\n",
10610                                isc_retval);
10611                         printf("CTL:opcode is %x\n",opcode);
10612                 } else {
10613 #if 0
10614                         printf("CTL:Precheck sent msg, opcode is %x\n",opcode);
10615 #endif
10616                 }
10617
10618                 /*
10619                  * XXX KDM this I/O is off the incoming queue, but hasn't
10620                  * been inserted on any other queue.  We may need to come
10621                  * up with a holding queue while we wait for serialization
10622                  * so that we have an idea of what we're waiting for from
10623                  * the other side.
10624                  */
10625                 goto bailout_unlock;
10626         }
10627
10628         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
10629                               (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
10630                               ctl_ooaq, ooa_links))) {
10631         case CTL_ACTION_BLOCK:
10632                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
10633                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
10634                                   blocked_links);
10635                 goto bailout_unlock;
10636                 break; /* NOTREACHED */
10637         case CTL_ACTION_PASS:
10638         case CTL_ACTION_SKIP:
10639                 goto queue_rtr;
10640                 break; /* NOTREACHED */
10641         case CTL_ACTION_OVERLAP:
10642                 ctl_set_overlapped_cmd(ctsio);
10643                 mtx_unlock(&ctl_softc->ctl_lock);
10644                 ctl_done((union ctl_io *)ctsio);
10645                 goto bailout;
10646                 break; /* NOTREACHED */
10647         case CTL_ACTION_OVERLAP_TAG:
10648                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
10649                 mtx_unlock(&ctl_softc->ctl_lock);
10650                 ctl_done((union ctl_io *)ctsio);
10651                 goto bailout;
10652                 break; /* NOTREACHED */
10653         case CTL_ACTION_ERROR:
10654         default:
10655                 ctl_set_internal_failure(ctsio,
10656                                          /*sks_valid*/ 0,
10657                                          /*retry_count*/ 0);
10658                 mtx_unlock(&ctl_softc->ctl_lock);
10659                 ctl_done((union ctl_io *)ctsio);
10660                 goto bailout;
10661                 break; /* NOTREACHED */
10662         }
10663
10664         goto bailout_unlock;
10665
10666 queue_rtr:
10667         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
10668         STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue, &ctsio->io_hdr, links);
10669
10670 bailout_unlock:
10671         mtx_unlock(&ctl_softc->ctl_lock);
10672
10673 bailout:
10674         return (retval);
10675 }
10676
10677 static int
10678 ctl_scsiio(struct ctl_scsiio *ctsio)
10679 {
10680         int retval;
10681         struct ctl_cmd_entry *entry;
10682
10683         retval = CTL_RETVAL_COMPLETE;
10684
10685         CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
10686
10687         entry = &ctl_cmd_table[ctsio->cdb[0]];
10688
10689         /*
10690          * If this I/O has been aborted, just send it straight to
10691          * ctl_done() without executing it.
10692          */
10693         if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
10694                 ctl_done((union ctl_io *)ctsio);
10695                 goto bailout;
10696         }
10697
10698         /*
10699          * All the checks should have been handled by ctl_scsiio_precheck().
10700          * We should be clear now to just execute the I/O.
10701          */
10702         retval = entry->execute(ctsio);
10703
10704 bailout:
10705         return (retval);
10706 }
10707
10708 /*
10709  * Since we only implement one target right now, a bus reset simply resets
10710  * our single target.
10711  */
10712 static int
10713 ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io)
10714 {
10715         return(ctl_target_reset(ctl_softc, io, CTL_UA_BUS_RESET));
10716 }
10717
10718 static int
10719 ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
10720                  ctl_ua_type ua_type)
10721 {
10722         struct ctl_lun *lun;
10723         int retval;
10724
10725         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
10726                 union ctl_ha_msg msg_info;
10727
10728                 io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
10729                 msg_info.hdr.nexus = io->io_hdr.nexus;
10730                 if (ua_type==CTL_UA_TARG_RESET)
10731                         msg_info.task.task_action = CTL_TASK_TARGET_RESET;
10732                 else
10733                         msg_info.task.task_action = CTL_TASK_BUS_RESET;
10734                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
10735                 msg_info.hdr.original_sc = NULL;
10736                 msg_info.hdr.serializing_sc = NULL;
10737                 if (CTL_HA_STATUS_SUCCESS != ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10738                     (void *)&msg_info, sizeof(msg_info), 0)) {
10739                 }
10740         }
10741         retval = 0;
10742
10743         STAILQ_FOREACH(lun, &ctl_softc->lun_list, links)
10744                 retval += ctl_lun_reset(lun, io, ua_type);
10745
10746         return (retval);
10747 }
10748
10749 /*
10750  * The LUN should always be set.  The I/O is optional, and is used to
10751  * distinguish between I/Os sent by this initiator, and by other
10752  * initiators.  We set unit attention for initiators other than this one.
10753  * SAM-3 is vague on this point.  It does say that a unit attention should
10754  * be established for other initiators when a LUN is reset (see section
10755  * 5.7.3), but it doesn't specifically say that the unit attention should
10756  * be established for this particular initiator when a LUN is reset.  Here
10757  * is the relevant text, from SAM-3 rev 8:
10758  *
10759  * 5.7.2 When a SCSI initiator port aborts its own tasks
10760  *
10761  * When a SCSI initiator port causes its own task(s) to be aborted, no
10762  * notification that the task(s) have been aborted shall be returned to
10763  * the SCSI initiator port other than the completion response for the
10764  * command or task management function action that caused the task(s) to
10765  * be aborted and notification(s) associated with related effects of the
10766  * action (e.g., a reset unit attention condition).
10767  *
10768  * XXX KDM for now, we're setting unit attention for all initiators.
10769  */
10770 static int
10771 ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
10772 {
10773         union ctl_io *xio;
10774 #if 0
10775         uint32_t initindex;
10776 #endif
10777         int i;
10778
10779         /*
10780          * Run through the OOA queue and abort each I/O.
10781          */
10782 #if 0
10783         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
10784 #endif
10785         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
10786              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
10787                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
10788         }
10789
10790         /*
10791          * This version sets unit attention for every
10792          */
10793 #if 0
10794         initindex = ctl_get_initindex(&io->io_hdr.nexus);
10795         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10796                 if (initindex == i)
10797                         continue;
10798                 lun->pending_sense[i].ua_pending |= ua_type;
10799         }
10800 #endif
10801
10802         /*
10803          * A reset (any kind, really) clears reservations established with
10804          * RESERVE/RELEASE.  It does not clear reservations established
10805          * with PERSISTENT RESERVE OUT, but we don't support that at the
10806          * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
10807          * reservations made with the RESERVE/RELEASE commands, because
10808          * those commands are obsolete in SPC-3.
10809          */
10810         lun->flags &= ~CTL_LUN_RESERVED;
10811
10812         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10813                 ctl_clear_mask(lun->have_ca, i);
10814                 lun->pending_sense[i].ua_pending |= ua_type;
10815         }
10816
10817         return (0);
10818 }
10819
10820 static int
10821 ctl_abort_task(union ctl_io *io)
10822 {
10823         union ctl_io *xio;
10824         struct ctl_lun *lun;
10825         struct ctl_softc *ctl_softc;
10826 #if 0
10827         struct sbuf sb;
10828         char printbuf[128];
10829 #endif
10830         int found;
10831         uint32_t targ_lun;
10832
10833         ctl_softc = control_softc;
10834         found = 0;
10835
10836         /*
10837          * Look up the LUN.
10838          */
10839         targ_lun = io->io_hdr.nexus.targ_lun;
10840         if (io->io_hdr.nexus.lun_map_fn != NULL)
10841                 targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
10842         if ((targ_lun < CTL_MAX_LUNS)
10843          && (ctl_softc->ctl_luns[targ_lun] != NULL))
10844                 lun = ctl_softc->ctl_luns[targ_lun];
10845         else
10846                 goto bailout;
10847
10848 #if 0
10849         printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
10850                lun->lun, io->taskio.tag_num, io->taskio.tag_type);
10851 #endif
10852
10853         /*
10854          * Run through the OOA queue and attempt to find the given I/O.
10855          * The target port, initiator ID, tag type and tag number have to
10856          * match the values that we got from the initiator.  If we have an
10857          * untagged command to abort, simply abort the first untagged command
10858          * we come to.  We only allow one untagged command at a time of course.
10859          */
10860 #if 0
10861         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
10862 #endif
10863         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
10864              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
10865 #if 0
10866                 sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
10867
10868                 sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
10869                             lun->lun, xio->scsiio.tag_num,
10870                             xio->scsiio.tag_type,
10871                             (xio->io_hdr.blocked_links.tqe_prev
10872                             == NULL) ? "" : " BLOCKED",
10873                             (xio->io_hdr.flags &
10874                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
10875                             (xio->io_hdr.flags &
10876                             CTL_FLAG_ABORT) ? " ABORT" : "",
10877                             (xio->io_hdr.flags &
10878                             CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
10879                 ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
10880                 sbuf_finish(&sb);
10881                 printf("%s\n", sbuf_data(&sb));
10882 #endif
10883
10884                 if ((xio->io_hdr.nexus.targ_port == io->io_hdr.nexus.targ_port)
10885                  && (xio->io_hdr.nexus.initid.id ==
10886                      io->io_hdr.nexus.initid.id)) {
10887                         /*
10888                          * If the abort says that the task is untagged, the
10889                          * task in the queue must be untagged.  Otherwise,
10890                          * we just check to see whether the tag numbers
10891                          * match.  This is because the QLogic firmware
10892                          * doesn't pass back the tag type in an abort
10893                          * request.
10894                          */
10895 #if 0
10896                         if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
10897                           && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
10898                          || (xio->scsiio.tag_num == io->taskio.tag_num)) {
10899 #endif
10900                         /*
10901                          * XXX KDM we've got problems with FC, because it
10902                          * doesn't send down a tag type with aborts.  So we
10903                          * can only really go by the tag number...
10904                          * This may cause problems with parallel SCSI.
10905                          * Need to figure that out!!
10906                          */
10907                         if (xio->scsiio.tag_num == io->taskio.tag_num) {
10908                                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
10909                                 found = 1;
10910                                 if ((io->io_hdr.flags &
10911                                      CTL_FLAG_FROM_OTHER_SC) == 0 &&
10912                                     !(lun->flags & CTL_LUN_PRIMARY_SC)) {
10913                                         union ctl_ha_msg msg_info;
10914
10915                                         io->io_hdr.flags |=
10916                                                         CTL_FLAG_SENT_2OTHER_SC;
10917                                         msg_info.hdr.nexus = io->io_hdr.nexus;
10918                                         msg_info.task.task_action =
10919                                                 CTL_TASK_ABORT_TASK;
10920                                         msg_info.task.tag_num =
10921                                                 io->taskio.tag_num;
10922                                         msg_info.task.tag_type =
10923                                                 io->taskio.tag_type;
10924                                         msg_info.hdr.msg_type =
10925                                                 CTL_MSG_MANAGE_TASKS;
10926                                         msg_info.hdr.original_sc = NULL;
10927                                         msg_info.hdr.serializing_sc = NULL;
10928 #if 0
10929                                         printf("Sent Abort to other side\n");
10930 #endif
10931                                         if (CTL_HA_STATUS_SUCCESS !=
10932                                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10933                                                 (void *)&msg_info,
10934                                                 sizeof(msg_info), 0)) {
10935                                         }
10936                                 }
10937 #if 0
10938                                 printf("ctl_abort_task: found I/O to abort\n");
10939 #endif
10940                                 break;
10941                         }
10942                 }
10943         }
10944
10945 bailout:
10946
10947         if (found == 0) {
10948                 /*
10949                  * This isn't really an error.  It's entirely possible for
10950                  * the abort and command completion to cross on the wire.
10951                  * This is more of an informative/diagnostic error.
10952                  */
10953 #if 0
10954                 printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
10955                        "%d:%d:%d:%d tag %d type %d\n",
10956                        io->io_hdr.nexus.initid.id,
10957                        io->io_hdr.nexus.targ_port,
10958                        io->io_hdr.nexus.targ_target.id,
10959                        io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
10960                        io->taskio.tag_type);
10961 #endif
10962                 return (1);
10963         } else
10964                 return (0);
10965 }
10966
10967 /*
10968  * This routine cannot block!  It must be callable from an interrupt
10969  * handler as well as from the work thread.
10970  */
10971 static void
10972 ctl_run_task_queue(struct ctl_softc *ctl_softc)
10973 {
10974         union ctl_io *io, *next_io;
10975
10976         mtx_assert(&ctl_softc->ctl_lock, MA_OWNED);
10977
10978         CTL_DEBUG_PRINT(("ctl_run_task_queue\n"));
10979
10980         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->task_queue);
10981              io != NULL; io = next_io) {
10982                 int retval;
10983                 const char *task_desc;
10984
10985                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
10986
10987                 retval = 0;
10988
10989                 switch (io->io_hdr.io_type) {
10990                 case CTL_IO_TASK: {
10991                         task_desc = ctl_scsi_task_string(&io->taskio);
10992                         if (task_desc != NULL) {
10993 #ifdef NEEDTOPORT
10994                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
10995                                             CTL_TASK_REPORT,
10996                                             csevent_LogType_Trace,
10997                                             csevent_Severity_Information,
10998                                             csevent_AlertLevel_Green,
10999                                             csevent_FRU_Firmware,
11000                                             csevent_FRU_Unknown,
11001                                             "CTL: received task: %s",task_desc);
11002 #endif
11003                         } else {
11004 #ifdef NEEDTOPORT
11005                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
11006                                             CTL_TASK_REPORT,
11007                                             csevent_LogType_Trace,
11008                                             csevent_Severity_Information,
11009                                             csevent_AlertLevel_Green,
11010                                             csevent_FRU_Firmware,
11011                                             csevent_FRU_Unknown,
11012                                             "CTL: received unknown task "
11013                                             "type: %d (%#x)",
11014                                             io->taskio.task_action,
11015                                             io->taskio.task_action);
11016 #endif
11017                         }
11018                         switch (io->taskio.task_action) {
11019                         case CTL_TASK_ABORT_TASK:
11020                                 retval = ctl_abort_task(io);
11021                                 break;
11022                         case CTL_TASK_ABORT_TASK_SET:
11023                                 break;
11024                         case CTL_TASK_CLEAR_ACA:
11025                                 break;
11026                         case CTL_TASK_CLEAR_TASK_SET:
11027                                 break;
11028                         case CTL_TASK_LUN_RESET: {
11029                                 struct ctl_lun *lun;
11030                                 uint32_t targ_lun;
11031                                 int retval;
11032
11033                                 targ_lun = io->io_hdr.nexus.targ_lun;
11034                                 if (io->io_hdr.nexus.lun_map_fn != NULL)
11035                                         targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
11036
11037                                 if ((targ_lun < CTL_MAX_LUNS)
11038                                  && (ctl_softc->ctl_luns[targ_lun] != NULL))
11039                                         lun = ctl_softc->ctl_luns[targ_lun];
11040                                 else {
11041                                         retval = 1;
11042                                         break;
11043                                 }
11044
11045                                 if (!(io->io_hdr.flags &
11046                                     CTL_FLAG_FROM_OTHER_SC)) {
11047                                         union ctl_ha_msg msg_info;
11048
11049                                         io->io_hdr.flags |=
11050                                                 CTL_FLAG_SENT_2OTHER_SC;
11051                                         msg_info.hdr.msg_type =
11052                                                 CTL_MSG_MANAGE_TASKS;
11053                                         msg_info.hdr.nexus = io->io_hdr.nexus;
11054                                         msg_info.task.task_action =
11055                                                 CTL_TASK_LUN_RESET;
11056                                         msg_info.hdr.original_sc = NULL;
11057                                         msg_info.hdr.serializing_sc = NULL;
11058                                         if (CTL_HA_STATUS_SUCCESS !=
11059                                             ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11060                                             (void *)&msg_info,
11061                                             sizeof(msg_info), 0)) {
11062                                         }
11063                                 }
11064
11065                                 retval = ctl_lun_reset(lun, io,
11066                                                        CTL_UA_LUN_RESET);
11067                                 break;
11068                         }
11069                         case CTL_TASK_TARGET_RESET:
11070                                 retval = ctl_target_reset(ctl_softc, io,
11071                                                           CTL_UA_TARG_RESET);
11072                                 break;
11073                         case CTL_TASK_BUS_RESET:
11074                                 retval = ctl_bus_reset(ctl_softc, io);
11075                                 break;
11076                         case CTL_TASK_PORT_LOGIN:
11077                                 break;
11078                         case CTL_TASK_PORT_LOGOUT:
11079                                 break;
11080                         default:
11081                                 printf("ctl_run_task_queue: got unknown task "
11082                                        "management event %d\n",
11083                                        io->taskio.task_action);
11084                                 break;
11085                         }
11086                         if (retval == 0)
11087                                 io->io_hdr.status = CTL_SUCCESS;
11088                         else
11089                                 io->io_hdr.status = CTL_ERROR;
11090
11091                         STAILQ_REMOVE(&ctl_softc->task_queue, &io->io_hdr,
11092                                       ctl_io_hdr, links);
11093                         /*
11094                          * This will queue this I/O to the done queue, but the
11095                          * work thread won't be able to process it until we
11096                          * return and the lock is released.
11097                          */
11098                         ctl_done_lock(io, /*have_lock*/ 1);
11099                         break;
11100                 }
11101                 default: {
11102
11103                         printf("%s: invalid I/O type %d msg %d cdb %x"
11104                                " iptl: %ju:%d:%ju:%d tag 0x%04x\n",
11105                                __func__, io->io_hdr.io_type,
11106                                io->io_hdr.msg_type, io->scsiio.cdb[0],
11107                                (uintmax_t)io->io_hdr.nexus.initid.id,
11108                                io->io_hdr.nexus.targ_port,
11109                                (uintmax_t)io->io_hdr.nexus.targ_target.id,
11110                                io->io_hdr.nexus.targ_lun /* XXX */,
11111                                (io->io_hdr.io_type == CTL_IO_TASK) ?
11112                                io->taskio.tag_num : io->scsiio.tag_num);
11113                         STAILQ_REMOVE(&ctl_softc->task_queue, &io->io_hdr,
11114                                       ctl_io_hdr, links);
11115                         ctl_free_io(io);
11116                         break;
11117                 }
11118                 }
11119         }
11120
11121         ctl_softc->flags &= ~CTL_FLAG_TASK_PENDING;
11122 }
11123
11124 /*
11125  * For HA operation.  Handle commands that come in from the other
11126  * controller.
11127  */
11128 static void
11129 ctl_handle_isc(union ctl_io *io)
11130 {
11131         int free_io;
11132         struct ctl_lun *lun;
11133         struct ctl_softc *ctl_softc;
11134         uint32_t targ_lun;
11135
11136         ctl_softc = control_softc;
11137
11138         targ_lun = io->io_hdr.nexus.targ_lun;
11139         if (io->io_hdr.nexus.lun_map_fn != NULL)
11140                 targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
11141         lun = ctl_softc->ctl_luns[targ_lun];
11142
11143         switch (io->io_hdr.msg_type) {
11144         case CTL_MSG_SERIALIZE:
11145                 free_io = ctl_serialize_other_sc_cmd(&io->scsiio,
11146                                                      /*have_lock*/ 0);
11147                 break;
11148         case CTL_MSG_R2R: {
11149                 uint8_t opcode;
11150                 struct ctl_cmd_entry *entry;
11151
11152                 /*
11153                  * This is only used in SER_ONLY mode.
11154                  */
11155                 free_io = 0;
11156                 opcode = io->scsiio.cdb[0];
11157                 entry = &ctl_cmd_table[opcode];
11158                 mtx_lock(&ctl_softc->ctl_lock);
11159                 if (ctl_scsiio_lun_check(ctl_softc, lun,
11160                     entry, (struct ctl_scsiio *)io) != 0) {
11161                         ctl_done_lock(io, /*have_lock*/ 1);
11162                         mtx_unlock(&ctl_softc->ctl_lock);
11163                         break;
11164                 }
11165                 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11166                 STAILQ_INSERT_TAIL(&ctl_softc->rtr_queue,
11167                                    &io->io_hdr, links);
11168                 mtx_unlock(&ctl_softc->ctl_lock);
11169                 break;
11170         }
11171         case CTL_MSG_FINISH_IO:
11172                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
11173                         free_io = 0;
11174                         ctl_done_lock(io, /*have_lock*/ 0);
11175                 } else {
11176                         free_io = 1;
11177                         mtx_lock(&ctl_softc->ctl_lock);
11178                         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
11179                                      ooa_links);
11180                         STAILQ_REMOVE(&ctl_softc->task_queue,
11181                                       &io->io_hdr, ctl_io_hdr, links);
11182                         ctl_check_blocked(lun);
11183                         mtx_unlock(&ctl_softc->ctl_lock);
11184                 }
11185                 break;
11186         case CTL_MSG_PERS_ACTION:
11187                 ctl_hndl_per_res_out_on_other_sc(
11188                         (union ctl_ha_msg *)&io->presio.pr_msg);
11189                 free_io = 1;
11190                 break;
11191         case CTL_MSG_BAD_JUJU:
11192                 free_io = 0;
11193                 ctl_done_lock(io, /*have_lock*/ 0);
11194                 break;
11195         case CTL_MSG_DATAMOVE:
11196                 /* Only used in XFER mode */
11197                 free_io = 0;
11198                 ctl_datamove_remote(io);
11199                 break;
11200         case CTL_MSG_DATAMOVE_DONE:
11201                 /* Only used in XFER mode */
11202                 free_io = 0;
11203                 io->scsiio.be_move_done(io);
11204                 break;
11205         default:
11206                 free_io = 1;
11207                 printf("%s: Invalid message type %d\n",
11208                        __func__, io->io_hdr.msg_type);
11209                 break;
11210         }
11211         if (free_io)
11212                 ctl_free_io(io);
11213
11214 }
11215
11216
11217 /*
11218  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
11219  * there is no match.
11220  */
11221 static ctl_lun_error_pattern
11222 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
11223 {
11224         struct ctl_cmd_entry *entry;
11225         ctl_lun_error_pattern filtered_pattern, pattern;
11226         uint8_t opcode;
11227
11228         pattern = desc->error_pattern;
11229
11230         /*
11231          * XXX KDM we need more data passed into this function to match a
11232          * custom pattern, and we actually need to implement custom pattern
11233          * matching.
11234          */
11235         if (pattern & CTL_LUN_PAT_CMD)
11236                 return (CTL_LUN_PAT_CMD);
11237
11238         if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
11239                 return (CTL_LUN_PAT_ANY);
11240
11241         opcode = ctsio->cdb[0];
11242         entry = &ctl_cmd_table[opcode];
11243
11244         filtered_pattern = entry->pattern & pattern;
11245
11246         /*
11247          * If the user requested specific flags in the pattern (e.g.
11248          * CTL_LUN_PAT_RANGE), make sure the command supports all of those
11249          * flags.
11250          *
11251          * If the user did not specify any flags, it doesn't matter whether
11252          * or not the command supports the flags.
11253          */
11254         if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
11255              (pattern & ~CTL_LUN_PAT_MASK))
11256                 return (CTL_LUN_PAT_NONE);
11257
11258         /*
11259          * If the user asked for a range check, see if the requested LBA
11260          * range overlaps with this command's LBA range.
11261          */
11262         if (filtered_pattern & CTL_LUN_PAT_RANGE) {
11263                 uint64_t lba1;
11264                 uint32_t len1;
11265                 ctl_action action;
11266                 int retval;
11267
11268                 retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
11269                 if (retval != 0)
11270                         return (CTL_LUN_PAT_NONE);
11271
11272                 action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
11273                                               desc->lba_range.len);
11274                 /*
11275                  * A "pass" means that the LBA ranges don't overlap, so
11276                  * this doesn't match the user's range criteria.
11277                  */
11278                 if (action == CTL_ACTION_PASS)
11279                         return (CTL_LUN_PAT_NONE);
11280         }
11281
11282         return (filtered_pattern);
11283 }
11284
11285 static void
11286 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
11287 {
11288         struct ctl_error_desc *desc, *desc2;
11289
11290         mtx_assert(&control_softc->ctl_lock, MA_OWNED);
11291
11292         STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
11293                 ctl_lun_error_pattern pattern;
11294                 /*
11295                  * Check to see whether this particular command matches
11296                  * the pattern in the descriptor.
11297                  */
11298                 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
11299                 if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
11300                         continue;
11301
11302                 switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
11303                 case CTL_LUN_INJ_ABORTED:
11304                         ctl_set_aborted(&io->scsiio);
11305                         break;
11306                 case CTL_LUN_INJ_MEDIUM_ERR:
11307                         ctl_set_medium_error(&io->scsiio);
11308                         break;
11309                 case CTL_LUN_INJ_UA:
11310                         /* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
11311                          * OCCURRED */
11312                         ctl_set_ua(&io->scsiio, 0x29, 0x00);
11313                         break;
11314                 case CTL_LUN_INJ_CUSTOM:
11315                         /*
11316                          * We're assuming the user knows what he is doing.
11317                          * Just copy the sense information without doing
11318                          * checks.
11319                          */
11320                         bcopy(&desc->custom_sense, &io->scsiio.sense_data,
11321                               ctl_min(sizeof(desc->custom_sense),
11322                                       sizeof(io->scsiio.sense_data)));
11323                         io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
11324                         io->scsiio.sense_len = SSD_FULL_SIZE;
11325                         io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
11326                         break;
11327                 case CTL_LUN_INJ_NONE:
11328                 default:
11329                         /*
11330                          * If this is an error injection type we don't know
11331                          * about, clear the continuous flag (if it is set)
11332                          * so it will get deleted below.
11333                          */
11334                         desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
11335                         break;
11336                 }
11337                 /*
11338                  * By default, each error injection action is a one-shot
11339                  */
11340                 if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
11341                         continue;
11342
11343                 STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
11344
11345                 free(desc, M_CTL);
11346         }
11347 }
11348
11349 #ifdef CTL_IO_DELAY
11350 static void
11351 ctl_datamove_timer_wakeup(void *arg)
11352 {
11353         union ctl_io *io;
11354
11355         io = (union ctl_io *)arg;
11356
11357         ctl_datamove(io);
11358 }
11359 #endif /* CTL_IO_DELAY */
11360
11361 void
11362 ctl_datamove(union ctl_io *io)
11363 {
11364         void (*fe_datamove)(union ctl_io *io);
11365
11366         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
11367
11368         CTL_DEBUG_PRINT(("ctl_datamove\n"));
11369
11370 #ifdef CTL_TIME_IO
11371         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
11372                 char str[256];
11373                 char path_str[64];
11374                 struct sbuf sb;
11375
11376                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
11377                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
11378
11379                 sbuf_cat(&sb, path_str);
11380                 switch (io->io_hdr.io_type) {
11381                 case CTL_IO_SCSI:
11382                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
11383                         sbuf_printf(&sb, "\n");
11384                         sbuf_cat(&sb, path_str);
11385                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
11386                                     io->scsiio.tag_num, io->scsiio.tag_type);
11387                         break;
11388                 case CTL_IO_TASK:
11389                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
11390                                     "Tag Type: %d\n", io->taskio.task_action,
11391                                     io->taskio.tag_num, io->taskio.tag_type);
11392                         break;
11393                 default:
11394                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
11395                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
11396                         break;
11397                 }
11398                 sbuf_cat(&sb, path_str);
11399                 sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
11400                             (intmax_t)time_uptime - io->io_hdr.start_time);
11401                 sbuf_finish(&sb);
11402                 printf("%s", sbuf_data(&sb));
11403         }
11404 #endif /* CTL_TIME_IO */
11405
11406         mtx_lock(&control_softc->ctl_lock);
11407 #ifdef CTL_IO_DELAY
11408         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
11409                 struct ctl_lun *lun;
11410
11411                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
11412
11413                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
11414         } else {
11415                 struct ctl_lun *lun;
11416
11417                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
11418                 if ((lun != NULL)
11419                  && (lun->delay_info.datamove_delay > 0)) {
11420                         struct callout *callout;
11421
11422                         callout = (struct callout *)&io->io_hdr.timer_bytes;
11423                         callout_init(callout, /*mpsafe*/ 1);
11424                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
11425                         callout_reset(callout,
11426                                       lun->delay_info.datamove_delay * hz,
11427                                       ctl_datamove_timer_wakeup, io);
11428                         if (lun->delay_info.datamove_type ==
11429                             CTL_DELAY_TYPE_ONESHOT)
11430                                 lun->delay_info.datamove_delay = 0;
11431                         mtx_unlock(&control_softc->ctl_lock);
11432                         return;
11433                 }
11434         }
11435 #endif
11436         /*
11437          * If we have any pending task management commands, process them
11438          * first.  This is necessary to eliminate a race condition with the
11439          * FETD:
11440          *
11441          * - FETD submits a task management command, like an abort.
11442          * - Back end calls fe_datamove() to move the data for the aborted
11443          *   command.  The FETD can't really accept it, but if it did, it
11444          *   would end up transmitting data for a command that the initiator
11445          *   told us to abort.
11446          *
11447          * We close the race by processing all pending task management
11448          * commands here (we can't block!), and then check this I/O to see
11449          * if it has been aborted.  If so, return it to the back end with
11450          * bad status, so the back end can say return an error to the back end
11451          * and then when the back end returns an error, we can return the
11452          * aborted command to the FETD, so it can clean up its resources.
11453          */
11454         if (control_softc->flags & CTL_FLAG_TASK_PENDING)
11455                 ctl_run_task_queue(control_softc);
11456
11457         /*
11458          * This command has been aborted.  Set the port status, so we fail
11459          * the data move.
11460          */
11461         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
11462                 printf("ctl_datamove: tag 0x%04x on (%ju:%d:%ju:%d) aborted\n",
11463                        io->scsiio.tag_num,(uintmax_t)io->io_hdr.nexus.initid.id,
11464                        io->io_hdr.nexus.targ_port,
11465                        (uintmax_t)io->io_hdr.nexus.targ_target.id,
11466                        io->io_hdr.nexus.targ_lun);
11467                 io->io_hdr.status = CTL_CMD_ABORTED;
11468                 io->io_hdr.port_status = 31337;
11469                 mtx_unlock(&control_softc->ctl_lock);
11470                 /*
11471                  * Note that the backend, in this case, will get the
11472                  * callback in its context.  In other cases it may get
11473                  * called in the frontend's interrupt thread context.
11474                  */
11475                 io->scsiio.be_move_done(io);
11476                 return;
11477         }
11478
11479         /*
11480          * If we're in XFER mode and this I/O is from the other shelf
11481          * controller, we need to send the DMA to the other side to
11482          * actually transfer the data to/from the host.  In serialize only
11483          * mode the transfer happens below CTL and ctl_datamove() is only
11484          * called on the machine that originally received the I/O.
11485          */
11486         if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
11487          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11488                 union ctl_ha_msg msg;
11489                 uint32_t sg_entries_sent;
11490                 int do_sg_copy;
11491                 int i;
11492
11493                 memset(&msg, 0, sizeof(msg));
11494                 msg.hdr.msg_type = CTL_MSG_DATAMOVE;
11495                 msg.hdr.original_sc = io->io_hdr.original_sc;
11496                 msg.hdr.serializing_sc = io;
11497                 msg.hdr.nexus = io->io_hdr.nexus;
11498                 msg.dt.flags = io->io_hdr.flags;
11499                 /*
11500                  * We convert everything into a S/G list here.  We can't
11501                  * pass by reference, only by value between controllers.
11502                  * So we can't pass a pointer to the S/G list, only as many
11503                  * S/G entries as we can fit in here.  If it's possible for
11504                  * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
11505                  * then we need to break this up into multiple transfers.
11506                  */
11507                 if (io->scsiio.kern_sg_entries == 0) {
11508                         msg.dt.kern_sg_entries = 1;
11509                         /*
11510                          * If this is in cached memory, flush the cache
11511                          * before we send the DMA request to the other
11512                          * controller.  We want to do this in either the
11513                          * read or the write case.  The read case is
11514                          * straightforward.  In the write case, we want to
11515                          * make sure nothing is in the local cache that
11516                          * could overwrite the DMAed data.
11517                          */
11518                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
11519                                 /*
11520                                  * XXX KDM use bus_dmamap_sync() here.
11521                                  */
11522                         }
11523
11524                         /*
11525                          * Convert to a physical address if this is a
11526                          * virtual address.
11527                          */
11528                         if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
11529                                 msg.dt.sg_list[0].addr =
11530                                         io->scsiio.kern_data_ptr;
11531                         } else {
11532                                 /*
11533                                  * XXX KDM use busdma here!
11534                                  */
11535 #if 0
11536                                 msg.dt.sg_list[0].addr = (void *)
11537                                         vtophys(io->scsiio.kern_data_ptr);
11538 #endif
11539                         }
11540
11541                         msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
11542                         do_sg_copy = 0;
11543                 } else {
11544                         struct ctl_sg_entry *sgl;
11545
11546                         do_sg_copy = 1;
11547                         msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
11548                         sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
11549                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
11550                                 /*
11551                                  * XXX KDM use bus_dmamap_sync() here.
11552                                  */
11553                         }
11554                 }
11555
11556                 msg.dt.kern_data_len = io->scsiio.kern_data_len;
11557                 msg.dt.kern_total_len = io->scsiio.kern_total_len;
11558                 msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
11559                 msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
11560                 msg.dt.sg_sequence = 0;
11561
11562                 /*
11563                  * Loop until we've sent all of the S/G entries.  On the
11564                  * other end, we'll recompose these S/G entries into one
11565                  * contiguous list before passing it to the
11566                  */
11567                 for (sg_entries_sent = 0; sg_entries_sent <
11568                      msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
11569                         msg.dt.cur_sg_entries = ctl_min((sizeof(msg.dt.sg_list)/
11570                                 sizeof(msg.dt.sg_list[0])),
11571                                 msg.dt.kern_sg_entries - sg_entries_sent);
11572
11573                         if (do_sg_copy != 0) {
11574                                 struct ctl_sg_entry *sgl;
11575                                 int j;
11576
11577                                 sgl = (struct ctl_sg_entry *)
11578                                         io->scsiio.kern_data_ptr;
11579                                 /*
11580                                  * If this is in cached memory, flush the cache
11581                                  * before we send the DMA request to the other
11582                                  * controller.  We want to do this in either
11583                                  * the * read or the write case.  The read
11584                                  * case is straightforward.  In the write
11585                                  * case, we want to make sure nothing is
11586                                  * in the local cache that could overwrite
11587                                  * the DMAed data.
11588                                  */
11589
11590                                 for (i = sg_entries_sent, j = 0;
11591                                      i < msg.dt.cur_sg_entries; i++, j++) {
11592                                         if ((io->io_hdr.flags &
11593                                              CTL_FLAG_NO_DATASYNC) == 0) {
11594                                                 /*
11595                                                  * XXX KDM use bus_dmamap_sync()
11596                                                  */
11597                                         }
11598                                         if ((io->io_hdr.flags &
11599                                              CTL_FLAG_BUS_ADDR) == 0) {
11600                                                 /*
11601                                                  * XXX KDM use busdma.
11602                                                  */
11603 #if 0
11604                                                 msg.dt.sg_list[j].addr =(void *)
11605                                                        vtophys(sgl[i].addr);
11606 #endif
11607                                         } else {
11608                                                 msg.dt.sg_list[j].addr =
11609                                                         sgl[i].addr;
11610                                         }
11611                                         msg.dt.sg_list[j].len = sgl[i].len;
11612                                 }
11613                         }
11614
11615                         sg_entries_sent += msg.dt.cur_sg_entries;
11616                         if (sg_entries_sent >= msg.dt.kern_sg_entries)
11617                                 msg.dt.sg_last = 1;
11618                         else
11619                                 msg.dt.sg_last = 0;
11620
11621                         /*
11622                          * XXX KDM drop and reacquire the lock here?
11623                          */
11624                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
11625                             sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
11626                                 /*
11627                                  * XXX do something here.
11628                                  */
11629                         }
11630
11631                         msg.dt.sent_sg_entries = sg_entries_sent;
11632                 }
11633                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11634                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER)
11635                         ctl_failover_io(io, /*have_lock*/ 1);
11636
11637         } else {
11638
11639                 /*
11640                  * Lookup the fe_datamove() function for this particular
11641                  * front end.
11642                  */
11643                 fe_datamove =
11644                     control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
11645                 mtx_unlock(&control_softc->ctl_lock);
11646
11647                 fe_datamove(io);
11648         }
11649 }
11650
11651 static void
11652 ctl_send_datamove_done(union ctl_io *io, int have_lock)
11653 {
11654         union ctl_ha_msg msg;
11655         int isc_status;
11656
11657         memset(&msg, 0, sizeof(msg));
11658
11659         msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
11660         msg.hdr.original_sc = io;
11661         msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
11662         msg.hdr.nexus = io->io_hdr.nexus;
11663         msg.hdr.status = io->io_hdr.status;
11664         msg.scsi.tag_num = io->scsiio.tag_num;
11665         msg.scsi.tag_type = io->scsiio.tag_type;
11666         msg.scsi.scsi_status = io->scsiio.scsi_status;
11667         memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
11668                sizeof(io->scsiio.sense_data));
11669         msg.scsi.sense_len = io->scsiio.sense_len;
11670         msg.scsi.sense_residual = io->scsiio.sense_residual;
11671         msg.scsi.fetd_status = io->io_hdr.port_status;
11672         msg.scsi.residual = io->scsiio.residual;
11673         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11674
11675         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
11676                 ctl_failover_io(io, /*have_lock*/ have_lock);
11677                 return;
11678         }
11679
11680         isc_status = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0);
11681         if (isc_status > CTL_HA_STATUS_SUCCESS) {
11682                 /* XXX do something if this fails */
11683         }
11684
11685 }
11686
11687 /*
11688  * The DMA to the remote side is done, now we need to tell the other side
11689  * we're done so it can continue with its data movement.
11690  */
11691 static void
11692 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
11693 {
11694         union ctl_io *io;
11695
11696         io = rq->context;
11697
11698         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
11699                 printf("%s: ISC DMA write failed with error %d", __func__,
11700                        rq->ret);
11701                 ctl_set_internal_failure(&io->scsiio,
11702                                          /*sks_valid*/ 1,
11703                                          /*retry_count*/ rq->ret);
11704         }
11705
11706         ctl_dt_req_free(rq);
11707
11708         /*
11709          * In this case, we had to malloc the memory locally.  Free it.
11710          */
11711         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
11712                 int i;
11713                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
11714                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
11715         }
11716         /*
11717          * The data is in local and remote memory, so now we need to send
11718          * status (good or back) back to the other side.
11719          */
11720         ctl_send_datamove_done(io, /*have_lock*/ 0);
11721 }
11722
11723 /*
11724  * We've moved the data from the host/controller into local memory.  Now we
11725  * need to push it over to the remote controller's memory.
11726  */
11727 static int
11728 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
11729 {
11730         int retval;
11731
11732         retval = 0;
11733
11734         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
11735                                           ctl_datamove_remote_write_cb);
11736
11737         return (retval);
11738 }
11739
11740 static void
11741 ctl_datamove_remote_write(union ctl_io *io)
11742 {
11743         int retval;
11744         void (*fe_datamove)(union ctl_io *io);
11745
11746         /*
11747          * - Get the data from the host/HBA into local memory.
11748          * - DMA memory from the local controller to the remote controller.
11749          * - Send status back to the remote controller.
11750          */
11751
11752         retval = ctl_datamove_remote_sgl_setup(io);
11753         if (retval != 0)
11754                 return;
11755
11756         /* Switch the pointer over so the FETD knows what to do */
11757         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
11758
11759         /*
11760          * Use a custom move done callback, since we need to send completion
11761          * back to the other controller, not to the backend on this side.
11762          */
11763         io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
11764
11765         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
11766
11767         fe_datamove(io);
11768
11769         return;
11770
11771 }
11772
11773 static int
11774 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
11775 {
11776 #if 0
11777         char str[256];
11778         char path_str[64];
11779         struct sbuf sb;
11780 #endif
11781
11782         /*
11783          * In this case, we had to malloc the memory locally.  Free it.
11784          */
11785         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
11786                 int i;
11787                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
11788                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
11789         }
11790
11791 #if 0
11792         scsi_path_string(io, path_str, sizeof(path_str));
11793         sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
11794         sbuf_cat(&sb, path_str);
11795         scsi_command_string(&io->scsiio, NULL, &sb);
11796         sbuf_printf(&sb, "\n");
11797         sbuf_cat(&sb, path_str);
11798         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
11799                     io->scsiio.tag_num, io->scsiio.tag_type);
11800         sbuf_cat(&sb, path_str);
11801         sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
11802                     io->io_hdr.flags, io->io_hdr.status);
11803         sbuf_finish(&sb);
11804         printk("%s", sbuf_data(&sb));
11805 #endif
11806
11807
11808         /*
11809          * The read is done, now we need to send status (good or bad) back
11810          * to the other side.
11811          */
11812         ctl_send_datamove_done(io, /*have_lock*/ 0);
11813
11814         return (0);
11815 }
11816
11817 static void
11818 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
11819 {
11820         union ctl_io *io;
11821         void (*fe_datamove)(union ctl_io *io);
11822
11823         io = rq->context;
11824
11825         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
11826                 printf("%s: ISC DMA read failed with error %d", __func__,
11827                        rq->ret);
11828                 ctl_set_internal_failure(&io->scsiio,
11829                                          /*sks_valid*/ 1,
11830                                          /*retry_count*/ rq->ret);
11831         }
11832
11833         ctl_dt_req_free(rq);
11834
11835         /* Switch the pointer over so the FETD knows what to do */
11836         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
11837
11838         /*
11839          * Use a custom move done callback, since we need to send completion
11840          * back to the other controller, not to the backend on this side.
11841          */
11842         io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
11843
11844         /* XXX KDM add checks like the ones in ctl_datamove? */
11845
11846         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
11847
11848         fe_datamove(io);
11849 }
11850
11851 static int
11852 ctl_datamove_remote_sgl_setup(union ctl_io *io)
11853 {
11854         struct ctl_sg_entry *local_sglist, *remote_sglist;
11855         struct ctl_sg_entry *local_dma_sglist, *remote_dma_sglist;
11856         struct ctl_softc *softc;
11857         int retval;
11858         int i;
11859
11860         retval = 0;
11861         softc = control_softc;
11862
11863         local_sglist = io->io_hdr.local_sglist;
11864         local_dma_sglist = io->io_hdr.local_dma_sglist;
11865         remote_sglist = io->io_hdr.remote_sglist;
11866         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
11867
11868         if (io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) {
11869                 for (i = 0; i < io->scsiio.kern_sg_entries; i++) {
11870                         local_sglist[i].len = remote_sglist[i].len;
11871
11872                         /*
11873                          * XXX Detect the situation where the RS-level I/O
11874                          * redirector on the other side has already read the
11875                          * data off of the AOR RS on this side, and
11876                          * transferred it to remote (mirror) memory on the
11877                          * other side.  Since we already have the data in
11878                          * memory here, we just need to use it.
11879                          *
11880                          * XXX KDM this can probably be removed once we
11881                          * get the cache device code in and take the
11882                          * current AOR implementation out.
11883                          */
11884 #ifdef NEEDTOPORT
11885                         if ((remote_sglist[i].addr >=
11886                              (void *)vtophys(softc->mirr->addr))
11887                          && (remote_sglist[i].addr <
11888                              ((void *)vtophys(softc->mirr->addr) +
11889                              CacheMirrorOffset))) {
11890                                 local_sglist[i].addr = remote_sglist[i].addr -
11891                                         CacheMirrorOffset;
11892                                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
11893                                      CTL_FLAG_DATA_IN)
11894                                         io->io_hdr.flags |= CTL_FLAG_REDIR_DONE;
11895                         } else {
11896                                 local_sglist[i].addr = remote_sglist[i].addr +
11897                                         CacheMirrorOffset;
11898                         }
11899 #endif
11900 #if 0
11901                         printf("%s: local %p, remote %p, len %d\n",
11902                                __func__, local_sglist[i].addr,
11903                                remote_sglist[i].addr, local_sglist[i].len);
11904 #endif
11905                 }
11906         } else {
11907                 uint32_t len_to_go;
11908
11909                 /*
11910                  * In this case, we don't have automatically allocated
11911                  * memory for this I/O on this controller.  This typically
11912                  * happens with internal CTL I/O -- e.g. inquiry, mode
11913                  * sense, etc.  Anything coming from RAIDCore will have
11914                  * a mirror area available.
11915                  */
11916                 len_to_go = io->scsiio.kern_data_len;
11917
11918                 /*
11919                  * Clear the no datasync flag, we have to use malloced
11920                  * buffers.
11921                  */
11922                 io->io_hdr.flags &= ~CTL_FLAG_NO_DATASYNC;
11923
11924                 /*
11925                  * The difficult thing here is that the size of the various
11926                  * S/G segments may be different than the size from the
11927                  * remote controller.  That'll make it harder when DMAing
11928                  * the data back to the other side.
11929                  */
11930                 for (i = 0; (i < sizeof(io->io_hdr.remote_sglist) /
11931                      sizeof(io->io_hdr.remote_sglist[0])) &&
11932                      (len_to_go > 0); i++) {
11933                         local_sglist[i].len = ctl_min(len_to_go, 131072);
11934                         CTL_SIZE_8B(local_dma_sglist[i].len,
11935                                     local_sglist[i].len);
11936                         local_sglist[i].addr =
11937                                 malloc(local_dma_sglist[i].len, M_CTL,M_WAITOK);
11938
11939                         local_dma_sglist[i].addr = local_sglist[i].addr;
11940
11941                         if (local_sglist[i].addr == NULL) {
11942                                 int j;
11943
11944                                 printf("malloc failed for %zd bytes!",
11945                                        local_dma_sglist[i].len);
11946                                 for (j = 0; j < i; j++) {
11947                                         free(local_sglist[j].addr, M_CTL);
11948                                 }
11949                                 ctl_set_internal_failure(&io->scsiio,
11950                                                          /*sks_valid*/ 1,
11951                                                          /*retry_count*/ 4857);
11952                                 retval = 1;
11953                                 goto bailout_error;
11954
11955                         }
11956                         /* XXX KDM do we need a sync here? */
11957
11958                         len_to_go -= local_sglist[i].len;
11959                 }
11960                 /*
11961                  * Reset the number of S/G entries accordingly.  The
11962                  * original number of S/G entries is available in
11963                  * rem_sg_entries.
11964                  */
11965                 io->scsiio.kern_sg_entries = i;
11966
11967 #if 0
11968                 printf("%s: kern_sg_entries = %d\n", __func__,
11969                        io->scsiio.kern_sg_entries);
11970                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
11971                         printf("%s: sg[%d] = %p, %d (DMA: %d)\n", __func__, i,
11972                                local_sglist[i].addr, local_sglist[i].len,
11973                                local_dma_sglist[i].len);
11974 #endif
11975         }
11976
11977
11978         return (retval);
11979
11980 bailout_error:
11981
11982         ctl_send_datamove_done(io, /*have_lock*/ 0);
11983
11984         return (retval);
11985 }
11986
11987 static int
11988 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
11989                          ctl_ha_dt_cb callback)
11990 {
11991         struct ctl_ha_dt_req *rq;
11992         struct ctl_sg_entry *remote_sglist, *local_sglist;
11993         struct ctl_sg_entry *remote_dma_sglist, *local_dma_sglist;
11994         uint32_t local_used, remote_used, total_used;
11995         int retval;
11996         int i, j;
11997
11998         retval = 0;
11999
12000         rq = ctl_dt_req_alloc();
12001
12002         /*
12003          * If we failed to allocate the request, and if the DMA didn't fail
12004          * anyway, set busy status.  This is just a resource allocation
12005          * failure.
12006          */
12007         if ((rq == NULL)
12008          && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
12009                 ctl_set_busy(&io->scsiio);
12010
12011         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
12012
12013                 if (rq != NULL)
12014                         ctl_dt_req_free(rq);
12015
12016                 /*
12017                  * The data move failed.  We need to return status back
12018                  * to the other controller.  No point in trying to DMA
12019                  * data to the remote controller.
12020                  */
12021
12022                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12023
12024                 retval = 1;
12025
12026                 goto bailout;
12027         }
12028
12029         local_sglist = io->io_hdr.local_sglist;
12030         local_dma_sglist = io->io_hdr.local_dma_sglist;
12031         remote_sglist = io->io_hdr.remote_sglist;
12032         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
12033         local_used = 0;
12034         remote_used = 0;
12035         total_used = 0;
12036
12037         if (io->io_hdr.flags & CTL_FLAG_REDIR_DONE) {
12038                 rq->ret = CTL_HA_STATUS_SUCCESS;
12039                 rq->context = io;
12040                 callback(rq);
12041                 goto bailout;
12042         }
12043
12044         /*
12045          * Pull/push the data over the wire from/to the other controller.
12046          * This takes into account the possibility that the local and
12047          * remote sglists may not be identical in terms of the size of
12048          * the elements and the number of elements.
12049          *
12050          * One fundamental assumption here is that the length allocated for
12051          * both the local and remote sglists is identical.  Otherwise, we've
12052          * essentially got a coding error of some sort.
12053          */
12054         for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
12055                 int isc_ret;
12056                 uint32_t cur_len, dma_length;
12057                 uint8_t *tmp_ptr;
12058
12059                 rq->id = CTL_HA_DATA_CTL;
12060                 rq->command = command;
12061                 rq->context = io;
12062
12063                 /*
12064                  * Both pointers should be aligned.  But it is possible
12065                  * that the allocation length is not.  They should both
12066                  * also have enough slack left over at the end, though,
12067                  * to round up to the next 8 byte boundary.
12068                  */
12069                 cur_len = ctl_min(local_sglist[i].len - local_used,
12070                                   remote_sglist[j].len - remote_used);
12071
12072                 /*
12073                  * In this case, we have a size issue and need to decrease
12074                  * the size, except in the case where we actually have less
12075                  * than 8 bytes left.  In that case, we need to increase
12076                  * the DMA length to get the last bit.
12077                  */
12078                 if ((cur_len & 0x7) != 0) {
12079                         if (cur_len > 0x7) {
12080                                 cur_len = cur_len - (cur_len & 0x7);
12081                                 dma_length = cur_len;
12082                         } else {
12083                                 CTL_SIZE_8B(dma_length, cur_len);
12084                         }
12085
12086                 } else
12087                         dma_length = cur_len;
12088
12089                 /*
12090                  * If we had to allocate memory for this I/O, instead of using
12091                  * the non-cached mirror memory, we'll need to flush the cache
12092                  * before trying to DMA to the other controller.
12093                  *
12094                  * We could end up doing this multiple times for the same
12095                  * segment if we have a larger local segment than remote
12096                  * segment.  That shouldn't be an issue.
12097                  */
12098                 if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12099                         /*
12100                          * XXX KDM use bus_dmamap_sync() here.
12101                          */
12102                 }
12103
12104                 rq->size = dma_length;
12105
12106                 tmp_ptr = (uint8_t *)local_sglist[i].addr;
12107                 tmp_ptr += local_used;
12108
12109                 /* Use physical addresses when talking to ISC hardware */
12110                 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
12111                         /* XXX KDM use busdma */
12112 #if 0
12113                         rq->local = vtophys(tmp_ptr);
12114 #endif
12115                 } else
12116                         rq->local = tmp_ptr;
12117
12118                 tmp_ptr = (uint8_t *)remote_sglist[j].addr;
12119                 tmp_ptr += remote_used;
12120                 rq->remote = tmp_ptr;
12121
12122                 rq->callback = NULL;
12123
12124                 local_used += cur_len;
12125                 if (local_used >= local_sglist[i].len) {
12126                         i++;
12127                         local_used = 0;
12128                 }
12129
12130                 remote_used += cur_len;
12131                 if (remote_used >= remote_sglist[j].len) {
12132                         j++;
12133                         remote_used = 0;
12134                 }
12135                 total_used += cur_len;
12136
12137                 if (total_used >= io->scsiio.kern_data_len)
12138                         rq->callback = callback;
12139
12140                 if ((rq->size & 0x7) != 0) {
12141                         printf("%s: warning: size %d is not on 8b boundary\n",
12142                                __func__, rq->size);
12143                 }
12144                 if (((uintptr_t)rq->local & 0x7) != 0) {
12145                         printf("%s: warning: local %p not on 8b boundary\n",
12146                                __func__, rq->local);
12147                 }
12148                 if (((uintptr_t)rq->remote & 0x7) != 0) {
12149                         printf("%s: warning: remote %p not on 8b boundary\n",
12150                                __func__, rq->local);
12151                 }
12152 #if 0
12153                 printf("%s: %s: local %#x remote %#x size %d\n", __func__,
12154                        (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
12155                        rq->local, rq->remote, rq->size);
12156 #endif
12157
12158                 isc_ret = ctl_dt_single(rq);
12159                 if (isc_ret == CTL_HA_STATUS_WAIT)
12160                         continue;
12161
12162                 if (isc_ret == CTL_HA_STATUS_DISCONNECT) {
12163                         rq->ret = CTL_HA_STATUS_SUCCESS;
12164                 } else {
12165                         rq->ret = isc_ret;
12166                 }
12167                 callback(rq);
12168                 goto bailout;
12169         }
12170
12171 bailout:
12172         return (retval);
12173
12174 }
12175
12176 static void
12177 ctl_datamove_remote_read(union ctl_io *io)
12178 {
12179         int retval;
12180         int i;
12181
12182         /*
12183          * This will send an error to the other controller in the case of a
12184          * failure.
12185          */
12186         retval = ctl_datamove_remote_sgl_setup(io);
12187         if (retval != 0)
12188                 return;
12189
12190         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
12191                                           ctl_datamove_remote_read_cb);
12192         if ((retval != 0)
12193          && ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0)) {
12194                 /*
12195                  * Make sure we free memory if there was an error..  The
12196                  * ctl_datamove_remote_xfer() function will send the
12197                  * datamove done message, or call the callback with an
12198                  * error if there is a problem.
12199                  */
12200                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12201                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12202         }
12203
12204         return;
12205 }
12206
12207 /*
12208  * Process a datamove request from the other controller.  This is used for
12209  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
12210  * first.  Once that is complete, the data gets DMAed into the remote
12211  * controller's memory.  For reads, we DMA from the remote controller's
12212  * memory into our memory first, and then move it out to the FETD.
12213  */
12214 static void
12215 ctl_datamove_remote(union ctl_io *io)
12216 {
12217         struct ctl_softc *softc;
12218
12219         softc = control_softc;
12220
12221         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
12222
12223         /*
12224          * Note that we look for an aborted I/O here, but don't do some of
12225          * the other checks that ctl_datamove() normally does.  We don't
12226          * need to run the task queue, because this I/O is on the ISC
12227          * queue, which is executed by the work thread after the task queue.
12228          * We don't need to run the datamove delay code, since that should
12229          * have been done if need be on the other controller.
12230          */
12231         mtx_lock(&softc->ctl_lock);
12232
12233         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12234
12235                 printf("%s: tag 0x%04x on (%d:%d:%d:%d) aborted\n", __func__,
12236                        io->scsiio.tag_num, io->io_hdr.nexus.initid.id,
12237                        io->io_hdr.nexus.targ_port,
12238                        io->io_hdr.nexus.targ_target.id,
12239                        io->io_hdr.nexus.targ_lun);
12240                 io->io_hdr.status = CTL_CMD_ABORTED;
12241                 io->io_hdr.port_status = 31338;
12242
12243                 mtx_unlock(&softc->ctl_lock);
12244
12245                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12246
12247                 return;
12248         }
12249
12250         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT) {
12251                 mtx_unlock(&softc->ctl_lock);
12252                 ctl_datamove_remote_write(io);
12253         } else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN){
12254                 mtx_unlock(&softc->ctl_lock);
12255                 ctl_datamove_remote_read(io);
12256         } else {
12257                 union ctl_ha_msg msg;
12258                 struct scsi_sense_data *sense;
12259                 uint8_t sks[3];
12260                 int retry_count;
12261
12262                 memset(&msg, 0, sizeof(msg));
12263
12264                 msg.hdr.msg_type = CTL_MSG_BAD_JUJU;
12265                 msg.hdr.status = CTL_SCSI_ERROR;
12266                 msg.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
12267
12268                 retry_count = 4243;
12269
12270                 sense = &msg.scsi.sense_data;
12271                 sks[0] = SSD_SCS_VALID;
12272                 sks[1] = (retry_count >> 8) & 0xff;
12273                 sks[2] = retry_count & 0xff;
12274
12275                 /* "Internal target failure" */
12276                 scsi_set_sense_data(sense,
12277                                     /*sense_format*/ SSD_TYPE_NONE,
12278                                     /*current_error*/ 1,
12279                                     /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
12280                                     /*asc*/ 0x44,
12281                                     /*ascq*/ 0x00,
12282                                     /*type*/ SSD_ELEM_SKS,
12283                                     /*size*/ sizeof(sks),
12284                                     /*data*/ sks,
12285                                     SSD_ELEM_NONE);
12286
12287                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12288                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12289                         ctl_failover_io(io, /*have_lock*/ 1);
12290                         mtx_unlock(&softc->ctl_lock);
12291                         return;
12292                 }
12293
12294                 mtx_unlock(&softc->ctl_lock);
12295
12296                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0) >
12297                     CTL_HA_STATUS_SUCCESS) {
12298                         /* XXX KDM what to do if this fails? */
12299                 }
12300                 return;
12301         }
12302
12303 }
12304
12305 static int
12306 ctl_process_done(union ctl_io *io, int have_lock)
12307 {
12308         struct ctl_lun *lun;
12309         struct ctl_softc *ctl_softc;
12310         void (*fe_done)(union ctl_io *io);
12311         uint32_t targ_port = ctl_port_idx(io->io_hdr.nexus.targ_port);
12312
12313         CTL_DEBUG_PRINT(("ctl_process_done\n"));
12314
12315         fe_done =
12316             control_softc->ctl_ports[targ_port]->fe_done;
12317
12318 #ifdef CTL_TIME_IO
12319         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12320                 char str[256];
12321                 char path_str[64];
12322                 struct sbuf sb;
12323
12324                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12325                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12326
12327                 sbuf_cat(&sb, path_str);
12328                 switch (io->io_hdr.io_type) {
12329                 case CTL_IO_SCSI:
12330                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12331                         sbuf_printf(&sb, "\n");
12332                         sbuf_cat(&sb, path_str);
12333                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12334                                     io->scsiio.tag_num, io->scsiio.tag_type);
12335                         break;
12336                 case CTL_IO_TASK:
12337                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12338                                     "Tag Type: %d\n", io->taskio.task_action,
12339                                     io->taskio.tag_num, io->taskio.tag_type);
12340                         break;
12341                 default:
12342                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12343                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12344                         break;
12345                 }
12346                 sbuf_cat(&sb, path_str);
12347                 sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
12348                             (intmax_t)time_uptime - io->io_hdr.start_time);
12349                 sbuf_finish(&sb);
12350                 printf("%s", sbuf_data(&sb));
12351         }
12352 #endif /* CTL_TIME_IO */
12353
12354         switch (io->io_hdr.io_type) {
12355         case CTL_IO_SCSI:
12356                 break;
12357         case CTL_IO_TASK:
12358                 if (bootverbose || verbose > 0)
12359                         ctl_io_error_print(io, NULL);
12360                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
12361                         ctl_free_io(io);
12362                 else
12363                         fe_done(io);
12364                 return (CTL_RETVAL_COMPLETE);
12365                 break;
12366         default:
12367                 printf("ctl_process_done: invalid io type %d\n",
12368                        io->io_hdr.io_type);
12369                 panic("ctl_process_done: invalid io type %d\n",
12370                       io->io_hdr.io_type);
12371                 break; /* NOTREACHED */
12372         }
12373
12374         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12375         if (lun == NULL) {
12376                 CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
12377                                  io->io_hdr.nexus.targ_lun));
12378                 fe_done(io);
12379                 goto bailout;
12380         }
12381         ctl_softc = lun->ctl_softc;
12382
12383         /*
12384          * Remove this from the OOA queue.
12385          */
12386         if (have_lock == 0)
12387                 mtx_lock(&ctl_softc->ctl_lock);
12388
12389         /*
12390          * Check to see if we have any errors to inject here.  We only
12391          * inject errors for commands that don't already have errors set.
12392          */
12393         if ((STAILQ_FIRST(&lun->error_list) != NULL)
12394          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))
12395                 ctl_inject_error(lun, io);
12396
12397         /*
12398          * XXX KDM how do we treat commands that aren't completed
12399          * successfully?
12400          *
12401          * XXX KDM should we also track I/O latency?
12402          */
12403         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
12404                 uint32_t blocksize;
12405 #ifdef CTL_TIME_IO
12406                 struct bintime cur_bt;
12407 #endif
12408
12409                 if ((lun->be_lun != NULL)
12410                  && (lun->be_lun->blocksize != 0))
12411                         blocksize = lun->be_lun->blocksize;
12412                 else
12413                         blocksize = 512;
12414
12415                 switch (io->io_hdr.io_type) {
12416                 case CTL_IO_SCSI: {
12417                         int isread;
12418                         struct ctl_lba_len lbalen;
12419
12420                         isread = 0;
12421                         switch (io->scsiio.cdb[0]) {
12422                         case READ_6:
12423                         case READ_10:
12424                         case READ_12:
12425                         case READ_16:
12426                                 isread = 1;
12427                                 /* FALLTHROUGH */
12428                         case WRITE_6:
12429                         case WRITE_10:
12430                         case WRITE_12:
12431                         case WRITE_16:
12432                         case WRITE_VERIFY_10:
12433                         case WRITE_VERIFY_12:
12434                         case WRITE_VERIFY_16:
12435                                 memcpy(&lbalen, io->io_hdr.ctl_private[
12436                                        CTL_PRIV_LBA_LEN].bytes, sizeof(lbalen));
12437
12438                                 if (isread) {
12439                                         lun->stats.ports[targ_port].bytes[CTL_STATS_READ] +=
12440                                                 lbalen.len * blocksize;
12441                                         lun->stats.ports[targ_port].operations[CTL_STATS_READ]++;
12442
12443 #ifdef CTL_TIME_IO
12444                                         bintime_add(
12445                                            &lun->stats.ports[targ_port].dma_time[CTL_STATS_READ],
12446                                            &io->io_hdr.dma_bt);
12447                                         lun->stats.ports[targ_port].num_dmas[CTL_STATS_READ] +=
12448                                                 io->io_hdr.num_dmas;
12449                                         getbintime(&cur_bt);
12450                                         bintime_sub(&cur_bt,
12451                                                     &io->io_hdr.start_bt);
12452
12453                                         bintime_add(
12454                                             &lun->stats.ports[targ_port].time[CTL_STATS_READ],
12455                                             &cur_bt);
12456
12457 #if 0
12458                                         cs_prof_gettime(&cur_ticks);
12459                                         lun->stats.time[CTL_STATS_READ] +=
12460                                                 cur_ticks -
12461                                                 io->io_hdr.start_ticks;
12462 #endif
12463 #if 0
12464                                         lun->stats.time[CTL_STATS_READ] +=
12465                                                 jiffies - io->io_hdr.start_time;
12466 #endif
12467 #endif /* CTL_TIME_IO */
12468                                 } else {
12469                                         lun->stats.ports[targ_port].bytes[CTL_STATS_WRITE] +=
12470                                                 lbalen.len * blocksize;
12471                                         lun->stats.ports[targ_port].operations[
12472                                                 CTL_STATS_WRITE]++;
12473
12474 #ifdef CTL_TIME_IO
12475                                         bintime_add(
12476                                           &lun->stats.ports[targ_port].dma_time[CTL_STATS_WRITE],
12477                                           &io->io_hdr.dma_bt);
12478                                         lun->stats.ports[targ_port].num_dmas[CTL_STATS_WRITE] +=
12479                                                 io->io_hdr.num_dmas;
12480                                         getbintime(&cur_bt);
12481                                         bintime_sub(&cur_bt,
12482                                                     &io->io_hdr.start_bt);
12483
12484                                         bintime_add(
12485                                             &lun->stats.ports[targ_port].time[CTL_STATS_WRITE],
12486                                             &cur_bt);
12487 #if 0
12488                                         cs_prof_gettime(&cur_ticks);
12489                                         lun->stats.ports[targ_port].time[CTL_STATS_WRITE] +=
12490                                                 cur_ticks -
12491                                                 io->io_hdr.start_ticks;
12492                                         lun->stats.ports[targ_port].time[CTL_STATS_WRITE] +=
12493                                                 jiffies - io->io_hdr.start_time;
12494 #endif
12495 #endif /* CTL_TIME_IO */
12496                                 }
12497                                 break;
12498                         default:
12499                                 lun->stats.ports[targ_port].operations[CTL_STATS_NO_IO]++;
12500
12501 #ifdef CTL_TIME_IO
12502                                 bintime_add(
12503                                   &lun->stats.ports[targ_port].dma_time[CTL_STATS_NO_IO],
12504                                   &io->io_hdr.dma_bt);
12505                                 lun->stats.ports[targ_port].num_dmas[CTL_STATS_NO_IO] +=
12506                                         io->io_hdr.num_dmas;
12507                                 getbintime(&cur_bt);
12508                                 bintime_sub(&cur_bt, &io->io_hdr.start_bt);
12509
12510                                 bintime_add(&lun->stats.ports[targ_port].time[CTL_STATS_NO_IO],
12511                                             &cur_bt);
12512
12513 #if 0
12514                                 cs_prof_gettime(&cur_ticks);
12515                                 lun->stats.ports[targ_port].time[CTL_STATS_NO_IO] +=
12516                                         cur_ticks -
12517                                         io->io_hdr.start_ticks;
12518                                 lun->stats.ports[targ_port].time[CTL_STATS_NO_IO] +=
12519                                         jiffies - io->io_hdr.start_time;
12520 #endif
12521 #endif /* CTL_TIME_IO */
12522                                 break;
12523                         }
12524                         break;
12525                 }
12526                 default:
12527                         break;
12528                 }
12529         }
12530
12531         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
12532
12533         /*
12534          * Run through the blocked queue on this LUN and see if anything
12535          * has become unblocked, now that this transaction is done.
12536          */
12537         ctl_check_blocked(lun);
12538
12539         /*
12540          * If the LUN has been invalidated, free it if there is nothing
12541          * left on its OOA queue.
12542          */
12543         if ((lun->flags & CTL_LUN_INVALID)
12544          && (TAILQ_FIRST(&lun->ooa_queue) == NULL))
12545                 ctl_free_lun(lun);
12546
12547         /*
12548          * If this command has been aborted, make sure we set the status
12549          * properly.  The FETD is responsible for freeing the I/O and doing
12550          * whatever it needs to do to clean up its state.
12551          */
12552         if (io->io_hdr.flags & CTL_FLAG_ABORT)
12553                 io->io_hdr.status = CTL_CMD_ABORTED;
12554
12555         /*
12556          * We print out status for every task management command.  For SCSI
12557          * commands, we filter out any unit attention errors; they happen
12558          * on every boot, and would clutter up the log.  Note:  task
12559          * management commands aren't printed here, they are printed above,
12560          * since they should never even make it down here.
12561          */
12562         switch (io->io_hdr.io_type) {
12563         case CTL_IO_SCSI: {
12564                 int error_code, sense_key, asc, ascq;
12565
12566                 sense_key = 0;
12567
12568                 if (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR)
12569                  && (io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)) {
12570                         /*
12571                          * Since this is just for printing, no need to
12572                          * show errors here.
12573                          */
12574                         scsi_extract_sense_len(&io->scsiio.sense_data,
12575                                                io->scsiio.sense_len,
12576                                                &error_code,
12577                                                &sense_key,
12578                                                &asc,
12579                                                &ascq,
12580                                                /*show_errors*/ 0);
12581                 }
12582
12583                 if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
12584                  && (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SCSI_ERROR)
12585                   || (io->scsiio.scsi_status != SCSI_STATUS_CHECK_COND)
12586                   || (sense_key != SSD_KEY_UNIT_ATTENTION))) {
12587
12588                         if ((time_uptime - ctl_softc->last_print_jiffies) <= 0){
12589                                 ctl_softc->skipped_prints++;
12590                                 if (have_lock == 0)
12591                                         mtx_unlock(&ctl_softc->ctl_lock);
12592                         } else {
12593                                 uint32_t skipped_prints;
12594
12595                                 skipped_prints = ctl_softc->skipped_prints;
12596
12597                                 ctl_softc->skipped_prints = 0;
12598                                 ctl_softc->last_print_jiffies = time_uptime;
12599
12600                                 if (have_lock == 0)
12601                                         mtx_unlock(&ctl_softc->ctl_lock);
12602                                 if (skipped_prints > 0) {
12603 #ifdef NEEDTOPORT
12604                                         csevent_log(CSC_CTL | CSC_SHELF_SW |
12605                                             CTL_ERROR_REPORT,
12606                                             csevent_LogType_Trace,
12607                                             csevent_Severity_Information,
12608                                             csevent_AlertLevel_Green,
12609                                             csevent_FRU_Firmware,
12610                                             csevent_FRU_Unknown,
12611                                             "High CTL error volume, %d prints "
12612                                             "skipped", skipped_prints);
12613 #endif
12614                                 }
12615                                 if (bootverbose || verbose > 0)
12616                                         ctl_io_error_print(io, NULL);
12617                         }
12618                 } else {
12619                         if (have_lock == 0)
12620                                 mtx_unlock(&ctl_softc->ctl_lock);
12621                 }
12622                 break;
12623         }
12624         case CTL_IO_TASK:
12625                 if (have_lock == 0)
12626                         mtx_unlock(&ctl_softc->ctl_lock);
12627                 if (bootverbose || verbose > 0)
12628                         ctl_io_error_print(io, NULL);
12629                 break;
12630         default:
12631                 if (have_lock == 0)
12632                         mtx_unlock(&ctl_softc->ctl_lock);
12633                 break;
12634         }
12635
12636         /*
12637          * Tell the FETD or the other shelf controller we're done with this
12638          * command.  Note that only SCSI commands get to this point.  Task
12639          * management commands are completed above.
12640          *
12641          * We only send status to the other controller if we're in XFER
12642          * mode.  In SER_ONLY mode, the I/O is done on the controller that
12643          * received the I/O (from CTL's perspective), and so the status is
12644          * generated there.
12645          *
12646          * XXX KDM if we hold the lock here, we could cause a deadlock
12647          * if the frontend comes back in in this context to queue
12648          * something.
12649          */
12650         if ((ctl_softc->ha_mode == CTL_HA_MODE_XFER)
12651          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12652                 union ctl_ha_msg msg;
12653
12654                 memset(&msg, 0, sizeof(msg));
12655                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
12656                 msg.hdr.original_sc = io->io_hdr.original_sc;
12657                 msg.hdr.nexus = io->io_hdr.nexus;
12658                 msg.hdr.status = io->io_hdr.status;
12659                 msg.scsi.scsi_status = io->scsiio.scsi_status;
12660                 msg.scsi.tag_num = io->scsiio.tag_num;
12661                 msg.scsi.tag_type = io->scsiio.tag_type;
12662                 msg.scsi.sense_len = io->scsiio.sense_len;
12663                 msg.scsi.sense_residual = io->scsiio.sense_residual;
12664                 msg.scsi.residual = io->scsiio.residual;
12665                 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12666                        sizeof(io->scsiio.sense_data));
12667                 /*
12668                  * We copy this whether or not this is an I/O-related
12669                  * command.  Otherwise, we'd have to go and check to see
12670                  * whether it's a read/write command, and it really isn't
12671                  * worth it.
12672                  */
12673                 memcpy(&msg.scsi.lbalen,
12674                        &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
12675                        sizeof(msg.scsi.lbalen));
12676
12677                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12678                                 sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
12679                         /* XXX do something here */
12680                 }
12681
12682                 ctl_free_io(io);
12683         } else
12684                 fe_done(io);
12685
12686 bailout:
12687
12688         return (CTL_RETVAL_COMPLETE);
12689 }
12690
12691 /*
12692  * Front end should call this if it doesn't do autosense.  When the request
12693  * sense comes back in from the initiator, we'll dequeue this and send it.
12694  */
12695 int
12696 ctl_queue_sense(union ctl_io *io)
12697 {
12698         struct ctl_lun *lun;
12699         struct ctl_softc *ctl_softc;
12700         uint32_t initidx, targ_lun;
12701
12702         ctl_softc = control_softc;
12703
12704         CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
12705
12706         /*
12707          * LUN lookup will likely move to the ctl_work_thread() once we
12708          * have our new queueing infrastructure (that doesn't put things on
12709          * a per-LUN queue initially).  That is so that we can handle
12710          * things like an INQUIRY to a LUN that we don't have enabled.  We
12711          * can't deal with that right now.
12712          */
12713         mtx_lock(&ctl_softc->ctl_lock);
12714
12715         /*
12716          * If we don't have a LUN for this, just toss the sense
12717          * information.
12718          */
12719         targ_lun = io->io_hdr.nexus.targ_lun;
12720         if (io->io_hdr.nexus.lun_map_fn != NULL)
12721                 targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
12722         if ((targ_lun < CTL_MAX_LUNS)
12723          && (ctl_softc->ctl_luns[targ_lun] != NULL))
12724                 lun = ctl_softc->ctl_luns[targ_lun];
12725         else
12726                 goto bailout;
12727
12728         initidx = ctl_get_initindex(&io->io_hdr.nexus);
12729
12730         /*
12731          * Already have CA set for this LUN...toss the sense information.
12732          */
12733         if (ctl_is_set(lun->have_ca, initidx))
12734                 goto bailout;
12735
12736         memcpy(&lun->pending_sense[initidx].sense, &io->scsiio.sense_data,
12737                ctl_min(sizeof(lun->pending_sense[initidx].sense),
12738                sizeof(io->scsiio.sense_data)));
12739         ctl_set_mask(lun->have_ca, initidx);
12740
12741 bailout:
12742         mtx_unlock(&ctl_softc->ctl_lock);
12743
12744         ctl_free_io(io);
12745
12746         return (CTL_RETVAL_COMPLETE);
12747 }
12748
12749 /*
12750  * Primary command inlet from frontend ports.  All SCSI and task I/O
12751  * requests must go through this function.
12752  */
12753 int
12754 ctl_queue(union ctl_io *io)
12755 {
12756         struct ctl_softc *ctl_softc;
12757
12758         CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
12759
12760         ctl_softc = control_softc;
12761
12762 #ifdef CTL_TIME_IO
12763         io->io_hdr.start_time = time_uptime;
12764         getbintime(&io->io_hdr.start_bt);
12765 #endif /* CTL_TIME_IO */
12766
12767         mtx_lock(&ctl_softc->ctl_lock);
12768
12769         switch (io->io_hdr.io_type) {
12770         case CTL_IO_SCSI:
12771                 STAILQ_INSERT_TAIL(&ctl_softc->incoming_queue, &io->io_hdr,
12772                                    links);
12773                 break;
12774         case CTL_IO_TASK:
12775                 STAILQ_INSERT_TAIL(&ctl_softc->task_queue, &io->io_hdr, links);
12776                 /*
12777                  * Set the task pending flag.  This is necessary to close a
12778                  * race condition with the FETD:
12779                  *
12780                  * - FETD submits a task management command, like an abort.
12781                  * - Back end calls fe_datamove() to move the data for the
12782                  *   aborted command.  The FETD can't really accept it, but
12783                  *   if it did, it would end up transmitting data for a
12784                  *   command that the initiator told us to abort.
12785                  *
12786                  * We close the race condition by setting the flag here,
12787                  * and checking it in ctl_datamove(), before calling the
12788                  * FETD's fe_datamove routine.  If we've got a task
12789                  * pending, we run the task queue and then check to see
12790                  * whether our particular I/O has been aborted.
12791                  */
12792                 ctl_softc->flags |= CTL_FLAG_TASK_PENDING;
12793                 break;
12794         default:
12795                 mtx_unlock(&ctl_softc->ctl_lock);
12796                 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
12797                 return (-EINVAL);
12798                 break; /* NOTREACHED */
12799         }
12800         mtx_unlock(&ctl_softc->ctl_lock);
12801
12802         ctl_wakeup_thread();
12803
12804         return (CTL_RETVAL_COMPLETE);
12805 }
12806
12807 #ifdef CTL_IO_DELAY
12808 static void
12809 ctl_done_timer_wakeup(void *arg)
12810 {
12811         union ctl_io *io;
12812
12813         io = (union ctl_io *)arg;
12814         ctl_done_lock(io, /*have_lock*/ 0);
12815 }
12816 #endif /* CTL_IO_DELAY */
12817
12818 void
12819 ctl_done_lock(union ctl_io *io, int have_lock)
12820 {
12821         struct ctl_softc *ctl_softc;
12822 #ifndef CTL_DONE_THREAD
12823         union ctl_io *xio;
12824 #endif /* !CTL_DONE_THREAD */
12825
12826         ctl_softc = control_softc;
12827
12828         if (have_lock == 0)
12829                 mtx_lock(&ctl_softc->ctl_lock);
12830
12831         /*
12832          * Enable this to catch duplicate completion issues.
12833          */
12834 #if 0
12835         if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
12836                 printf("%s: type %d msg %d cdb %x iptl: "
12837                        "%d:%d:%d:%d tag 0x%04x "
12838                        "flag %#x status %x\n",
12839                         __func__,
12840                         io->io_hdr.io_type,
12841                         io->io_hdr.msg_type,
12842                         io->scsiio.cdb[0],
12843                         io->io_hdr.nexus.initid.id,
12844                         io->io_hdr.nexus.targ_port,
12845                         io->io_hdr.nexus.targ_target.id,
12846                         io->io_hdr.nexus.targ_lun,
12847                         (io->io_hdr.io_type ==
12848                         CTL_IO_TASK) ?
12849                         io->taskio.tag_num :
12850                         io->scsiio.tag_num,
12851                         io->io_hdr.flags,
12852                         io->io_hdr.status);
12853         } else
12854                 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
12855 #endif
12856
12857         /*
12858          * This is an internal copy of an I/O, and should not go through
12859          * the normal done processing logic.
12860          */
12861         if (io->io_hdr.flags & CTL_FLAG_INT_COPY) {
12862                 if (have_lock == 0)
12863                         mtx_unlock(&ctl_softc->ctl_lock);
12864                 return;
12865         }
12866
12867         /*
12868          * We need to send a msg to the serializing shelf to finish the IO
12869          * as well.  We don't send a finish message to the other shelf if
12870          * this is a task management command.  Task management commands
12871          * aren't serialized in the OOA queue, but rather just executed on
12872          * both shelf controllers for commands that originated on that
12873          * controller.
12874          */
12875         if ((io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)
12876          && (io->io_hdr.io_type != CTL_IO_TASK)) {
12877                 union ctl_ha_msg msg_io;
12878
12879                 msg_io.hdr.msg_type = CTL_MSG_FINISH_IO;
12880                 msg_io.hdr.serializing_sc = io->io_hdr.serializing_sc;
12881                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_io,
12882                     sizeof(msg_io), 0 ) != CTL_HA_STATUS_SUCCESS) {
12883                 }
12884                 /* continue on to finish IO */
12885         }
12886 #ifdef CTL_IO_DELAY
12887         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12888                 struct ctl_lun *lun;
12889
12890                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12891
12892                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12893         } else {
12894                 struct ctl_lun *lun;
12895
12896                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12897
12898                 if ((lun != NULL)
12899                  && (lun->delay_info.done_delay > 0)) {
12900                         struct callout *callout;
12901
12902                         callout = (struct callout *)&io->io_hdr.timer_bytes;
12903                         callout_init(callout, /*mpsafe*/ 1);
12904                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12905                         callout_reset(callout,
12906                                       lun->delay_info.done_delay * hz,
12907                                       ctl_done_timer_wakeup, io);
12908                         if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
12909                                 lun->delay_info.done_delay = 0;
12910                         if (have_lock == 0)
12911                                 mtx_unlock(&ctl_softc->ctl_lock);
12912                         return;
12913                 }
12914         }
12915 #endif /* CTL_IO_DELAY */
12916
12917         STAILQ_INSERT_TAIL(&ctl_softc->done_queue, &io->io_hdr, links);
12918
12919 #ifdef CTL_DONE_THREAD
12920         if (have_lock == 0)
12921                 mtx_unlock(&ctl_softc->ctl_lock);
12922
12923         ctl_wakeup_thread();
12924 #else /* CTL_DONE_THREAD */
12925         for (xio = (union ctl_io *)STAILQ_FIRST(&ctl_softc->done_queue);
12926              xio != NULL;
12927              xio =(union ctl_io *)STAILQ_FIRST(&ctl_softc->done_queue)) {
12928
12929                 STAILQ_REMOVE_HEAD(&ctl_softc->done_queue, links);
12930
12931                 ctl_process_done(xio, /*have_lock*/ 1);
12932         }
12933         if (have_lock == 0)
12934                 mtx_unlock(&ctl_softc->ctl_lock);
12935 #endif /* CTL_DONE_THREAD */
12936 }
12937
12938 void
12939 ctl_done(union ctl_io *io)
12940 {
12941         ctl_done_lock(io, /*have_lock*/ 0);
12942 }
12943
12944 int
12945 ctl_isc(struct ctl_scsiio *ctsio)
12946 {
12947         struct ctl_lun *lun;
12948         int retval;
12949
12950         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12951
12952         CTL_DEBUG_PRINT(("ctl_isc: command: %02x\n", ctsio->cdb[0]));
12953
12954         CTL_DEBUG_PRINT(("ctl_isc: calling data_submit()\n"));
12955
12956         retval = lun->backend->data_submit((union ctl_io *)ctsio);
12957
12958         return (retval);
12959 }
12960
12961
12962 static void
12963 ctl_work_thread(void *arg)
12964 {
12965         struct ctl_softc *softc;
12966         union ctl_io *io;
12967         struct ctl_be_lun *be_lun;
12968         int retval;
12969
12970         CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
12971
12972         softc = (struct ctl_softc *)arg;
12973         if (softc == NULL)
12974                 return;
12975
12976         mtx_lock(&softc->ctl_lock);
12977         for (;;) {
12978                 retval = 0;
12979
12980                 /*
12981                  * We handle the queues in this order:
12982                  * - task management
12983                  * - ISC
12984                  * - done queue (to free up resources, unblock other commands)
12985                  * - RtR queue
12986                  * - incoming queue
12987                  *
12988                  * If those queues are empty, we break out of the loop and
12989                  * go to sleep.
12990                  */
12991                 io = (union ctl_io *)STAILQ_FIRST(&softc->task_queue);
12992                 if (io != NULL) {
12993                         ctl_run_task_queue(softc);
12994                         continue;
12995                 }
12996                 io = (union ctl_io *)STAILQ_FIRST(&softc->isc_queue);
12997                 if (io != NULL) {
12998                         STAILQ_REMOVE_HEAD(&softc->isc_queue, links);
12999                         ctl_handle_isc(io);
13000                         continue;
13001                 }
13002                 io = (union ctl_io *)STAILQ_FIRST(&softc->done_queue);
13003                 if (io != NULL) {
13004                         STAILQ_REMOVE_HEAD(&softc->done_queue, links);
13005                         /* clear any blocked commands, call fe_done */
13006                         mtx_unlock(&softc->ctl_lock);
13007                         /*
13008                          * XXX KDM
13009                          * Call this without a lock for now.  This will
13010                          * depend on whether there is any way the FETD can
13011                          * sleep or deadlock if called with the CTL lock
13012                          * held.
13013                          */
13014                         retval = ctl_process_done(io, /*have_lock*/ 0);
13015                         mtx_lock(&softc->ctl_lock);
13016                         continue;
13017                 }
13018                 if (!ctl_pause_rtr) {
13019                         io = (union ctl_io *)STAILQ_FIRST(&softc->rtr_queue);
13020                         if (io != NULL) {
13021                                 STAILQ_REMOVE_HEAD(&softc->rtr_queue, links);
13022                                 mtx_unlock(&softc->ctl_lock);
13023                                 retval = ctl_scsiio(&io->scsiio);
13024                                 if (retval != CTL_RETVAL_COMPLETE)
13025                                         CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
13026                                 mtx_lock(&softc->ctl_lock);
13027                                 continue;
13028                         }
13029                 }
13030                 io = (union ctl_io *)STAILQ_FIRST(&softc->incoming_queue);
13031                 if (io != NULL) {
13032                         STAILQ_REMOVE_HEAD(&softc->incoming_queue, links);
13033                         mtx_unlock(&softc->ctl_lock);
13034                         ctl_scsiio_precheck(softc, &io->scsiio);
13035                         mtx_lock(&softc->ctl_lock);
13036                         continue;
13037                 }
13038                 /*
13039                  * We might want to move this to a separate thread, so that
13040                  * configuration requests (in this case LUN creations)
13041                  * won't impact the I/O path.
13042                  */
13043                 be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
13044                 if (be_lun != NULL) {
13045                         STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
13046                         mtx_unlock(&softc->ctl_lock);
13047                         ctl_create_lun(be_lun);
13048                         mtx_lock(&softc->ctl_lock);
13049                         continue;
13050                 }
13051
13052                 /* XXX KDM use the PDROP flag?? */
13053                 /* Sleep until we have something to do. */
13054                 mtx_sleep(softc, &softc->ctl_lock, PRIBIO, "ctl_work", 0);
13055
13056                 /* Back to the top of the loop to see what woke us up. */
13057                 continue;
13058         }
13059 }
13060
13061 void
13062 ctl_wakeup_thread()
13063 {
13064         struct ctl_softc *softc;
13065
13066         softc = control_softc;
13067
13068         wakeup(softc);
13069 }
13070
13071 /* Initialization and failover */
13072
13073 void
13074 ctl_init_isc_msg(void)
13075 {
13076         printf("CTL: Still calling this thing\n");
13077 }
13078
13079 /*
13080  * Init component
13081  *      Initializes component into configuration defined by bootMode
13082  *      (see hasc-sv.c)
13083  *      returns hasc_Status:
13084  *              OK
13085  *              ERROR - fatal error
13086  */
13087 static ctl_ha_comp_status
13088 ctl_isc_init(struct ctl_ha_component *c)
13089 {
13090         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13091
13092         c->status = ret;
13093         return ret;
13094 }
13095
13096 /* Start component
13097  *      Starts component in state requested. If component starts successfully,
13098  *      it must set its own state to the requestrd state
13099  *      When requested state is HASC_STATE_HA, the component may refine it
13100  *      by adding _SLAVE or _MASTER flags.
13101  *      Currently allowed state transitions are:
13102  *      UNKNOWN->HA             - initial startup
13103  *      UNKNOWN->SINGLE - initial startup when no parter detected
13104  *      HA->SINGLE              - failover
13105  * returns ctl_ha_comp_status:
13106  *              OK      - component successfully started in requested state
13107  *              FAILED  - could not start the requested state, failover may
13108  *                        be possible
13109  *              ERROR   - fatal error detected, no future startup possible
13110  */
13111 static ctl_ha_comp_status
13112 ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
13113 {
13114         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13115
13116         printf("%s: go\n", __func__);
13117
13118         // UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
13119         if (c->state == CTL_HA_STATE_UNKNOWN ) {
13120                 ctl_is_single = 0;
13121                 if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
13122                     != CTL_HA_STATUS_SUCCESS) {
13123                         printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
13124                         ret = CTL_HA_COMP_STATUS_ERROR;
13125                 }
13126         } else if (CTL_HA_STATE_IS_HA(c->state)
13127                 && CTL_HA_STATE_IS_SINGLE(state)){
13128                 // HA->SINGLE transition
13129                 ctl_failover();
13130                 ctl_is_single = 1;
13131         } else {
13132                 printf("ctl_isc_start:Invalid state transition %X->%X\n",
13133                        c->state, state);
13134                 ret = CTL_HA_COMP_STATUS_ERROR;
13135         }
13136         if (CTL_HA_STATE_IS_SINGLE(state))
13137                 ctl_is_single = 1;
13138
13139         c->state = state;
13140         c->status = ret;
13141         return ret;
13142 }
13143
13144 /*
13145  * Quiesce component
13146  * The component must clear any error conditions (set status to OK) and
13147  * prepare itself to another Start call
13148  * returns ctl_ha_comp_status:
13149  *      OK
13150  *      ERROR
13151  */
13152 static ctl_ha_comp_status
13153 ctl_isc_quiesce(struct ctl_ha_component *c)
13154 {
13155         int ret = CTL_HA_COMP_STATUS_OK;
13156
13157         ctl_pause_rtr = 1;
13158         c->status = ret;
13159         return ret;
13160 }
13161
13162 struct ctl_ha_component ctl_ha_component_ctlisc =
13163 {
13164         .name = "CTL ISC",
13165         .state = CTL_HA_STATE_UNKNOWN,
13166         .init = ctl_isc_init,
13167         .start = ctl_isc_start,
13168         .quiesce = ctl_isc_quiesce
13169 };
13170
13171 /*
13172  *  vim: ts=8
13173  */