sys/cam/ctl/ctl.c

   1 /*-
   2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
   3  * Copyright (c) 2012 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Edward Tomasz Napierala
   7  * under sponsorship from the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  16  *    substantially similar to the "NO WARRANTY" disclaimer below
  17  *    ("Disclaimer") and any redistribution must be conditioned upon
  18  *    including a substantially similar Disclaimer requirement for further
  19  *    binary redistribution.
  20  *
  21  * NO WARRANTY
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  30  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  31  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGES.
  33  *
  34  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl.c#8 $
  35  */
  36 /*
  37  * CAM Target Layer, a SCSI device emulation subsystem.
  38  *
  39  * Author: Ken Merry <ken@FreeBSD.org>
  40  */
  41
  42 #define _CTL_C
  43
  44 #include <sys/cdefs.h>
  45 __FBSDID("$FreeBSD$");
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/types.h>
  51 #include <sys/kthread.h>
  52 #include <sys/bio.h>
  53 #include <sys/fcntl.h>
  54 #include <sys/lock.h>
  55 #include <sys/module.h>
  56 #include <sys/mutex.h>
  57 #include <sys/condvar.h>
  58 #include <sys/malloc.h>
  59 #include <sys/conf.h>
  60 #include <sys/ioccom.h>
  61 #include <sys/queue.h>
  62 #include <sys/sbuf.h>
  63 #include <sys/smp.h>
  64 #include <sys/endian.h>
  65 #include <sys/sysctl.h>
  66
  67 #include <cam/cam.h>
  68 #include <cam/scsi/scsi_all.h>
  69 #include <cam/scsi/scsi_da.h>
  70 #include <cam/ctl/ctl_io.h>
  71 #include <cam/ctl/ctl.h>
  72 #include <cam/ctl/ctl_frontend.h>
  73 #include <cam/ctl/ctl_frontend_internal.h>
  74 #include <cam/ctl/ctl_util.h>
  75 #include <cam/ctl/ctl_backend.h>
  76 #include <cam/ctl/ctl_ioctl.h>
  77 #include <cam/ctl/ctl_ha.h>
  78 #include <cam/ctl/ctl_private.h>
  79 #include <cam/ctl/ctl_debug.h>
  80 #include <cam/ctl/ctl_scsi_all.h>
  81 #include <cam/ctl/ctl_error.h>
  82
  83 struct ctl_softc *control_softc = NULL;
  84
  85 /*
  86  * Size and alignment macros needed for Copan-specific HA hardware.  These
  87  * can go away when the HA code is re-written, and uses busdma for any
  88  * hardware.
  89  */
  90 #define CTL_ALIGN_8B(target, source, type)                              \
  91         if (((uint32_t)source & 0x7) != 0)                              \
  92                 target = (type)(source + (0x8 - ((uint32_t)source & 0x7)));\
  93         else                                                            \
  94                 target = (type)source;
  95
  96 #define CTL_SIZE_8B(target, size)                                       \
  97         if ((size & 0x7) != 0)                                          \
  98                 target = size + (0x8 - (size & 0x7));                   \
  99         else                                                            \
 100                 target = size;
 101
 102 #define CTL_ALIGN_8B_MARGIN     16
 103
 104 /*
 105  * Template mode pages.
 106  */
 107
 108 /*
 109  * Note that these are default values only.  The actual values will be
 110  * filled in when the user does a mode sense.
 111  */
 112 static struct copan_power_subpage power_page_default = {
 113         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 114         /*subpage*/ PWR_SUBPAGE_CODE,
 115         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 116                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 117         /*page_version*/ PWR_VERSION,
 118         /* total_luns */ 26,
 119         /* max_active_luns*/ PWR_DFLT_MAX_LUNS,
 120         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 121                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 122                       0, 0, 0, 0, 0, 0}
 123 };
 124
 125 static struct copan_power_subpage power_page_changeable = {
 126         /*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
 127         /*subpage*/ PWR_SUBPAGE_CODE,
 128         /*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
 129                          (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
 130         /*page_version*/ 0,
 131         /* total_luns */ 0,
 132         /* max_active_luns*/ 0,
 133         /*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
 134                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 135                       0, 0, 0, 0, 0, 0}
 136 };
 137
 138 static struct copan_aps_subpage aps_page_default = {
 139         APS_PAGE_CODE | SMPH_SPF, //page_code
 140         APS_SUBPAGE_CODE, //subpage
 141         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 142          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 143         APS_VERSION, //page_version
 144         0, //lock_active
 145         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 146         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 147         0, 0, 0, 0, 0} //reserved
 148 };
 149
 150 static struct copan_aps_subpage aps_page_changeable = {
 151         APS_PAGE_CODE | SMPH_SPF, //page_code
 152         APS_SUBPAGE_CODE, //subpage
 153         {(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
 154          (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
 155         0, //page_version
 156         0, //lock_active
 157         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 158         0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 159         0, 0, 0, 0, 0} //reserved
 160 };
 161
 162 static struct copan_debugconf_subpage debugconf_page_default = {
 163         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 164         DBGCNF_SUBPAGE_CODE,            /* subpage */
 165         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 166          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 167         DBGCNF_VERSION,                 /* page_version */
 168         {CTL_TIME_IO_DEFAULT_SECS>>8,
 169          CTL_TIME_IO_DEFAULT_SECS>>0},  /* ctl_time_io_secs */
 170 };
 171
 172 static struct copan_debugconf_subpage debugconf_page_changeable = {
 173         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 174         DBGCNF_SUBPAGE_CODE,            /* subpage */
 175         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 176          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 177         0,                              /* page_version */
 178         {0xff,0xff},                    /* ctl_time_io_secs */
 179 };
 180
 181 static struct scsi_format_page format_page_default = {
 182         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 183         /*page_length*/sizeof(struct scsi_format_page) - 2,
 184         /*tracks_per_zone*/ {0, 0},
 185         /*alt_sectors_per_zone*/ {0, 0},
 186         /*alt_tracks_per_zone*/ {0, 0},
 187         /*alt_tracks_per_lun*/ {0, 0},
 188         /*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
 189                                 CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
 190         /*bytes_per_sector*/ {0, 0},
 191         /*interleave*/ {0, 0},
 192         /*track_skew*/ {0, 0},
 193         /*cylinder_skew*/ {0, 0},
 194         /*flags*/ SFP_HSEC,
 195         /*reserved*/ {0, 0, 0}
 196 };
 197
 198 static struct scsi_format_page format_page_changeable = {
 199         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 200         /*page_length*/sizeof(struct scsi_format_page) - 2,
 201         /*tracks_per_zone*/ {0, 0},
 202         /*alt_sectors_per_zone*/ {0, 0},
 203         /*alt_tracks_per_zone*/ {0, 0},
 204         /*alt_tracks_per_lun*/ {0, 0},
 205         /*sectors_per_track*/ {0, 0},
 206         /*bytes_per_sector*/ {0, 0},
 207         /*interleave*/ {0, 0},
 208         /*track_skew*/ {0, 0},
 209         /*cylinder_skew*/ {0, 0},
 210         /*flags*/ 0,
 211         /*reserved*/ {0, 0, 0}
 212 };
 213
 214 static struct scsi_rigid_disk_page rigid_disk_page_default = {
 215         /*page_code*/SMS_RIGID_DISK_PAGE,
 216         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 217         /*cylinders*/ {0, 0, 0},
 218         /*heads*/ CTL_DEFAULT_HEADS,
 219         /*start_write_precomp*/ {0, 0, 0},
 220         /*start_reduced_current*/ {0, 0, 0},
 221         /*step_rate*/ {0, 0},
 222         /*landing_zone_cylinder*/ {0, 0, 0},
 223         /*rpl*/ SRDP_RPL_DISABLED,
 224         /*rotational_offset*/ 0,
 225         /*reserved1*/ 0,
 226         /*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
 227                            CTL_DEFAULT_ROTATION_RATE & 0xff},
 228         /*reserved2*/ {0, 0}
 229 };
 230
 231 static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
 232         /*page_code*/SMS_RIGID_DISK_PAGE,
 233         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 234         /*cylinders*/ {0, 0, 0},
 235         /*heads*/ 0,
 236         /*start_write_precomp*/ {0, 0, 0},
 237         /*start_reduced_current*/ {0, 0, 0},
 238         /*step_rate*/ {0, 0},
 239         /*landing_zone_cylinder*/ {0, 0, 0},
 240         /*rpl*/ 0,
 241         /*rotational_offset*/ 0,
 242         /*reserved1*/ 0,
 243         /*rotation_rate*/ {0, 0},
 244         /*reserved2*/ {0, 0}
 245 };
 246
 247 static struct scsi_caching_page caching_page_default = {
 248         /*page_code*/SMS_CACHING_PAGE,
 249         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 250         /*flags1*/ SCP_DISC | SCP_WCE,
 251         /*ret_priority*/ 0,
 252         /*disable_pf_transfer_len*/ {0xff, 0xff},
 253         /*min_prefetch*/ {0, 0},
 254         /*max_prefetch*/ {0xff, 0xff},
 255         /*max_pf_ceiling*/ {0xff, 0xff},
 256         /*flags2*/ 0,
 257         /*cache_segments*/ 0,
 258         /*cache_seg_size*/ {0, 0},
 259         /*reserved*/ 0,
 260         /*non_cache_seg_size*/ {0, 0, 0}
 261 };
 262
 263 static struct scsi_caching_page caching_page_changeable = {
 264         /*page_code*/SMS_CACHING_PAGE,
 265         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 266         /*flags1*/ 0,
 267         /*ret_priority*/ 0,
 268         /*disable_pf_transfer_len*/ {0, 0},
 269         /*min_prefetch*/ {0, 0},
 270         /*max_prefetch*/ {0, 0},
 271         /*max_pf_ceiling*/ {0, 0},
 272         /*flags2*/ 0,
 273         /*cache_segments*/ 0,
 274         /*cache_seg_size*/ {0, 0},
 275         /*reserved*/ 0,
 276         /*non_cache_seg_size*/ {0, 0, 0}
 277 };
 278
 279 static struct scsi_control_page control_page_default = {
 280         /*page_code*/SMS_CONTROL_MODE_PAGE,
 281         /*page_length*/sizeof(struct scsi_control_page) - 2,
 282         /*rlec*/0,
 283         /*queue_flags*/0,
 284         /*eca_and_aen*/0,
 285         /*reserved*/0,
 286         /*aen_holdoff_period*/{0, 0}
 287 };
 288
 289 static struct scsi_control_page control_page_changeable = {
 290         /*page_code*/SMS_CONTROL_MODE_PAGE,
 291         /*page_length*/sizeof(struct scsi_control_page) - 2,
 292         /*rlec*/SCP_DSENSE,
 293         /*queue_flags*/0,
 294         /*eca_and_aen*/0,
 295         /*reserved*/0,
 296         /*aen_holdoff_period*/{0, 0}
 297 };
 298
 299
 300 /*
 301  * XXX KDM move these into the softc.
 302  */
 303 static int rcv_sync_msg;
 304 static int persis_offset;
 305 static uint8_t ctl_pause_rtr;
 306 static int     ctl_is_single = 1;
 307 static int     index_to_aps_page;
 308
 309 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 310 static int worker_threads = -1;
 311 TUNABLE_INT("kern.cam.ctl.worker_threads", &worker_threads);
 312 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
 313     &worker_threads, 1, "Number of worker threads");
 314 static int verbose = 0;
 315 TUNABLE_INT("kern.cam.ctl.verbose", &verbose);
 316 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, verbose, CTLFLAG_RWTUN,
 317     &verbose, 0, "Show SCSI errors returned to initiator");
 318
 319 /*
 320  * Supported pages (0x00), Serial number (0x80), Device ID (0x83),
 321  * SCSI Ports (0x88), Block limits (0xB0) and
 322  * Logical Block Provisioning (0xB2)
 323  */
 324 #define SCSI_EVPD_NUM_SUPPORTED_PAGES   6
 325
 326 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 327                                   int param);
 328 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 329 static int ctl_init(void);
 330 void ctl_shutdown(void);
 331 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 332 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 333 static void ctl_ioctl_online(void *arg);
 334 static void ctl_ioctl_offline(void *arg);
 335 static int ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id);
 336 static int ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id);
 337 static int ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio);
 338 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
 339 static int ctl_ioctl_submit_wait(union ctl_io *io);
 340 static void ctl_ioctl_datamove(union ctl_io *io);
 341 static void ctl_ioctl_done(union ctl_io *io);
 342 static void ctl_ioctl_hard_startstop_callback(void *arg,
 343                                               struct cfi_metatask *metatask);
 344 static void ctl_ioctl_bbrread_callback(void *arg,struct cfi_metatask *metatask);
 345 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 346                               struct ctl_ooa *ooa_hdr,
 347                               struct ctl_ooa_entry *kern_entries);
 348 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 349                      struct thread *td);
 350 uint32_t ctl_get_resindex(struct ctl_nexus *nexus);
 351 uint32_t ctl_port_idx(int port_num);
 352 static uint32_t ctl_map_lun(int port_num, uint32_t lun);
 353 static uint32_t ctl_map_lun_back(int port_num, uint32_t lun);
 354 #ifdef unused
 355 static union ctl_io *ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port,
 356                                    uint32_t targ_target, uint32_t targ_lun,
 357                                    int can_wait);
 358 static void ctl_kfree_io(union ctl_io *io);
 359 #endif /* unused */
 360 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
 361                          struct ctl_be_lun *be_lun, struct ctl_id target_id);
 362 static int ctl_free_lun(struct ctl_lun *lun);
 363 static void ctl_create_lun(struct ctl_be_lun *be_lun);
 364 /**
 365 static void ctl_failover_change_pages(struct ctl_softc *softc,
 366                                       struct ctl_scsiio *ctsio, int master);
 367 **/
 368
 369 static int ctl_do_mode_select(union ctl_io *io);
 370 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
 371                            uint64_t res_key, uint64_t sa_res_key,
 372                            uint8_t type, uint32_t residx,
 373                            struct ctl_scsiio *ctsio,
 374                            struct scsi_per_res_out *cdb,
 375                            struct scsi_per_res_out_parms* param);
 376 static void ctl_pro_preempt_other(struct ctl_lun *lun,
 377                                   union ctl_ha_msg *msg);
 378 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
 379 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
 380 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
 381 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
 382 static int ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio,
 383                                          int alloc_len);
 384 static int ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio,
 385                                          int alloc_len);
 386 static int ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len);
 387 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
 388 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
 389 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len);
 390 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2);
 391 static ctl_action ctl_check_for_blockage(union ctl_io *pending_io,
 392                                          union ctl_io *ooa_io);
 393 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
 394                                 union ctl_io *starting_io);
 395 static int ctl_check_blocked(struct ctl_lun *lun);
 396 static int ctl_scsiio_lun_check(struct ctl_softc *ctl_softc,
 397                                 struct ctl_lun *lun,
 398                                 const struct ctl_cmd_entry *entry,
 399                                 struct ctl_scsiio *ctsio);
 400 //static int ctl_check_rtr(union ctl_io *pending_io, struct ctl_softc *softc);
 401 static void ctl_failover(void);
 402 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 403                                struct ctl_scsiio *ctsio);
 404 static int ctl_scsiio(struct ctl_scsiio *ctsio);
 405
 406 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 407 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 408                             ctl_ua_type ua_type);
 409 static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 410                          ctl_ua_type ua_type);
 411 static int ctl_abort_task(union ctl_io *io);
 412 static int ctl_abort_task_set(union ctl_io *io);
 413 static int ctl_i_t_nexus_reset(union ctl_io *io);
 414 static void ctl_run_task(union ctl_io *io);
 415 #ifdef CTL_IO_DELAY
 416 static void ctl_datamove_timer_wakeup(void *arg);
 417 static void ctl_done_timer_wakeup(void *arg);
 418 #endif /* CTL_IO_DELAY */
 419
 420 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 421 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
 422 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
 423 static void ctl_datamove_remote_write(union ctl_io *io);
 424 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
 425 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 426 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 427 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 428                                     ctl_ha_dt_cb callback);
 429 static void ctl_datamove_remote_read(union ctl_io *io);
 430 static void ctl_datamove_remote(union ctl_io *io);
 431 static int ctl_process_done(union ctl_io *io);
 432 static void ctl_lun_thread(void *arg);
 433 static void ctl_work_thread(void *arg);
 434 static void ctl_enqueue_incoming(union ctl_io *io);
 435 static void ctl_enqueue_rtr(union ctl_io *io);
 436 static void ctl_enqueue_done(union ctl_io *io);
 437 static void ctl_enqueue_isc(union ctl_io *io);
 438 static const struct ctl_cmd_entry *
 439     ctl_get_cmd_entry(struct ctl_scsiio *ctsio);
 440 static const struct ctl_cmd_entry *
 441     ctl_validate_command(struct ctl_scsiio *ctsio);
 442 static int ctl_cmd_applicable(uint8_t lun_type,
 443     const struct ctl_cmd_entry *entry);
 444
 445 /*
 446  * Load the serialization table.  This isn't very pretty, but is probably
 447  * the easiest way to do it.
 448  */
 449 #include "ctl_ser_table.c"
 450
 451 /*
 452  * We only need to define open, close and ioctl routines for this driver.
 453  */
 454 static struct cdevsw ctl_cdevsw = {
 455         .d_version =    D_VERSION,
 456         .d_flags =      0,
 457         .d_open =       ctl_open,
 458         .d_close =      ctl_close,
 459         .d_ioctl =      ctl_ioctl,
 460         .d_name =       "ctl",
 461 };
 462
 463
 464 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 465 MALLOC_DEFINE(M_CTLIO, "ctlio", "Memory used for CTL requests");
 466
 467 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 468
 469 static moduledata_t ctl_moduledata = {
 470         "ctl",
 471         ctl_module_event_handler,
 472         NULL
 473 };
 474
 475 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
 476 MODULE_VERSION(ctl, 1);
 477
 478 static struct ctl_frontend ioctl_frontend =
 479 {
 480         .name = "ioctl",
 481 };
 482
 483 static void
 484 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 485                             union ctl_ha_msg *msg_info)
 486 {
 487         struct ctl_scsiio *ctsio;
 488
 489         if (msg_info->hdr.original_sc == NULL) {
 490                 printf("%s: original_sc == NULL!\n", __func__);
 491                 /* XXX KDM now what? */
 492                 return;
 493         }
 494
 495         ctsio = &msg_info->hdr.original_sc->scsiio;
 496         ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 497         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 498         ctsio->io_hdr.status = msg_info->hdr.status;
 499         ctsio->scsi_status = msg_info->scsi.scsi_status;
 500         ctsio->sense_len = msg_info->scsi.sense_len;
 501         ctsio->sense_residual = msg_info->scsi.sense_residual;
 502         ctsio->residual = msg_info->scsi.residual;
 503         memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 504                sizeof(ctsio->sense_data));
 505         memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
 506                &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));
 507         ctl_enqueue_isc((union ctl_io *)ctsio);
 508 }
 509
 510 static void
 511 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
 512                                 union ctl_ha_msg *msg_info)
 513 {
 514         struct ctl_scsiio *ctsio;
 515
 516         if (msg_info->hdr.serializing_sc == NULL) {
 517                 printf("%s: serializing_sc == NULL!\n", __func__);
 518                 /* XXX KDM now what? */
 519                 return;
 520         }
 521
 522         ctsio = &msg_info->hdr.serializing_sc->scsiio;
 523 #if 0
 524         /*
 525          * Attempt to catch the situation where an I/O has
 526          * been freed, and we're using it again.
 527          */
 528         if (ctsio->io_hdr.io_type == 0xff) {
 529                 union ctl_io *tmp_io;
 530                 tmp_io = (union ctl_io *)ctsio;
 531                 printf("%s: %p use after free!\n", __func__,
 532                        ctsio);
 533                 printf("%s: type %d msg %d cdb %x iptl: "
 534                        "%d:%d:%d:%d tag 0x%04x "
 535                        "flag %#x status %x\n",
 536                         __func__,
 537                         tmp_io->io_hdr.io_type,
 538                         tmp_io->io_hdr.msg_type,
 539                         tmp_io->scsiio.cdb[0],
 540                         tmp_io->io_hdr.nexus.initid.id,
 541                         tmp_io->io_hdr.nexus.targ_port,
 542                         tmp_io->io_hdr.nexus.targ_target.id,
 543                         tmp_io->io_hdr.nexus.targ_lun,
 544                         (tmp_io->io_hdr.io_type ==
 545                         CTL_IO_TASK) ?
 546                         tmp_io->taskio.tag_num :
 547                         tmp_io->scsiio.tag_num,
 548                         tmp_io->io_hdr.flags,
 549                         tmp_io->io_hdr.status);
 550         }
 551 #endif
 552         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 553         ctl_enqueue_isc((union ctl_io *)ctsio);
 554 }
 555
 556 /*
 557  * ISC (Inter Shelf Communication) event handler.  Events from the HA
 558  * subsystem come in here.
 559  */
 560 static void
 561 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 562 {
 563         struct ctl_softc *ctl_softc;
 564         union ctl_io *io;
 565         struct ctl_prio *presio;
 566         ctl_ha_status isc_status;
 567
 568         ctl_softc = control_softc;
 569         io = NULL;
 570
 571
 572 #if 0
 573         printf("CTL: Isc Msg event %d\n", event);
 574 #endif
 575         if (event == CTL_HA_EVT_MSG_RECV) {
 576                 union ctl_ha_msg msg_info;
 577
 578                 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
 579                                              sizeof(msg_info), /*wait*/ 0);
 580 #if 0
 581                 printf("CTL: msg_type %d\n", msg_info.msg_type);
 582 #endif
 583                 if (isc_status != 0) {
 584                         printf("Error receiving message, status = %d\n",
 585                                isc_status);
 586                         return;
 587                 }
 588
 589                 switch (msg_info.hdr.msg_type) {
 590                 case CTL_MSG_SERIALIZE:
 591 #if 0
 592                         printf("Serialize\n");
 593 #endif
 594                         io = ctl_alloc_io((void *)ctl_softc->othersc_pool);
 595                         if (io == NULL) {
 596                                 printf("ctl_isc_event_handler: can't allocate "
 597                                        "ctl_io!\n");
 598                                 /* Bad Juju */
 599                                 /* Need to set busy and send msg back */
 600                                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
 601                                 msg_info.hdr.status = CTL_SCSI_ERROR;
 602                                 msg_info.scsi.scsi_status = SCSI_STATUS_BUSY;
 603                                 msg_info.scsi.sense_len = 0;
 604                                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
 605                                     sizeof(msg_info), 0) > CTL_HA_STATUS_SUCCESS){
 606                                 }
 607                                 goto bailout;
 608                         }
 609                         ctl_zero_io(io);
 610                         // populate ctsio from msg_info
 611                         io->io_hdr.io_type = CTL_IO_SCSI;
 612                         io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
 613                         io->io_hdr.original_sc = msg_info.hdr.original_sc;
 614 #if 0
 615                         printf("pOrig %x\n", (int)msg_info.original_sc);
 616 #endif
 617                         io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
 618                                             CTL_FLAG_IO_ACTIVE;
 619                         /*
 620                          * If we're in serialization-only mode, we don't
 621                          * want to go through full done processing.  Thus
 622                          * the COPY flag.
 623                          *
 624                          * XXX KDM add another flag that is more specific.
 625                          */
 626                         if (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)
 627                                 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
 628                         io->io_hdr.nexus = msg_info.hdr.nexus;
 629 #if 0
 630                         printf("targ %d, port %d, iid %d, lun %d\n",
 631                                io->io_hdr.nexus.targ_target.id,
 632                                io->io_hdr.nexus.targ_port,
 633                                io->io_hdr.nexus.initid.id,
 634                                io->io_hdr.nexus.targ_lun);
 635 #endif
 636                         io->scsiio.tag_num = msg_info.scsi.tag_num;
 637                         io->scsiio.tag_type = msg_info.scsi.tag_type;
 638                         memcpy(io->scsiio.cdb, msg_info.scsi.cdb,
 639                                CTL_MAX_CDBLEN);
 640                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
 641                                 const struct ctl_cmd_entry *entry;
 642
 643                                 entry = ctl_get_cmd_entry(&io->scsiio);
 644                                 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
 645                                 io->io_hdr.flags |=
 646                                         entry->flags & CTL_FLAG_DATA_MASK;
 647                         }
 648                         ctl_enqueue_isc(io);
 649                         break;
 650
 651                 /* Performed on the Originating SC, XFER mode only */
 652                 case CTL_MSG_DATAMOVE: {
 653                         struct ctl_sg_entry *sgl;
 654                         int i, j;
 655
 656                         io = msg_info.hdr.original_sc;
 657                         if (io == NULL) {
 658                                 printf("%s: original_sc == NULL!\n", __func__);
 659                                 /* XXX KDM do something here */
 660                                 break;
 661                         }
 662                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
 663                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 664                         /*
 665                          * Keep track of this, we need to send it back over
 666                          * when the datamove is complete.
 667                          */
 668                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 669
 670                         if (msg_info.dt.sg_sequence == 0) {
 671                                 /*
 672                                  * XXX KDM we use the preallocated S/G list
 673                                  * here, but we'll need to change this to
 674                                  * dynamic allocation if we need larger S/G
 675                                  * lists.
 676                                  */
 677                                 if (msg_info.dt.kern_sg_entries >
 678                                     sizeof(io->io_hdr.remote_sglist) /
 679                                     sizeof(io->io_hdr.remote_sglist[0])) {
 680                                         printf("%s: number of S/G entries "
 681                                             "needed %u > allocated num %zd\n",
 682                                             __func__,
 683                                             msg_info.dt.kern_sg_entries,
 684                                             sizeof(io->io_hdr.remote_sglist)/
 685                                             sizeof(io->io_hdr.remote_sglist[0]));
 686
 687                                         /*
 688                                          * XXX KDM send a message back to
 689                                          * the other side to shut down the
 690                                          * DMA.  The error will come back
 691                                          * through via the normal channel.
 692                                          */
 693                                         break;
 694                                 }
 695                                 sgl = io->io_hdr.remote_sglist;
 696                                 memset(sgl, 0,
 697                                        sizeof(io->io_hdr.remote_sglist));
 698
 699                                 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
 700
 701                                 io->scsiio.kern_sg_entries =
 702                                         msg_info.dt.kern_sg_entries;
 703                                 io->scsiio.rem_sg_entries =
 704                                         msg_info.dt.kern_sg_entries;
 705                                 io->scsiio.kern_data_len =
 706                                         msg_info.dt.kern_data_len;
 707                                 io->scsiio.kern_total_len =
 708                                         msg_info.dt.kern_total_len;
 709                                 io->scsiio.kern_data_resid =
 710                                         msg_info.dt.kern_data_resid;
 711                                 io->scsiio.kern_rel_offset =
 712                                         msg_info.dt.kern_rel_offset;
 713                                 /*
 714                                  * Clear out per-DMA flags.
 715                                  */
 716                                 io->io_hdr.flags &= ~CTL_FLAG_RDMA_MASK;
 717                                 /*
 718                                  * Add per-DMA flags that are set for this
 719                                  * particular DMA request.
 720                                  */
 721                                 io->io_hdr.flags |= msg_info.dt.flags &
 722                                                     CTL_FLAG_RDMA_MASK;
 723                         } else
 724                                 sgl = (struct ctl_sg_entry *)
 725                                         io->scsiio.kern_data_ptr;
 726
 727                         for (i = msg_info.dt.sent_sg_entries, j = 0;
 728                              i < (msg_info.dt.sent_sg_entries +
 729                              msg_info.dt.cur_sg_entries); i++, j++) {
 730                                 sgl[i].addr = msg_info.dt.sg_list[j].addr;
 731                                 sgl[i].len = msg_info.dt.sg_list[j].len;
 732
 733 #if 0
 734                                 printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
 735                                        __func__,
 736                                        msg_info.dt.sg_list[j].addr,
 737                                        msg_info.dt.sg_list[j].len,
 738                                        sgl[i].addr, sgl[i].len, j, i);
 739 #endif
 740                         }
 741 #if 0
 742                         memcpy(&sgl[msg_info.dt.sent_sg_entries],
 743                                msg_info.dt.sg_list,
 744                                sizeof(*sgl) * msg_info.dt.cur_sg_entries);
 745 #endif
 746
 747                         /*
 748                          * If this is the last piece of the I/O, we've got
 749                          * the full S/G list.  Queue processing in the thread.
 750                          * Otherwise wait for the next piece.
 751                          */
 752                         if (msg_info.dt.sg_last != 0)
 753                                 ctl_enqueue_isc(io);
 754                         break;
 755                 }
 756                 /* Performed on the Serializing (primary) SC, XFER mode only */
 757                 case CTL_MSG_DATAMOVE_DONE: {
 758                         if (msg_info.hdr.serializing_sc == NULL) {
 759                                 printf("%s: serializing_sc == NULL!\n",
 760                                        __func__);
 761                                 /* XXX KDM now what? */
 762                                 break;
 763                         }
 764                         /*
 765                          * We grab the sense information here in case
 766                          * there was a failure, so we can return status
 767                          * back to the initiator.
 768                          */
 769                         io = msg_info.hdr.serializing_sc;
 770                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
 771                         io->io_hdr.status = msg_info.hdr.status;
 772                         io->scsiio.scsi_status = msg_info.scsi.scsi_status;
 773                         io->scsiio.sense_len = msg_info.scsi.sense_len;
 774                         io->scsiio.sense_residual =msg_info.scsi.sense_residual;
 775                         io->io_hdr.port_status = msg_info.scsi.fetd_status;
 776                         io->scsiio.residual = msg_info.scsi.residual;
 777                         memcpy(&io->scsiio.sense_data,&msg_info.scsi.sense_data,
 778                                sizeof(io->scsiio.sense_data));
 779                         ctl_enqueue_isc(io);
 780                         break;
 781                 }
 782
 783                 /* Preformed on Originating SC, SER_ONLY mode */
 784                 case CTL_MSG_R2R:
 785                         io = msg_info.hdr.original_sc;
 786                         if (io == NULL) {
 787                                 printf("%s: Major Bummer\n", __func__);
 788                                 return;
 789                         } else {
 790 #if 0
 791                                 printf("pOrig %x\n",(int) ctsio);
 792 #endif
 793                         }
 794                         io->io_hdr.msg_type = CTL_MSG_R2R;
 795                         io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
 796                         ctl_enqueue_isc(io);
 797                         break;
 798
 799                 /*
 800                  * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
 801                  * mode.
 802                  * Performed on the Originating (i.e. secondary) SC in XFER
 803                  * mode
 804                  */
 805                 case CTL_MSG_FINISH_IO:
 806                         if (ctl_softc->ha_mode == CTL_HA_MODE_XFER)
 807                                 ctl_isc_handler_finish_xfer(ctl_softc,
 808                                                             &msg_info);
 809                         else
 810                                 ctl_isc_handler_finish_ser_only(ctl_softc,
 811                                                                 &msg_info);
 812                         break;
 813
 814                 /* Preformed on Originating SC */
 815                 case CTL_MSG_BAD_JUJU:
 816                         io = msg_info.hdr.original_sc;
 817                         if (io == NULL) {
 818                                 printf("%s: Bad JUJU!, original_sc is NULL!\n",
 819                                        __func__);
 820                                 break;
 821                         }
 822                         ctl_copy_sense_data(&msg_info, io);
 823                         /*
 824                          * IO should have already been cleaned up on other
 825                          * SC so clear this flag so we won't send a message
 826                          * back to finish the IO there.
 827                          */
 828                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
 829                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 830
 831                         /* io = msg_info.hdr.serializing_sc; */
 832                         io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
 833                         ctl_enqueue_isc(io);
 834                         break;
 835
 836                 /* Handle resets sent from the other side */
 837                 case CTL_MSG_MANAGE_TASKS: {
 838                         struct ctl_taskio *taskio;
 839                         taskio = (struct ctl_taskio *)ctl_alloc_io(
 840                                 (void *)ctl_softc->othersc_pool);
 841                         if (taskio == NULL) {
 842                                 printf("ctl_isc_event_handler: can't allocate "
 843                                        "ctl_io!\n");
 844                                 /* Bad Juju */
 845                                 /* should I just call the proper reset func
 846                                    here??? */
 847                                 goto bailout;
 848                         }
 849                         ctl_zero_io((union ctl_io *)taskio);
 850                         taskio->io_hdr.io_type = CTL_IO_TASK;
 851                         taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
 852                         taskio->io_hdr.nexus = msg_info.hdr.nexus;
 853                         taskio->task_action = msg_info.task.task_action;
 854                         taskio->tag_num = msg_info.task.tag_num;
 855                         taskio->tag_type = msg_info.task.tag_type;
 856 #ifdef CTL_TIME_IO
 857                         taskio->io_hdr.start_time = time_uptime;
 858                         getbintime(&taskio->io_hdr.start_bt);
 859 #if 0
 860                         cs_prof_gettime(&taskio->io_hdr.start_ticks);
 861 #endif
 862 #endif /* CTL_TIME_IO */
 863                         ctl_run_task((union ctl_io *)taskio);
 864                         break;
 865                 }
 866                 /* Persistent Reserve action which needs attention */
 867                 case CTL_MSG_PERS_ACTION:
 868                         presio = (struct ctl_prio *)ctl_alloc_io(
 869                                 (void *)ctl_softc->othersc_pool);
 870                         if (presio == NULL) {
 871                                 printf("ctl_isc_event_handler: can't allocate "
 872                                        "ctl_io!\n");
 873                                 /* Bad Juju */
 874                                 /* Need to set busy and send msg back */
 875                                 goto bailout;
 876                         }
 877                         ctl_zero_io((union ctl_io *)presio);
 878                         presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
 879                         presio->pr_msg = msg_info.pr;
 880                         ctl_enqueue_isc((union ctl_io *)presio);
 881                         break;
 882                 case CTL_MSG_SYNC_FE:
 883                         rcv_sync_msg = 1;
 884                         break;
 885                 case CTL_MSG_APS_LOCK: {
 886                         // It's quicker to execute this then to
 887                         // queue it.
 888                         struct ctl_lun *lun;
 889                         struct ctl_page_index *page_index;
 890                         struct copan_aps_subpage *current_sp;
 891                         uint32_t targ_lun;
 892
 893                         targ_lun = msg_info.hdr.nexus.targ_mapped_lun;
 894                         lun = ctl_softc->ctl_luns[targ_lun];
 895                         mtx_lock(&lun->lun_lock);
 896                         page_index = &lun->mode_pages.index[index_to_aps_page];
 897                         current_sp = (struct copan_aps_subpage *)
 898                                      (page_index->page_data +
 899                                      (page_index->page_len * CTL_PAGE_CURRENT));
 900
 901                         current_sp->lock_active = msg_info.aps.lock_flag;
 902                         mtx_unlock(&lun->lun_lock);
 903                         break;
 904                 }
 905                 default:
 906                         printf("How did I get here?\n");
 907                 }
 908         } else if (event == CTL_HA_EVT_MSG_SENT) {
 909                 if (param != CTL_HA_STATUS_SUCCESS) {
 910                         printf("Bad status from ctl_ha_msg_send status %d\n",
 911                                param);
 912                 }
 913                 return;
 914         } else if (event == CTL_HA_EVT_DISCONNECT) {
 915                 printf("CTL: Got a disconnect from Isc\n");
 916                 return;
 917         } else {
 918                 printf("ctl_isc_event_handler: Unknown event %d\n", event);
 919                 return;
 920         }
 921
 922 bailout:
 923         return;
 924 }
 925
 926 static void
 927 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
 928 {
 929         struct scsi_sense_data *sense;
 930
 931         sense = &dest->scsiio.sense_data;
 932         bcopy(&src->scsi.sense_data, sense, sizeof(*sense));
 933         dest->scsiio.scsi_status = src->scsi.scsi_status;
 934         dest->scsiio.sense_len = src->scsi.sense_len;
 935         dest->io_hdr.status = src->hdr.status;
 936 }
 937
 938 static int
 939 ctl_init(void)
 940 {
 941         struct ctl_softc *softc;
 942         struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
 943         struct ctl_port *port;
 944         uint8_t sc_id =0;
 945         int i, error, retval;
 946         //int isc_retval;
 947
 948         retval = 0;
 949         ctl_pause_rtr = 0;
 950         rcv_sync_msg = 0;
 951
 952         control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
 953                                M_WAITOK | M_ZERO);
 954         softc = control_softc;
 955
 956         softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
 957                               "cam/ctl");
 958
 959         softc->dev->si_drv1 = softc;
 960
 961         /*
 962          * By default, return a "bad LUN" peripheral qualifier for unknown
 963          * LUNs.  The user can override this default using the tunable or
 964          * sysctl.  See the comment in ctl_inquiry_std() for more details.
 965          */
 966         softc->inquiry_pq_no_lun = 1;
 967         TUNABLE_INT_FETCH("kern.cam.ctl.inquiry_pq_no_lun",
 968                           &softc->inquiry_pq_no_lun);
 969         sysctl_ctx_init(&softc->sysctl_ctx);
 970         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
 971                 SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
 972                 CTLFLAG_RD, 0, "CAM Target Layer");
 973
 974         if (softc->sysctl_tree == NULL) {
 975                 printf("%s: unable to allocate sysctl tree\n", __func__);
 976                 destroy_dev(softc->dev);
 977                 free(control_softc, M_DEVBUF);
 978                 control_softc = NULL;
 979                 return (ENOMEM);
 980         }
 981
 982         SYSCTL_ADD_INT(&softc->sysctl_ctx,
 983                        SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
 984                        "inquiry_pq_no_lun", CTLFLAG_RW,
 985                        &softc->inquiry_pq_no_lun, 0,
 986                        "Report no lun possible for invalid LUNs");
 987
 988         mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
 989         mtx_init(&softc->pool_lock, "CTL pool mutex", NULL, MTX_DEF);
 990         softc->open_count = 0;
 991
 992         /*
 993          * Default to actually sending a SYNCHRONIZE CACHE command down to
 994          * the drive.
 995          */
 996         softc->flags = CTL_FLAG_REAL_SYNC;
 997
 998         /*
 999          * In Copan's HA scheme, the "master" and "slave" roles are
1000          * figured out through the slot the controller is in.  Although it
1001          * is an active/active system, someone has to be in charge.
1002          */
1003 #ifdef NEEDTOPORT
1004         scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
1005 #endif
1006
1007         if (sc_id == 0) {
1008                 softc->flags |= CTL_FLAG_MASTER_SHELF;
1009                 persis_offset = 0;
1010         } else
1011                 persis_offset = CTL_MAX_INITIATORS;
1012
1013         /*
1014          * XXX KDM need to figure out where we want to get our target ID
1015          * and WWID.  Is it different on each port?
1016          */
1017         softc->target.id = 0;
1018         softc->target.wwid[0] = 0x12345678;
1019         softc->target.wwid[1] = 0x87654321;
1020         STAILQ_INIT(&softc->lun_list);
1021         STAILQ_INIT(&softc->pending_lun_queue);
1022         STAILQ_INIT(&softc->fe_list);
1023         STAILQ_INIT(&softc->port_list);
1024         STAILQ_INIT(&softc->be_list);
1025         STAILQ_INIT(&softc->io_pools);
1026
1027         if (ctl_pool_create(softc, CTL_POOL_INTERNAL, CTL_POOL_ENTRIES_INTERNAL,
1028                             &internal_pool)!= 0){
1029                 printf("ctl: can't allocate %d entry internal pool, "
1030                        "exiting\n", CTL_POOL_ENTRIES_INTERNAL);
1031                 return (ENOMEM);
1032         }
1033
1034         if (ctl_pool_create(softc, CTL_POOL_EMERGENCY,
1035                             CTL_POOL_ENTRIES_EMERGENCY, &emergency_pool) != 0) {
1036                 printf("ctl: can't allocate %d entry emergency pool, "
1037                        "exiting\n", CTL_POOL_ENTRIES_EMERGENCY);
1038                 ctl_pool_free(internal_pool);
1039                 return (ENOMEM);
1040         }
1041
1042         if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC,
1043                             &other_pool) != 0)
1044         {
1045                 printf("ctl: can't allocate %d entry other SC pool, "
1046                        "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1047                 ctl_pool_free(internal_pool);
1048                 ctl_pool_free(emergency_pool);
1049                 return (ENOMEM);
1050         }
1051
1052         softc->internal_pool = internal_pool;
1053         softc->emergency_pool = emergency_pool;
1054         softc->othersc_pool = other_pool;
1055
1056         if (worker_threads <= 0)
1057                 worker_threads = max(1, mp_ncpus / 4);
1058         if (worker_threads > CTL_MAX_THREADS)
1059                 worker_threads = CTL_MAX_THREADS;
1060
1061         for (i = 0; i < worker_threads; i++) {
1062                 struct ctl_thread *thr = &softc->threads[i];
1063
1064                 mtx_init(&thr->queue_lock, "CTL queue mutex", NULL, MTX_DEF);
1065                 thr->ctl_softc = softc;
1066                 STAILQ_INIT(&thr->incoming_queue);
1067                 STAILQ_INIT(&thr->rtr_queue);
1068                 STAILQ_INIT(&thr->done_queue);
1069                 STAILQ_INIT(&thr->isc_queue);
1070
1071                 error = kproc_kthread_add(ctl_work_thread, thr,
1072                     &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i);
1073                 if (error != 0) {
1074                         printf("error creating CTL work thread!\n");
1075                         ctl_pool_free(internal_pool);
1076                         ctl_pool_free(emergency_pool);
1077                         ctl_pool_free(other_pool);
1078                         return (error);
1079                 }
1080         }
1081         error = kproc_kthread_add(ctl_lun_thread, softc,
1082             &softc->ctl_proc, NULL, 0, 0, "ctl", "lun");
1083         if (error != 0) {
1084                 printf("error creating CTL lun thread!\n");
1085                 ctl_pool_free(internal_pool);
1086                 ctl_pool_free(emergency_pool);
1087                 ctl_pool_free(other_pool);
1088                 return (error);
1089         }
1090         if (bootverbose)
1091                 printf("ctl: CAM Target Layer loaded\n");
1092
1093         /*
1094          * Initialize the ioctl front end.
1095          */
1096         ctl_frontend_register(&ioctl_frontend);
1097         port = &softc->ioctl_info.port;
1098         port->frontend = &ioctl_frontend;
1099         sprintf(softc->ioctl_info.port_name, "ioctl");
1100         port->port_type = CTL_PORT_IOCTL;
1101         port->num_requested_ctl_io = 100;
1102         port->port_name = softc->ioctl_info.port_name;
1103         port->port_online = ctl_ioctl_online;
1104         port->port_offline = ctl_ioctl_offline;
1105         port->onoff_arg = &softc->ioctl_info;
1106         port->lun_enable = ctl_ioctl_lun_enable;
1107         port->lun_disable = ctl_ioctl_lun_disable;
1108         port->targ_lun_arg = &softc->ioctl_info;
1109         port->fe_datamove = ctl_ioctl_datamove;
1110         port->fe_done = ctl_ioctl_done;
1111         port->max_targets = 15;
1112         port->max_target_id = 15;
1113
1114         if (ctl_port_register(&softc->ioctl_info.port,
1115                           (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
1116                 printf("ctl: ioctl front end registration failed, will "
1117                        "continue anyway\n");
1118         }
1119
1120 #ifdef CTL_IO_DELAY
1121         if (sizeof(struct callout) > CTL_TIMER_BYTES) {
1122                 printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
1123                        sizeof(struct callout), CTL_TIMER_BYTES);
1124                 return (EINVAL);
1125         }
1126 #endif /* CTL_IO_DELAY */
1127
1128         return (0);
1129 }
1130
1131 void
1132 ctl_shutdown(void)
1133 {
1134         struct ctl_softc *softc;
1135         struct ctl_lun *lun, *next_lun;
1136         struct ctl_io_pool *pool;
1137
1138         softc = (struct ctl_softc *)control_softc;
1139
1140         if (ctl_port_deregister(&softc->ioctl_info.port) != 0)
1141                 printf("ctl: ioctl front end deregistration failed\n");
1142
1143         mtx_lock(&softc->ctl_lock);
1144
1145         /*
1146          * Free up each LUN.
1147          */
1148         for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1149                 next_lun = STAILQ_NEXT(lun, links);
1150                 ctl_free_lun(lun);
1151         }
1152
1153         mtx_unlock(&softc->ctl_lock);
1154
1155         ctl_frontend_deregister(&ioctl_frontend);
1156
1157         /*
1158          * This will rip the rug out from under any FETDs or anyone else
1159          * that has a pool allocated.  Since we increment our module
1160          * refcount any time someone outside the main CTL module allocates
1161          * a pool, we shouldn't have any problems here.  The user won't be
1162          * able to unload the CTL module until client modules have
1163          * successfully unloaded.
1164          */
1165         while ((pool = STAILQ_FIRST(&softc->io_pools)) != NULL)
1166                 ctl_pool_free(pool);
1167
1168 #if 0
1169         ctl_shutdown_thread(softc->work_thread);
1170         mtx_destroy(&softc->queue_lock);
1171 #endif
1172
1173         mtx_destroy(&softc->pool_lock);
1174         mtx_destroy(&softc->ctl_lock);
1175
1176         destroy_dev(softc->dev);
1177
1178         sysctl_ctx_free(&softc->sysctl_ctx);
1179
1180         free(control_softc, M_DEVBUF);
1181         control_softc = NULL;
1182
1183         if (bootverbose)
1184                 printf("ctl: CAM Target Layer unloaded\n");
1185 }
1186
1187 static int
1188 ctl_module_event_handler(module_t mod, int what, void *arg)
1189 {
1190
1191         switch (what) {
1192         case MOD_LOAD:
1193                 return (ctl_init());
1194         case MOD_UNLOAD:
1195                 return (EBUSY);
1196         default:
1197                 return (EOPNOTSUPP);
1198         }
1199 }
1200
1201 /*
1202  * XXX KDM should we do some access checks here?  Bump a reference count to
1203  * prevent a CTL module from being unloaded while someone has it open?
1204  */
1205 static int
1206 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1207 {
1208         return (0);
1209 }
1210
1211 static int
1212 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1213 {
1214         return (0);
1215 }
1216
1217 int
1218 ctl_port_enable(ctl_port_type port_type)
1219 {
1220         struct ctl_softc *softc;
1221         struct ctl_port *port;
1222
1223         if (ctl_is_single == 0) {
1224                 union ctl_ha_msg msg_info;
1225                 int isc_retval;
1226
1227 #if 0
1228                 printf("%s: HA mode, synchronizing frontend enable\n",
1229                         __func__);
1230 #endif
1231                 msg_info.hdr.msg_type = CTL_MSG_SYNC_FE;
1232                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1233                         sizeof(msg_info), 1 )) > CTL_HA_STATUS_SUCCESS) {
1234                         printf("Sync msg send error retval %d\n", isc_retval);
1235                 }
1236                 if (!rcv_sync_msg) {
1237                         isc_retval=ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
1238                                 sizeof(msg_info), 1);
1239                 }
1240 #if 0
1241                 printf("CTL:Frontend Enable\n");
1242         } else {
1243                 printf("%s: single mode, skipping frontend synchronization\n",
1244                         __func__);
1245 #endif
1246         }
1247
1248         softc = control_softc;
1249
1250         STAILQ_FOREACH(port, &softc->port_list, links) {
1251                 if (port_type & port->port_type)
1252                 {
1253 #if 0
1254                         printf("port %d\n", port->targ_port);
1255 #endif
1256                         ctl_port_online(port);
1257                 }
1258         }
1259
1260         return (0);
1261 }
1262
1263 int
1264 ctl_port_disable(ctl_port_type port_type)
1265 {
1266         struct ctl_softc *softc;
1267         struct ctl_port *port;
1268
1269         softc = control_softc;
1270
1271         STAILQ_FOREACH(port, &softc->port_list, links) {
1272                 if (port_type & port->port_type)
1273                         ctl_port_offline(port);
1274         }
1275
1276         return (0);
1277 }
1278
1279 /*
1280  * Returns 0 for success, 1 for failure.
1281  * Currently the only failure mode is if there aren't enough entries
1282  * allocated.  So, in case of a failure, look at num_entries_dropped,
1283  * reallocate and try again.
1284  */
1285 int
1286 ctl_port_list(struct ctl_port_entry *entries, int num_entries_alloced,
1287               int *num_entries_filled, int *num_entries_dropped,
1288               ctl_port_type port_type, int no_virtual)
1289 {
1290         struct ctl_softc *softc;
1291         struct ctl_port *port;
1292         int entries_dropped, entries_filled;
1293         int retval;
1294         int i;
1295
1296         softc = control_softc;
1297
1298         retval = 0;
1299         entries_filled = 0;
1300         entries_dropped = 0;
1301
1302         i = 0;
1303         mtx_lock(&softc->ctl_lock);
1304         STAILQ_FOREACH(port, &softc->port_list, links) {
1305                 struct ctl_port_entry *entry;
1306
1307                 if ((port->port_type & port_type) == 0)
1308                         continue;
1309
1310                 if ((no_virtual != 0)
1311                  && (port->virtual_port != 0))
1312                         continue;
1313
1314                 if (entries_filled >= num_entries_alloced) {
1315                         entries_dropped++;
1316                         continue;
1317                 }
1318                 entry = &entries[i];
1319
1320                 entry->port_type = port->port_type;
1321                 strlcpy(entry->port_name, port->port_name,
1322                         sizeof(entry->port_name));
1323                 entry->physical_port = port->physical_port;
1324                 entry->virtual_port = port->virtual_port;
1325                 entry->wwnn = port->wwnn;
1326                 entry->wwpn = port->wwpn;
1327
1328                 i++;
1329                 entries_filled++;
1330         }
1331
1332         mtx_unlock(&softc->ctl_lock);
1333
1334         if (entries_dropped > 0)
1335                 retval = 1;
1336
1337         *num_entries_dropped = entries_dropped;
1338         *num_entries_filled = entries_filled;
1339
1340         return (retval);
1341 }
1342
1343 static void
1344 ctl_ioctl_online(void *arg)
1345 {
1346         struct ctl_ioctl_info *ioctl_info;
1347
1348         ioctl_info = (struct ctl_ioctl_info *)arg;
1349
1350         ioctl_info->flags |= CTL_IOCTL_FLAG_ENABLED;
1351 }
1352
1353 static void
1354 ctl_ioctl_offline(void *arg)
1355 {
1356         struct ctl_ioctl_info *ioctl_info;
1357
1358         ioctl_info = (struct ctl_ioctl_info *)arg;
1359
1360         ioctl_info->flags &= ~CTL_IOCTL_FLAG_ENABLED;
1361 }
1362
1363 /*
1364  * Remove an initiator by port number and initiator ID.
1365  * Returns 0 for success, -1 for failure.
1366  */
1367 int
1368 ctl_remove_initiator(struct ctl_port *port, int iid)
1369 {
1370         struct ctl_softc *softc = control_softc;
1371
1372         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1373
1374         if (iid > CTL_MAX_INIT_PER_PORT) {
1375                 printf("%s: initiator ID %u > maximun %u!\n",
1376                        __func__, iid, CTL_MAX_INIT_PER_PORT);
1377                 return (-1);
1378         }
1379
1380         mtx_lock(&softc->ctl_lock);
1381         port->wwpn_iid[iid].in_use--;
1382         port->wwpn_iid[iid].last_use = time_uptime;
1383         mtx_unlock(&softc->ctl_lock);
1384
1385         return (0);
1386 }
1387
1388 /*
1389  * Add an initiator to the initiator map.
1390  * Returns iid for success, < 0 for failure.
1391  */
1392 int
1393 ctl_add_initiator(struct ctl_port *port, int iid, uint64_t wwpn, char *name)
1394 {
1395         struct ctl_softc *softc = control_softc;
1396         time_t best_time;
1397         int i, best;
1398
1399         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1400
1401         if (iid >= CTL_MAX_INIT_PER_PORT) {
1402                 printf("%s: WWPN %#jx initiator ID %u > maximum %u!\n",
1403                        __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1404                 free(name, M_CTL);
1405                 return (-1);
1406         }
1407
1408         mtx_lock(&softc->ctl_lock);
1409
1410         if (iid < 0 && (wwpn != 0 || name != NULL)) {
1411                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1412                         if (wwpn != 0 && wwpn == port->wwpn_iid[i].wwpn) {
1413                                 iid = i;
1414                                 break;
1415                         }
1416                         if (name != NULL && port->wwpn_iid[i].name != NULL &&
1417                             strcmp(name, port->wwpn_iid[i].name) == 0) {
1418                                 iid = i;
1419                                 break;
1420                         }
1421                 }
1422         }
1423
1424         if (iid < 0) {
1425                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1426                         if (port->wwpn_iid[i].in_use == 0 &&
1427                             port->wwpn_iid[i].wwpn == 0 &&
1428                             port->wwpn_iid[i].name == NULL) {
1429                                 iid = i;
1430                                 break;
1431                         }
1432                 }
1433         }
1434
1435         if (iid < 0) {
1436                 best = -1;
1437                 best_time = INT32_MAX;
1438                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1439                         if (port->wwpn_iid[i].in_use == 0) {
1440                                 if (port->wwpn_iid[i].last_use < best_time) {
1441                                         best = i;
1442                                         best_time = port->wwpn_iid[i].last_use;
1443                                 }
1444                         }
1445                 }
1446                 iid = best;
1447         }
1448
1449         if (iid < 0) {
1450                 mtx_unlock(&softc->ctl_lock);
1451                 free(name, M_CTL);
1452                 return (-2);
1453         }
1454
1455         if (port->wwpn_iid[iid].in_use > 0 && (wwpn != 0 || name != NULL)) {
1456                 /*
1457                  * This is not an error yet.
1458                  */
1459                 if (wwpn != 0 && wwpn == port->wwpn_iid[iid].wwpn) {
1460 #if 0
1461                         printf("%s: port %d iid %u WWPN %#jx arrived"
1462                             " again\n", __func__, port->targ_port,
1463                             iid, (uintmax_t)wwpn);
1464 #endif
1465                         goto take;
1466                 }
1467                 if (name != NULL && port->wwpn_iid[iid].name != NULL &&
1468                     strcmp(name, port->wwpn_iid[iid].name) == 0) {
1469 #if 0
1470                         printf("%s: port %d iid %u name '%s' arrived"
1471                             " again\n", __func__, port->targ_port,
1472                             iid, name);
1473 #endif
1474                         goto take;
1475                 }
1476
1477                 /*
1478                  * This is an error, but what do we do about it?  The
1479                  * driver is telling us we have a new WWPN for this
1480                  * initiator ID, so we pretty much need to use it.
1481                  */
1482                 printf("%s: port %d iid %u WWPN %#jx '%s' arrived,"
1483                     " but WWPN %#jx '%s' is still at that address\n",
1484                     __func__, port->targ_port, iid, wwpn, name,
1485                     (uintmax_t)port->wwpn_iid[iid].wwpn,
1486                     port->wwpn_iid[iid].name);
1487
1488                 /*
1489                  * XXX KDM clear have_ca and ua_pending on each LUN for
1490                  * this initiator.
1491                  */
1492         }
1493 take:
1494         free(port->wwpn_iid[iid].name, M_CTL);
1495         port->wwpn_iid[iid].name = name;
1496         port->wwpn_iid[iid].wwpn = wwpn;
1497         port->wwpn_iid[iid].in_use++;
1498         mtx_unlock(&softc->ctl_lock);
1499
1500         return (iid);
1501 }
1502
1503 static int
1504 ctl_create_iid(struct ctl_port *port, int iid, uint8_t *buf)
1505 {
1506         int len;
1507
1508         switch (port->port_type) {
1509         case CTL_PORT_FC:
1510         {
1511                 struct scsi_transportid_fcp *id =
1512                     (struct scsi_transportid_fcp *)buf;
1513                 if (port->wwpn_iid[iid].wwpn == 0)
1514                         return (0);
1515                 memset(id, 0, sizeof(*id));
1516                 id->format_protocol = SCSI_PROTO_FC;
1517                 scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->n_port_name);
1518                 return (sizeof(*id));
1519         }
1520         case CTL_PORT_ISCSI:
1521         {
1522                 struct scsi_transportid_iscsi_port *id =
1523                     (struct scsi_transportid_iscsi_port *)buf;
1524                 if (port->wwpn_iid[iid].name == NULL)
1525                         return (0);
1526                 memset(id, 0, 256);
1527                 id->format_protocol = SCSI_TRN_ISCSI_FORMAT_PORT |
1528                     SCSI_PROTO_ISCSI;
1529                 len = strlcpy(id->iscsi_name, port->wwpn_iid[iid].name, 252) + 1;
1530                 len = roundup2(min(len, 252), 4);
1531                 scsi_ulto2b(len, id->additional_length);
1532                 return (sizeof(*id) + len);
1533         }
1534         case CTL_PORT_SAS:
1535         {
1536                 struct scsi_transportid_sas *id =
1537                     (struct scsi_transportid_sas *)buf;
1538                 if (port->wwpn_iid[iid].wwpn == 0)
1539                         return (0);
1540                 memset(id, 0, sizeof(*id));
1541                 id->format_protocol = SCSI_PROTO_SAS;
1542                 scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->sas_address);
1543                 return (sizeof(*id));
1544         }
1545         default:
1546         {
1547                 struct scsi_transportid_spi *id =
1548                     (struct scsi_transportid_spi *)buf;
1549                 memset(id, 0, sizeof(*id));
1550                 id->format_protocol = SCSI_PROTO_SPI;
1551                 scsi_ulto2b(iid, id->scsi_addr);
1552                 scsi_ulto2b(port->targ_port, id->rel_trgt_port_id);
1553                 return (sizeof(*id));
1554         }
1555         }
1556 }
1557
1558 static int
1559 ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
1560 {
1561         return (0);
1562 }
1563
1564 static int
1565 ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
1566 {
1567         return (0);
1568 }
1569
1570 /*
1571  * Data movement routine for the CTL ioctl frontend port.
1572  */
1573 static int
1574 ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
1575 {
1576         struct ctl_sg_entry *ext_sglist, *kern_sglist;
1577         struct ctl_sg_entry ext_entry, kern_entry;
1578         int ext_sglen, ext_sg_entries, kern_sg_entries;
1579         int ext_sg_start, ext_offset;
1580         int len_to_copy, len_copied;
1581         int kern_watermark, ext_watermark;
1582         int ext_sglist_malloced;
1583         int i, j;
1584
1585         ext_sglist_malloced = 0;
1586         ext_sg_start = 0;
1587         ext_offset = 0;
1588
1589         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove\n"));
1590
1591         /*
1592          * If this flag is set, fake the data transfer.
1593          */
1594         if (ctsio->io_hdr.flags & CTL_FLAG_NO_DATAMOVE) {
1595                 ctsio->ext_data_filled = ctsio->ext_data_len;
1596                 goto bailout;
1597         }
1598
1599         /*
1600          * To simplify things here, if we have a single buffer, stick it in
1601          * a S/G entry and just make it a single entry S/G list.
1602          */
1603         if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
1604                 int len_seen;
1605
1606                 ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
1607
1608                 ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
1609                                                            M_WAITOK);
1610                 ext_sglist_malloced = 1;
1611                 if (copyin(ctsio->ext_data_ptr, ext_sglist,
1612                                    ext_sglen) != 0) {
1613                         ctl_set_internal_failure(ctsio,
1614                                                  /*sks_valid*/ 0,
1615                                                  /*retry_count*/ 0);
1616                         goto bailout;
1617                 }
1618                 ext_sg_entries = ctsio->ext_sg_entries;
1619                 len_seen = 0;
1620                 for (i = 0; i < ext_sg_entries; i++) {
1621                         if ((len_seen + ext_sglist[i].len) >=
1622                              ctsio->ext_data_filled) {
1623                                 ext_sg_start = i;
1624                                 ext_offset = ctsio->ext_data_filled - len_seen;
1625                                 break;
1626                         }
1627                         len_seen += ext_sglist[i].len;
1628                 }
1629         } else {
1630                 ext_sglist = &ext_entry;
1631                 ext_sglist->addr = ctsio->ext_data_ptr;
1632                 ext_sglist->len = ctsio->ext_data_len;
1633                 ext_sg_entries = 1;
1634                 ext_sg_start = 0;
1635                 ext_offset = ctsio->ext_data_filled;
1636         }
1637
1638         if (ctsio->kern_sg_entries > 0) {
1639                 kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
1640                 kern_sg_entries = ctsio->kern_sg_entries;
1641         } else {
1642                 kern_sglist = &kern_entry;
1643                 kern_sglist->addr = ctsio->kern_data_ptr;
1644                 kern_sglist->len = ctsio->kern_data_len;
1645                 kern_sg_entries = 1;
1646         }
1647
1648
1649         kern_watermark = 0;
1650         ext_watermark = ext_offset;
1651         len_copied = 0;
1652         for (i = ext_sg_start, j = 0;
1653              i < ext_sg_entries && j < kern_sg_entries;) {
1654                 uint8_t *ext_ptr, *kern_ptr;
1655
1656                 len_to_copy = ctl_min(ext_sglist[i].len - ext_watermark,
1657                                       kern_sglist[j].len - kern_watermark);
1658
1659                 ext_ptr = (uint8_t *)ext_sglist[i].addr;
1660                 ext_ptr = ext_ptr + ext_watermark;
1661                 if (ctsio->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
1662                         /*
1663                          * XXX KDM fix this!
1664                          */
1665                         panic("need to implement bus address support");
1666 #if 0
1667                         kern_ptr = bus_to_virt(kern_sglist[j].addr);
1668 #endif
1669                 } else
1670                         kern_ptr = (uint8_t *)kern_sglist[j].addr;
1671                 kern_ptr = kern_ptr + kern_watermark;
1672
1673                 kern_watermark += len_to_copy;
1674                 ext_watermark += len_to_copy;
1675
1676                 if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
1677                      CTL_FLAG_DATA_IN) {
1678                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1679                                          "bytes to user\n", len_to_copy));
1680                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1681                                          "to %p\n", kern_ptr, ext_ptr));
1682                         if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
1683                                 ctl_set_internal_failure(ctsio,
1684                                                          /*sks_valid*/ 0,
1685                                                          /*retry_count*/ 0);
1686                                 goto bailout;
1687                         }
1688                 } else {
1689                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1690                                          "bytes from user\n", len_to_copy));
1691                         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1692                                          "to %p\n", ext_ptr, kern_ptr));
1693                         if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
1694                                 ctl_set_internal_failure(ctsio,
1695                                                          /*sks_valid*/ 0,
1696                                                          /*retry_count*/0);
1697                                 goto bailout;
1698                         }
1699                 }
1700
1701                 len_copied += len_to_copy;
1702
1703                 if (ext_sglist[i].len == ext_watermark) {
1704                         i++;
1705                         ext_watermark = 0;
1706                 }
1707
1708                 if (kern_sglist[j].len == kern_watermark) {
1709                         j++;
1710                         kern_watermark = 0;
1711                 }
1712         }
1713
1714         ctsio->ext_data_filled += len_copied;
1715
1716         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_sg_entries: %d, "
1717                          "kern_sg_entries: %d\n", ext_sg_entries,
1718                          kern_sg_entries));
1719         CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_data_len = %d, "
1720                          "kern_data_len = %d\n", ctsio->ext_data_len,
1721                          ctsio->kern_data_len));
1722
1723
1724         /* XXX KDM set residual?? */
1725 bailout:
1726
1727         if (ext_sglist_malloced != 0)
1728                 free(ext_sglist, M_CTL);
1729
1730         return (CTL_RETVAL_COMPLETE);
1731 }
1732
1733 /*
1734  * Serialize a command that went down the "wrong" side, and so was sent to
1735  * this controller for execution.  The logic is a little different than the
1736  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1737  * sent back to the other side, but in the success case, we execute the
1738  * command on this side (XFER mode) or tell the other side to execute it
1739  * (SER_ONLY mode).
1740  */
1741 static int
1742 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
1743 {
1744         struct ctl_softc *ctl_softc;
1745         union ctl_ha_msg msg_info;
1746         struct ctl_lun *lun;
1747         int retval = 0;
1748         uint32_t targ_lun;
1749
1750         ctl_softc = control_softc;
1751
1752         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
1753         lun = ctl_softc->ctl_luns[targ_lun];
1754         if (lun==NULL)
1755         {
1756                 /*
1757                  * Why isn't LUN defined? The other side wouldn't
1758                  * send a cmd if the LUN is undefined.
1759                  */
1760                 printf("%s: Bad JUJU!, LUN is NULL!\n", __func__);
1761
1762                 /* "Logical unit not supported" */
1763                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1764                                    lun,
1765                                    /*sense_format*/SSD_TYPE_NONE,
1766                                    /*current_error*/ 1,
1767                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1768                                    /*asc*/ 0x25,
1769                                    /*ascq*/ 0x00,
1770                                    SSD_ELEM_NONE);
1771
1772                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1773                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1774                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1775                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1776                 msg_info.hdr.serializing_sc = NULL;
1777                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1778                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1779                                 sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1780                 }
1781                 return(1);
1782
1783         }
1784
1785         mtx_lock(&lun->lun_lock);
1786         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1787
1788         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1789                 (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1790                  ooa_links))) {
1791         case CTL_ACTION_BLOCK:
1792                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
1793                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
1794                                   blocked_links);
1795                 break;
1796         case CTL_ACTION_PASS:
1797         case CTL_ACTION_SKIP:
1798                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
1799                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
1800                         ctl_enqueue_rtr((union ctl_io *)ctsio);
1801                 } else {
1802
1803                         /* send msg back to other side */
1804                         msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1805                         msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
1806                         msg_info.hdr.msg_type = CTL_MSG_R2R;
1807 #if 0
1808                         printf("2. pOrig %x\n", (int)msg_info.hdr.original_sc);
1809 #endif
1810                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1811                             sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1812                         }
1813                 }
1814                 break;
1815         case CTL_ACTION_OVERLAP:
1816                 /* OVERLAPPED COMMANDS ATTEMPTED */
1817                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1818                                    lun,
1819                                    /*sense_format*/SSD_TYPE_NONE,
1820                                    /*current_error*/ 1,
1821                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1822                                    /*asc*/ 0x4E,
1823                                    /*ascq*/ 0x00,
1824                                    SSD_ELEM_NONE);
1825
1826                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1827                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1828                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1829                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1830                 msg_info.hdr.serializing_sc = NULL;
1831                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1832 #if 0
1833                 printf("BAD JUJU:Major Bummer Overlap\n");
1834 #endif
1835                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1836                 retval = 1;
1837                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1838                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1839                 }
1840                 break;
1841         case CTL_ACTION_OVERLAP_TAG:
1842                 /* TAGGED OVERLAPPED COMMANDS (NN = QUEUE TAG) */
1843                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1844                                    lun,
1845                                    /*sense_format*/SSD_TYPE_NONE,
1846                                    /*current_error*/ 1,
1847                                    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1848                                    /*asc*/ 0x4D,
1849                                    /*ascq*/ ctsio->tag_num & 0xff,
1850                                    SSD_ELEM_NONE);
1851
1852                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1853                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1854                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1855                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1856                 msg_info.hdr.serializing_sc = NULL;
1857                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1858 #if 0
1859                 printf("BAD JUJU:Major Bummer Overlap Tag\n");
1860 #endif
1861                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1862                 retval = 1;
1863                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1864                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1865                 }
1866                 break;
1867         case CTL_ACTION_ERROR:
1868         default:
1869                 /* "Internal target failure" */
1870                 ctl_set_sense_data(&msg_info.scsi.sense_data,
1871                                    lun,
1872                                    /*sense_format*/SSD_TYPE_NONE,
1873                                    /*current_error*/ 1,
1874                                    /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
1875                                    /*asc*/ 0x44,
1876                                    /*ascq*/ 0x00,
1877                                    SSD_ELEM_NONE);
1878
1879                 msg_info.scsi.sense_len = SSD_FULL_SIZE;
1880                 msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1881                 msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1882                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1883                 msg_info.hdr.serializing_sc = NULL;
1884                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1885 #if 0
1886                 printf("BAD JUJU:Major Bummer HW Error\n");
1887 #endif
1888                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1889                 retval = 1;
1890                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1891                     sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1892                 }
1893                 break;
1894         }
1895         mtx_unlock(&lun->lun_lock);
1896         return (retval);
1897 }
1898
1899 static int
1900 ctl_ioctl_submit_wait(union ctl_io *io)
1901 {
1902         struct ctl_fe_ioctl_params params;
1903         ctl_fe_ioctl_state last_state;
1904         int done, retval;
1905
1906         retval = 0;
1907
1908         bzero(&params, sizeof(params));
1909
1910         mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
1911         cv_init(&params.sem, "ctlioccv");
1912         params.state = CTL_IOCTL_INPROG;
1913         last_state = params.state;
1914
1915         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = &params;
1916
1917         CTL_DEBUG_PRINT(("ctl_ioctl_submit_wait\n"));
1918
1919         /* This shouldn't happen */
1920         if ((retval = ctl_queue(io)) != CTL_RETVAL_COMPLETE)
1921                 return (retval);
1922
1923         done = 0;
1924
1925         do {
1926                 mtx_lock(&params.ioctl_mtx);
1927                 /*
1928                  * Check the state here, and don't sleep if the state has
1929                  * already changed (i.e. wakeup has already occured, but we
1930                  * weren't waiting yet).
1931                  */
1932                 if (params.state == last_state) {
1933                         /* XXX KDM cv_wait_sig instead? */
1934                         cv_wait(&params.sem, &params.ioctl_mtx);
1935                 }
1936                 last_state = params.state;
1937
1938                 switch (params.state) {
1939                 case CTL_IOCTL_INPROG:
1940                         /* Why did we wake up? */
1941                         /* XXX KDM error here? */
1942                         mtx_unlock(&params.ioctl_mtx);
1943                         break;
1944                 case CTL_IOCTL_DATAMOVE:
1945                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DATAMOVE\n"));
1946
1947                         /*
1948                          * change last_state back to INPROG to avoid
1949                          * deadlock on subsequent data moves.
1950                          */
1951                         params.state = last_state = CTL_IOCTL_INPROG;
1952
1953                         mtx_unlock(&params.ioctl_mtx);
1954                         ctl_ioctl_do_datamove(&io->scsiio);
1955                         /*
1956                          * Note that in some cases, most notably writes,
1957                          * this will queue the I/O and call us back later.
1958                          * In other cases, generally reads, this routine
1959                          * will immediately call back and wake us up,
1960                          * probably using our own context.
1961                          */
1962                         io->scsiio.be_move_done(io);
1963                         break;
1964                 case CTL_IOCTL_DONE:
1965                         mtx_unlock(&params.ioctl_mtx);
1966                         CTL_DEBUG_PRINT(("got CTL_IOCTL_DONE\n"));
1967                         done = 1;
1968                         break;
1969                 default:
1970                         mtx_unlock(&params.ioctl_mtx);
1971                         /* XXX KDM error here? */
1972                         break;
1973                 }
1974         } while (done == 0);
1975
1976         mtx_destroy(&params.ioctl_mtx);
1977         cv_destroy(&params.sem);
1978
1979         return (CTL_RETVAL_COMPLETE);
1980 }
1981
1982 static void
1983 ctl_ioctl_datamove(union ctl_io *io)
1984 {
1985         struct ctl_fe_ioctl_params *params;
1986
1987         params = (struct ctl_fe_ioctl_params *)
1988                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1989
1990         mtx_lock(&params->ioctl_mtx);
1991         params->state = CTL_IOCTL_DATAMOVE;
1992         cv_broadcast(&params->sem);
1993         mtx_unlock(&params->ioctl_mtx);
1994 }
1995
1996 static void
1997 ctl_ioctl_done(union ctl_io *io)
1998 {
1999         struct ctl_fe_ioctl_params *params;
2000
2001         params = (struct ctl_fe_ioctl_params *)
2002                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2003
2004         mtx_lock(&params->ioctl_mtx);
2005         params->state = CTL_IOCTL_DONE;
2006         cv_broadcast(&params->sem);
2007         mtx_unlock(&params->ioctl_mtx);
2008 }
2009
2010 static void
2011 ctl_ioctl_hard_startstop_callback(void *arg, struct cfi_metatask *metatask)
2012 {
2013         struct ctl_fe_ioctl_startstop_info *sd_info;
2014
2015         sd_info = (struct ctl_fe_ioctl_startstop_info *)arg;
2016
2017         sd_info->hs_info.status = metatask->status;
2018         sd_info->hs_info.total_luns = metatask->taskinfo.startstop.total_luns;
2019         sd_info->hs_info.luns_complete =
2020                 metatask->taskinfo.startstop.luns_complete;
2021         sd_info->hs_info.luns_failed = metatask->taskinfo.startstop.luns_failed;
2022
2023         cv_broadcast(&sd_info->sem);
2024 }
2025
2026 static void
2027 ctl_ioctl_bbrread_callback(void *arg, struct cfi_metatask *metatask)
2028 {
2029         struct ctl_fe_ioctl_bbrread_info *fe_bbr_info;
2030
2031         fe_bbr_info = (struct ctl_fe_ioctl_bbrread_info *)arg;
2032
2033         mtx_lock(fe_bbr_info->lock);
2034         fe_bbr_info->bbr_info->status = metatask->status;
2035         fe_bbr_info->bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2036         fe_bbr_info->wakeup_done = 1;
2037         mtx_unlock(fe_bbr_info->lock);
2038
2039         cv_broadcast(&fe_bbr_info->sem);
2040 }
2041
2042 /*
2043  * Returns 0 for success, errno for failure.
2044  */
2045 static int
2046 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
2047                    struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
2048 {
2049         union ctl_io *io;
2050         int retval;
2051
2052         retval = 0;
2053
2054         mtx_lock(&lun->lun_lock);
2055         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
2056              (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2057              ooa_links)) {
2058                 struct ctl_ooa_entry *entry;
2059
2060                 /*
2061                  * If we've got more than we can fit, just count the
2062                  * remaining entries.
2063                  */
2064                 if (*cur_fill_num >= ooa_hdr->alloc_num)
2065                         continue;
2066
2067                 entry = &kern_entries[*cur_fill_num];
2068
2069                 entry->tag_num = io->scsiio.tag_num;
2070                 entry->lun_num = lun->lun;
2071 #ifdef CTL_TIME_IO
2072                 entry->start_bt = io->io_hdr.start_bt;
2073 #endif
2074                 bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
2075                 entry->cdb_len = io->scsiio.cdb_len;
2076                 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
2077                         entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
2078
2079                 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
2080                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
2081
2082                 if (io->io_hdr.flags & CTL_FLAG_ABORT)
2083                         entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
2084
2085                 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
2086                         entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
2087
2088                 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
2089                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
2090         }
2091         mtx_unlock(&lun->lun_lock);
2092
2093         return (retval);
2094 }
2095
2096 static void *
2097 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2098                  size_t error_str_len)
2099 {
2100         void *kptr;
2101
2102         kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2103
2104         if (copyin(user_addr, kptr, len) != 0) {
2105                 snprintf(error_str, error_str_len, "Error copying %d bytes "
2106                          "from user address %p to kernel address %p", len,
2107                          user_addr, kptr);
2108                 free(kptr, M_CTL);
2109                 return (NULL);
2110         }
2111
2112         return (kptr);
2113 }
2114
2115 static void
2116 ctl_free_args(int num_args, struct ctl_be_arg *args)
2117 {
2118         int i;
2119
2120         if (args == NULL)
2121                 return;
2122
2123         for (i = 0; i < num_args; i++) {
2124                 free(args[i].kname, M_CTL);
2125                 free(args[i].kvalue, M_CTL);
2126         }
2127
2128         free(args, M_CTL);
2129 }
2130
2131 static struct ctl_be_arg *
2132 ctl_copyin_args(int num_args, struct ctl_be_arg *uargs,
2133                 char *error_str, size_t error_str_len)
2134 {
2135         struct ctl_be_arg *args;
2136         int i;
2137
2138         args = ctl_copyin_alloc(uargs, num_args * sizeof(*args),
2139                                 error_str, error_str_len);
2140
2141         if (args == NULL)
2142                 goto bailout;
2143
2144         for (i = 0; i < num_args; i++) {
2145                 args[i].kname = NULL;
2146                 args[i].kvalue = NULL;
2147         }
2148
2149         for (i = 0; i < num_args; i++) {
2150                 uint8_t *tmpptr;
2151
2152                 args[i].kname = ctl_copyin_alloc(args[i].name,
2153                         args[i].namelen, error_str, error_str_len);
2154                 if (args[i].kname == NULL)
2155                         goto bailout;
2156
2157                 if (args[i].kname[args[i].namelen - 1] != '\0') {
2158                         snprintf(error_str, error_str_len, "Argument %d "
2159                                  "name is not NUL-terminated", i);
2160                         goto bailout;
2161                 }
2162
2163                 if (args[i].flags & CTL_BEARG_RD) {
2164                         tmpptr = ctl_copyin_alloc(args[i].value,
2165                                 args[i].vallen, error_str, error_str_len);
2166                         if (tmpptr == NULL)
2167                                 goto bailout;
2168                         if ((args[i].flags & CTL_BEARG_ASCII)
2169                          && (tmpptr[args[i].vallen - 1] != '\0')) {
2170                                 snprintf(error_str, error_str_len, "Argument "
2171                                     "%d value is not NUL-terminated", i);
2172                                 goto bailout;
2173                         }
2174                         args[i].kvalue = tmpptr;
2175                 } else {
2176                         args[i].kvalue = malloc(args[i].vallen,
2177                             M_CTL, M_WAITOK | M_ZERO);
2178                 }
2179         }
2180
2181         return (args);
2182 bailout:
2183
2184         ctl_free_args(num_args, args);
2185
2186         return (NULL);
2187 }
2188
2189 static void
2190 ctl_copyout_args(int num_args, struct ctl_be_arg *args)
2191 {
2192         int i;
2193
2194         for (i = 0; i < num_args; i++) {
2195                 if (args[i].flags & CTL_BEARG_WR)
2196                         copyout(args[i].kvalue, args[i].value, args[i].vallen);
2197         }
2198 }
2199
2200 /*
2201  * Escape characters that are illegal or not recommended in XML.
2202  */
2203 int
2204 ctl_sbuf_printf_esc(struct sbuf *sb, char *str)
2205 {
2206         int retval;
2207
2208         retval = 0;
2209
2210         for (; *str; str++) {
2211                 switch (*str) {
2212                 case '&':
2213                         retval = sbuf_printf(sb, "&amp;");
2214                         break;
2215                 case '>':
2216                         retval = sbuf_printf(sb, "&gt;");
2217                         break;
2218                 case '<':
2219                         retval = sbuf_printf(sb, "&lt;");
2220                         break;
2221                 default:
2222                         retval = sbuf_putc(sb, *str);
2223                         break;
2224                 }
2225
2226                 if (retval != 0)
2227                         break;
2228
2229         }
2230
2231         return (retval);
2232 }
2233
2234 static int
2235 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2236           struct thread *td)
2237 {
2238         struct ctl_softc *softc;
2239         int retval;
2240
2241         softc = control_softc;
2242
2243         retval = 0;
2244
2245         switch (cmd) {
2246         case CTL_IO: {
2247                 union ctl_io *io;
2248                 void *pool_tmp;
2249
2250                 /*
2251                  * If we haven't been "enabled", don't allow any SCSI I/O
2252                  * to this FETD.
2253                  */
2254                 if ((softc->ioctl_info.flags & CTL_IOCTL_FLAG_ENABLED) == 0) {
2255                         retval = EPERM;
2256                         break;
2257                 }
2258
2259                 io = ctl_alloc_io(softc->ioctl_info.port.ctl_pool_ref);
2260                 if (io == NULL) {
2261                         printf("ctl_ioctl: can't allocate ctl_io!\n");
2262                         retval = ENOSPC;
2263                         break;
2264                 }
2265
2266                 /*
2267                  * Need to save the pool reference so it doesn't get
2268                  * spammed by the user's ctl_io.
2269                  */
2270                 pool_tmp = io->io_hdr.pool;
2271
2272                 memcpy(io, (void *)addr, sizeof(*io));
2273
2274                 io->io_hdr.pool = pool_tmp;
2275                 /*
2276                  * No status yet, so make sure the status is set properly.
2277                  */
2278                 io->io_hdr.status = CTL_STATUS_NONE;
2279
2280                 /*
2281                  * The user sets the initiator ID, target and LUN IDs.
2282                  */
2283                 io->io_hdr.nexus.targ_port = softc->ioctl_info.port.targ_port;
2284                 io->io_hdr.flags |= CTL_FLAG_USER_REQ;
2285                 if ((io->io_hdr.io_type == CTL_IO_SCSI)
2286                  && (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
2287                         io->scsiio.tag_num = softc->ioctl_info.cur_tag_num++;
2288
2289                 retval = ctl_ioctl_submit_wait(io);
2290
2291                 if (retval != 0) {
2292                         ctl_free_io(io);
2293                         break;
2294                 }
2295
2296                 memcpy((void *)addr, io, sizeof(*io));
2297
2298                 /* return this to our pool */
2299                 ctl_free_io(io);
2300
2301                 break;
2302         }
2303         case CTL_ENABLE_PORT:
2304         case CTL_DISABLE_PORT:
2305         case CTL_SET_PORT_WWNS: {
2306                 struct ctl_port *port;
2307                 struct ctl_port_entry *entry;
2308
2309                 entry = (struct ctl_port_entry *)addr;
2310
2311                 mtx_lock(&softc->ctl_lock);
2312                 STAILQ_FOREACH(port, &softc->port_list, links) {
2313                         int action, done;
2314
2315                         action = 0;
2316                         done = 0;
2317
2318                         if ((entry->port_type == CTL_PORT_NONE)
2319                          && (entry->targ_port == port->targ_port)) {
2320                                 /*
2321                                  * If the user only wants to enable or
2322                                  * disable or set WWNs on a specific port,
2323                                  * do the operation and we're done.
2324                                  */
2325                                 action = 1;
2326                                 done = 1;
2327                         } else if (entry->port_type & port->port_type) {
2328                                 /*
2329                                  * Compare the user's type mask with the
2330                                  * particular frontend type to see if we
2331                                  * have a match.
2332                                  */
2333                                 action = 1;
2334                                 done = 0;
2335
2336                                 /*
2337                                  * Make sure the user isn't trying to set
2338                                  * WWNs on multiple ports at the same time.
2339                                  */
2340                                 if (cmd == CTL_SET_PORT_WWNS) {
2341                                         printf("%s: Can't set WWNs on "
2342                                                "multiple ports\n", __func__);
2343                                         retval = EINVAL;
2344                                         break;
2345                                 }
2346                         }
2347                         if (action != 0) {
2348                                 /*
2349                                  * XXX KDM we have to drop the lock here,
2350                                  * because the online/offline operations
2351                                  * can potentially block.  We need to
2352                                  * reference count the frontends so they
2353                                  * can't go away,
2354                                  */
2355                                 mtx_unlock(&softc->ctl_lock);
2356
2357                                 if (cmd == CTL_ENABLE_PORT) {
2358                                         struct ctl_lun *lun;
2359
2360                                         STAILQ_FOREACH(lun, &softc->lun_list,
2361                                                        links) {
2362                                                 port->lun_enable(port->targ_lun_arg,
2363                                                     lun->target,
2364                                                     lun->lun);
2365                                         }
2366
2367                                         ctl_port_online(port);
2368                                 } else if (cmd == CTL_DISABLE_PORT) {
2369                                         struct ctl_lun *lun;
2370
2371                                         ctl_port_offline(port);
2372
2373                                         STAILQ_FOREACH(lun, &softc->lun_list,
2374                                                        links) {
2375                                                 port->lun_disable(
2376                                                     port->targ_lun_arg,
2377                                                     lun->target,
2378                                                     lun->lun);
2379                                         }
2380                                 }
2381
2382                                 mtx_lock(&softc->ctl_lock);
2383
2384                                 if (cmd == CTL_SET_PORT_WWNS)
2385                                         ctl_port_set_wwns(port,
2386                                             (entry->flags & CTL_PORT_WWNN_VALID) ?
2387                                             1 : 0, entry->wwnn,
2388                                             (entry->flags & CTL_PORT_WWPN_VALID) ?
2389                                             1 : 0, entry->wwpn);
2390                         }
2391                         if (done != 0)
2392                                 break;
2393                 }
2394                 mtx_unlock(&softc->ctl_lock);
2395                 break;
2396         }
2397         case CTL_GET_PORT_LIST: {
2398                 struct ctl_port *port;
2399                 struct ctl_port_list *list;
2400                 int i;
2401
2402                 list = (struct ctl_port_list *)addr;
2403
2404                 if (list->alloc_len != (list->alloc_num *
2405                     sizeof(struct ctl_port_entry))) {
2406                         printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2407                                "alloc_num %u * sizeof(struct ctl_port_entry) "
2408                                "%zu\n", __func__, list->alloc_len,
2409                                list->alloc_num, sizeof(struct ctl_port_entry));
2410                         retval = EINVAL;
2411                         break;
2412                 }
2413                 list->fill_len = 0;
2414                 list->fill_num = 0;
2415                 list->dropped_num = 0;
2416                 i = 0;
2417                 mtx_lock(&softc->ctl_lock);
2418                 STAILQ_FOREACH(port, &softc->port_list, links) {
2419                         struct ctl_port_entry entry, *list_entry;
2420
2421                         if (list->fill_num >= list->alloc_num) {
2422                                 list->dropped_num++;
2423                                 continue;
2424                         }
2425
2426                         entry.port_type = port->port_type;
2427                         strlcpy(entry.port_name, port->port_name,
2428                                 sizeof(entry.port_name));
2429                         entry.targ_port = port->targ_port;
2430                         entry.physical_port = port->physical_port;
2431                         entry.virtual_port = port->virtual_port;
2432                         entry.wwnn = port->wwnn;
2433                         entry.wwpn = port->wwpn;
2434                         if (port->status & CTL_PORT_STATUS_ONLINE)
2435                                 entry.online = 1;
2436                         else
2437                                 entry.online = 0;
2438
2439                         list_entry = &list->entries[i];
2440
2441                         retval = copyout(&entry, list_entry, sizeof(entry));
2442                         if (retval != 0) {
2443                                 printf("%s: CTL_GET_PORT_LIST: copyout "
2444                                        "returned %d\n", __func__, retval);
2445                                 break;
2446                         }
2447                         i++;
2448                         list->fill_num++;
2449                         list->fill_len += sizeof(entry);
2450                 }
2451                 mtx_unlock(&softc->ctl_lock);
2452
2453                 /*
2454                  * If this is non-zero, we had a copyout fault, so there's
2455                  * probably no point in attempting to set the status inside
2456                  * the structure.
2457                  */
2458                 if (retval != 0)
2459                         break;
2460
2461                 if (list->dropped_num > 0)
2462                         list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2463                 else
2464                         list->status = CTL_PORT_LIST_OK;
2465                 break;
2466         }
2467         case CTL_DUMP_OOA: {
2468                 struct ctl_lun *lun;
2469                 union ctl_io *io;
2470                 char printbuf[128];
2471                 struct sbuf sb;
2472
2473                 mtx_lock(&softc->ctl_lock);
2474                 printf("Dumping OOA queues:\n");
2475                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2476                         mtx_lock(&lun->lun_lock);
2477                         for (io = (union ctl_io *)TAILQ_FIRST(
2478                              &lun->ooa_queue); io != NULL;
2479                              io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2480                              ooa_links)) {
2481                                 sbuf_new(&sb, printbuf, sizeof(printbuf),
2482                                          SBUF_FIXEDLEN);
2483                                 sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2484                                             (intmax_t)lun->lun,
2485                                             io->scsiio.tag_num,
2486                                             (io->io_hdr.flags &
2487                                             CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2488                                             (io->io_hdr.flags &
2489                                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2490                                             (io->io_hdr.flags &
2491                                             CTL_FLAG_ABORT) ? " ABORT" : "",
2492                                             (io->io_hdr.flags &
2493                                         CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2494                                 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2495                                 sbuf_finish(&sb);
2496                                 printf("%s\n", sbuf_data(&sb));
2497                         }
2498                         mtx_unlock(&lun->lun_lock);
2499                 }
2500                 printf("OOA queues dump done\n");
2501                 mtx_unlock(&softc->ctl_lock);
2502                 break;
2503         }
2504         case CTL_GET_OOA: {
2505                 struct ctl_lun *lun;
2506                 struct ctl_ooa *ooa_hdr;
2507                 struct ctl_ooa_entry *entries;
2508                 uint32_t cur_fill_num;
2509
2510                 ooa_hdr = (struct ctl_ooa *)addr;
2511
2512                 if ((ooa_hdr->alloc_len == 0)
2513                  || (ooa_hdr->alloc_num == 0)) {
2514                         printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2515                                "must be non-zero\n", __func__,
2516                                ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2517                         retval = EINVAL;
2518                         break;
2519                 }
2520
2521                 if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2522                     sizeof(struct ctl_ooa_entry))) {
2523                         printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2524                                "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2525                                __func__, ooa_hdr->alloc_len,
2526                                ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2527                         retval = EINVAL;
2528                         break;
2529                 }
2530
2531                 entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2532                 if (entries == NULL) {
2533                         printf("%s: could not allocate %d bytes for OOA "
2534                                "dump\n", __func__, ooa_hdr->alloc_len);
2535                         retval = ENOMEM;
2536                         break;
2537                 }
2538
2539                 mtx_lock(&softc->ctl_lock);
2540                 if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2541                  && ((ooa_hdr->lun_num > CTL_MAX_LUNS)
2542                   || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2543                         mtx_unlock(&softc->ctl_lock);
2544                         free(entries, M_CTL);
2545                         printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2546                                __func__, (uintmax_t)ooa_hdr->lun_num);
2547                         retval = EINVAL;
2548                         break;
2549                 }
2550
2551                 cur_fill_num = 0;
2552
2553                 if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2554                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
2555                                 retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2556                                         ooa_hdr, entries);
2557                                 if (retval != 0)
2558                                         break;
2559                         }
2560                         if (retval != 0) {
2561                                 mtx_unlock(&softc->ctl_lock);
2562                                 free(entries, M_CTL);
2563                                 break;
2564                         }
2565                 } else {
2566                         lun = softc->ctl_luns[ooa_hdr->lun_num];
2567
2568                         retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
2569                                                     entries);
2570                 }
2571                 mtx_unlock(&softc->ctl_lock);
2572
2573                 ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2574                 ooa_hdr->fill_len = ooa_hdr->fill_num *
2575                         sizeof(struct ctl_ooa_entry);
2576                 retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2577                 if (retval != 0) {
2578                         printf("%s: error copying out %d bytes for OOA dump\n",
2579                                __func__, ooa_hdr->fill_len);
2580                 }
2581
2582                 getbintime(&ooa_hdr->cur_bt);
2583
2584                 if (cur_fill_num > ooa_hdr->alloc_num) {
2585                         ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2586                         ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2587                 } else {
2588                         ooa_hdr->dropped_num = 0;
2589                         ooa_hdr->status = CTL_OOA_OK;
2590                 }
2591
2592                 free(entries, M_CTL);
2593                 break;
2594         }
2595         case CTL_CHECK_OOA: {
2596                 union ctl_io *io;
2597                 struct ctl_lun *lun;
2598                 struct ctl_ooa_info *ooa_info;
2599
2600
2601                 ooa_info = (struct ctl_ooa_info *)addr;
2602
2603                 if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2604                         ooa_info->status = CTL_OOA_INVALID_LUN;
2605                         break;
2606                 }
2607                 mtx_lock(&softc->ctl_lock);
2608                 lun = softc->ctl_luns[ooa_info->lun_id];
2609                 if (lun == NULL) {
2610                         mtx_unlock(&softc->ctl_lock);
2611                         ooa_info->status = CTL_OOA_INVALID_LUN;
2612                         break;
2613                 }
2614                 mtx_lock(&lun->lun_lock);
2615                 mtx_unlock(&softc->ctl_lock);
2616                 ooa_info->num_entries = 0;
2617                 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2618                      io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2619                      &io->io_hdr, ooa_links)) {
2620                         ooa_info->num_entries++;
2621                 }
2622                 mtx_unlock(&lun->lun_lock);
2623
2624                 ooa_info->status = CTL_OOA_SUCCESS;
2625
2626                 break;
2627         }
2628         case CTL_HARD_START:
2629         case CTL_HARD_STOP: {
2630                 struct ctl_fe_ioctl_startstop_info ss_info;
2631                 struct cfi_metatask *metatask;
2632                 struct mtx hs_mtx;
2633
2634                 mtx_init(&hs_mtx, "HS Mutex", NULL, MTX_DEF);
2635
2636                 cv_init(&ss_info.sem, "hard start/stop cv" );
2637
2638                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2639                 if (metatask == NULL) {
2640                         retval = ENOMEM;
2641                         mtx_destroy(&hs_mtx);
2642                         break;
2643                 }
2644
2645                 if (cmd == CTL_HARD_START)
2646                         metatask->tasktype = CFI_TASK_STARTUP;
2647                 else
2648                         metatask->tasktype = CFI_TASK_SHUTDOWN;
2649
2650                 metatask->callback = ctl_ioctl_hard_startstop_callback;
2651                 metatask->callback_arg = &ss_info;
2652
2653                 cfi_action(metatask);
2654
2655                 /* Wait for the callback */
2656                 mtx_lock(&hs_mtx);
2657                 cv_wait_sig(&ss_info.sem, &hs_mtx);
2658                 mtx_unlock(&hs_mtx);
2659
2660                 /*
2661                  * All information has been copied from the metatask by the
2662                  * time cv_broadcast() is called, so we free the metatask here.
2663                  */
2664                 cfi_free_metatask(metatask);
2665
2666                 memcpy((void *)addr, &ss_info.hs_info, sizeof(ss_info.hs_info));
2667
2668                 mtx_destroy(&hs_mtx);
2669                 break;
2670         }
2671         case CTL_BBRREAD: {
2672                 struct ctl_bbrread_info *bbr_info;
2673                 struct ctl_fe_ioctl_bbrread_info fe_bbr_info;
2674                 struct mtx bbr_mtx;
2675                 struct cfi_metatask *metatask;
2676
2677                 bbr_info = (struct ctl_bbrread_info *)addr;
2678
2679                 bzero(&fe_bbr_info, sizeof(fe_bbr_info));
2680
2681                 bzero(&bbr_mtx, sizeof(bbr_mtx));
2682                 mtx_init(&bbr_mtx, "BBR Mutex", NULL, MTX_DEF);
2683
2684                 fe_bbr_info.bbr_info = bbr_info;
2685                 fe_bbr_info.lock = &bbr_mtx;
2686
2687                 cv_init(&fe_bbr_info.sem, "BBR read cv");
2688                 metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2689
2690                 if (metatask == NULL) {
2691                         mtx_destroy(&bbr_mtx);
2692                         cv_destroy(&fe_bbr_info.sem);
2693                         retval = ENOMEM;
2694                         break;
2695                 }
2696                 metatask->tasktype = CFI_TASK_BBRREAD;
2697                 metatask->callback = ctl_ioctl_bbrread_callback;
2698                 metatask->callback_arg = &fe_bbr_info;
2699                 metatask->taskinfo.bbrread.lun_num = bbr_info->lun_num;
2700                 metatask->taskinfo.bbrread.lba = bbr_info->lba;
2701                 metatask->taskinfo.bbrread.len = bbr_info->len;
2702
2703                 cfi_action(metatask);
2704
2705                 mtx_lock(&bbr_mtx);
2706                 while (fe_bbr_info.wakeup_done == 0)
2707                         cv_wait_sig(&fe_bbr_info.sem, &bbr_mtx);
2708                 mtx_unlock(&bbr_mtx);
2709
2710                 bbr_info->status = metatask->status;
2711                 bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2712                 bbr_info->scsi_status = metatask->taskinfo.bbrread.scsi_status;
2713                 memcpy(&bbr_info->sense_data,
2714                        &metatask->taskinfo.bbrread.sense_data,
2715                        ctl_min(sizeof(bbr_info->sense_data),
2716                                sizeof(metatask->taskinfo.bbrread.sense_data)));
2717
2718                 cfi_free_metatask(metatask);
2719
2720                 mtx_destroy(&bbr_mtx);
2721                 cv_destroy(&fe_bbr_info.sem);
2722
2723                 break;
2724         }
2725         case CTL_DELAY_IO: {
2726                 struct ctl_io_delay_info *delay_info;
2727 #ifdef CTL_IO_DELAY
2728                 struct ctl_lun *lun;
2729 #endif /* CTL_IO_DELAY */
2730
2731                 delay_info = (struct ctl_io_delay_info *)addr;
2732
2733 #ifdef CTL_IO_DELAY
2734                 mtx_lock(&softc->ctl_lock);
2735
2736                 if ((delay_info->lun_id > CTL_MAX_LUNS)
2737                  || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2738                         delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2739                 } else {
2740                         lun = softc->ctl_luns[delay_info->lun_id];
2741                         mtx_lock(&lun->lun_lock);
2742
2743                         delay_info->status = CTL_DELAY_STATUS_OK;
2744
2745                         switch (delay_info->delay_type) {
2746                         case CTL_DELAY_TYPE_CONT:
2747                                 break;
2748                         case CTL_DELAY_TYPE_ONESHOT:
2749                                 break;
2750                         default:
2751                                 delay_info->status =
2752                                         CTL_DELAY_STATUS_INVALID_TYPE;
2753                                 break;
2754                         }
2755
2756                         switch (delay_info->delay_loc) {
2757                         case CTL_DELAY_LOC_DATAMOVE:
2758                                 lun->delay_info.datamove_type =
2759                                         delay_info->delay_type;
2760                                 lun->delay_info.datamove_delay =
2761                                         delay_info->delay_secs;
2762                                 break;
2763                         case CTL_DELAY_LOC_DONE:
2764                                 lun->delay_info.done_type =
2765                                         delay_info->delay_type;
2766                                 lun->delay_info.done_delay =
2767                                         delay_info->delay_secs;
2768                                 break;
2769                         default:
2770                                 delay_info->status =
2771                                         CTL_DELAY_STATUS_INVALID_LOC;
2772                                 break;
2773                         }
2774                         mtx_unlock(&lun->lun_lock);
2775                 }
2776
2777                 mtx_unlock(&softc->ctl_lock);
2778 #else
2779                 delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2780 #endif /* CTL_IO_DELAY */
2781                 break;
2782         }
2783         case CTL_REALSYNC_SET: {
2784                 int *syncstate;
2785
2786                 syncstate = (int *)addr;
2787
2788                 mtx_lock(&softc->ctl_lock);
2789                 switch (*syncstate) {
2790                 case 0:
2791                         softc->flags &= ~CTL_FLAG_REAL_SYNC;
2792                         break;
2793                 case 1:
2794                         softc->flags |= CTL_FLAG_REAL_SYNC;
2795                         break;
2796                 default:
2797                         retval = EINVAL;
2798                         break;
2799                 }
2800                 mtx_unlock(&softc->ctl_lock);
2801                 break;
2802         }
2803         case CTL_REALSYNC_GET: {
2804                 int *syncstate;
2805
2806                 syncstate = (int*)addr;
2807
2808                 mtx_lock(&softc->ctl_lock);
2809                 if (softc->flags & CTL_FLAG_REAL_SYNC)
2810                         *syncstate = 1;
2811                 else
2812                         *syncstate = 0;
2813                 mtx_unlock(&softc->ctl_lock);
2814
2815                 break;
2816         }
2817         case CTL_SETSYNC:
2818         case CTL_GETSYNC: {
2819                 struct ctl_sync_info *sync_info;
2820                 struct ctl_lun *lun;
2821
2822                 sync_info = (struct ctl_sync_info *)addr;
2823
2824                 mtx_lock(&softc->ctl_lock);
2825                 lun = softc->ctl_luns[sync_info->lun_id];
2826                 if (lun == NULL) {
2827                         mtx_unlock(&softc->ctl_lock);
2828                         sync_info->status = CTL_GS_SYNC_NO_LUN;
2829                 }
2830                 /*
2831                  * Get or set the sync interval.  We're not bounds checking
2832                  * in the set case, hopefully the user won't do something
2833                  * silly.
2834                  */
2835                 mtx_lock(&lun->lun_lock);
2836                 mtx_unlock(&softc->ctl_lock);
2837                 if (cmd == CTL_GETSYNC)
2838                         sync_info->sync_interval = lun->sync_interval;
2839                 else
2840                         lun->sync_interval = sync_info->sync_interval;
2841                 mtx_unlock(&lun->lun_lock);
2842
2843                 sync_info->status = CTL_GS_SYNC_OK;
2844
2845                 break;
2846         }
2847         case CTL_GETSTATS: {
2848                 struct ctl_stats *stats;
2849                 struct ctl_lun *lun;
2850                 int i;
2851
2852                 stats = (struct ctl_stats *)addr;
2853
2854                 if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2855                      stats->alloc_len) {
2856                         stats->status = CTL_SS_NEED_MORE_SPACE;
2857                         stats->num_luns = softc->num_luns;
2858                         break;
2859                 }
2860                 /*
2861                  * XXX KDM no locking here.  If the LUN list changes,
2862                  * things can blow up.
2863                  */
2864                 for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2865                      i++, lun = STAILQ_NEXT(lun, links)) {
2866                         retval = copyout(&lun->stats, &stats->lun_stats[i],
2867                                          sizeof(lun->stats));
2868                         if (retval != 0)
2869                                 break;
2870                 }
2871                 stats->num_luns = softc->num_luns;
2872                 stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2873                                  softc->num_luns;
2874                 stats->status = CTL_SS_OK;
2875 #ifdef CTL_TIME_IO
2876                 stats->flags = CTL_STATS_FLAG_TIME_VALID;
2877 #else
2878                 stats->flags = CTL_STATS_FLAG_NONE;
2879 #endif
2880                 getnanouptime(&stats->timestamp);
2881                 break;
2882         }
2883         case CTL_ERROR_INJECT: {
2884                 struct ctl_error_desc *err_desc, *new_err_desc;
2885                 struct ctl_lun *lun;
2886
2887                 err_desc = (struct ctl_error_desc *)addr;
2888
2889                 new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2890                                       M_WAITOK | M_ZERO);
2891                 bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2892
2893                 mtx_lock(&softc->ctl_lock);
2894                 lun = softc->ctl_luns[err_desc->lun_id];
2895                 if (lun == NULL) {
2896                         mtx_unlock(&softc->ctl_lock);
2897                         printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2898                                __func__, (uintmax_t)err_desc->lun_id);
2899                         retval = EINVAL;
2900                         break;
2901                 }
2902                 mtx_lock(&lun->lun_lock);
2903                 mtx_unlock(&softc->ctl_lock);
2904
2905                 /*
2906                  * We could do some checking here to verify the validity
2907                  * of the request, but given the complexity of error
2908                  * injection requests, the checking logic would be fairly
2909                  * complex.
2910                  *
2911                  * For now, if the request is invalid, it just won't get
2912                  * executed and might get deleted.
2913                  */
2914                 STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2915
2916                 /*
2917                  * XXX KDM check to make sure the serial number is unique,
2918                  * in case we somehow manage to wrap.  That shouldn't
2919                  * happen for a very long time, but it's the right thing to
2920                  * do.
2921                  */
2922                 new_err_desc->serial = lun->error_serial;
2923                 err_desc->serial = lun->error_serial;
2924                 lun->error_serial++;
2925
2926                 mtx_unlock(&lun->lun_lock);
2927                 break;
2928         }
2929         case CTL_ERROR_INJECT_DELETE: {
2930                 struct ctl_error_desc *delete_desc, *desc, *desc2;
2931                 struct ctl_lun *lun;
2932                 int delete_done;
2933
2934                 delete_desc = (struct ctl_error_desc *)addr;
2935                 delete_done = 0;
2936
2937                 mtx_lock(&softc->ctl_lock);
2938                 lun = softc->ctl_luns[delete_desc->lun_id];
2939                 if (lun == NULL) {
2940                         mtx_unlock(&softc->ctl_lock);
2941                         printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
2942                                __func__, (uintmax_t)delete_desc->lun_id);
2943                         retval = EINVAL;
2944                         break;
2945                 }
2946                 mtx_lock(&lun->lun_lock);
2947                 mtx_unlock(&softc->ctl_lock);
2948                 STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
2949                         if (desc->serial != delete_desc->serial)
2950                                 continue;
2951
2952                         STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
2953                                       links);
2954                         free(desc, M_CTL);
2955                         delete_done = 1;
2956                 }
2957                 mtx_unlock(&lun->lun_lock);
2958                 if (delete_done == 0) {
2959                         printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
2960                                "error serial %ju on LUN %u\n", __func__,
2961                                delete_desc->serial, delete_desc->lun_id);
2962                         retval = EINVAL;
2963                         break;
2964                 }
2965                 break;
2966         }
2967         case CTL_DUMP_STRUCTS: {
2968                 int i, j, k, idx;
2969                 struct ctl_port *port;
2970                 struct ctl_frontend *fe;
2971
2972                 mtx_lock(&softc->ctl_lock);
2973                 printf("CTL Persistent Reservation information start:\n");
2974                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2975                         struct ctl_lun *lun;
2976
2977                         lun = softc->ctl_luns[i];
2978
2979                         if ((lun == NULL)
2980                          || ((lun->flags & CTL_LUN_DISABLED) != 0))
2981                                 continue;
2982
2983                         for (j = 0; j < (CTL_MAX_PORTS * 2); j++) {
2984                                 for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
2985                                         idx = j * CTL_MAX_INIT_PER_PORT + k;
2986                                         if (lun->per_res[idx].registered == 0)
2987                                                 continue;
2988                                         printf("  LUN %d port %d iid %d key "
2989                                                "%#jx\n", i, j, k,
2990                                                (uintmax_t)scsi_8btou64(
2991                                                lun->per_res[idx].res_key.key));
2992                                 }
2993                         }
2994                 }
2995                 printf("CTL Persistent Reservation information end\n");
2996                 printf("CTL Ports:\n");
2997                 STAILQ_FOREACH(port, &softc->port_list, links) {
2998                         printf("  Port %d '%s' Frontend '%s' Type %u pp %d vp %d WWNN "
2999                                "%#jx WWPN %#jx\n", port->targ_port, port->port_name,
3000                                port->frontend->name, port->port_type,
3001                                port->physical_port, port->virtual_port,
3002                                (uintmax_t)port->wwnn, (uintmax_t)port->wwpn);
3003                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
3004                                 if (port->wwpn_iid[j].in_use == 0 &&
3005                                     port->wwpn_iid[j].wwpn == 0 &&
3006                                     port->wwpn_iid[j].name == NULL)
3007                                         continue;
3008
3009                                 printf("    iid %u use %d WWPN %#jx '%s'\n",
3010                                     j, port->wwpn_iid[j].in_use,
3011                                     (uintmax_t)port->wwpn_iid[j].wwpn,
3012                                     port->wwpn_iid[j].name);
3013                         }
3014                 }
3015                 printf("CTL Port information end\n");
3016                 mtx_unlock(&softc->ctl_lock);
3017                 /*
3018                  * XXX KDM calling this without a lock.  We'd likely want
3019                  * to drop the lock before calling the frontend's dump
3020                  * routine anyway.
3021                  */
3022                 printf("CTL Frontends:\n");
3023                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
3024                         printf("  Frontend '%s'\n", fe->name);
3025                         if (fe->fe_dump != NULL)
3026                                 fe->fe_dump();
3027                 }
3028                 printf("CTL Frontend information end\n");
3029                 break;
3030         }
3031         case CTL_LUN_REQ: {
3032                 struct ctl_lun_req *lun_req;
3033                 struct ctl_backend_driver *backend;
3034
3035                 lun_req = (struct ctl_lun_req *)addr;
3036
3037                 backend = ctl_backend_find(lun_req->backend);
3038                 if (backend == NULL) {
3039                         lun_req->status = CTL_LUN_ERROR;
3040                         snprintf(lun_req->error_str,
3041                                  sizeof(lun_req->error_str),
3042                                  "Backend \"%s\" not found.",
3043                                  lun_req->backend);
3044                         break;
3045                 }
3046                 if (lun_req->num_be_args > 0) {
3047                         lun_req->kern_be_args = ctl_copyin_args(
3048                                 lun_req->num_be_args,
3049                                 lun_req->be_args,
3050                                 lun_req->error_str,
3051                                 sizeof(lun_req->error_str));
3052                         if (lun_req->kern_be_args == NULL) {
3053                                 lun_req->status = CTL_LUN_ERROR;
3054                                 break;
3055                         }
3056                 }
3057
3058                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3059
3060                 if (lun_req->num_be_args > 0) {
3061                         ctl_copyout_args(lun_req->num_be_args,
3062                                       lun_req->kern_be_args);
3063                         ctl_free_args(lun_req->num_be_args,
3064                                       lun_req->kern_be_args);
3065                 }
3066                 break;
3067         }
3068         case CTL_LUN_LIST: {
3069                 struct sbuf *sb;
3070                 struct ctl_lun *lun;
3071                 struct ctl_lun_list *list;
3072                 struct ctl_option *opt;
3073
3074                 list = (struct ctl_lun_list *)addr;
3075
3076                 /*
3077                  * Allocate a fixed length sbuf here, based on the length
3078                  * of the user's buffer.  We could allocate an auto-extending
3079                  * buffer, and then tell the user how much larger our
3080                  * amount of data is than his buffer, but that presents
3081                  * some problems:
3082                  *
3083                  * 1.  The sbuf(9) routines use a blocking malloc, and so
3084                  *     we can't hold a lock while calling them with an
3085                  *     auto-extending buffer.
3086                  *
3087                  * 2.  There is not currently a LUN reference counting
3088                  *     mechanism, outside of outstanding transactions on
3089                  *     the LUN's OOA queue.  So a LUN could go away on us
3090                  *     while we're getting the LUN number, backend-specific
3091                  *     information, etc.  Thus, given the way things
3092                  *     currently work, we need to hold the CTL lock while
3093                  *     grabbing LUN information.
3094                  *
3095                  * So, from the user's standpoint, the best thing to do is
3096                  * allocate what he thinks is a reasonable buffer length,
3097                  * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
3098                  * double the buffer length and try again.  (And repeat
3099                  * that until he succeeds.)
3100                  */
3101                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3102                 if (sb == NULL) {
3103                         list->status = CTL_LUN_LIST_ERROR;
3104                         snprintf(list->error_str, sizeof(list->error_str),
3105                                  "Unable to allocate %d bytes for LUN list",
3106                                  list->alloc_len);
3107                         break;
3108                 }
3109
3110                 sbuf_printf(sb, "<ctllunlist>\n");
3111
3112                 mtx_lock(&softc->ctl_lock);
3113                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
3114                         mtx_lock(&lun->lun_lock);
3115                         retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3116                                              (uintmax_t)lun->lun);
3117
3118                         /*
3119                          * Bail out as soon as we see that we've overfilled
3120                          * the buffer.
3121                          */
3122                         if (retval != 0)
3123                                 break;
3124
3125                         retval = sbuf_printf(sb, "\t<backend_type>%s"
3126                                              "</backend_type>\n",
3127                                              (lun->backend == NULL) ?  "none" :
3128                                              lun->backend->name);
3129
3130                         if (retval != 0)
3131                                 break;
3132
3133                         retval = sbuf_printf(sb, "\t<lun_type>%d</lun_type>\n",
3134                                              lun->be_lun->lun_type);
3135
3136                         if (retval != 0)
3137                                 break;
3138
3139                         if (lun->backend == NULL) {
3140                                 retval = sbuf_printf(sb, "</lun>\n");
3141                                 if (retval != 0)
3142                                         break;
3143                                 continue;
3144                         }
3145
3146                         retval = sbuf_printf(sb, "\t<size>%ju</size>\n",
3147                                              (lun->be_lun->maxlba > 0) ?
3148                                              lun->be_lun->maxlba + 1 : 0);
3149
3150                         if (retval != 0)
3151                                 break;
3152
3153                         retval = sbuf_printf(sb, "\t<blocksize>%u</blocksize>\n",
3154                                              lun->be_lun->blocksize);
3155
3156                         if (retval != 0)
3157                                 break;
3158
3159                         retval = sbuf_printf(sb, "\t<serial_number>");
3160
3161                         if (retval != 0)
3162                                 break;
3163
3164                         retval = ctl_sbuf_printf_esc(sb,
3165                                                      lun->be_lun->serial_num);
3166
3167                         if (retval != 0)
3168                                 break;
3169
3170                         retval = sbuf_printf(sb, "</serial_number>\n");
3171
3172                         if (retval != 0)
3173                                 break;
3174
3175                         retval = sbuf_printf(sb, "\t<device_id>");
3176
3177                         if (retval != 0)
3178                                 break;
3179
3180                         retval = ctl_sbuf_printf_esc(sb,lun->be_lun->device_id);
3181
3182                         if (retval != 0)
3183                                 break;
3184
3185                         retval = sbuf_printf(sb, "</device_id>\n");
3186
3187                         if (retval != 0)
3188                                 break;
3189
3190                         if (lun->backend->lun_info != NULL) {
3191                                 retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3192                                 if (retval != 0)
3193                                         break;
3194                         }
3195                         STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3196                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3197                                     opt->name, opt->value, opt->name);
3198                                 if (retval != 0)
3199                                         break;
3200                         }
3201
3202                         retval = sbuf_printf(sb, "</lun>\n");
3203
3204                         if (retval != 0)
3205                                 break;
3206                         mtx_unlock(&lun->lun_lock);
3207                 }
3208                 if (lun != NULL)
3209                         mtx_unlock(&lun->lun_lock);
3210                 mtx_unlock(&softc->ctl_lock);
3211
3212                 if ((retval != 0)
3213                  || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3214                         retval = 0;
3215                         sbuf_delete(sb);
3216                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3217                         snprintf(list->error_str, sizeof(list->error_str),
3218                                  "Out of space, %d bytes is too small",
3219                                  list->alloc_len);
3220                         break;
3221                 }
3222
3223                 sbuf_finish(sb);
3224
3225                 retval = copyout(sbuf_data(sb), list->lun_xml,
3226                                  sbuf_len(sb) + 1);
3227
3228                 list->fill_len = sbuf_len(sb) + 1;
3229                 list->status = CTL_LUN_LIST_OK;
3230                 sbuf_delete(sb);
3231                 break;
3232         }
3233         case CTL_ISCSI: {
3234                 struct ctl_iscsi *ci;
3235                 struct ctl_frontend *fe;
3236
3237                 ci = (struct ctl_iscsi *)addr;
3238
3239                 fe = ctl_frontend_find("iscsi");
3240                 if (fe == NULL) {
3241                         ci->status = CTL_ISCSI_ERROR;
3242                         snprintf(ci->error_str, sizeof(ci->error_str),
3243                             "Frontend \"iscsi\" not found.");
3244                         break;
3245                 }
3246
3247                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3248                 break;
3249         }
3250         case CTL_PORT_REQ: {
3251                 struct ctl_req *req;
3252                 struct ctl_frontend *fe;
3253
3254                 req = (struct ctl_req *)addr;
3255
3256                 fe = ctl_frontend_find(req->driver);
3257                 if (fe == NULL) {
3258                         req->status = CTL_LUN_ERROR;
3259                         snprintf(req->error_str, sizeof(req->error_str),
3260                             "Frontend \"%s\" not found.", req->driver);
3261                         break;
3262                 }
3263                 if (req->num_args > 0) {
3264                         req->kern_args = ctl_copyin_args(req->num_args,
3265                             req->args, req->error_str, sizeof(req->error_str));
3266                         if (req->kern_args == NULL) {
3267                                 req->status = CTL_LUN_ERROR;
3268                                 break;
3269                         }
3270                 }
3271
3272                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3273
3274                 if (req->num_args > 0) {
3275                         ctl_copyout_args(req->num_args, req->kern_args);
3276                         ctl_free_args(req->num_args, req->kern_args);
3277                 }
3278                 break;
3279         }
3280         case CTL_PORT_LIST: {
3281                 struct sbuf *sb;
3282                 struct ctl_port *port;
3283                 struct ctl_lun_list *list;
3284                 struct ctl_option *opt;
3285
3286                 list = (struct ctl_lun_list *)addr;
3287
3288                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3289                 if (sb == NULL) {
3290                         list->status = CTL_LUN_LIST_ERROR;
3291                         snprintf(list->error_str, sizeof(list->error_str),
3292                                  "Unable to allocate %d bytes for LUN list",
3293                                  list->alloc_len);
3294                         break;
3295                 }
3296
3297                 sbuf_printf(sb, "<ctlportlist>\n");
3298
3299                 mtx_lock(&softc->ctl_lock);
3300                 STAILQ_FOREACH(port, &softc->port_list, links) {
3301                         retval = sbuf_printf(sb, "<targ_port id=\"%ju\">\n",
3302                                              (uintmax_t)port->targ_port);
3303
3304                         /*
3305                          * Bail out as soon as we see that we've overfilled
3306                          * the buffer.
3307                          */
3308                         if (retval != 0)
3309                                 break;
3310
3311                         retval = sbuf_printf(sb, "\t<frontend_type>%s"
3312                             "</frontend_type>\n", port->frontend->name);
3313                         if (retval != 0)
3314                                 break;
3315
3316                         retval = sbuf_printf(sb, "\t<port_type>%d</port_type>\n",
3317                                              port->port_type);
3318                         if (retval != 0)
3319                                 break;
3320
3321                         retval = sbuf_printf(sb, "\t<online>%s</online>\n",
3322                             (port->status & CTL_PORT_STATUS_ONLINE) ? "YES" : "NO");
3323                         if (retval != 0)
3324                                 break;
3325
3326                         retval = sbuf_printf(sb, "\t<port_name>%s</port_name>\n",
3327                             port->port_name);
3328                         if (retval != 0)
3329                                 break;
3330
3331                         retval = sbuf_printf(sb, "\t<physical_port>%d</physical_port>\n",
3332                             port->physical_port);
3333                         if (retval != 0)
3334                                 break;
3335
3336                         retval = sbuf_printf(sb, "\t<virtual_port>%d</virtual_port>\n",
3337                             port->virtual_port);
3338                         if (retval != 0)
3339                                 break;
3340
3341                         retval = sbuf_printf(sb, "\t<wwnn>%#jx</wwnn>\n",
3342                             (uintmax_t)port->wwnn);
3343                         if (retval != 0)
3344                                 break;
3345
3346                         retval = sbuf_printf(sb, "\t<wwpn>%#jx</wwpn>\n",
3347                             (uintmax_t)port->wwpn);
3348                         if (retval != 0)
3349                                 break;
3350
3351                         if (port->port_info != NULL) {
3352                                 retval = port->port_info(port->onoff_arg, sb);
3353                                 if (retval != 0)
3354                                         break;
3355                         }
3356                         STAILQ_FOREACH(opt, &port->options, links) {
3357                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3358                                     opt->name, opt->value, opt->name);
3359                                 if (retval != 0)
3360                                         break;
3361                         }
3362
3363                         retval = sbuf_printf(sb, "</targ_port>\n");
3364                         if (retval != 0)
3365                                 break;
3366                 }
3367                 mtx_unlock(&softc->ctl_lock);
3368
3369                 if ((retval != 0)
3370                  || ((retval = sbuf_printf(sb, "</ctlportlist>\n")) != 0)) {
3371                         retval = 0;
3372                         sbuf_delete(sb);
3373                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3374                         snprintf(list->error_str, sizeof(list->error_str),
3375                                  "Out of space, %d bytes is too small",
3376                                  list->alloc_len);
3377                         break;
3378                 }
3379
3380                 sbuf_finish(sb);
3381
3382                 retval = copyout(sbuf_data(sb), list->lun_xml,
3383                                  sbuf_len(sb) + 1);
3384
3385                 list->fill_len = sbuf_len(sb) + 1;
3386                 list->status = CTL_LUN_LIST_OK;
3387                 sbuf_delete(sb);
3388                 break;
3389         }
3390         default: {
3391                 /* XXX KDM should we fix this? */
3392 #if 0
3393                 struct ctl_backend_driver *backend;
3394                 unsigned int type;
3395                 int found;
3396
3397                 found = 0;
3398
3399                 /*
3400                  * We encode the backend type as the ioctl type for backend
3401                  * ioctls.  So parse it out here, and then search for a
3402                  * backend of this type.
3403                  */
3404                 type = _IOC_TYPE(cmd);
3405
3406                 STAILQ_FOREACH(backend, &softc->be_list, links) {
3407                         if (backend->type == type) {
3408                                 found = 1;
3409                                 break;
3410                         }
3411                 }
3412                 if (found == 0) {
3413                         printf("ctl: unknown ioctl command %#lx or backend "
3414                                "%d\n", cmd, type);
3415                         retval = EINVAL;
3416                         break;
3417                 }
3418                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3419 #endif
3420                 retval = ENOTTY;
3421                 break;
3422         }
3423         }
3424         return (retval);
3425 }
3426
3427 uint32_t
3428 ctl_get_initindex(struct ctl_nexus *nexus)
3429 {
3430         if (nexus->targ_port < CTL_MAX_PORTS)
3431                 return (nexus->initid.id +
3432                         (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3433         else
3434                 return (nexus->initid.id +
3435                        ((nexus->targ_port - CTL_MAX_PORTS) *
3436                         CTL_MAX_INIT_PER_PORT));
3437 }
3438
3439 uint32_t
3440 ctl_get_resindex(struct ctl_nexus *nexus)
3441 {
3442         return (nexus->initid.id + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3443 }
3444
3445 uint32_t
3446 ctl_port_idx(int port_num)
3447 {
3448         if (port_num < CTL_MAX_PORTS)
3449                 return(port_num);
3450         else
3451                 return(port_num - CTL_MAX_PORTS);
3452 }
3453
3454 static uint32_t
3455 ctl_map_lun(int port_num, uint32_t lun_id)
3456 {
3457         struct ctl_port *port;
3458
3459         port = control_softc->ctl_ports[ctl_port_idx(port_num)];
3460         if (port == NULL)
3461                 return (UINT32_MAX);
3462         if (port->lun_map == NULL)
3463                 return (lun_id);
3464         return (port->lun_map(port->targ_lun_arg, lun_id));
3465 }
3466
3467 static uint32_t
3468 ctl_map_lun_back(int port_num, uint32_t lun_id)
3469 {
3470         struct ctl_port *port;
3471         uint32_t i;
3472
3473         port = control_softc->ctl_ports[ctl_port_idx(port_num)];
3474         if (port->lun_map == NULL)
3475                 return (lun_id);
3476         for (i = 0; i < CTL_MAX_LUNS; i++) {
3477                 if (port->lun_map(port->targ_lun_arg, i) == lun_id)
3478                         return (i);
3479         }
3480         return (UINT32_MAX);
3481 }
3482
3483 /*
3484  * Note:  This only works for bitmask sizes that are at least 32 bits, and
3485  * that are a power of 2.
3486  */
3487 int
3488 ctl_ffz(uint32_t *mask, uint32_t size)
3489 {
3490         uint32_t num_chunks, num_pieces;
3491         int i, j;
3492
3493         num_chunks = (size >> 5);
3494         if (num_chunks == 0)
3495                 num_chunks++;
3496         num_pieces = ctl_min((sizeof(uint32_t) * 8), size);
3497
3498         for (i = 0; i < num_chunks; i++) {
3499                 for (j = 0; j < num_pieces; j++) {
3500                         if ((mask[i] & (1 << j)) == 0)
3501                                 return ((i << 5) + j);
3502                 }
3503         }
3504
3505         return (-1);
3506 }
3507
3508 int
3509 ctl_set_mask(uint32_t *mask, uint32_t bit)
3510 {
3511         uint32_t chunk, piece;
3512
3513         chunk = bit >> 5;
3514         piece = bit % (sizeof(uint32_t) * 8);
3515
3516         if ((mask[chunk] & (1 << piece)) != 0)
3517                 return (-1);
3518         else
3519                 mask[chunk] |= (1 << piece);
3520
3521         return (0);
3522 }
3523
3524 int
3525 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3526 {
3527         uint32_t chunk, piece;
3528
3529         chunk = bit >> 5;
3530         piece = bit % (sizeof(uint32_t) * 8);
3531
3532         if ((mask[chunk] & (1 << piece)) == 0)
3533                 return (-1);
3534         else
3535                 mask[chunk] &= ~(1 << piece);
3536
3537         return (0);
3538 }
3539
3540 int
3541 ctl_is_set(uint32_t *mask, uint32_t bit)
3542 {
3543         uint32_t chunk, piece;
3544
3545         chunk = bit >> 5;
3546         piece = bit % (sizeof(uint32_t) * 8);
3547
3548         if ((mask[chunk] & (1 << piece)) == 0)
3549                 return (0);
3550         else
3551                 return (1);
3552 }
3553
3554 #ifdef unused
3555 /*
3556  * The bus, target and lun are optional, they can be filled in later.
3557  * can_wait is used to determine whether we can wait on the malloc or not.
3558  */
3559 union ctl_io*
3560 ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port, uint32_t targ_target,
3561               uint32_t targ_lun, int can_wait)
3562 {
3563         union ctl_io *io;
3564
3565         if (can_wait)
3566                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_WAITOK);
3567         else
3568                 io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3569
3570         if (io != NULL) {
3571                 io->io_hdr.io_type = io_type;
3572                 io->io_hdr.targ_port = targ_port;
3573                 /*
3574                  * XXX KDM this needs to change/go away.  We need to move
3575                  * to a preallocated pool of ctl_scsiio structures.
3576                  */
3577                 io->io_hdr.nexus.targ_target.id = targ_target;
3578                 io->io_hdr.nexus.targ_lun = targ_lun;
3579         }
3580
3581         return (io);
3582 }
3583
3584 void
3585 ctl_kfree_io(union ctl_io *io)
3586 {
3587         free(io, M_CTL);
3588 }
3589 #endif /* unused */
3590
3591 /*
3592  * ctl_softc, pool_type, total_ctl_io are passed in.
3593  * npool is passed out.
3594  */
3595 int
3596 ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type,
3597                 uint32_t total_ctl_io, struct ctl_io_pool **npool)
3598 {
3599         uint32_t i;
3600         union ctl_io *cur_io, *next_io;
3601         struct ctl_io_pool *pool;
3602         int retval;
3603
3604         retval = 0;
3605
3606         pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3607                                             M_NOWAIT | M_ZERO);
3608         if (pool == NULL) {
3609                 retval = ENOMEM;
3610                 goto bailout;
3611         }
3612
3613         pool->type = pool_type;
3614         pool->ctl_softc = ctl_softc;
3615
3616         mtx_lock(&ctl_softc->pool_lock);
3617         pool->id = ctl_softc->cur_pool_id++;
3618         mtx_unlock(&ctl_softc->pool_lock);
3619
3620         pool->flags = CTL_POOL_FLAG_NONE;
3621         pool->refcount = 1;             /* Reference for validity. */
3622         STAILQ_INIT(&pool->free_queue);
3623
3624         /*
3625          * XXX KDM other options here:
3626          * - allocate a page at a time
3627          * - allocate one big chunk of memory.
3628          * Page allocation might work well, but would take a little more
3629          * tracking.
3630          */
3631         for (i = 0; i < total_ctl_io; i++) {
3632                 cur_io = (union ctl_io *)malloc(sizeof(*cur_io), M_CTLIO,
3633                                                 M_NOWAIT);
3634                 if (cur_io == NULL) {
3635                         retval = ENOMEM;
3636                         break;
3637                 }
3638                 cur_io->io_hdr.pool = pool;
3639                 STAILQ_INSERT_TAIL(&pool->free_queue, &cur_io->io_hdr, links);
3640                 pool->total_ctl_io++;
3641                 pool->free_ctl_io++;
3642         }
3643
3644         if (retval != 0) {
3645                 for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3646                      cur_io != NULL; cur_io = next_io) {
3647                         next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr,
3648                                                               links);
3649                         STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr,
3650                                       ctl_io_hdr, links);
3651                         free(cur_io, M_CTLIO);
3652                 }
3653
3654                 free(pool, M_CTL);
3655                 goto bailout;
3656         }
3657         mtx_lock(&ctl_softc->pool_lock);
3658         ctl_softc->num_pools++;
3659         STAILQ_INSERT_TAIL(&ctl_softc->io_pools, pool, links);
3660         /*
3661          * Increment our usage count if this is an external consumer, so we
3662          * can't get unloaded until the external consumer (most likely a
3663          * FETD) unloads and frees his pool.
3664          *
3665          * XXX KDM will this increment the caller's module use count, or
3666          * mine?
3667          */
3668 #if 0
3669         if ((pool_type != CTL_POOL_EMERGENCY)
3670          && (pool_type != CTL_POOL_INTERNAL)
3671          && (pool_type != CTL_POOL_4OTHERSC))
3672                 MOD_INC_USE_COUNT;
3673 #endif
3674
3675         mtx_unlock(&ctl_softc->pool_lock);
3676
3677         *npool = pool;
3678
3679 bailout:
3680
3681         return (retval);
3682 }
3683
3684 static int
3685 ctl_pool_acquire(struct ctl_io_pool *pool)
3686 {
3687
3688         mtx_assert(&pool->ctl_softc->pool_lock, MA_OWNED);
3689
3690         if (pool->flags & CTL_POOL_FLAG_INVALID)
3691                 return (EINVAL);
3692
3693         pool->refcount++;
3694
3695         return (0);
3696 }
3697
3698 static void
3699 ctl_pool_release(struct ctl_io_pool *pool)
3700 {
3701         struct ctl_softc *ctl_softc = pool->ctl_softc;
3702         union ctl_io *io;
3703
3704         mtx_assert(&ctl_softc->pool_lock, MA_OWNED);
3705
3706         if (--pool->refcount != 0)
3707                 return;
3708
3709         while ((io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue)) != NULL) {
3710                 STAILQ_REMOVE(&pool->free_queue, &io->io_hdr, ctl_io_hdr,
3711                               links);
3712                 free(io, M_CTLIO);
3713         }
3714
3715         STAILQ_REMOVE(&ctl_softc->io_pools, pool, ctl_io_pool, links);
3716         ctl_softc->num_pools--;
3717
3718         /*
3719          * XXX KDM will this decrement the caller's usage count or mine?
3720          */
3721 #if 0
3722         if ((pool->type != CTL_POOL_EMERGENCY)
3723          && (pool->type != CTL_POOL_INTERNAL)
3724          && (pool->type != CTL_POOL_4OTHERSC))
3725                 MOD_DEC_USE_COUNT;
3726 #endif
3727
3728         free(pool, M_CTL);
3729 }
3730
3731 void
3732 ctl_pool_free(struct ctl_io_pool *pool)
3733 {
3734         struct ctl_softc *ctl_softc;
3735
3736         if (pool == NULL)
3737                 return;
3738
3739         ctl_softc = pool->ctl_softc;
3740         mtx_lock(&ctl_softc->pool_lock);
3741         pool->flags |= CTL_POOL_FLAG_INVALID;
3742         ctl_pool_release(pool);
3743         mtx_unlock(&ctl_softc->pool_lock);
3744 }
3745
3746 /*
3747  * This routine does not block (except for spinlocks of course).
3748  * It tries to allocate a ctl_io union from the caller's pool as quickly as
3749  * possible.
3750  */
3751 union ctl_io *
3752 ctl_alloc_io(void *pool_ref)
3753 {
3754         union ctl_io *io;
3755         struct ctl_softc *ctl_softc;
3756         struct ctl_io_pool *pool, *npool;
3757         struct ctl_io_pool *emergency_pool;
3758
3759         pool = (struct ctl_io_pool *)pool_ref;
3760
3761         if (pool == NULL) {
3762                 printf("%s: pool is NULL\n", __func__);
3763                 return (NULL);
3764         }
3765
3766         emergency_pool = NULL;
3767
3768         ctl_softc = pool->ctl_softc;
3769
3770         mtx_lock(&ctl_softc->pool_lock);
3771         /*
3772          * First, try to get the io structure from the user's pool.
3773          */
3774         if (ctl_pool_acquire(pool) == 0) {
3775                 io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3776                 if (io != NULL) {
3777                         STAILQ_REMOVE_HEAD(&pool->free_queue, links);
3778                         pool->total_allocated++;
3779                         pool->free_ctl_io--;
3780                         mtx_unlock(&ctl_softc->pool_lock);
3781                         return (io);
3782                 } else
3783                         ctl_pool_release(pool);
3784         }
3785         /*
3786          * If he doesn't have any io structures left, search for an
3787          * emergency pool and grab one from there.
3788          */
3789         STAILQ_FOREACH(npool, &ctl_softc->io_pools, links) {
3790                 if (npool->type != CTL_POOL_EMERGENCY)
3791                         continue;
3792
3793                 if (ctl_pool_acquire(npool) != 0)
3794                         continue;
3795
3796                 emergency_pool = npool;
3797
3798                 io = (union ctl_io *)STAILQ_FIRST(&npool->free_queue);
3799                 if (io != NULL) {
3800                         STAILQ_REMOVE_HEAD(&npool->free_queue, links);
3801                         npool->total_allocated++;
3802                         npool->free_ctl_io--;
3803                         mtx_unlock(&ctl_softc->pool_lock);
3804                         return (io);
3805                 } else
3806                         ctl_pool_release(npool);
3807         }
3808
3809         /* Drop the spinlock before we malloc */
3810         mtx_unlock(&ctl_softc->pool_lock);
3811
3812         /*
3813          * The emergency pool (if it exists) didn't have one, so try an
3814          * atomic (i.e. nonblocking) malloc and see if we get lucky.
3815          */
3816         io = (union ctl_io *)malloc(sizeof(*io), M_CTLIO, M_NOWAIT);
3817         if (io != NULL) {
3818                 /*
3819                  * If the emergency pool exists but is empty, add this
3820                  * ctl_io to its list when it gets freed.
3821                  */
3822                 if (emergency_pool != NULL) {
3823                         mtx_lock(&ctl_softc->pool_lock);
3824                         if (ctl_pool_acquire(emergency_pool) == 0) {
3825                                 io->io_hdr.pool = emergency_pool;
3826                                 emergency_pool->total_ctl_io++;
3827                                 /*
3828                                  * Need to bump this, otherwise
3829                                  * total_allocated and total_freed won't
3830                                  * match when we no longer have anything
3831                                  * outstanding.
3832                                  */
3833                                 emergency_pool->total_allocated++;
3834                         }
3835                         mtx_unlock(&ctl_softc->pool_lock);
3836                 } else
3837                         io->io_hdr.pool = NULL;
3838         }
3839
3840         return (io);
3841 }
3842
3843 void
3844 ctl_free_io(union ctl_io *io)
3845 {
3846         if (io == NULL)
3847                 return;
3848
3849         /*
3850          * If this ctl_io has a pool, return it to that pool.
3851          */
3852         if (io->io_hdr.pool != NULL) {
3853                 struct ctl_io_pool *pool;
3854
3855                 pool = (struct ctl_io_pool *)io->io_hdr.pool;
3856                 mtx_lock(&pool->ctl_softc->pool_lock);
3857                 io->io_hdr.io_type = 0xff;
3858                 STAILQ_INSERT_TAIL(&pool->free_queue, &io->io_hdr, links);
3859                 pool->total_freed++;
3860                 pool->free_ctl_io++;
3861                 ctl_pool_release(pool);
3862                 mtx_unlock(&pool->ctl_softc->pool_lock);
3863         } else {
3864                 /*
3865                  * Otherwise, just free it.  We probably malloced it and
3866                  * the emergency pool wasn't available.
3867                  */
3868                 free(io, M_CTLIO);
3869         }
3870
3871 }
3872
3873 void
3874 ctl_zero_io(union ctl_io *io)
3875 {
3876         void *pool_ref;
3877
3878         if (io == NULL)
3879                 return;
3880
3881         /*
3882          * May need to preserve linked list pointers at some point too.
3883          */
3884         pool_ref = io->io_hdr.pool;
3885
3886         memset(io, 0, sizeof(*io));
3887
3888         io->io_hdr.pool = pool_ref;
3889 }
3890
3891 /*
3892  * This routine is currently used for internal copies of ctl_ios that need
3893  * to persist for some reason after we've already returned status to the
3894  * FETD.  (Thus the flag set.)
3895  *
3896  * XXX XXX
3897  * Note that this makes a blind copy of all fields in the ctl_io, except
3898  * for the pool reference.  This includes any memory that has been
3899  * allocated!  That memory will no longer be valid after done has been
3900  * called, so this would be VERY DANGEROUS for command that actually does
3901  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3902  * start and stop commands, which don't transfer any data, so this is not a
3903  * problem.  If it is used for anything else, the caller would also need to
3904  * allocate data buffer space and this routine would need to be modified to
3905  * copy the data buffer(s) as well.
3906  */
3907 void
3908 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3909 {
3910         void *pool_ref;
3911
3912         if ((src == NULL)
3913          || (dest == NULL))
3914                 return;
3915
3916         /*
3917          * May need to preserve linked list pointers at some point too.
3918          */
3919         pool_ref = dest->io_hdr.pool;
3920
3921         memcpy(dest, src, ctl_min(sizeof(*src), sizeof(*dest)));
3922
3923         dest->io_hdr.pool = pool_ref;
3924         /*
3925          * We need to know that this is an internal copy, and doesn't need
3926          * to get passed back to the FETD that allocated it.
3927          */
3928         dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3929 }
3930
3931 #ifdef NEEDTOPORT
3932 static void
3933 ctl_update_power_subpage(struct copan_power_subpage *page)
3934 {
3935         int num_luns, num_partitions, config_type;
3936         struct ctl_softc *softc;
3937         cs_BOOL_t aor_present, shelf_50pct_power;
3938         cs_raidset_personality_t rs_type;
3939         int max_active_luns;
3940
3941         softc = control_softc;
3942
3943         /* subtract out the processor LUN */
3944         num_luns = softc->num_luns - 1;
3945         /*
3946          * Default to 7 LUNs active, which was the only number we allowed
3947          * in the past.
3948          */
3949         max_active_luns = 7;
3950
3951         num_partitions = config_GetRsPartitionInfo();
3952         config_type = config_GetConfigType();
3953         shelf_50pct_power = config_GetShelfPowerMode();
3954         aor_present = config_IsAorRsPresent();
3955
3956         rs_type = ddb_GetRsRaidType(1);
3957         if ((rs_type != CS_RAIDSET_PERSONALITY_RAID5)
3958          && (rs_type != CS_RAIDSET_PERSONALITY_RAID1)) {
3959                 EPRINT(0, "Unsupported RS type %d!", rs_type);
3960         }
3961
3962
3963         page->total_luns = num_luns;
3964
3965         switch (config_type) {
3966         case 40:
3967                 /*
3968                  * In a 40 drive configuration, it doesn't matter what DC
3969                  * cards we have, whether we have AOR enabled or not,
3970                  * partitioning or not, or what type of RAIDset we have.
3971                  * In that scenario, we can power up every LUN we present
3972                  * to the user.
3973                  */
3974                 max_active_luns = num_luns;
3975
3976                 break;
3977         case 64:
3978                 if (shelf_50pct_power == CS_FALSE) {
3979                         /* 25% power */
3980                         if (aor_present == CS_TRUE) {
3981                                 if (rs_type ==
3982                                      CS_RAIDSET_PERSONALITY_RAID5) {
3983                                         max_active_luns = 7;
3984                                 } else if (rs_type ==
3985                                          CS_RAIDSET_PERSONALITY_RAID1){
3986                                         max_active_luns = 14;
3987                                 } else {
3988                                         /* XXX KDM now what?? */
3989                                 }
3990                         } else {
3991                                 if (rs_type ==
3992                                      CS_RAIDSET_PERSONALITY_RAID5) {
3993                                         max_active_luns = 8;
3994                                 } else if (rs_type ==
3995                                          CS_RAIDSET_PERSONALITY_RAID1){
3996                                         max_active_luns = 16;
3997                                 } else {
3998                                         /* XXX KDM now what?? */
3999                                 }
4000                         }
4001                 } else {
4002                         /* 50% power */
4003                         /*
4004                          * With 50% power in a 64 drive configuration, we
4005                          * can power all LUNs we present.
4006                          */
4007                         max_active_luns = num_luns;
4008                 }
4009                 break;
4010         case 112:
4011                 if (shelf_50pct_power == CS_FALSE) {
4012                         /* 25% power */
4013                         if (aor_present == CS_TRUE) {
4014                                 if (rs_type ==
4015                                      CS_RAIDSET_PERSONALITY_RAID5) {
4016                                         max_active_luns = 7;
4017                                 } else if (rs_type ==
4018                                          CS_RAIDSET_PERSONALITY_RAID1){
4019                                         max_active_luns = 14;
4020                                 } else {
4021                                         /* XXX KDM now what?? */
4022                                 }
4023                         } else {
4024                                 if (rs_type ==
4025                                      CS_RAIDSET_PERSONALITY_RAID5) {
4026                                         max_active_luns = 8;
4027                                 } else if (rs_type ==
4028                                          CS_RAIDSET_PERSONALITY_RAID1){
4029                                         max_active_luns = 16;
4030                                 } else {
4031                                         /* XXX KDM now what?? */
4032                                 }
4033                         }
4034                 } else {
4035                         /* 50% power */
4036                         if (aor_present == CS_TRUE) {
4037                                 if (rs_type ==
4038                                      CS_RAIDSET_PERSONALITY_RAID5) {
4039                                         max_active_luns = 14;
4040                                 } else if (rs_type ==
4041                                          CS_RAIDSET_PERSONALITY_RAID1){
4042                                         /*
4043                                          * We're assuming here that disk
4044                                          * caching is enabled, and so we're
4045                                          * able to power up half of each
4046                                          * LUN, and cache all writes.
4047                                          */
4048                                         max_active_luns = num_luns;
4049                                 } else {
4050                                         /* XXX KDM now what?? */
4051                                 }
4052                         } else {
4053                                 if (rs_type ==
4054                                      CS_RAIDSET_PERSONALITY_RAID5) {
4055                                         max_active_luns = 15;
4056                                 } else if (rs_type ==
4057                                          CS_RAIDSET_PERSONALITY_RAID1){
4058                                         max_active_luns = 30;
4059                                 } else {
4060                                         /* XXX KDM now what?? */
4061                                 }
4062                         }
4063                 }
4064                 break;
4065         default:
4066                 /*
4067                  * In this case, we have an unknown configuration, so we
4068                  * just use the default from above.
4069                  */
4070                 break;
4071         }
4072
4073         page->max_active_luns = max_active_luns;
4074 #if 0
4075         printk("%s: total_luns = %d, max_active_luns = %d\n", __func__,
4076                page->total_luns, page->max_active_luns);
4077 #endif
4078 }
4079 #endif /* NEEDTOPORT */
4080
4081 /*
4082  * This routine could be used in the future to load default and/or saved
4083  * mode page parameters for a particuar lun.
4084  */
4085 static int
4086 ctl_init_page_index(struct ctl_lun *lun)
4087 {
4088         int i;
4089         struct ctl_page_index *page_index;
4090         struct ctl_softc *softc;
4091
4092         memcpy(&lun->mode_pages.index, page_index_template,
4093                sizeof(page_index_template));
4094
4095         softc = lun->ctl_softc;
4096
4097         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4098
4099                 page_index = &lun->mode_pages.index[i];
4100                 /*
4101                  * If this is a disk-only mode page, there's no point in
4102                  * setting it up.  For some pages, we have to have some
4103                  * basic information about the disk in order to calculate the
4104                  * mode page data.
4105                  */
4106                 if ((lun->be_lun->lun_type != T_DIRECT)
4107                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
4108                         continue;
4109
4110                 switch (page_index->page_code & SMPH_PC_MASK) {
4111                 case SMS_FORMAT_DEVICE_PAGE: {
4112                         struct scsi_format_page *format_page;
4113
4114                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4115                                 panic("subpage is incorrect!");
4116
4117                         /*
4118                          * Sectors per track are set above.  Bytes per
4119                          * sector need to be set here on a per-LUN basis.
4120                          */
4121                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
4122                                &format_page_default,
4123                                sizeof(format_page_default));
4124                         memcpy(&lun->mode_pages.format_page[
4125                                CTL_PAGE_CHANGEABLE], &format_page_changeable,
4126                                sizeof(format_page_changeable));
4127                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
4128                                &format_page_default,
4129                                sizeof(format_page_default));
4130                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
4131                                &format_page_default,
4132                                sizeof(format_page_default));
4133
4134                         format_page = &lun->mode_pages.format_page[
4135                                 CTL_PAGE_CURRENT];
4136                         scsi_ulto2b(lun->be_lun->blocksize,
4137                                     format_page->bytes_per_sector);
4138
4139                         format_page = &lun->mode_pages.format_page[
4140                                 CTL_PAGE_DEFAULT];
4141                         scsi_ulto2b(lun->be_lun->blocksize,
4142                                     format_page->bytes_per_sector);
4143
4144                         format_page = &lun->mode_pages.format_page[
4145                                 CTL_PAGE_SAVED];
4146                         scsi_ulto2b(lun->be_lun->blocksize,
4147                                     format_page->bytes_per_sector);
4148
4149                         page_index->page_data =
4150                                 (uint8_t *)lun->mode_pages.format_page;
4151                         break;
4152                 }
4153                 case SMS_RIGID_DISK_PAGE: {
4154                         struct scsi_rigid_disk_page *rigid_disk_page;
4155                         uint32_t sectors_per_cylinder;
4156                         uint64_t cylinders;
4157 #ifndef __XSCALE__
4158                         int shift;
4159 #endif /* !__XSCALE__ */
4160
4161                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4162                                 panic("invalid subpage value %d",
4163                                       page_index->subpage);
4164
4165                         /*
4166                          * Rotation rate and sectors per track are set
4167                          * above.  We calculate the cylinders here based on
4168                          * capacity.  Due to the number of heads and
4169                          * sectors per track we're using, smaller arrays
4170                          * may turn out to have 0 cylinders.  Linux and
4171                          * FreeBSD don't pay attention to these mode pages
4172                          * to figure out capacity, but Solaris does.  It
4173                          * seems to deal with 0 cylinders just fine, and
4174                          * works out a fake geometry based on the capacity.
4175                          */
4176                         memcpy(&lun->mode_pages.rigid_disk_page[
4177                                CTL_PAGE_CURRENT], &rigid_disk_page_default,
4178                                sizeof(rigid_disk_page_default));
4179                         memcpy(&lun->mode_pages.rigid_disk_page[
4180                                CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
4181                                sizeof(rigid_disk_page_changeable));
4182                         memcpy(&lun->mode_pages.rigid_disk_page[
4183                                CTL_PAGE_DEFAULT], &rigid_disk_page_default,
4184                                sizeof(rigid_disk_page_default));
4185                         memcpy(&lun->mode_pages.rigid_disk_page[
4186                                CTL_PAGE_SAVED], &rigid_disk_page_default,
4187                                sizeof(rigid_disk_page_default));
4188
4189                         sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
4190                                 CTL_DEFAULT_HEADS;
4191
4192                         /*
4193                          * The divide method here will be more accurate,
4194                          * probably, but results in floating point being
4195                          * used in the kernel on i386 (__udivdi3()).  On the
4196                          * XScale, though, __udivdi3() is implemented in
4197                          * software.
4198                          *
4199                          * The shift method for cylinder calculation is
4200                          * accurate if sectors_per_cylinder is a power of
4201                          * 2.  Otherwise it might be slightly off -- you
4202                          * might have a bit of a truncation problem.
4203                          */
4204 #ifdef  __XSCALE__
4205                         cylinders = (lun->be_lun->maxlba + 1) /
4206                                 sectors_per_cylinder;
4207 #else
4208                         for (shift = 31; shift > 0; shift--) {
4209                                 if (sectors_per_cylinder & (1 << shift))
4210                                         break;
4211                         }
4212                         cylinders = (lun->be_lun->maxlba + 1) >> shift;
4213 #endif
4214
4215                         /*
4216                          * We've basically got 3 bytes, or 24 bits for the
4217                          * cylinder size in the mode page.  If we're over,
4218                          * just round down to 2^24.
4219                          */
4220                         if (cylinders > 0xffffff)
4221                                 cylinders = 0xffffff;
4222
4223                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4224                                 CTL_PAGE_CURRENT];
4225                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4226
4227                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4228                                 CTL_PAGE_DEFAULT];
4229                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4230
4231                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4232                                 CTL_PAGE_SAVED];
4233                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4234
4235                         page_index->page_data =
4236                                 (uint8_t *)lun->mode_pages.rigid_disk_page;
4237                         break;
4238                 }
4239                 case SMS_CACHING_PAGE: {
4240
4241                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4242                                 panic("invalid subpage value %d",
4243                                       page_index->subpage);
4244                         /*
4245                          * Defaults should be okay here, no calculations
4246                          * needed.
4247                          */
4248                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
4249                                &caching_page_default,
4250                                sizeof(caching_page_default));
4251                         memcpy(&lun->mode_pages.caching_page[
4252                                CTL_PAGE_CHANGEABLE], &caching_page_changeable,
4253                                sizeof(caching_page_changeable));
4254                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
4255                                &caching_page_default,
4256                                sizeof(caching_page_default));
4257                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4258                                &caching_page_default,
4259                                sizeof(caching_page_default));
4260                         page_index->page_data =
4261                                 (uint8_t *)lun->mode_pages.caching_page;
4262                         break;
4263                 }
4264                 case SMS_CONTROL_MODE_PAGE: {
4265
4266                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4267                                 panic("invalid subpage value %d",
4268                                       page_index->subpage);
4269
4270                         /*
4271                          * Defaults should be okay here, no calculations
4272                          * needed.
4273                          */
4274                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
4275                                &control_page_default,
4276                                sizeof(control_page_default));
4277                         memcpy(&lun->mode_pages.control_page[
4278                                CTL_PAGE_CHANGEABLE], &control_page_changeable,
4279                                sizeof(control_page_changeable));
4280                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
4281                                &control_page_default,
4282                                sizeof(control_page_default));
4283                         memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
4284                                &control_page_default,
4285                                sizeof(control_page_default));
4286                         page_index->page_data =
4287                                 (uint8_t *)lun->mode_pages.control_page;
4288                         break;
4289
4290                 }
4291                 case SMS_VENDOR_SPECIFIC_PAGE:{
4292                         switch (page_index->subpage) {
4293                         case PWR_SUBPAGE_CODE: {
4294                                 struct copan_power_subpage *current_page,
4295                                                            *saved_page;
4296
4297                                 memcpy(&lun->mode_pages.power_subpage[
4298                                        CTL_PAGE_CURRENT],
4299                                        &power_page_default,
4300                                        sizeof(power_page_default));
4301                                 memcpy(&lun->mode_pages.power_subpage[
4302                                        CTL_PAGE_CHANGEABLE],
4303                                        &power_page_changeable,
4304                                        sizeof(power_page_changeable));
4305                                 memcpy(&lun->mode_pages.power_subpage[
4306                                        CTL_PAGE_DEFAULT],
4307                                        &power_page_default,
4308                                        sizeof(power_page_default));
4309                                 memcpy(&lun->mode_pages.power_subpage[
4310                                        CTL_PAGE_SAVED],
4311                                        &power_page_default,
4312                                        sizeof(power_page_default));
4313                                 page_index->page_data =
4314                                     (uint8_t *)lun->mode_pages.power_subpage;
4315
4316                                 current_page = (struct copan_power_subpage *)
4317                                         (page_index->page_data +
4318                                          (page_index->page_len *
4319                                           CTL_PAGE_CURRENT));
4320                                 saved_page = (struct copan_power_subpage *)
4321                                         (page_index->page_data +
4322                                          (page_index->page_len *
4323                                           CTL_PAGE_SAVED));
4324                                 break;
4325                         }
4326                         case APS_SUBPAGE_CODE: {
4327                                 struct copan_aps_subpage *current_page,
4328                                                          *saved_page;
4329
4330                                 // This gets set multiple times but
4331                                 // it should always be the same. It's
4332                                 // only done during init so who cares.
4333                                 index_to_aps_page = i;
4334
4335                                 memcpy(&lun->mode_pages.aps_subpage[
4336                                        CTL_PAGE_CURRENT],
4337                                        &aps_page_default,
4338                                        sizeof(aps_page_default));
4339                                 memcpy(&lun->mode_pages.aps_subpage[
4340                                        CTL_PAGE_CHANGEABLE],
4341                                        &aps_page_changeable,
4342                                        sizeof(aps_page_changeable));
4343                                 memcpy(&lun->mode_pages.aps_subpage[
4344                                        CTL_PAGE_DEFAULT],
4345                                        &aps_page_default,
4346                                        sizeof(aps_page_default));
4347                                 memcpy(&lun->mode_pages.aps_subpage[
4348                                        CTL_PAGE_SAVED],
4349                                        &aps_page_default,
4350                                        sizeof(aps_page_default));
4351                                 page_index->page_data =
4352                                         (uint8_t *)lun->mode_pages.aps_subpage;
4353
4354                                 current_page = (struct copan_aps_subpage *)
4355                                         (page_index->page_data +
4356                                          (page_index->page_len *
4357                                           CTL_PAGE_CURRENT));
4358                                 saved_page = (struct copan_aps_subpage *)
4359                                         (page_index->page_data +
4360                                          (page_index->page_len *
4361                                           CTL_PAGE_SAVED));
4362                                 break;
4363                         }
4364                         case DBGCNF_SUBPAGE_CODE: {
4365                                 struct copan_debugconf_subpage *current_page,
4366                                                                *saved_page;
4367
4368                                 memcpy(&lun->mode_pages.debugconf_subpage[
4369                                        CTL_PAGE_CURRENT],
4370                                        &debugconf_page_default,
4371                                        sizeof(debugconf_page_default));
4372                                 memcpy(&lun->mode_pages.debugconf_subpage[
4373                                        CTL_PAGE_CHANGEABLE],
4374                                        &debugconf_page_changeable,
4375                                        sizeof(debugconf_page_changeable));
4376                                 memcpy(&lun->mode_pages.debugconf_subpage[
4377                                        CTL_PAGE_DEFAULT],
4378                                        &debugconf_page_default,
4379                                        sizeof(debugconf_page_default));
4380                                 memcpy(&lun->mode_pages.debugconf_subpage[
4381                                        CTL_PAGE_SAVED],
4382                                        &debugconf_page_default,
4383                                        sizeof(debugconf_page_default));
4384                                 page_index->page_data =
4385                                         (uint8_t *)lun->mode_pages.debugconf_subpage;
4386
4387                                 current_page = (struct copan_debugconf_subpage *)
4388                                         (page_index->page_data +
4389                                          (page_index->page_len *
4390                                           CTL_PAGE_CURRENT));
4391                                 saved_page = (struct copan_debugconf_subpage *)
4392                                         (page_index->page_data +
4393                                          (page_index->page_len *
4394                                           CTL_PAGE_SAVED));
4395                                 break;
4396                         }
4397                         default:
4398                                 panic("invalid subpage value %d",
4399                                       page_index->subpage);
4400                                 break;
4401                         }
4402                         break;
4403                 }
4404                 default:
4405                         panic("invalid page value %d",
4406                               page_index->page_code & SMPH_PC_MASK);
4407                         break;
4408         }
4409         }
4410
4411         return (CTL_RETVAL_COMPLETE);
4412 }
4413
4414 /*
4415  * LUN allocation.
4416  *
4417  * Requirements:
4418  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4419  *   wants us to allocate the LUN and he can block.
4420  * - ctl_softc is always set
4421  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4422  *
4423  * Returns 0 for success, non-zero (errno) for failure.
4424  */
4425 static int
4426 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4427               struct ctl_be_lun *const be_lun, struct ctl_id target_id)
4428 {
4429         struct ctl_lun *nlun, *lun;
4430         struct ctl_port *port;
4431         struct scsi_vpd_id_descriptor *desc;
4432         struct scsi_vpd_id_t10 *t10id;
4433         const char *scsiname, *vendor;
4434         int lun_number, i, lun_malloced;
4435         int devidlen, idlen1, idlen2 = 0, len;
4436
4437         if (be_lun == NULL)
4438                 return (EINVAL);
4439
4440         /*
4441          * We currently only support Direct Access or Processor LUN types.
4442          */
4443         switch (be_lun->lun_type) {
4444         case T_DIRECT:
4445                 break;
4446         case T_PROCESSOR:
4447                 break;
4448         case T_SEQUENTIAL:
4449         case T_CHANGER:
4450         default:
4451                 be_lun->lun_config_status(be_lun->be_lun,
4452                                           CTL_LUN_CONFIG_FAILURE);
4453                 break;
4454         }
4455         if (ctl_lun == NULL) {
4456                 lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4457                 lun_malloced = 1;
4458         } else {
4459                 lun_malloced = 0;
4460                 lun = ctl_lun;
4461         }
4462
4463         memset(lun, 0, sizeof(*lun));
4464         if (lun_malloced)
4465                 lun->flags = CTL_LUN_MALLOCED;
4466
4467         /* Generate LUN ID. */
4468         devidlen = max(CTL_DEVID_MIN_LEN,
4469             strnlen(be_lun->device_id, CTL_DEVID_LEN));
4470         idlen1 = sizeof(*t10id) + devidlen;
4471         len = sizeof(struct scsi_vpd_id_descriptor) + idlen1;
4472         scsiname = ctl_get_opt(&be_lun->options, "scsiname");
4473         if (scsiname != NULL) {
4474                 idlen2 = roundup2(strlen(scsiname) + 1, 4);
4475                 len += sizeof(struct scsi_vpd_id_descriptor) + idlen2;
4476         }
4477         lun->lun_devid = malloc(sizeof(struct ctl_devid) + len,
4478             M_CTL, M_WAITOK | M_ZERO);
4479         lun->lun_devid->len = len;
4480         desc = (struct scsi_vpd_id_descriptor *)lun->lun_devid->data;
4481         desc->proto_codeset = SVPD_ID_CODESET_ASCII;
4482         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
4483         desc->length = idlen1;
4484         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
4485         memset(t10id->vendor, ' ', sizeof(t10id->vendor));
4486         if ((vendor = ctl_get_opt(&be_lun->options, "vendor")) == NULL) {
4487                 strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
4488         } else {
4489                 strncpy(t10id->vendor, vendor,
4490                     min(sizeof(t10id->vendor), strlen(vendor)));
4491         }
4492         strncpy((char *)t10id->vendor_spec_id,
4493             (char *)be_lun->device_id, devidlen);
4494         if (scsiname != NULL) {
4495                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4496                     desc->length);
4497                 desc->proto_codeset = SVPD_ID_CODESET_UTF8;
4498                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4499                     SVPD_ID_TYPE_SCSI_NAME;
4500                 desc->length = idlen2;
4501                 strlcpy(desc->identifier, scsiname, idlen2);
4502         }
4503
4504         mtx_lock(&ctl_softc->ctl_lock);
4505         /*
4506          * See if the caller requested a particular LUN number.  If so, see
4507          * if it is available.  Otherwise, allocate the first available LUN.
4508          */
4509         if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4510                 if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4511                  || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4512                         mtx_unlock(&ctl_softc->ctl_lock);
4513                         if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4514                                 printf("ctl: requested LUN ID %d is higher "
4515                                        "than CTL_MAX_LUNS - 1 (%d)\n",
4516                                        be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4517                         } else {
4518                                 /*
4519                                  * XXX KDM return an error, or just assign
4520                                  * another LUN ID in this case??
4521                                  */
4522                                 printf("ctl: requested LUN ID %d is already "
4523                                        "in use\n", be_lun->req_lun_id);
4524                         }
4525                         if (lun->flags & CTL_LUN_MALLOCED)
4526                                 free(lun, M_CTL);
4527                         be_lun->lun_config_status(be_lun->be_lun,
4528                                                   CTL_LUN_CONFIG_FAILURE);
4529                         return (ENOSPC);
4530                 }
4531                 lun_number = be_lun->req_lun_id;
4532         } else {
4533                 lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, CTL_MAX_LUNS);
4534                 if (lun_number == -1) {
4535                         mtx_unlock(&ctl_softc->ctl_lock);
4536                         printf("ctl: can't allocate LUN on target %ju, out of "
4537                                "LUNs\n", (uintmax_t)target_id.id);
4538                         if (lun->flags & CTL_LUN_MALLOCED)
4539                                 free(lun, M_CTL);
4540                         be_lun->lun_config_status(be_lun->be_lun,
4541                                                   CTL_LUN_CONFIG_FAILURE);
4542                         return (ENOSPC);
4543                 }
4544         }
4545         ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4546
4547         mtx_init(&lun->lun_lock, "CTL LUN", NULL, MTX_DEF);
4548         lun->target = target_id;
4549         lun->lun = lun_number;
4550         lun->be_lun = be_lun;
4551         /*
4552          * The processor LUN is always enabled.  Disk LUNs come on line
4553          * disabled, and must be enabled by the backend.
4554          */
4555         lun->flags |= CTL_LUN_DISABLED;
4556         lun->backend = be_lun->be;
4557         be_lun->ctl_lun = lun;
4558         be_lun->lun_id = lun_number;
4559         atomic_add_int(&be_lun->be->num_luns, 1);
4560         if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4561                 lun->flags |= CTL_LUN_STOPPED;
4562
4563         if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4564                 lun->flags |= CTL_LUN_INOPERABLE;
4565
4566         if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4567                 lun->flags |= CTL_LUN_PRIMARY_SC;
4568
4569         lun->ctl_softc = ctl_softc;
4570         TAILQ_INIT(&lun->ooa_queue);
4571         TAILQ_INIT(&lun->blocked_queue);
4572         STAILQ_INIT(&lun->error_list);
4573
4574         /*
4575          * Initialize the mode page index.
4576          */
4577         ctl_init_page_index(lun);
4578
4579         /*
4580          * Set the poweron UA for all initiators on this LUN only.
4581          */
4582         for (i = 0; i < CTL_MAX_INITIATORS; i++)
4583                 lun->pending_sense[i].ua_pending = CTL_UA_POWERON;
4584
4585         /*
4586          * Now, before we insert this lun on the lun list, set the lun
4587          * inventory changed UA for all other luns.
4588          */
4589         STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4590                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4591                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4592                 }
4593         }
4594
4595         STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4596
4597         ctl_softc->ctl_luns[lun_number] = lun;
4598
4599         ctl_softc->num_luns++;
4600
4601         /* Setup statistics gathering */
4602         lun->stats.device_type = be_lun->lun_type;
4603         lun->stats.lun_number = lun_number;
4604         if (lun->stats.device_type == T_DIRECT)
4605                 lun->stats.blocksize = be_lun->blocksize;
4606         else
4607                 lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4608         for (i = 0;i < CTL_MAX_PORTS;i++)
4609                 lun->stats.ports[i].targ_port = i;
4610
4611         mtx_unlock(&ctl_softc->ctl_lock);
4612
4613         lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4614
4615         /*
4616          * Run through each registered FETD and bring it online if it isn't
4617          * already.  Enable the target ID if it hasn't been enabled, and
4618          * enable this particular LUN.
4619          */
4620         STAILQ_FOREACH(port, &ctl_softc->port_list, links) {
4621                 int retval;
4622
4623                 retval = port->lun_enable(port->targ_lun_arg, target_id,lun_number);
4624                 if (retval != 0) {
4625                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4626                                "%d for lun_enable on target %ju lun %d\n",
4627                                port->port_name, port->targ_port, retval,
4628                                (uintmax_t)target_id.id, lun_number);
4629                 } else
4630                         port->status |= CTL_PORT_STATUS_LUN_ONLINE;
4631         }
4632         return (0);
4633 }
4634
4635 /*
4636  * Delete a LUN.
4637  * Assumptions:
4638  * - LUN has already been marked invalid and any pending I/O has been taken
4639  *   care of.
4640  */
4641 static int
4642 ctl_free_lun(struct ctl_lun *lun)
4643 {
4644         struct ctl_softc *softc;
4645 #if 0
4646         struct ctl_port *port;
4647 #endif
4648         struct ctl_lun *nlun;
4649         int i;
4650
4651         softc = lun->ctl_softc;
4652
4653         mtx_assert(&softc->ctl_lock, MA_OWNED);
4654
4655         STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4656
4657         ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4658
4659         softc->ctl_luns[lun->lun] = NULL;
4660
4661         if (!TAILQ_EMPTY(&lun->ooa_queue))
4662                 panic("Freeing a LUN %p with outstanding I/O!!\n", lun);
4663
4664         softc->num_luns--;
4665
4666         /*
4667          * XXX KDM this scheme only works for a single target/multiple LUN
4668          * setup.  It needs to be revamped for a multiple target scheme.
4669          *
4670          * XXX KDM this results in port->lun_disable() getting called twice,
4671          * once when ctl_disable_lun() is called, and a second time here.
4672          * We really need to re-think the LUN disable semantics.  There
4673          * should probably be several steps/levels to LUN removal:
4674          *  - disable
4675          *  - invalidate
4676          *  - free
4677          *
4678          * Right now we only have a disable method when communicating to
4679          * the front end ports, at least for individual LUNs.
4680          */
4681 #if 0
4682         STAILQ_FOREACH(port, &softc->port_list, links) {
4683                 int retval;
4684
4685                 retval = port->lun_disable(port->targ_lun_arg, lun->target,
4686                                          lun->lun);
4687                 if (retval != 0) {
4688                         printf("ctl_free_lun: FETD %s port %d returned error "
4689                                "%d for lun_disable on target %ju lun %jd\n",
4690                                port->port_name, port->targ_port, retval,
4691                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4692                 }
4693
4694                 if (STAILQ_FIRST(&softc->lun_list) == NULL) {
4695                         port->status &= ~CTL_PORT_STATUS_LUN_ONLINE;
4696
4697                         retval = port->targ_disable(port->targ_lun_arg,lun->target);
4698                         if (retval != 0) {
4699                                 printf("ctl_free_lun: FETD %s port %d "
4700                                        "returned error %d for targ_disable on "
4701                                        "target %ju\n", port->port_name,
4702                                        port->targ_port, retval,
4703                                        (uintmax_t)lun->target.id);
4704                         } else
4705                                 port->status &= ~CTL_PORT_STATUS_TARG_ONLINE;
4706
4707                         if ((port->status & CTL_PORT_STATUS_TARG_ONLINE) != 0)
4708                                 continue;
4709
4710 #if 0
4711                         port->port_offline(port->onoff_arg);
4712                         port->status &= ~CTL_PORT_STATUS_ONLINE;
4713 #endif
4714                 }
4715         }
4716 #endif
4717
4718         /*
4719          * Tell the backend to free resources, if this LUN has a backend.
4720          */
4721         atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4722         lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4723
4724         mtx_destroy(&lun->lun_lock);
4725         free(lun->lun_devid, M_CTL);
4726         if (lun->flags & CTL_LUN_MALLOCED)
4727                 free(lun, M_CTL);
4728
4729         STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4730                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4731                         nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4732                 }
4733         }
4734
4735         return (0);
4736 }
4737
4738 static void
4739 ctl_create_lun(struct ctl_be_lun *be_lun)
4740 {
4741         struct ctl_softc *ctl_softc;
4742
4743         ctl_softc = control_softc;
4744
4745         /*
4746          * ctl_alloc_lun() should handle all potential failure cases.
4747          */
4748         ctl_alloc_lun(ctl_softc, NULL, be_lun, ctl_softc->target);
4749 }
4750
4751 int
4752 ctl_add_lun(struct ctl_be_lun *be_lun)
4753 {
4754         struct ctl_softc *ctl_softc = control_softc;
4755
4756         mtx_lock(&ctl_softc->ctl_lock);
4757         STAILQ_INSERT_TAIL(&ctl_softc->pending_lun_queue, be_lun, links);
4758         mtx_unlock(&ctl_softc->ctl_lock);
4759         wakeup(&ctl_softc->pending_lun_queue);
4760
4761         return (0);
4762 }
4763
4764 int
4765 ctl_enable_lun(struct ctl_be_lun *be_lun)
4766 {
4767         struct ctl_softc *ctl_softc;
4768         struct ctl_port *port, *nport;
4769         struct ctl_lun *lun;
4770         int retval;
4771
4772         ctl_softc = control_softc;
4773
4774         lun = (struct ctl_lun *)be_lun->ctl_lun;
4775
4776         mtx_lock(&ctl_softc->ctl_lock);
4777         mtx_lock(&lun->lun_lock);
4778         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4779                 /*
4780                  * eh?  Why did we get called if the LUN is already
4781                  * enabled?
4782                  */
4783                 mtx_unlock(&lun->lun_lock);
4784                 mtx_unlock(&ctl_softc->ctl_lock);
4785                 return (0);
4786         }
4787         lun->flags &= ~CTL_LUN_DISABLED;
4788         mtx_unlock(&lun->lun_lock);
4789
4790         for (port = STAILQ_FIRST(&ctl_softc->port_list); port != NULL; port = nport) {
4791                 nport = STAILQ_NEXT(port, links);
4792
4793                 /*
4794                  * Drop the lock while we call the FETD's enable routine.
4795                  * This can lead to a callback into CTL (at least in the
4796                  * case of the internal initiator frontend.
4797                  */
4798                 mtx_unlock(&ctl_softc->ctl_lock);
4799                 retval = port->lun_enable(port->targ_lun_arg, lun->target,lun->lun);
4800                 mtx_lock(&ctl_softc->ctl_lock);
4801                 if (retval != 0) {
4802                         printf("%s: FETD %s port %d returned error "
4803                                "%d for lun_enable on target %ju lun %jd\n",
4804                                __func__, port->port_name, port->targ_port, retval,
4805                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4806                 }
4807 #if 0
4808                  else {
4809             /* NOTE:  TODO:  why does lun enable affect port status? */
4810                         port->status |= CTL_PORT_STATUS_LUN_ONLINE;
4811                 }
4812 #endif
4813         }
4814
4815         mtx_unlock(&ctl_softc->ctl_lock);
4816
4817         return (0);
4818 }
4819
4820 int
4821 ctl_disable_lun(struct ctl_be_lun *be_lun)
4822 {
4823         struct ctl_softc *ctl_softc;
4824         struct ctl_port *port;
4825         struct ctl_lun *lun;
4826         int retval;
4827
4828         ctl_softc = control_softc;
4829
4830         lun = (struct ctl_lun *)be_lun->ctl_lun;
4831
4832         mtx_lock(&ctl_softc->ctl_lock);
4833         mtx_lock(&lun->lun_lock);
4834         if (lun->flags & CTL_LUN_DISABLED) {
4835                 mtx_unlock(&lun->lun_lock);
4836                 mtx_unlock(&ctl_softc->ctl_lock);
4837                 return (0);
4838         }
4839         lun->flags |= CTL_LUN_DISABLED;
4840         mtx_unlock(&lun->lun_lock);
4841
4842         STAILQ_FOREACH(port, &ctl_softc->port_list, links) {
4843                 mtx_unlock(&ctl_softc->ctl_lock);
4844                 /*
4845                  * Drop the lock before we call the frontend's disable
4846                  * routine, to avoid lock order reversals.
4847                  *
4848                  * XXX KDM what happens if the frontend list changes while
4849                  * we're traversing it?  It's unlikely, but should be handled.
4850                  */
4851                 retval = port->lun_disable(port->targ_lun_arg, lun->target,
4852                                          lun->lun);
4853                 mtx_lock(&ctl_softc->ctl_lock);
4854                 if (retval != 0) {
4855                         printf("ctl_alloc_lun: FETD %s port %d returned error "
4856                                "%d for lun_disable on target %ju lun %jd\n",
4857                                port->port_name, port->targ_port, retval,
4858                                (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4859                 }
4860         }
4861
4862         mtx_unlock(&ctl_softc->ctl_lock);
4863
4864         return (0);
4865 }
4866
4867 int
4868 ctl_start_lun(struct ctl_be_lun *be_lun)
4869 {
4870         struct ctl_softc *ctl_softc;
4871         struct ctl_lun *lun;
4872
4873         ctl_softc = control_softc;
4874
4875         lun = (struct ctl_lun *)be_lun->ctl_lun;
4876
4877         mtx_lock(&lun->lun_lock);
4878         lun->flags &= ~CTL_LUN_STOPPED;
4879         mtx_unlock(&lun->lun_lock);
4880
4881         return (0);
4882 }
4883
4884 int
4885 ctl_stop_lun(struct ctl_be_lun *be_lun)
4886 {
4887         struct ctl_softc *ctl_softc;
4888         struct ctl_lun *lun;
4889
4890         ctl_softc = control_softc;
4891
4892         lun = (struct ctl_lun *)be_lun->ctl_lun;
4893
4894         mtx_lock(&lun->lun_lock);
4895         lun->flags |= CTL_LUN_STOPPED;
4896         mtx_unlock(&lun->lun_lock);
4897
4898         return (0);
4899 }
4900
4901 int
4902 ctl_lun_offline(struct ctl_be_lun *be_lun)
4903 {
4904         struct ctl_softc *ctl_softc;
4905         struct ctl_lun *lun;
4906
4907         ctl_softc = control_softc;
4908
4909         lun = (struct ctl_lun *)be_lun->ctl_lun;
4910
4911         mtx_lock(&lun->lun_lock);
4912         lun->flags |= CTL_LUN_OFFLINE;
4913         mtx_unlock(&lun->lun_lock);
4914
4915         return (0);
4916 }
4917
4918 int
4919 ctl_lun_online(struct ctl_be_lun *be_lun)
4920 {
4921         struct ctl_softc *ctl_softc;
4922         struct ctl_lun *lun;
4923
4924         ctl_softc = control_softc;
4925
4926         lun = (struct ctl_lun *)be_lun->ctl_lun;
4927
4928         mtx_lock(&lun->lun_lock);
4929         lun->flags &= ~CTL_LUN_OFFLINE;
4930         mtx_unlock(&lun->lun_lock);
4931
4932         return (0);
4933 }
4934
4935 int
4936 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4937 {
4938         struct ctl_softc *ctl_softc;
4939         struct ctl_lun *lun;
4940
4941         ctl_softc = control_softc;
4942
4943         lun = (struct ctl_lun *)be_lun->ctl_lun;
4944
4945         mtx_lock(&lun->lun_lock);
4946
4947         /*
4948          * The LUN needs to be disabled before it can be marked invalid.
4949          */
4950         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4951                 mtx_unlock(&lun->lun_lock);
4952                 return (-1);
4953         }
4954         /*
4955          * Mark the LUN invalid.
4956          */
4957         lun->flags |= CTL_LUN_INVALID;
4958
4959         /*
4960          * If there is nothing in the OOA queue, go ahead and free the LUN.
4961          * If we have something in the OOA queue, we'll free it when the
4962          * last I/O completes.
4963          */
4964         if (TAILQ_EMPTY(&lun->ooa_queue)) {
4965                 mtx_unlock(&lun->lun_lock);
4966                 mtx_lock(&ctl_softc->ctl_lock);
4967                 ctl_free_lun(lun);
4968                 mtx_unlock(&ctl_softc->ctl_lock);
4969         } else
4970                 mtx_unlock(&lun->lun_lock);
4971
4972         return (0);
4973 }
4974
4975 int
4976 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
4977 {
4978         struct ctl_softc *ctl_softc;
4979         struct ctl_lun *lun;
4980
4981         ctl_softc = control_softc;
4982         lun = (struct ctl_lun *)be_lun->ctl_lun;
4983
4984         mtx_lock(&lun->lun_lock);
4985         lun->flags |= CTL_LUN_INOPERABLE;
4986         mtx_unlock(&lun->lun_lock);
4987
4988         return (0);
4989 }
4990
4991 int
4992 ctl_lun_operable(struct ctl_be_lun *be_lun)
4993 {
4994         struct ctl_softc *ctl_softc;
4995         struct ctl_lun *lun;
4996
4997         ctl_softc = control_softc;
4998         lun = (struct ctl_lun *)be_lun->ctl_lun;
4999
5000         mtx_lock(&lun->lun_lock);
5001         lun->flags &= ~CTL_LUN_INOPERABLE;
5002         mtx_unlock(&lun->lun_lock);
5003
5004         return (0);
5005 }
5006
5007 int
5008 ctl_lun_power_lock(struct ctl_be_lun *be_lun, struct ctl_nexus *nexus,
5009                    int lock)
5010 {
5011         struct ctl_softc *softc;
5012         struct ctl_lun *lun;
5013         struct copan_aps_subpage *current_sp;
5014         struct ctl_page_index *page_index;
5015         int i;
5016
5017         softc = control_softc;
5018
5019         mtx_lock(&softc->ctl_lock);
5020
5021         lun = (struct ctl_lun *)be_lun->ctl_lun;
5022         mtx_lock(&lun->lun_lock);
5023
5024         page_index = NULL;
5025         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
5026                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
5027                      APS_PAGE_CODE)
5028                         continue;
5029
5030                 if (lun->mode_pages.index[i].subpage != APS_SUBPAGE_CODE)
5031                         continue;
5032                 page_index = &lun->mode_pages.index[i];
5033         }
5034
5035         if (page_index == NULL) {
5036                 mtx_unlock(&lun->lun_lock);
5037                 mtx_unlock(&softc->ctl_lock);
5038                 printf("%s: APS subpage not found for lun %ju!\n", __func__,
5039                        (uintmax_t)lun->lun);
5040                 return (1);
5041         }
5042 #if 0
5043         if ((softc->aps_locked_lun != 0)
5044          && (softc->aps_locked_lun != lun->lun)) {
5045                 printf("%s: attempt to lock LUN %llu when %llu is already "
5046                        "locked\n");
5047                 mtx_unlock(&lun->lun_lock);
5048                 mtx_unlock(&softc->ctl_lock);
5049                 return (1);
5050         }
5051 #endif
5052
5053         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
5054                 (page_index->page_len * CTL_PAGE_CURRENT));
5055
5056         if (lock != 0) {
5057                 current_sp->lock_active = APS_LOCK_ACTIVE;
5058                 softc->aps_locked_lun = lun->lun;
5059         } else {
5060                 current_sp->lock_active = 0;
5061                 softc->aps_locked_lun = 0;
5062         }
5063
5064
5065         /*
5066          * If we're in HA mode, try to send the lock message to the other
5067          * side.
5068          */
5069         if (ctl_is_single == 0) {
5070                 int isc_retval;
5071                 union ctl_ha_msg lock_msg;
5072
5073                 lock_msg.hdr.nexus = *nexus;
5074                 lock_msg.hdr.msg_type = CTL_MSG_APS_LOCK;
5075                 if (lock != 0)
5076                         lock_msg.aps.lock_flag = 1;
5077                 else
5078                         lock_msg.aps.lock_flag = 0;
5079                 isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &lock_msg,
5080                                          sizeof(lock_msg), 0);
5081                 if (isc_retval > CTL_HA_STATUS_SUCCESS) {
5082                         printf("%s: APS (lock=%d) error returned from "
5083                                "ctl_ha_msg_send: %d\n", __func__, lock, isc_retval);
5084                         mtx_unlock(&lun->lun_lock);
5085                         mtx_unlock(&softc->ctl_lock);
5086                         return (1);
5087                 }
5088         }
5089
5090         mtx_unlock(&lun->lun_lock);
5091         mtx_unlock(&softc->ctl_lock);
5092
5093         return (0);
5094 }
5095
5096 void
5097 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
5098 {
5099         struct ctl_lun *lun;
5100         struct ctl_softc *softc;
5101         int i;
5102
5103         softc = control_softc;
5104
5105         lun = (struct ctl_lun *)be_lun->ctl_lun;
5106
5107         mtx_lock(&lun->lun_lock);
5108
5109         for (i = 0; i < CTL_MAX_INITIATORS; i++)
5110                 lun->pending_sense[i].ua_pending |= CTL_UA_CAPACITY_CHANGED;
5111
5112         mtx_unlock(&lun->lun_lock);
5113 }
5114
5115 /*
5116  * Backend "memory move is complete" callback for requests that never
5117  * make it down to say RAIDCore's configuration code.
5118  */
5119 int
5120 ctl_config_move_done(union ctl_io *io)
5121 {
5122         int retval;
5123
5124         retval = CTL_RETVAL_COMPLETE;
5125
5126
5127         CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
5128         /*
5129          * XXX KDM this shouldn't happen, but what if it does?
5130          */
5131         if (io->io_hdr.io_type != CTL_IO_SCSI)
5132                 panic("I/O type isn't CTL_IO_SCSI!");
5133
5134         if ((io->io_hdr.port_status == 0)
5135          && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
5136          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
5137                 io->io_hdr.status = CTL_SUCCESS;
5138         else if ((io->io_hdr.port_status != 0)
5139               && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
5140               && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)){
5141                 /*
5142                  * For hardware error sense keys, the sense key
5143                  * specific value is defined to be a retry count,
5144                  * but we use it to pass back an internal FETD
5145                  * error code.  XXX KDM  Hopefully the FETD is only
5146                  * using 16 bits for an error code, since that's
5147                  * all the space we have in the sks field.
5148                  */
5149                 ctl_set_internal_failure(&io->scsiio,
5150                                          /*sks_valid*/ 1,
5151                                          /*retry_count*/
5152                                          io->io_hdr.port_status);
5153                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5154                         free(io->scsiio.kern_data_ptr, M_CTL);
5155                 ctl_done(io);
5156                 goto bailout;
5157         }
5158
5159         if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
5160          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
5161          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
5162                 /*
5163                  * XXX KDM just assuming a single pointer here, and not a
5164                  * S/G list.  If we start using S/G lists for config data,
5165                  * we'll need to know how to clean them up here as well.
5166                  */
5167                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5168                         free(io->scsiio.kern_data_ptr, M_CTL);
5169                 /* Hopefully the user has already set the status... */
5170                 ctl_done(io);
5171         } else {
5172                 /*
5173                  * XXX KDM now we need to continue data movement.  Some
5174                  * options:
5175                  * - call ctl_scsiio() again?  We don't do this for data
5176                  *   writes, because for those at least we know ahead of
5177                  *   time where the write will go and how long it is.  For
5178                  *   config writes, though, that information is largely
5179                  *   contained within the write itself, thus we need to
5180                  *   parse out the data again.
5181                  *
5182                  * - Call some other function once the data is in?
5183                  */
5184
5185                 /*
5186                  * XXX KDM call ctl_scsiio() again for now, and check flag
5187                  * bits to see whether we're allocated or not.
5188                  */
5189                 retval = ctl_scsiio(&io->scsiio);
5190         }
5191 bailout:
5192         return (retval);
5193 }
5194
5195 /*
5196  * This gets called by a backend driver when it is done with a
5197  * data_submit method.
5198  */
5199 void
5200 ctl_data_submit_done(union ctl_io *io)
5201 {
5202         /*
5203          * If the IO_CONT flag is set, we need to call the supplied
5204          * function to continue processing the I/O, instead of completing
5205          * the I/O just yet.
5206          *
5207          * If there is an error, though, we don't want to keep processing.
5208          * Instead, just send status back to the initiator.
5209          */
5210         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5211             (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5212             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5213              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5214                 io->scsiio.io_cont(io);
5215                 return;
5216         }
5217         ctl_done(io);
5218 }
5219
5220 /*
5221  * This gets called by a backend driver when it is done with a
5222  * configuration write.
5223  */
5224 void
5225 ctl_config_write_done(union ctl_io *io)
5226 {
5227         /*
5228          * If the IO_CONT flag is set, we need to call the supplied
5229          * function to continue processing the I/O, instead of completing
5230          * the I/O just yet.
5231          *
5232          * If there is an error, though, we don't want to keep processing.
5233          * Instead, just send status back to the initiator.
5234          */
5235         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT)
5236          && (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)
5237           || ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))) {
5238                 io->scsiio.io_cont(io);
5239                 return;
5240         }
5241         /*
5242          * Since a configuration write can be done for commands that actually
5243          * have data allocated, like write buffer, and commands that have
5244          * no data, like start/stop unit, we need to check here.
5245          */
5246         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
5247                 free(io->scsiio.kern_data_ptr, M_CTL);
5248         ctl_done(io);
5249 }
5250
5251 /*
5252  * SCSI release command.
5253  */
5254 int
5255 ctl_scsi_release(struct ctl_scsiio *ctsio)
5256 {
5257         int length, longid, thirdparty_id, resv_id;
5258         struct ctl_softc *ctl_softc;
5259         struct ctl_lun *lun;
5260
5261         length = 0;
5262         resv_id = 0;
5263
5264         CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
5265
5266         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5267         ctl_softc = control_softc;
5268
5269         switch (ctsio->cdb[0]) {
5270         case RELEASE_10: {
5271                 struct scsi_release_10 *cdb;
5272
5273                 cdb = (struct scsi_release_10 *)ctsio->cdb;
5274
5275                 if (cdb->byte2 & SR10_LONGID)
5276                         longid = 1;
5277                 else
5278                         thirdparty_id = cdb->thirdparty_id;
5279
5280                 resv_id = cdb->resv_id;
5281                 length = scsi_2btoul(cdb->length);
5282                 break;
5283         }
5284         }
5285
5286
5287         /*
5288          * XXX KDM right now, we only support LUN reservation.  We don't
5289          * support 3rd party reservations, or extent reservations, which
5290          * might actually need the parameter list.  If we've gotten this
5291          * far, we've got a LUN reservation.  Anything else got kicked out
5292          * above.  So, according to SPC, ignore the length.
5293          */
5294         length = 0;
5295
5296         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5297          && (length > 0)) {
5298                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5299                 ctsio->kern_data_len = length;
5300                 ctsio->kern_total_len = length;
5301                 ctsio->kern_data_resid = 0;
5302                 ctsio->kern_rel_offset = 0;
5303                 ctsio->kern_sg_entries = 0;
5304                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5305                 ctsio->be_move_done = ctl_config_move_done;
5306                 ctl_datamove((union ctl_io *)ctsio);
5307
5308                 return (CTL_RETVAL_COMPLETE);
5309         }
5310
5311         if (length > 0)
5312                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5313
5314         mtx_lock(&lun->lun_lock);
5315
5316         /*
5317          * According to SPC, it is not an error for an intiator to attempt
5318          * to release a reservation on a LUN that isn't reserved, or that
5319          * is reserved by another initiator.  The reservation can only be
5320          * released, though, by the initiator who made it or by one of
5321          * several reset type events.
5322          */
5323         if (lun->flags & CTL_LUN_RESERVED) {
5324                 if ((ctsio->io_hdr.nexus.initid.id == lun->rsv_nexus.initid.id)
5325                  && (ctsio->io_hdr.nexus.targ_port == lun->rsv_nexus.targ_port)
5326                  && (ctsio->io_hdr.nexus.targ_target.id ==
5327                      lun->rsv_nexus.targ_target.id)) {
5328                         lun->flags &= ~CTL_LUN_RESERVED;
5329                 }
5330         }
5331
5332         mtx_unlock(&lun->lun_lock);
5333
5334         ctsio->scsi_status = SCSI_STATUS_OK;
5335         ctsio->io_hdr.status = CTL_SUCCESS;
5336
5337         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5338                 free(ctsio->kern_data_ptr, M_CTL);
5339                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5340         }
5341
5342         ctl_done((union ctl_io *)ctsio);
5343         return (CTL_RETVAL_COMPLETE);
5344 }
5345
5346 int
5347 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5348 {
5349         int extent, thirdparty, longid;
5350         int resv_id, length;
5351         uint64_t thirdparty_id;
5352         struct ctl_softc *ctl_softc;
5353         struct ctl_lun *lun;
5354
5355         extent = 0;
5356         thirdparty = 0;
5357         longid = 0;
5358         resv_id = 0;
5359         length = 0;
5360         thirdparty_id = 0;
5361
5362         CTL_DEBUG_PRINT(("ctl_reserve\n"));
5363
5364         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5365         ctl_softc = control_softc;
5366
5367         switch (ctsio->cdb[0]) {
5368         case RESERVE_10: {
5369                 struct scsi_reserve_10 *cdb;
5370
5371                 cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5372
5373                 if (cdb->byte2 & SR10_LONGID)
5374                         longid = 1;
5375                 else
5376                         thirdparty_id = cdb->thirdparty_id;
5377
5378                 resv_id = cdb->resv_id;
5379                 length = scsi_2btoul(cdb->length);
5380                 break;
5381         }
5382         }
5383
5384         /*
5385          * XXX KDM right now, we only support LUN reservation.  We don't
5386          * support 3rd party reservations, or extent reservations, which
5387          * might actually need the parameter list.  If we've gotten this
5388          * far, we've got a LUN reservation.  Anything else got kicked out
5389          * above.  So, according to SPC, ignore the length.
5390          */
5391         length = 0;
5392
5393         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5394          && (length > 0)) {
5395                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5396                 ctsio->kern_data_len = length;
5397                 ctsio->kern_total_len = length;
5398                 ctsio->kern_data_resid = 0;
5399                 ctsio->kern_rel_offset = 0;
5400                 ctsio->kern_sg_entries = 0;
5401                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5402                 ctsio->be_move_done = ctl_config_move_done;
5403                 ctl_datamove((union ctl_io *)ctsio);
5404
5405                 return (CTL_RETVAL_COMPLETE);
5406         }
5407
5408         if (length > 0)
5409                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5410
5411         mtx_lock(&lun->lun_lock);
5412         if (lun->flags & CTL_LUN_RESERVED) {
5413                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
5414                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
5415                  || (ctsio->io_hdr.nexus.targ_target.id !=
5416                      lun->rsv_nexus.targ_target.id)) {
5417                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
5418                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
5419                         goto bailout;
5420                 }
5421         }
5422
5423         lun->flags |= CTL_LUN_RESERVED;
5424         lun->rsv_nexus = ctsio->io_hdr.nexus;
5425
5426         ctsio->scsi_status = SCSI_STATUS_OK;
5427         ctsio->io_hdr.status = CTL_SUCCESS;
5428
5429 bailout:
5430         mtx_unlock(&lun->lun_lock);
5431
5432         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5433                 free(ctsio->kern_data_ptr, M_CTL);
5434                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5435         }
5436
5437         ctl_done((union ctl_io *)ctsio);
5438         return (CTL_RETVAL_COMPLETE);
5439 }
5440
5441 int
5442 ctl_start_stop(struct ctl_scsiio *ctsio)
5443 {
5444         struct scsi_start_stop_unit *cdb;
5445         struct ctl_lun *lun;
5446         struct ctl_softc *ctl_softc;
5447         int retval;
5448
5449         CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5450
5451         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5452         ctl_softc = control_softc;
5453         retval = 0;
5454
5455         cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5456
5457         /*
5458          * XXX KDM
5459          * We don't support the immediate bit on a stop unit.  In order to
5460          * do that, we would need to code up a way to know that a stop is
5461          * pending, and hold off any new commands until it completes, one
5462          * way or another.  Then we could accept or reject those commands
5463          * depending on its status.  We would almost need to do the reverse
5464          * of what we do below for an immediate start -- return the copy of
5465          * the ctl_io to the FETD with status to send to the host (and to
5466          * free the copy!) and then free the original I/O once the stop
5467          * actually completes.  That way, the OOA queue mechanism can work
5468          * to block commands that shouldn't proceed.  Another alternative
5469          * would be to put the copy in the queue in place of the original,
5470          * and return the original back to the caller.  That could be
5471          * slightly safer..
5472          */
5473         if ((cdb->byte2 & SSS_IMMED)
5474          && ((cdb->how & SSS_START) == 0)) {
5475                 ctl_set_invalid_field(ctsio,
5476                                       /*sks_valid*/ 1,
5477                                       /*command*/ 1,
5478                                       /*field*/ 1,
5479                                       /*bit_valid*/ 1,
5480                                       /*bit*/ 0);
5481                 ctl_done((union ctl_io *)ctsio);
5482                 return (CTL_RETVAL_COMPLETE);
5483         }
5484
5485         if ((lun->flags & CTL_LUN_PR_RESERVED)
5486          && ((cdb->how & SSS_START)==0)) {
5487                 uint32_t residx;
5488
5489                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5490                 if (!lun->per_res[residx].registered
5491                  || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5492
5493                         ctl_set_reservation_conflict(ctsio);
5494                         ctl_done((union ctl_io *)ctsio);
5495                         return (CTL_RETVAL_COMPLETE);
5496                 }
5497         }
5498
5499         /*
5500          * If there is no backend on this device, we can't start or stop
5501          * it.  In theory we shouldn't get any start/stop commands in the
5502          * first place at this level if the LUN doesn't have a backend.
5503          * That should get stopped by the command decode code.
5504          */
5505         if (lun->backend == NULL) {
5506                 ctl_set_invalid_opcode(ctsio);
5507                 ctl_done((union ctl_io *)ctsio);
5508                 return (CTL_RETVAL_COMPLETE);
5509         }
5510
5511         /*
5512          * XXX KDM Copan-specific offline behavior.
5513          * Figure out a reasonable way to port this?
5514          */
5515 #ifdef NEEDTOPORT
5516         mtx_lock(&lun->lun_lock);
5517
5518         if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5519          && (lun->flags & CTL_LUN_OFFLINE)) {
5520                 /*
5521                  * If the LUN is offline, and the on/offline bit isn't set,
5522                  * reject the start or stop.  Otherwise, let it through.
5523                  */
5524                 mtx_unlock(&lun->lun_lock);
5525                 ctl_set_lun_not_ready(ctsio);
5526                 ctl_done((union ctl_io *)ctsio);
5527         } else {
5528                 mtx_unlock(&lun->lun_lock);
5529 #endif /* NEEDTOPORT */
5530                 /*
5531                  * This could be a start or a stop when we're online,
5532                  * or a stop/offline or start/online.  A start or stop when
5533                  * we're offline is covered in the case above.
5534                  */
5535                 /*
5536                  * In the non-immediate case, we send the request to
5537                  * the backend and return status to the user when
5538                  * it is done.
5539                  *
5540                  * In the immediate case, we allocate a new ctl_io
5541                  * to hold a copy of the request, and send that to
5542                  * the backend.  We then set good status on the
5543                  * user's request and return it immediately.
5544                  */
5545                 if (cdb->byte2 & SSS_IMMED) {
5546                         union ctl_io *new_io;
5547
5548                         new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5549                         if (new_io == NULL) {
5550                                 ctl_set_busy(ctsio);
5551                                 ctl_done((union ctl_io *)ctsio);
5552                         } else {
5553                                 ctl_copy_io((union ctl_io *)ctsio,
5554                                             new_io);
5555                                 retval = lun->backend->config_write(new_io);
5556                                 ctl_set_success(ctsio);
5557                                 ctl_done((union ctl_io *)ctsio);
5558                         }
5559                 } else {
5560                         retval = lun->backend->config_write(
5561                                 (union ctl_io *)ctsio);
5562                 }
5563 #ifdef NEEDTOPORT
5564         }
5565 #endif
5566         return (retval);
5567 }
5568
5569 /*
5570  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5571  * we don't really do anything with the LBA and length fields if the user
5572  * passes them in.  Instead we'll just flush out the cache for the entire
5573  * LUN.
5574  */
5575 int
5576 ctl_sync_cache(struct ctl_scsiio *ctsio)
5577 {
5578         struct ctl_lun *lun;
5579         struct ctl_softc *ctl_softc;
5580         uint64_t starting_lba;
5581         uint32_t block_count;
5582         int retval;
5583
5584         CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5585
5586         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5587         ctl_softc = control_softc;
5588         retval = 0;
5589
5590         switch (ctsio->cdb[0]) {
5591         case SYNCHRONIZE_CACHE: {
5592                 struct scsi_sync_cache *cdb;
5593                 cdb = (struct scsi_sync_cache *)ctsio->cdb;
5594
5595                 starting_lba = scsi_4btoul(cdb->begin_lba);
5596                 block_count = scsi_2btoul(cdb->lb_count);
5597                 break;
5598         }
5599         case SYNCHRONIZE_CACHE_16: {
5600                 struct scsi_sync_cache_16 *cdb;
5601                 cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5602
5603                 starting_lba = scsi_8btou64(cdb->begin_lba);
5604                 block_count = scsi_4btoul(cdb->lb_count);
5605                 break;
5606         }
5607         default:
5608                 ctl_set_invalid_opcode(ctsio);
5609                 ctl_done((union ctl_io *)ctsio);
5610                 goto bailout;
5611                 break; /* NOTREACHED */
5612         }
5613
5614         /*
5615          * We check the LBA and length, but don't do anything with them.
5616          * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5617          * get flushed.  This check will just help satisfy anyone who wants
5618          * to see an error for an out of range LBA.
5619          */
5620         if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5621                 ctl_set_lba_out_of_range(ctsio);
5622                 ctl_done((union ctl_io *)ctsio);
5623                 goto bailout;
5624         }
5625
5626         /*
5627          * If this LUN has no backend, we can't flush the cache anyway.
5628          */
5629         if (lun->backend == NULL) {
5630                 ctl_set_invalid_opcode(ctsio);
5631                 ctl_done((union ctl_io *)ctsio);
5632                 goto bailout;
5633         }
5634
5635         /*
5636          * Check to see whether we're configured to send the SYNCHRONIZE
5637          * CACHE command directly to the back end.
5638          */
5639         mtx_lock(&lun->lun_lock);
5640         if ((ctl_softc->flags & CTL_FLAG_REAL_SYNC)
5641          && (++(lun->sync_count) >= lun->sync_interval)) {
5642                 lun->sync_count = 0;
5643                 mtx_unlock(&lun->lun_lock);
5644                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5645         } else {
5646                 mtx_unlock(&lun->lun_lock);
5647                 ctl_set_success(ctsio);
5648                 ctl_done((union ctl_io *)ctsio);
5649         }
5650
5651 bailout:
5652
5653         return (retval);
5654 }
5655
5656 int
5657 ctl_format(struct ctl_scsiio *ctsio)
5658 {
5659         struct scsi_format *cdb;
5660         struct ctl_lun *lun;
5661         struct ctl_softc *ctl_softc;
5662         int length, defect_list_len;
5663
5664         CTL_DEBUG_PRINT(("ctl_format\n"));
5665
5666         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5667         ctl_softc = control_softc;
5668
5669         cdb = (struct scsi_format *)ctsio->cdb;
5670
5671         length = 0;
5672         if (cdb->byte2 & SF_FMTDATA) {
5673                 if (cdb->byte2 & SF_LONGLIST)
5674                         length = sizeof(struct scsi_format_header_long);
5675                 else
5676                         length = sizeof(struct scsi_format_header_short);
5677         }
5678
5679         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5680          && (length > 0)) {
5681                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5682                 ctsio->kern_data_len = length;
5683                 ctsio->kern_total_len = length;
5684                 ctsio->kern_data_resid = 0;
5685                 ctsio->kern_rel_offset = 0;
5686                 ctsio->kern_sg_entries = 0;
5687                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5688                 ctsio->be_move_done = ctl_config_move_done;
5689                 ctl_datamove((union ctl_io *)ctsio);
5690
5691                 return (CTL_RETVAL_COMPLETE);
5692         }
5693
5694         defect_list_len = 0;
5695
5696         if (cdb->byte2 & SF_FMTDATA) {
5697                 if (cdb->byte2 & SF_LONGLIST) {
5698                         struct scsi_format_header_long *header;
5699
5700                         header = (struct scsi_format_header_long *)
5701                                 ctsio->kern_data_ptr;
5702
5703                         defect_list_len = scsi_4btoul(header->defect_list_len);
5704                         if (defect_list_len != 0) {
5705                                 ctl_set_invalid_field(ctsio,
5706                                                       /*sks_valid*/ 1,
5707                                                       /*command*/ 0,
5708                                                       /*field*/ 2,
5709                                                       /*bit_valid*/ 0,
5710                                                       /*bit*/ 0);
5711                                 goto bailout;
5712                         }
5713                 } else {
5714                         struct scsi_format_header_short *header;
5715
5716                         header = (struct scsi_format_header_short *)
5717                                 ctsio->kern_data_ptr;
5718
5719                         defect_list_len = scsi_2btoul(header->defect_list_len);
5720                         if (defect_list_len != 0) {
5721                                 ctl_set_invalid_field(ctsio,
5722                                                       /*sks_valid*/ 1,
5723                                                       /*command*/ 0,
5724                                                       /*field*/ 2,
5725                                                       /*bit_valid*/ 0,
5726                                                       /*bit*/ 0);
5727                                 goto bailout;
5728                         }
5729                 }
5730         }
5731
5732         /*
5733          * The format command will clear out the "Medium format corrupted"
5734          * status if set by the configuration code.  That status is really
5735          * just a way to notify the host that we have lost the media, and
5736          * get them to issue a command that will basically make them think
5737          * they're blowing away the media.
5738          */
5739         mtx_lock(&lun->lun_lock);
5740         lun->flags &= ~CTL_LUN_INOPERABLE;
5741         mtx_unlock(&lun->lun_lock);
5742
5743         ctsio->scsi_status = SCSI_STATUS_OK;
5744         ctsio->io_hdr.status = CTL_SUCCESS;
5745 bailout:
5746
5747         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5748                 free(ctsio->kern_data_ptr, M_CTL);
5749                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5750         }
5751
5752         ctl_done((union ctl_io *)ctsio);
5753         return (CTL_RETVAL_COMPLETE);
5754 }
5755
5756 int
5757 ctl_read_buffer(struct ctl_scsiio *ctsio)
5758 {
5759         struct scsi_read_buffer *cdb;
5760         struct ctl_lun *lun;
5761         int buffer_offset, len;
5762         static uint8_t descr[4];
5763         static uint8_t echo_descr[4] = { 0 };
5764
5765         CTL_DEBUG_PRINT(("ctl_read_buffer\n"));
5766
5767         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5768         cdb = (struct scsi_read_buffer *)ctsio->cdb;
5769
5770         if (lun->flags & CTL_LUN_PR_RESERVED) {
5771                 uint32_t residx;
5772
5773                 /*
5774                  * XXX KDM need a lock here.
5775                  */
5776                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5777                 if ((lun->res_type == SPR_TYPE_EX_AC
5778                   && residx != lun->pr_res_idx)
5779                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
5780                    || lun->res_type == SPR_TYPE_EX_AC_AR)
5781                   && !lun->per_res[residx].registered)) {
5782                         ctl_set_reservation_conflict(ctsio);
5783                         ctl_done((union ctl_io *)ctsio);
5784                         return (CTL_RETVAL_COMPLETE);
5785                 }
5786         }
5787
5788         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA &&
5789             (cdb->byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
5790             (cdb->byte2 & RWB_MODE) != RWB_MODE_DESCR) {
5791                 ctl_set_invalid_field(ctsio,
5792                                       /*sks_valid*/ 1,
5793                                       /*command*/ 1,
5794                                       /*field*/ 1,
5795                                       /*bit_valid*/ 1,
5796                                       /*bit*/ 4);
5797                 ctl_done((union ctl_io *)ctsio);
5798                 return (CTL_RETVAL_COMPLETE);
5799         }
5800
5801         len = scsi_3btoul(cdb->length);
5802         buffer_offset = scsi_3btoul(cdb->offset);
5803
5804         if (buffer_offset + len > sizeof(lun->write_buffer)) {
5805                 ctl_set_invalid_field(ctsio,
5806                                       /*sks_valid*/ 1,
5807                                       /*command*/ 1,
5808                                       /*field*/ 6,
5809                                       /*bit_valid*/ 0,
5810                                       /*bit*/ 0);
5811                 ctl_done((union ctl_io *)ctsio);
5812                 return (CTL_RETVAL_COMPLETE);
5813         }
5814
5815         if ((cdb->byte2 & RWB_MODE) == RWB_MODE_DESCR) {
5816                 descr[0] = 0;
5817                 scsi_ulto3b(sizeof(lun->write_buffer), &descr[1]);
5818                 ctsio->kern_data_ptr = descr;
5819                 len = min(len, sizeof(descr));
5820         } else if ((cdb->byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
5821                 ctsio->kern_data_ptr = echo_descr;
5822                 len = min(len, sizeof(echo_descr));
5823         } else
5824                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5825         ctsio->kern_data_len = len;
5826         ctsio->kern_total_len = len;
5827         ctsio->kern_data_resid = 0;
5828         ctsio->kern_rel_offset = 0;
5829         ctsio->kern_sg_entries = 0;
5830         ctsio->be_move_done = ctl_config_move_done;
5831         ctl_datamove((union ctl_io *)ctsio);
5832
5833         return (CTL_RETVAL_COMPLETE);
5834 }
5835
5836 int
5837 ctl_write_buffer(struct ctl_scsiio *ctsio)
5838 {
5839         struct scsi_write_buffer *cdb;
5840         struct ctl_lun *lun;
5841         int buffer_offset, len;
5842
5843         CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5844
5845         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5846         cdb = (struct scsi_write_buffer *)ctsio->cdb;
5847
5848         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5849                 ctl_set_invalid_field(ctsio,
5850                                       /*sks_valid*/ 1,
5851                                       /*command*/ 1,
5852                                       /*field*/ 1,
5853                                       /*bit_valid*/ 1,
5854                                       /*bit*/ 4);
5855                 ctl_done((union ctl_io *)ctsio);
5856                 return (CTL_RETVAL_COMPLETE);
5857         }
5858
5859         len = scsi_3btoul(cdb->length);
5860         buffer_offset = scsi_3btoul(cdb->offset);
5861
5862         if (buffer_offset + len > sizeof(lun->write_buffer)) {
5863                 ctl_set_invalid_field(ctsio,
5864                                       /*sks_valid*/ 1,
5865                                       /*command*/ 1,
5866                                       /*field*/ 6,
5867                                       /*bit_valid*/ 0,
5868                                       /*bit*/ 0);
5869                 ctl_done((union ctl_io *)ctsio);
5870                 return (CTL_RETVAL_COMPLETE);
5871         }
5872
5873         /*
5874          * If we've got a kernel request that hasn't been malloced yet,
5875          * malloc it and tell the caller the data buffer is here.
5876          */
5877         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5878                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5879                 ctsio->kern_data_len = len;
5880                 ctsio->kern_total_len = len;
5881                 ctsio->kern_data_resid = 0;
5882                 ctsio->kern_rel_offset = 0;
5883                 ctsio->kern_sg_entries = 0;
5884                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5885                 ctsio->be_move_done = ctl_config_move_done;
5886                 ctl_datamove((union ctl_io *)ctsio);
5887
5888                 return (CTL_RETVAL_COMPLETE);
5889         }
5890
5891         ctl_done((union ctl_io *)ctsio);
5892
5893         return (CTL_RETVAL_COMPLETE);
5894 }
5895
5896 int
5897 ctl_write_same(struct ctl_scsiio *ctsio)
5898 {
5899         struct ctl_lun *lun;
5900         struct ctl_lba_len_flags *lbalen;
5901         uint64_t lba;
5902         uint32_t num_blocks;
5903         int len, retval;
5904         uint8_t byte2;
5905
5906         retval = CTL_RETVAL_COMPLETE;
5907
5908         CTL_DEBUG_PRINT(("ctl_write_same\n"));
5909
5910         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5911
5912         switch (ctsio->cdb[0]) {
5913         case WRITE_SAME_10: {
5914                 struct scsi_write_same_10 *cdb;
5915
5916                 cdb = (struct scsi_write_same_10 *)ctsio->cdb;
5917
5918                 lba = scsi_4btoul(cdb->addr);
5919                 num_blocks = scsi_2btoul(cdb->length);
5920                 byte2 = cdb->byte2;
5921                 break;
5922         }
5923         case WRITE_SAME_16: {
5924                 struct scsi_write_same_16 *cdb;
5925
5926                 cdb = (struct scsi_write_same_16 *)ctsio->cdb;
5927
5928                 lba = scsi_8btou64(cdb->addr);
5929                 num_blocks = scsi_4btoul(cdb->length);
5930                 byte2 = cdb->byte2;
5931                 break;
5932         }
5933         default:
5934                 /*
5935                  * We got a command we don't support.  This shouldn't
5936                  * happen, commands should be filtered out above us.
5937                  */
5938                 ctl_set_invalid_opcode(ctsio);
5939                 ctl_done((union ctl_io *)ctsio);
5940
5941                 return (CTL_RETVAL_COMPLETE);
5942                 break; /* NOTREACHED */
5943         }
5944
5945         /*
5946          * The first check is to make sure we're in bounds, the second
5947          * check is to catch wrap-around problems.  If the lba + num blocks
5948          * is less than the lba, then we've wrapped around and the block
5949          * range is invalid anyway.
5950          */
5951         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5952          || ((lba + num_blocks) < lba)) {
5953                 ctl_set_lba_out_of_range(ctsio);
5954                 ctl_done((union ctl_io *)ctsio);
5955                 return (CTL_RETVAL_COMPLETE);
5956         }
5957
5958         /* Zero number of blocks means "to the last logical block" */
5959         if (num_blocks == 0) {
5960                 if ((lun->be_lun->maxlba + 1) - lba > UINT32_MAX) {
5961                         ctl_set_invalid_field(ctsio,
5962                                               /*sks_valid*/ 0,
5963                                               /*command*/ 1,
5964                                               /*field*/ 0,
5965                                               /*bit_valid*/ 0,
5966                                               /*bit*/ 0);
5967                         ctl_done((union ctl_io *)ctsio);
5968                         return (CTL_RETVAL_COMPLETE);
5969                 }
5970                 num_blocks = (lun->be_lun->maxlba + 1) - lba;
5971         }
5972
5973         len = lun->be_lun->blocksize;
5974
5975         /*
5976          * If we've got a kernel request that hasn't been malloced yet,
5977          * malloc it and tell the caller the data buffer is here.
5978          */
5979         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5980                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5981                 ctsio->kern_data_len = len;
5982                 ctsio->kern_total_len = len;
5983                 ctsio->kern_data_resid = 0;
5984                 ctsio->kern_rel_offset = 0;
5985                 ctsio->kern_sg_entries = 0;
5986                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5987                 ctsio->be_move_done = ctl_config_move_done;
5988                 ctl_datamove((union ctl_io *)ctsio);
5989
5990                 return (CTL_RETVAL_COMPLETE);
5991         }
5992
5993         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5994         lbalen->lba = lba;
5995         lbalen->len = num_blocks;
5996         lbalen->flags = byte2;
5997         retval = lun->backend->config_write((union ctl_io *)ctsio);
5998
5999         return (retval);
6000 }
6001
6002 int
6003 ctl_unmap(struct ctl_scsiio *ctsio)
6004 {
6005         struct ctl_lun *lun;
6006         struct scsi_unmap *cdb;
6007         struct ctl_ptr_len_flags *ptrlen;
6008         struct scsi_unmap_header *hdr;
6009         struct scsi_unmap_desc *buf, *end;
6010         uint64_t lba;
6011         uint32_t num_blocks;
6012         int len, retval;
6013         uint8_t byte2;
6014
6015         retval = CTL_RETVAL_COMPLETE;
6016
6017         CTL_DEBUG_PRINT(("ctl_unmap\n"));
6018
6019         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6020         cdb = (struct scsi_unmap *)ctsio->cdb;
6021
6022         len = scsi_2btoul(cdb->length);
6023         byte2 = cdb->byte2;
6024
6025         /*
6026          * If we've got a kernel request that hasn't been malloced yet,
6027          * malloc it and tell the caller the data buffer is here.
6028          */
6029         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6030                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
6031                 ctsio->kern_data_len = len;
6032                 ctsio->kern_total_len = len;
6033                 ctsio->kern_data_resid = 0;
6034                 ctsio->kern_rel_offset = 0;
6035                 ctsio->kern_sg_entries = 0;
6036                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6037                 ctsio->be_move_done = ctl_config_move_done;
6038                 ctl_datamove((union ctl_io *)ctsio);
6039
6040                 return (CTL_RETVAL_COMPLETE);
6041         }
6042
6043         len = ctsio->kern_total_len - ctsio->kern_data_resid;
6044         hdr = (struct scsi_unmap_header *)ctsio->kern_data_ptr;
6045         if (len < sizeof (*hdr) ||
6046             len < (scsi_2btoul(hdr->length) + sizeof(hdr->length)) ||
6047             len < (scsi_2btoul(hdr->desc_length) + sizeof (*hdr)) ||
6048             scsi_2btoul(hdr->desc_length) % sizeof(*buf) != 0) {
6049                 ctl_set_invalid_field(ctsio,
6050                                       /*sks_valid*/ 0,
6051                                       /*command*/ 0,
6052                                       /*field*/ 0,
6053                                       /*bit_valid*/ 0,
6054                                       /*bit*/ 0);
6055                 ctl_done((union ctl_io *)ctsio);
6056                 return (CTL_RETVAL_COMPLETE);
6057         }
6058         len = scsi_2btoul(hdr->desc_length);
6059         buf = (struct scsi_unmap_desc *)(hdr + 1);
6060         end = buf + len / sizeof(*buf);
6061
6062         ptrlen = (struct ctl_ptr_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
6063         ptrlen->ptr = (void *)buf;
6064         ptrlen->len = len;
6065         ptrlen->flags = byte2;
6066
6067         for (; buf < end; buf++) {
6068                 lba = scsi_8btou64(buf->lba);
6069                 num_blocks = scsi_4btoul(buf->length);
6070                 if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
6071                  || ((lba + num_blocks) < lba)) {
6072                         ctl_set_lba_out_of_range(ctsio);
6073                         ctl_done((union ctl_io *)ctsio);
6074                         return (CTL_RETVAL_COMPLETE);
6075                 }
6076         }
6077
6078         retval = lun->backend->config_write((union ctl_io *)ctsio);
6079
6080         return (retval);
6081 }
6082
6083 /*
6084  * Note that this function currently doesn't actually do anything inside
6085  * CTL to enforce things if the DQue bit is turned on.
6086  *
6087  * Also note that this function can't be used in the default case, because
6088  * the DQue bit isn't set in the changeable mask for the control mode page
6089  * anyway.  This is just here as an example for how to implement a page
6090  * handler, and a placeholder in case we want to allow the user to turn
6091  * tagged queueing on and off.
6092  *
6093  * The D_SENSE bit handling is functional, however, and will turn
6094  * descriptor sense on and off for a given LUN.
6095  */
6096 int
6097 ctl_control_page_handler(struct ctl_scsiio *ctsio,
6098                          struct ctl_page_index *page_index, uint8_t *page_ptr)
6099 {
6100         struct scsi_control_page *current_cp, *saved_cp, *user_cp;
6101         struct ctl_lun *lun;
6102         struct ctl_softc *softc;
6103         int set_ua;
6104         uint32_t initidx;
6105
6106         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6107         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
6108         set_ua = 0;
6109
6110         user_cp = (struct scsi_control_page *)page_ptr;
6111         current_cp = (struct scsi_control_page *)
6112                 (page_index->page_data + (page_index->page_len *
6113                 CTL_PAGE_CURRENT));
6114         saved_cp = (struct scsi_control_page *)
6115                 (page_index->page_data + (page_index->page_len *
6116                 CTL_PAGE_SAVED));
6117
6118         softc = control_softc;
6119
6120         mtx_lock(&lun->lun_lock);
6121         if (((current_cp->rlec & SCP_DSENSE) == 0)
6122          && ((user_cp->rlec & SCP_DSENSE) != 0)) {
6123                 /*
6124                  * Descriptor sense is currently turned off and the user
6125                  * wants to turn it on.
6126                  */
6127                 current_cp->rlec |= SCP_DSENSE;
6128                 saved_cp->rlec |= SCP_DSENSE;
6129                 lun->flags |= CTL_LUN_SENSE_DESC;
6130                 set_ua = 1;
6131         } else if (((current_cp->rlec & SCP_DSENSE) != 0)
6132                 && ((user_cp->rlec & SCP_DSENSE) == 0)) {
6133                 /*
6134                  * Descriptor sense is currently turned on, and the user
6135                  * wants to turn it off.
6136                  */
6137                 current_cp->rlec &= ~SCP_DSENSE;
6138                 saved_cp->rlec &= ~SCP_DSENSE;
6139                 lun->flags &= ~CTL_LUN_SENSE_DESC;
6140                 set_ua = 1;
6141         }
6142         if (current_cp->queue_flags & SCP_QUEUE_DQUE) {
6143                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
6144 #ifdef NEEDTOPORT
6145                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6146                                     CTL_UNTAG_TO_UNTAG,
6147                                     csevent_LogType_Trace,
6148                                     csevent_Severity_Information,
6149                                     csevent_AlertLevel_Green,
6150                                     csevent_FRU_Firmware,
6151                                     csevent_FRU_Unknown,
6152                                     "Received untagged to untagged transition");
6153 #endif /* NEEDTOPORT */
6154                 } else {
6155 #ifdef NEEDTOPORT
6156                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6157                                     CTL_UNTAG_TO_TAG,
6158                                     csevent_LogType_ConfigChange,
6159                                     csevent_Severity_Information,
6160                                     csevent_AlertLevel_Green,
6161                                     csevent_FRU_Firmware,
6162                                     csevent_FRU_Unknown,
6163                                     "Received untagged to tagged "
6164                                     "queueing transition");
6165 #endif /* NEEDTOPORT */
6166
6167                         current_cp->queue_flags &= ~SCP_QUEUE_DQUE;
6168                         saved_cp->queue_flags &= ~SCP_QUEUE_DQUE;
6169                         set_ua = 1;
6170                 }
6171         } else {
6172                 if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
6173 #ifdef NEEDTOPORT
6174                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6175                                     CTL_TAG_TO_UNTAG,
6176                                     csevent_LogType_ConfigChange,
6177                                     csevent_Severity_Warning,
6178                                     csevent_AlertLevel_Yellow,
6179                                     csevent_FRU_Firmware,
6180                                     csevent_FRU_Unknown,
6181                                     "Received tagged queueing to untagged "
6182                                     "transition");
6183 #endif /* NEEDTOPORT */
6184
6185                         current_cp->queue_flags |= SCP_QUEUE_DQUE;
6186                         saved_cp->queue_flags |= SCP_QUEUE_DQUE;
6187                         set_ua = 1;
6188                 } else {
6189 #ifdef NEEDTOPORT
6190                         csevent_log(CSC_CTL | CSC_SHELF_SW |
6191                                     CTL_TAG_TO_TAG,
6192                                     csevent_LogType_Trace,
6193                                     csevent_Severity_Information,
6194                                     csevent_AlertLevel_Green,
6195                                     csevent_FRU_Firmware,
6196                                     csevent_FRU_Unknown,
6197                                     "Received tagged queueing to tagged "
6198                                     "queueing transition");
6199 #endif /* NEEDTOPORT */
6200                 }
6201         }
6202         if (set_ua != 0) {
6203                 int i;
6204                 /*
6205                  * Let other initiators know that the mode
6206                  * parameters for this LUN have changed.
6207                  */
6208                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
6209                         if (i == initidx)
6210                                 continue;
6211
6212                         lun->pending_sense[i].ua_pending |=
6213                                 CTL_UA_MODE_CHANGE;
6214                 }
6215         }
6216         mtx_unlock(&lun->lun_lock);
6217
6218         return (0);
6219 }
6220
6221 int
6222 ctl_power_sp_handler(struct ctl_scsiio *ctsio,
6223                      struct ctl_page_index *page_index, uint8_t *page_ptr)
6224 {
6225         return (0);
6226 }
6227
6228 int
6229 ctl_power_sp_sense_handler(struct ctl_scsiio *ctsio,
6230                            struct ctl_page_index *page_index, int pc)
6231 {
6232         struct copan_power_subpage *page;
6233
6234         page = (struct copan_power_subpage *)page_index->page_data +
6235                 (page_index->page_len * pc);
6236
6237         switch (pc) {
6238         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6239                 /*
6240                  * We don't update the changable bits for this page.
6241                  */
6242                 break;
6243         case SMS_PAGE_CTRL_CURRENT >> 6:
6244         case SMS_PAGE_CTRL_DEFAULT >> 6:
6245         case SMS_PAGE_CTRL_SAVED >> 6:
6246 #ifdef NEEDTOPORT
6247                 ctl_update_power_subpage(page);
6248 #endif
6249                 break;
6250         default:
6251 #ifdef NEEDTOPORT
6252                 EPRINT(0, "Invalid PC %d!!", pc);
6253 #endif
6254                 break;
6255         }
6256         return (0);
6257 }
6258
6259
6260 int
6261 ctl_aps_sp_handler(struct ctl_scsiio *ctsio,
6262                    struct ctl_page_index *page_index, uint8_t *page_ptr)
6263 {
6264         struct copan_aps_subpage *user_sp;
6265         struct copan_aps_subpage *current_sp;
6266         union ctl_modepage_info *modepage_info;
6267         struct ctl_softc *softc;
6268         struct ctl_lun *lun;
6269         int retval;
6270
6271         retval = CTL_RETVAL_COMPLETE;
6272         current_sp = (struct copan_aps_subpage *)(page_index->page_data +
6273                      (page_index->page_len * CTL_PAGE_CURRENT));
6274         softc = control_softc;
6275         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6276
6277         user_sp = (struct copan_aps_subpage *)page_ptr;
6278
6279         modepage_info = (union ctl_modepage_info *)
6280                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6281
6282         modepage_info->header.page_code = page_index->page_code & SMPH_PC_MASK;
6283         modepage_info->header.subpage = page_index->subpage;
6284         modepage_info->aps.lock_active = user_sp->lock_active;
6285
6286         mtx_lock(&softc->ctl_lock);
6287
6288         /*
6289          * If there is a request to lock the LUN and another LUN is locked
6290          * this is an error. If the requested LUN is already locked ignore
6291          * the request. If no LUN is locked attempt to lock it.
6292          * if there is a request to unlock the LUN and the LUN is currently
6293          * locked attempt to unlock it. Otherwise ignore the request. i.e.
6294          * if another LUN is locked or no LUN is locked.
6295          */
6296         if (user_sp->lock_active & APS_LOCK_ACTIVE) {
6297                 if (softc->aps_locked_lun == lun->lun) {
6298                         /*
6299                          * This LUN is already locked, so we're done.
6300                          */
6301                         retval = CTL_RETVAL_COMPLETE;
6302                 } else if (softc->aps_locked_lun == 0) {
6303                         /*
6304                          * No one has the lock, pass the request to the
6305                          * backend.
6306                          */
6307                         retval = lun->backend->config_write(
6308                                 (union ctl_io *)ctsio);
6309                 } else {
6310                         /*
6311                          * Someone else has the lock, throw out the request.
6312                          */
6313                         ctl_set_already_locked(ctsio);
6314                         free(ctsio->kern_data_ptr, M_CTL);
6315                         ctl_done((union ctl_io *)ctsio);
6316
6317                         /*
6318                          * Set the return value so that ctl_do_mode_select()
6319                          * won't try to complete the command.  We already
6320                          * completed it here.
6321                          */
6322                         retval = CTL_RETVAL_ERROR;
6323                 }
6324         } else if (softc->aps_locked_lun == lun->lun) {
6325                 /*
6326                  * This LUN is locked, so pass the unlock request to the
6327                  * backend.
6328                  */
6329                 retval = lun->backend->config_write((union ctl_io *)ctsio);
6330         }
6331         mtx_unlock(&softc->ctl_lock);
6332
6333         return (retval);
6334 }
6335
6336 int
6337 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6338                                 struct ctl_page_index *page_index,
6339                                 uint8_t *page_ptr)
6340 {
6341         uint8_t *c;
6342         int i;
6343
6344         c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6345         ctl_time_io_secs =
6346                 (c[0] << 8) |
6347                 (c[1] << 0) |
6348                 0;
6349         CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6350         printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6351         printf("page data:");
6352         for (i=0; i<8; i++)
6353                 printf(" %.2x",page_ptr[i]);
6354         printf("\n");
6355         return (0);
6356 }
6357
6358 int
6359 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6360                                struct ctl_page_index *page_index,
6361                                int pc)
6362 {
6363         struct copan_debugconf_subpage *page;
6364
6365         page = (struct copan_debugconf_subpage *)page_index->page_data +
6366                 (page_index->page_len * pc);
6367
6368         switch (pc) {
6369         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6370         case SMS_PAGE_CTRL_DEFAULT >> 6:
6371         case SMS_PAGE_CTRL_SAVED >> 6:
6372                 /*
6373                  * We don't update the changable or default bits for this page.
6374                  */
6375                 break;
6376         case SMS_PAGE_CTRL_CURRENT >> 6:
6377                 page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6378                 page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6379                 break;
6380         default:
6381 #ifdef NEEDTOPORT
6382                 EPRINT(0, "Invalid PC %d!!", pc);
6383 #endif /* NEEDTOPORT */
6384                 break;
6385         }
6386         return (0);
6387 }
6388
6389
6390 static int
6391 ctl_do_mode_select(union ctl_io *io)
6392 {
6393         struct scsi_mode_page_header *page_header;
6394         struct ctl_page_index *page_index;
6395         struct ctl_scsiio *ctsio;
6396         int control_dev, page_len;
6397         int page_len_offset, page_len_size;
6398         union ctl_modepage_info *modepage_info;
6399         struct ctl_lun *lun;
6400         int *len_left, *len_used;
6401         int retval, i;
6402
6403         ctsio = &io->scsiio;
6404         page_index = NULL;
6405         page_len = 0;
6406         retval = CTL_RETVAL_COMPLETE;
6407
6408         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6409
6410         if (lun->be_lun->lun_type != T_DIRECT)
6411                 control_dev = 1;
6412         else
6413                 control_dev = 0;
6414
6415         modepage_info = (union ctl_modepage_info *)
6416                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6417         len_left = &modepage_info->header.len_left;
6418         len_used = &modepage_info->header.len_used;
6419
6420 do_next_page:
6421
6422         page_header = (struct scsi_mode_page_header *)
6423                 (ctsio->kern_data_ptr + *len_used);
6424
6425         if (*len_left == 0) {
6426                 free(ctsio->kern_data_ptr, M_CTL);
6427                 ctl_set_success(ctsio);
6428                 ctl_done((union ctl_io *)ctsio);
6429                 return (CTL_RETVAL_COMPLETE);
6430         } else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6431
6432                 free(ctsio->kern_data_ptr, M_CTL);
6433                 ctl_set_param_len_error(ctsio);
6434                 ctl_done((union ctl_io *)ctsio);
6435                 return (CTL_RETVAL_COMPLETE);
6436
6437         } else if ((page_header->page_code & SMPH_SPF)
6438                 && (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6439
6440                 free(ctsio->kern_data_ptr, M_CTL);
6441                 ctl_set_param_len_error(ctsio);
6442                 ctl_done((union ctl_io *)ctsio);
6443                 return (CTL_RETVAL_COMPLETE);
6444         }
6445
6446
6447         /*
6448          * XXX KDM should we do something with the block descriptor?
6449          */
6450         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6451
6452                 if ((control_dev != 0)
6453                  && (lun->mode_pages.index[i].page_flags &
6454                      CTL_PAGE_FLAG_DISK_ONLY))
6455                         continue;
6456
6457                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6458                     (page_header->page_code & SMPH_PC_MASK))
6459                         continue;
6460
6461                 /*
6462                  * If neither page has a subpage code, then we've got a
6463                  * match.
6464                  */
6465                 if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6466                  && ((page_header->page_code & SMPH_SPF) == 0)) {
6467                         page_index = &lun->mode_pages.index[i];
6468                         page_len = page_header->page_length;
6469                         break;
6470                 }
6471
6472                 /*
6473                  * If both pages have subpages, then the subpage numbers
6474                  * have to match.
6475                  */
6476                 if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6477                   && (page_header->page_code & SMPH_SPF)) {
6478                         struct scsi_mode_page_header_sp *sph;
6479
6480                         sph = (struct scsi_mode_page_header_sp *)page_header;
6481
6482                         if (lun->mode_pages.index[i].subpage ==
6483                             sph->subpage) {
6484                                 page_index = &lun->mode_pages.index[i];
6485                                 page_len = scsi_2btoul(sph->page_length);
6486                                 break;
6487                         }
6488                 }
6489         }
6490
6491         /*
6492          * If we couldn't find the page, or if we don't have a mode select
6493          * handler for it, send back an error to the user.
6494          */
6495         if ((page_index == NULL)
6496          || (page_index->select_handler == NULL)) {
6497                 ctl_set_invalid_field(ctsio,
6498                                       /*sks_valid*/ 1,
6499                                       /*command*/ 0,
6500                                       /*field*/ *len_used,
6501                                       /*bit_valid*/ 0,
6502                                       /*bit*/ 0);
6503                 free(ctsio->kern_data_ptr, M_CTL);
6504                 ctl_done((union ctl_io *)ctsio);
6505                 return (CTL_RETVAL_COMPLETE);
6506         }
6507
6508         if (page_index->page_code & SMPH_SPF) {
6509                 page_len_offset = 2;
6510                 page_len_size = 2;
6511         } else {
6512                 page_len_size = 1;
6513                 page_len_offset = 1;
6514         }
6515
6516         /*
6517          * If the length the initiator gives us isn't the one we specify in
6518          * the mode page header, or if they didn't specify enough data in
6519          * the CDB to avoid truncating this page, kick out the request.
6520          */
6521         if ((page_len != (page_index->page_len - page_len_offset -
6522                           page_len_size))
6523          || (*len_left < page_index->page_len)) {
6524
6525
6526                 ctl_set_invalid_field(ctsio,
6527                                       /*sks_valid*/ 1,
6528                                       /*command*/ 0,
6529                                       /*field*/ *len_used + page_len_offset,
6530                                       /*bit_valid*/ 0,
6531                                       /*bit*/ 0);
6532                 free(ctsio->kern_data_ptr, M_CTL);
6533                 ctl_done((union ctl_io *)ctsio);
6534                 return (CTL_RETVAL_COMPLETE);
6535         }
6536
6537         /*
6538          * Run through the mode page, checking to make sure that the bits
6539          * the user changed are actually legal for him to change.
6540          */
6541         for (i = 0; i < page_index->page_len; i++) {
6542                 uint8_t *user_byte, *change_mask, *current_byte;
6543                 int bad_bit;
6544                 int j;
6545
6546                 user_byte = (uint8_t *)page_header + i;
6547                 change_mask = page_index->page_data +
6548                               (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6549                 current_byte = page_index->page_data +
6550                                (page_index->page_len * CTL_PAGE_CURRENT) + i;
6551
6552                 /*
6553                  * Check to see whether the user set any bits in this byte
6554                  * that he is not allowed to set.
6555                  */
6556                 if ((*user_byte & ~(*change_mask)) ==
6557                     (*current_byte & ~(*change_mask)))
6558                         continue;
6559
6560                 /*
6561                  * Go through bit by bit to determine which one is illegal.
6562                  */
6563                 bad_bit = 0;
6564                 for (j = 7; j >= 0; j--) {
6565                         if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6566                             (((1 << i) & ~(*change_mask)) & *current_byte)) {
6567                                 bad_bit = i;
6568                                 break;
6569                         }
6570                 }
6571                 ctl_set_invalid_field(ctsio,
6572                                       /*sks_valid*/ 1,
6573                                       /*command*/ 0,
6574                                       /*field*/ *len_used + i,
6575                                       /*bit_valid*/ 1,
6576                                       /*bit*/ bad_bit);
6577                 free(ctsio->kern_data_ptr, M_CTL);
6578                 ctl_done((union ctl_io *)ctsio);
6579                 return (CTL_RETVAL_COMPLETE);
6580         }
6581
6582         /*
6583          * Decrement these before we call the page handler, since we may
6584          * end up getting called back one way or another before the handler
6585          * returns to this context.
6586          */
6587         *len_left -= page_index->page_len;
6588         *len_used += page_index->page_len;
6589
6590         retval = page_index->select_handler(ctsio, page_index,
6591                                             (uint8_t *)page_header);
6592
6593         /*
6594          * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6595          * wait until this queued command completes to finish processing
6596          * the mode page.  If it returns anything other than
6597          * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6598          * already set the sense information, freed the data pointer, and
6599          * completed the io for us.
6600          */
6601         if (retval != CTL_RETVAL_COMPLETE)
6602                 goto bailout_no_done;
6603
6604         /*
6605          * If the initiator sent us more than one page, parse the next one.
6606          */
6607         if (*len_left > 0)
6608                 goto do_next_page;
6609
6610         ctl_set_success(ctsio);
6611         free(ctsio->kern_data_ptr, M_CTL);
6612         ctl_done((union ctl_io *)ctsio);
6613
6614 bailout_no_done:
6615
6616         return (CTL_RETVAL_COMPLETE);
6617
6618 }
6619
6620 int
6621 ctl_mode_select(struct ctl_scsiio *ctsio)
6622 {
6623         int param_len, pf, sp;
6624         int header_size, bd_len;
6625         int len_left, len_used;
6626         struct ctl_page_index *page_index;
6627         struct ctl_lun *lun;
6628         int control_dev, page_len;
6629         union ctl_modepage_info *modepage_info;
6630         int retval;
6631
6632         pf = 0;
6633         sp = 0;
6634         page_len = 0;
6635         len_used = 0;
6636         len_left = 0;
6637         retval = 0;
6638         bd_len = 0;
6639         page_index = NULL;
6640
6641         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6642
6643         if (lun->be_lun->lun_type != T_DIRECT)
6644                 control_dev = 1;
6645         else
6646                 control_dev = 0;
6647
6648         switch (ctsio->cdb[0]) {
6649         case MODE_SELECT_6: {
6650                 struct scsi_mode_select_6 *cdb;
6651
6652                 cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6653
6654                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6655                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6656
6657                 param_len = cdb->length;
6658                 header_size = sizeof(struct scsi_mode_header_6);
6659                 break;
6660         }
6661         case MODE_SELECT_10: {
6662                 struct scsi_mode_select_10 *cdb;
6663
6664                 cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6665
6666                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6667                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6668
6669                 param_len = scsi_2btoul(cdb->length);
6670                 header_size = sizeof(struct scsi_mode_header_10);
6671                 break;
6672         }
6673         default:
6674                 ctl_set_invalid_opcode(ctsio);
6675                 ctl_done((union ctl_io *)ctsio);
6676                 return (CTL_RETVAL_COMPLETE);
6677                 break; /* NOTREACHED */
6678         }
6679
6680         /*
6681          * From SPC-3:
6682          * "A parameter list length of zero indicates that the Data-Out Buffer
6683          * shall be empty. This condition shall not be considered as an error."
6684          */
6685         if (param_len == 0) {
6686                 ctl_set_success(ctsio);
6687                 ctl_done((union ctl_io *)ctsio);
6688                 return (CTL_RETVAL_COMPLETE);
6689         }
6690
6691         /*
6692          * Since we'll hit this the first time through, prior to
6693          * allocation, we don't need to free a data buffer here.
6694          */
6695         if (param_len < header_size) {
6696                 ctl_set_param_len_error(ctsio);
6697                 ctl_done((union ctl_io *)ctsio);
6698                 return (CTL_RETVAL_COMPLETE);
6699         }
6700
6701         /*
6702          * Allocate the data buffer and grab the user's data.  In theory,
6703          * we shouldn't have to sanity check the parameter list length here
6704          * because the maximum size is 64K.  We should be able to malloc
6705          * that much without too many problems.
6706          */
6707         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6708                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6709                 ctsio->kern_data_len = param_len;
6710                 ctsio->kern_total_len = param_len;
6711                 ctsio->kern_data_resid = 0;
6712                 ctsio->kern_rel_offset = 0;
6713                 ctsio->kern_sg_entries = 0;
6714                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6715                 ctsio->be_move_done = ctl_config_move_done;
6716                 ctl_datamove((union ctl_io *)ctsio);
6717
6718                 return (CTL_RETVAL_COMPLETE);
6719         }
6720
6721         switch (ctsio->cdb[0]) {
6722         case MODE_SELECT_6: {
6723                 struct scsi_mode_header_6 *mh6;
6724
6725                 mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6726                 bd_len = mh6->blk_desc_len;
6727                 break;
6728         }
6729         case MODE_SELECT_10: {
6730                 struct scsi_mode_header_10 *mh10;
6731
6732                 mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6733                 bd_len = scsi_2btoul(mh10->blk_desc_len);
6734                 break;
6735         }
6736         default:
6737                 panic("Invalid CDB type %#x", ctsio->cdb[0]);
6738                 break;
6739         }
6740
6741         if (param_len < (header_size + bd_len)) {
6742                 free(ctsio->kern_data_ptr, M_CTL);
6743                 ctl_set_param_len_error(ctsio);
6744                 ctl_done((union ctl_io *)ctsio);
6745                 return (CTL_RETVAL_COMPLETE);
6746         }
6747
6748         /*
6749          * Set the IO_CONT flag, so that if this I/O gets passed to
6750          * ctl_config_write_done(), it'll get passed back to
6751          * ctl_do_mode_select() for further processing, or completion if
6752          * we're all done.
6753          */
6754         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6755         ctsio->io_cont = ctl_do_mode_select;
6756
6757         modepage_info = (union ctl_modepage_info *)
6758                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6759
6760         memset(modepage_info, 0, sizeof(*modepage_info));
6761
6762         len_left = param_len - header_size - bd_len;
6763         len_used = header_size + bd_len;
6764
6765         modepage_info->header.len_left = len_left;
6766         modepage_info->header.len_used = len_used;
6767
6768         return (ctl_do_mode_select((union ctl_io *)ctsio));
6769 }
6770
6771 int
6772 ctl_mode_sense(struct ctl_scsiio *ctsio)
6773 {
6774         struct ctl_lun *lun;
6775         int pc, page_code, dbd, llba, subpage;
6776         int alloc_len, page_len, header_len, total_len;
6777         struct scsi_mode_block_descr *block_desc;
6778         struct ctl_page_index *page_index;
6779         int control_dev;
6780
6781         dbd = 0;
6782         llba = 0;
6783         block_desc = NULL;
6784         page_index = NULL;
6785
6786         CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6787
6788         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6789
6790         if (lun->be_lun->lun_type != T_DIRECT)
6791                 control_dev = 1;
6792         else
6793                 control_dev = 0;
6794
6795         if (lun->flags & CTL_LUN_PR_RESERVED) {
6796                 uint32_t residx;
6797
6798                 /*
6799                  * XXX KDM need a lock here.
6800                  */
6801                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
6802                 if ((lun->res_type == SPR_TYPE_EX_AC
6803                   && residx != lun->pr_res_idx)
6804                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
6805                    || lun->res_type == SPR_TYPE_EX_AC_AR)
6806                   && !lun->per_res[residx].registered)) {
6807                         ctl_set_reservation_conflict(ctsio);
6808                         ctl_done((union ctl_io *)ctsio);
6809                         return (CTL_RETVAL_COMPLETE);
6810                 }
6811         }
6812
6813         switch (ctsio->cdb[0]) {
6814         case MODE_SENSE_6: {
6815                 struct scsi_mode_sense_6 *cdb;
6816
6817                 cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6818
6819                 header_len = sizeof(struct scsi_mode_hdr_6);
6820                 if (cdb->byte2 & SMS_DBD)
6821                         dbd = 1;
6822                 else
6823                         header_len += sizeof(struct scsi_mode_block_descr);
6824
6825                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6826                 page_code = cdb->page & SMS_PAGE_CODE;
6827                 subpage = cdb->subpage;
6828                 alloc_len = cdb->length;
6829                 break;
6830         }
6831         case MODE_SENSE_10: {
6832                 struct scsi_mode_sense_10 *cdb;
6833
6834                 cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6835
6836                 header_len = sizeof(struct scsi_mode_hdr_10);
6837
6838                 if (cdb->byte2 & SMS_DBD)
6839                         dbd = 1;
6840                 else
6841                         header_len += sizeof(struct scsi_mode_block_descr);
6842                 if (cdb->byte2 & SMS10_LLBAA)
6843                         llba = 1;
6844                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6845                 page_code = cdb->page & SMS_PAGE_CODE;
6846                 subpage = cdb->subpage;
6847                 alloc_len = scsi_2btoul(cdb->length);
6848                 break;
6849         }
6850         default:
6851                 ctl_set_invalid_opcode(ctsio);
6852                 ctl_done((union ctl_io *)ctsio);
6853                 return (CTL_RETVAL_COMPLETE);
6854                 break; /* NOTREACHED */
6855         }
6856
6857         /*
6858          * We have to make a first pass through to calculate the size of
6859          * the pages that match the user's query.  Then we allocate enough
6860          * memory to hold it, and actually copy the data into the buffer.
6861          */
6862         switch (page_code) {
6863         case SMS_ALL_PAGES_PAGE: {
6864                 int i;
6865
6866                 page_len = 0;
6867
6868                 /*
6869                  * At the moment, values other than 0 and 0xff here are
6870                  * reserved according to SPC-3.
6871                  */
6872                 if ((subpage != SMS_SUBPAGE_PAGE_0)
6873                  && (subpage != SMS_SUBPAGE_ALL)) {
6874                         ctl_set_invalid_field(ctsio,
6875                                               /*sks_valid*/ 1,
6876                                               /*command*/ 1,
6877                                               /*field*/ 3,
6878                                               /*bit_valid*/ 0,
6879                                               /*bit*/ 0);
6880                         ctl_done((union ctl_io *)ctsio);
6881                         return (CTL_RETVAL_COMPLETE);
6882                 }
6883
6884                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6885                         if ((control_dev != 0)
6886                          && (lun->mode_pages.index[i].page_flags &
6887                              CTL_PAGE_FLAG_DISK_ONLY))
6888                                 continue;
6889
6890                         /*
6891                          * We don't use this subpage if the user didn't
6892                          * request all subpages.
6893                          */
6894                         if ((lun->mode_pages.index[i].subpage != 0)
6895                          && (subpage == SMS_SUBPAGE_PAGE_0))
6896                                 continue;
6897
6898 #if 0
6899                         printf("found page %#x len %d\n",
6900                                lun->mode_pages.index[i].page_code &
6901                                SMPH_PC_MASK,
6902                                lun->mode_pages.index[i].page_len);
6903 #endif
6904                         page_len += lun->mode_pages.index[i].page_len;
6905                 }
6906                 break;
6907         }
6908         default: {
6909                 int i;
6910
6911                 page_len = 0;
6912
6913                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6914                         /* Look for the right page code */
6915                         if ((lun->mode_pages.index[i].page_code &
6916                              SMPH_PC_MASK) != page_code)
6917                                 continue;
6918
6919                         /* Look for the right subpage or the subpage wildcard*/
6920                         if ((lun->mode_pages.index[i].subpage != subpage)
6921                          && (subpage != SMS_SUBPAGE_ALL))
6922                                 continue;
6923
6924                         /* Make sure the page is supported for this dev type */
6925                         if ((control_dev != 0)
6926                          && (lun->mode_pages.index[i].page_flags &
6927                              CTL_PAGE_FLAG_DISK_ONLY))
6928                                 continue;
6929
6930 #if 0
6931                         printf("found page %#x len %d\n",
6932                                lun->mode_pages.index[i].page_code &
6933                                SMPH_PC_MASK,
6934                                lun->mode_pages.index[i].page_len);
6935 #endif
6936
6937                         page_len += lun->mode_pages.index[i].page_len;
6938                 }
6939
6940                 if (page_len == 0) {
6941                         ctl_set_invalid_field(ctsio,
6942                                               /*sks_valid*/ 1,
6943                                               /*command*/ 1,
6944                                               /*field*/ 2,
6945                                               /*bit_valid*/ 1,
6946                                               /*bit*/ 5);
6947                         ctl_done((union ctl_io *)ctsio);
6948                         return (CTL_RETVAL_COMPLETE);
6949                 }
6950                 break;
6951         }
6952         }
6953
6954         total_len = header_len + page_len;
6955 #if 0
6956         printf("header_len = %d, page_len = %d, total_len = %d\n",
6957                header_len, page_len, total_len);
6958 #endif
6959
6960         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6961         ctsio->kern_sg_entries = 0;
6962         ctsio->kern_data_resid = 0;
6963         ctsio->kern_rel_offset = 0;
6964         if (total_len < alloc_len) {
6965                 ctsio->residual = alloc_len - total_len;
6966                 ctsio->kern_data_len = total_len;
6967                 ctsio->kern_total_len = total_len;
6968         } else {
6969                 ctsio->residual = 0;
6970                 ctsio->kern_data_len = alloc_len;
6971                 ctsio->kern_total_len = alloc_len;
6972         }
6973
6974         switch (ctsio->cdb[0]) {
6975         case MODE_SENSE_6: {
6976                 struct scsi_mode_hdr_6 *header;
6977
6978                 header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6979
6980                 header->datalen = ctl_min(total_len - 1, 254);
6981
6982                 if (dbd)
6983                         header->block_descr_len = 0;
6984                 else
6985                         header->block_descr_len =
6986                                 sizeof(struct scsi_mode_block_descr);
6987                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6988                 break;
6989         }
6990         case MODE_SENSE_10: {
6991                 struct scsi_mode_hdr_10 *header;
6992                 int datalen;
6993
6994                 header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6995
6996                 datalen = ctl_min(total_len - 2, 65533);
6997                 scsi_ulto2b(datalen, header->datalen);
6998                 if (dbd)
6999                         scsi_ulto2b(0, header->block_descr_len);
7000                 else
7001                         scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
7002                                     header->block_descr_len);
7003                 block_desc = (struct scsi_mode_block_descr *)&header[1];
7004                 break;
7005         }
7006         default:
7007                 panic("invalid CDB type %#x", ctsio->cdb[0]);
7008                 break; /* NOTREACHED */
7009         }
7010
7011         /*
7012          * If we've got a disk, use its blocksize in the block
7013          * descriptor.  Otherwise, just set it to 0.
7014          */
7015         if (dbd == 0) {
7016                 if (control_dev != 0)
7017                         scsi_ulto3b(lun->be_lun->blocksize,
7018                                     block_desc->block_len);
7019                 else
7020                         scsi_ulto3b(0, block_desc->block_len);
7021         }
7022
7023         switch (page_code) {
7024         case SMS_ALL_PAGES_PAGE: {
7025                 int i, data_used;
7026
7027                 data_used = header_len;
7028                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
7029                         struct ctl_page_index *page_index;
7030
7031                         page_index = &lun->mode_pages.index[i];
7032
7033                         if ((control_dev != 0)
7034                          && (page_index->page_flags &
7035                             CTL_PAGE_FLAG_DISK_ONLY))
7036                                 continue;
7037
7038                         /*
7039                          * We don't use this subpage if the user didn't
7040                          * request all subpages.  We already checked (above)
7041                          * to make sure the user only specified a subpage
7042                          * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
7043                          */
7044                         if ((page_index->subpage != 0)
7045                          && (subpage == SMS_SUBPAGE_PAGE_0))
7046                                 continue;
7047
7048                         /*
7049                          * Call the handler, if it exists, to update the
7050                          * page to the latest values.
7051                          */
7052                         if (page_index->sense_handler != NULL)
7053                                 page_index->sense_handler(ctsio, page_index,pc);
7054
7055                         memcpy(ctsio->kern_data_ptr + data_used,
7056                                page_index->page_data +
7057                                (page_index->page_len * pc),
7058                                page_index->page_len);
7059                         data_used += page_index->page_len;
7060                 }
7061                 break;
7062         }
7063         default: {
7064                 int i, data_used;
7065
7066                 data_used = header_len;
7067
7068                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
7069                         struct ctl_page_index *page_index;
7070
7071                         page_index = &lun->mode_pages.index[i];
7072
7073                         /* Look for the right page code */
7074                         if ((page_index->page_code & SMPH_PC_MASK) != page_code)
7075                                 continue;
7076
7077                         /* Look for the right subpage or the subpage wildcard*/
7078                         if ((page_index->subpage != subpage)
7079                          && (subpage != SMS_SUBPAGE_ALL))
7080                                 continue;
7081
7082                         /* Make sure the page is supported for this dev type */
7083                         if ((control_dev != 0)
7084                          && (page_index->page_flags &
7085                              CTL_PAGE_FLAG_DISK_ONLY))
7086                                 continue;
7087
7088                         /*
7089                          * Call the handler, if it exists, to update the
7090                          * page to the latest values.
7091                          */
7092                         if (page_index->sense_handler != NULL)
7093                                 page_index->sense_handler(ctsio, page_index,pc);
7094
7095                         memcpy(ctsio->kern_data_ptr + data_used,
7096                                page_index->page_data +
7097                                (page_index->page_len * pc),
7098                                page_index->page_len);
7099                         data_used += page_index->page_len;
7100                 }
7101                 break;
7102         }
7103         }
7104
7105         ctsio->scsi_status = SCSI_STATUS_OK;
7106
7107         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7108         ctsio->be_move_done = ctl_config_move_done;
7109         ctl_datamove((union ctl_io *)ctsio);
7110
7111         return (CTL_RETVAL_COMPLETE);
7112 }
7113
7114 int
7115 ctl_read_capacity(struct ctl_scsiio *ctsio)
7116 {
7117         struct scsi_read_capacity *cdb;
7118         struct scsi_read_capacity_data *data;
7119         struct ctl_lun *lun;
7120         uint32_t lba;
7121
7122         CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
7123
7124         cdb = (struct scsi_read_capacity *)ctsio->cdb;
7125
7126         lba = scsi_4btoul(cdb->addr);
7127         if (((cdb->pmi & SRC_PMI) == 0)
7128          && (lba != 0)) {
7129                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7130                                       /*sks_valid*/ 1,
7131                                       /*command*/ 1,
7132                                       /*field*/ 2,
7133                                       /*bit_valid*/ 0,
7134                                       /*bit*/ 0);
7135                 ctl_done((union ctl_io *)ctsio);
7136                 return (CTL_RETVAL_COMPLETE);
7137         }
7138
7139         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7140
7141         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7142         data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
7143         ctsio->residual = 0;
7144         ctsio->kern_data_len = sizeof(*data);
7145         ctsio->kern_total_len = sizeof(*data);
7146         ctsio->kern_data_resid = 0;
7147         ctsio->kern_rel_offset = 0;
7148         ctsio->kern_sg_entries = 0;
7149
7150         /*
7151          * If the maximum LBA is greater than 0xfffffffe, the user must
7152          * issue a SERVICE ACTION IN (16) command, with the read capacity
7153          * serivce action set.
7154          */
7155         if (lun->be_lun->maxlba > 0xfffffffe)
7156                 scsi_ulto4b(0xffffffff, data->addr);
7157         else
7158                 scsi_ulto4b(lun->be_lun->maxlba, data->addr);
7159
7160         /*
7161          * XXX KDM this may not be 512 bytes...
7162          */
7163         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7164
7165         ctsio->scsi_status = SCSI_STATUS_OK;
7166
7167         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7168         ctsio->be_move_done = ctl_config_move_done;
7169         ctl_datamove((union ctl_io *)ctsio);
7170
7171         return (CTL_RETVAL_COMPLETE);
7172 }
7173
7174 int
7175 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
7176 {
7177         struct scsi_read_capacity_16 *cdb;
7178         struct scsi_read_capacity_data_long *data;
7179         struct ctl_lun *lun;
7180         uint64_t lba;
7181         uint32_t alloc_len;
7182
7183         CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
7184
7185         cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
7186
7187         alloc_len = scsi_4btoul(cdb->alloc_len);
7188         lba = scsi_8btou64(cdb->addr);
7189
7190         if ((cdb->reladr & SRC16_PMI)
7191          && (lba != 0)) {
7192                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7193                                       /*sks_valid*/ 1,
7194                                       /*command*/ 1,
7195                                       /*field*/ 2,
7196                                       /*bit_valid*/ 0,
7197                                       /*bit*/ 0);
7198                 ctl_done((union ctl_io *)ctsio);
7199                 return (CTL_RETVAL_COMPLETE);
7200         }
7201
7202         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7203
7204         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7205         data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
7206
7207         if (sizeof(*data) < alloc_len) {
7208                 ctsio->residual = alloc_len - sizeof(*data);
7209                 ctsio->kern_data_len = sizeof(*data);
7210                 ctsio->kern_total_len = sizeof(*data);
7211         } else {
7212                 ctsio->residual = 0;
7213                 ctsio->kern_data_len = alloc_len;
7214                 ctsio->kern_total_len = alloc_len;
7215         }
7216         ctsio->kern_data_resid = 0;
7217         ctsio->kern_rel_offset = 0;
7218         ctsio->kern_sg_entries = 0;
7219
7220         scsi_u64to8b(lun->be_lun->maxlba, data->addr);
7221         /* XXX KDM this may not be 512 bytes... */
7222         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7223         data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
7224         scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
7225         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
7226                 data->lalba_lbp[0] |= SRC16_LBPME;
7227
7228         ctsio->scsi_status = SCSI_STATUS_OK;
7229
7230         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7231         ctsio->be_move_done = ctl_config_move_done;
7232         ctl_datamove((union ctl_io *)ctsio);
7233
7234         return (CTL_RETVAL_COMPLETE);
7235 }
7236
7237 int
7238 ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
7239 {
7240         struct scsi_maintenance_in *cdb;
7241         int retval;
7242         int alloc_len, ext, total_len = 0, g, p, pc, pg;
7243         int num_target_port_groups, num_target_ports, single;
7244         struct ctl_lun *lun;
7245         struct ctl_softc *softc;
7246         struct ctl_port *port;
7247         struct scsi_target_group_data *rtg_ptr;
7248         struct scsi_target_group_data_extended *rtg_ext_ptr;
7249         struct scsi_target_port_group_descriptor *tpg_desc;
7250
7251         CTL_DEBUG_PRINT(("ctl_report_tagret_port_groups\n"));
7252
7253         cdb = (struct scsi_maintenance_in *)ctsio->cdb;
7254         softc = control_softc;
7255         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7256
7257         retval = CTL_RETVAL_COMPLETE;
7258
7259         switch (cdb->byte2 & STG_PDF_MASK) {
7260         case STG_PDF_LENGTH:
7261                 ext = 0;
7262                 break;
7263         case STG_PDF_EXTENDED:
7264                 ext = 1;
7265                 break;
7266         default:
7267                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7268                                       /*sks_valid*/ 1,
7269                                       /*command*/ 1,
7270                                       /*field*/ 2,
7271                                       /*bit_valid*/ 1,
7272                                       /*bit*/ 5);
7273                 ctl_done((union ctl_io *)ctsio);
7274                 return(retval);
7275         }
7276
7277         single = ctl_is_single;
7278         if (single)
7279                 num_target_port_groups = 1;
7280         else
7281                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
7282         num_target_ports = 0;
7283         mtx_lock(&softc->ctl_lock);
7284         STAILQ_FOREACH(port, &softc->port_list, links) {
7285                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7286                         continue;
7287                 if (ctl_map_lun_back(port->targ_port, lun->lun) >= CTL_MAX_LUNS)
7288                         continue;
7289                 num_target_ports++;
7290         }
7291         mtx_unlock(&softc->ctl_lock);
7292
7293         if (ext)
7294                 total_len = sizeof(struct scsi_target_group_data_extended);
7295         else
7296                 total_len = sizeof(struct scsi_target_group_data);
7297         total_len += sizeof(struct scsi_target_port_group_descriptor) *
7298                 num_target_port_groups +
7299             sizeof(struct scsi_target_port_descriptor) *
7300                 num_target_ports * num_target_port_groups;
7301
7302         alloc_len = scsi_4btoul(cdb->length);
7303
7304         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7305
7306         ctsio->kern_sg_entries = 0;
7307
7308         if (total_len < alloc_len) {
7309                 ctsio->residual = alloc_len - total_len;
7310                 ctsio->kern_data_len = total_len;
7311                 ctsio->kern_total_len = total_len;
7312         } else {
7313                 ctsio->residual = 0;
7314                 ctsio->kern_data_len = alloc_len;
7315                 ctsio->kern_total_len = alloc_len;
7316         }
7317         ctsio->kern_data_resid = 0;
7318         ctsio->kern_rel_offset = 0;
7319
7320         if (ext) {
7321                 rtg_ext_ptr = (struct scsi_target_group_data_extended *)
7322                     ctsio->kern_data_ptr;
7323                 scsi_ulto4b(total_len - 4, rtg_ext_ptr->length);
7324                 rtg_ext_ptr->format_type = 0x10;
7325                 rtg_ext_ptr->implicit_transition_time = 0;
7326                 tpg_desc = &rtg_ext_ptr->groups[0];
7327         } else {
7328                 rtg_ptr = (struct scsi_target_group_data *)
7329                     ctsio->kern_data_ptr;
7330                 scsi_ulto4b(total_len - 4, rtg_ptr->length);
7331                 tpg_desc = &rtg_ptr->groups[0];
7332         }
7333
7334         pg = ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS;
7335         mtx_lock(&softc->ctl_lock);
7336         for (g = 0; g < num_target_port_groups; g++) {
7337                 if (g == pg)
7338                         tpg_desc->pref_state = TPG_PRIMARY |
7339                             TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7340                 else
7341                         tpg_desc->pref_state =
7342                             TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7343                 tpg_desc->support = TPG_AO_SUP;
7344                 if (!single)
7345                         tpg_desc->support |= TPG_AN_SUP;
7346                 scsi_ulto2b(g + 1, tpg_desc->target_port_group);
7347                 tpg_desc->status = TPG_IMPLICIT;
7348                 pc = 0;
7349                 STAILQ_FOREACH(port, &softc->port_list, links) {
7350                         if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7351                                 continue;
7352                         if (ctl_map_lun_back(port->targ_port, lun->lun) >=
7353                             CTL_MAX_LUNS)
7354                                 continue;
7355                         p = port->targ_port % CTL_MAX_PORTS + g * CTL_MAX_PORTS;
7356                         scsi_ulto2b(p, tpg_desc->descriptors[pc].
7357                             relative_target_port_identifier);
7358                         pc++;
7359                 }
7360                 tpg_desc->target_port_count = pc;
7361                 tpg_desc = (struct scsi_target_port_group_descriptor *)
7362                     &tpg_desc->descriptors[pc];
7363         }
7364         mtx_unlock(&softc->ctl_lock);
7365
7366         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7367         ctsio->be_move_done = ctl_config_move_done;
7368
7369         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7370                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7371                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7372                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7373                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7374
7375         ctl_datamove((union ctl_io *)ctsio);
7376         return(retval);
7377 }
7378
7379 int
7380 ctl_report_supported_opcodes(struct ctl_scsiio *ctsio)
7381 {
7382         struct ctl_lun *lun;
7383         struct scsi_report_supported_opcodes *cdb;
7384         const struct ctl_cmd_entry *entry, *sentry;
7385         struct scsi_report_supported_opcodes_all *all;
7386         struct scsi_report_supported_opcodes_descr *descr;
7387         struct scsi_report_supported_opcodes_one *one;
7388         int retval;
7389         int alloc_len, total_len;
7390         int opcode, service_action, i, j, num;
7391
7392         CTL_DEBUG_PRINT(("ctl_report_supported_opcodes\n"));
7393
7394         cdb = (struct scsi_report_supported_opcodes *)ctsio->cdb;
7395         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7396
7397         retval = CTL_RETVAL_COMPLETE;
7398
7399         opcode = cdb->requested_opcode;
7400         service_action = scsi_2btoul(cdb->requested_service_action);
7401         switch (cdb->options & RSO_OPTIONS_MASK) {
7402         case RSO_OPTIONS_ALL:
7403                 num = 0;
7404                 for (i = 0; i < 256; i++) {
7405                         entry = &ctl_cmd_table[i];
7406                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7407                                 for (j = 0; j < 32; j++) {
7408                                         sentry = &((const struct ctl_cmd_entry *)
7409                                             entry->execute)[j];
7410                                         if (ctl_cmd_applicable(
7411                                             lun->be_lun->lun_type, sentry))
7412                                                 num++;
7413                                 }
7414                         } else {
7415                                 if (ctl_cmd_applicable(lun->be_lun->lun_type,
7416                                     entry))
7417                                         num++;
7418                         }
7419                 }
7420                 total_len = sizeof(struct scsi_report_supported_opcodes_all) +
7421                     num * sizeof(struct scsi_report_supported_opcodes_descr);
7422                 break;
7423         case RSO_OPTIONS_OC:
7424                 if (ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) {
7425                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7426                                               /*sks_valid*/ 1,
7427                                               /*command*/ 1,
7428                                               /*field*/ 2,
7429                                               /*bit_valid*/ 1,
7430                                               /*bit*/ 2);
7431                         ctl_done((union ctl_io *)ctsio);
7432                         return (CTL_RETVAL_COMPLETE);
7433                 }
7434                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7435                 break;
7436         case RSO_OPTIONS_OC_SA:
7437                 if ((ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) == 0 ||
7438                     service_action >= 32) {
7439                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7440                                               /*sks_valid*/ 1,
7441                                               /*command*/ 1,
7442                                               /*field*/ 2,
7443                                               /*bit_valid*/ 1,
7444                                               /*bit*/ 2);
7445                         ctl_done((union ctl_io *)ctsio);
7446                         return (CTL_RETVAL_COMPLETE);
7447                 }
7448                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7449                 break;
7450         default:
7451                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7452                                       /*sks_valid*/ 1,
7453                                       /*command*/ 1,
7454                                       /*field*/ 2,
7455                                       /*bit_valid*/ 1,
7456                                       /*bit*/ 2);
7457                 ctl_done((union ctl_io *)ctsio);
7458                 return (CTL_RETVAL_COMPLETE);
7459         }
7460
7461         alloc_len = scsi_4btoul(cdb->length);
7462
7463         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7464
7465         ctsio->kern_sg_entries = 0;
7466
7467         if (total_len < alloc_len) {
7468                 ctsio->residual = alloc_len - total_len;
7469                 ctsio->kern_data_len = total_len;
7470                 ctsio->kern_total_len = total_len;
7471         } else {
7472                 ctsio->residual = 0;
7473                 ctsio->kern_data_len = alloc_len;
7474                 ctsio->kern_total_len = alloc_len;
7475         }
7476         ctsio->kern_data_resid = 0;
7477         ctsio->kern_rel_offset = 0;
7478
7479         switch (cdb->options & RSO_OPTIONS_MASK) {
7480         case RSO_OPTIONS_ALL:
7481                 all = (struct scsi_report_supported_opcodes_all *)
7482                     ctsio->kern_data_ptr;
7483                 num = 0;
7484                 for (i = 0; i < 256; i++) {
7485                         entry = &ctl_cmd_table[i];
7486                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7487                                 for (j = 0; j < 32; j++) {
7488                                         sentry = &((const struct ctl_cmd_entry *)
7489                                             entry->execute)[j];
7490                                         if (!ctl_cmd_applicable(
7491                                             lun->be_lun->lun_type, sentry))
7492                                                 continue;
7493                                         descr = &all->descr[num++];
7494                                         descr->opcode = i;
7495                                         scsi_ulto2b(j, descr->service_action);
7496                                         descr->flags = RSO_SERVACTV;
7497                                         scsi_ulto2b(sentry->length,
7498                                             descr->cdb_length);
7499                                 }
7500                         } else {
7501                                 if (!ctl_cmd_applicable(lun->be_lun->lun_type,
7502                                     entry))
7503                                         continue;
7504                                 descr = &all->descr[num++];
7505                                 descr->opcode = i;
7506                                 scsi_ulto2b(0, descr->service_action);
7507                                 descr->flags = 0;
7508                                 scsi_ulto2b(entry->length, descr->cdb_length);
7509                         }
7510                 }
7511                 scsi_ulto4b(
7512                     num * sizeof(struct scsi_report_supported_opcodes_descr),
7513                     all->length);
7514                 break;
7515         case RSO_OPTIONS_OC:
7516                 one = (struct scsi_report_supported_opcodes_one *)
7517                     ctsio->kern_data_ptr;
7518                 entry = &ctl_cmd_table[opcode];
7519                 goto fill_one;
7520         case RSO_OPTIONS_OC_SA:
7521                 one = (struct scsi_report_supported_opcodes_one *)
7522                     ctsio->kern_data_ptr;
7523                 entry = &ctl_cmd_table[opcode];
7524                 entry = &((const struct ctl_cmd_entry *)
7525                     entry->execute)[service_action];
7526 fill_one:
7527                 if (ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
7528                         one->support = 3;
7529                         scsi_ulto2b(entry->length, one->cdb_length);
7530                         one->cdb_usage[0] = opcode;
7531                         memcpy(&one->cdb_usage[1], entry->usage,
7532                             entry->length - 1);
7533                 } else
7534                         one->support = 1;
7535                 break;
7536         }
7537
7538         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7539         ctsio->be_move_done = ctl_config_move_done;
7540
7541         ctl_datamove((union ctl_io *)ctsio);
7542         return(retval);
7543 }
7544
7545 int
7546 ctl_report_supported_tmf(struct ctl_scsiio *ctsio)
7547 {
7548         struct ctl_lun *lun;
7549         struct scsi_report_supported_tmf *cdb;
7550         struct scsi_report_supported_tmf_data *data;
7551         int retval;
7552         int alloc_len, total_len;
7553
7554         CTL_DEBUG_PRINT(("ctl_report_supported_tmf\n"));
7555
7556         cdb = (struct scsi_report_supported_tmf *)ctsio->cdb;
7557         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7558
7559         retval = CTL_RETVAL_COMPLETE;
7560
7561         total_len = sizeof(struct scsi_report_supported_tmf_data);
7562         alloc_len = scsi_4btoul(cdb->length);
7563
7564         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7565
7566         ctsio->kern_sg_entries = 0;
7567
7568         if (total_len < alloc_len) {
7569                 ctsio->residual = alloc_len - total_len;
7570                 ctsio->kern_data_len = total_len;
7571                 ctsio->kern_total_len = total_len;
7572         } else {
7573                 ctsio->residual = 0;
7574                 ctsio->kern_data_len = alloc_len;
7575                 ctsio->kern_total_len = alloc_len;
7576         }
7577         ctsio->kern_data_resid = 0;
7578         ctsio->kern_rel_offset = 0;
7579
7580         data = (struct scsi_report_supported_tmf_data *)ctsio->kern_data_ptr;
7581         data->byte1 |= RST_ATS | RST_ATSS | RST_LURS | RST_TRS;
7582         data->byte2 |= RST_ITNRS;
7583
7584         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7585         ctsio->be_move_done = ctl_config_move_done;
7586
7587         ctl_datamove((union ctl_io *)ctsio);
7588         return (retval);
7589 }
7590
7591 int
7592 ctl_report_timestamp(struct ctl_scsiio *ctsio)
7593 {
7594         struct ctl_lun *lun;
7595         struct scsi_report_timestamp *cdb;
7596         struct scsi_report_timestamp_data *data;
7597         struct timeval tv;
7598         int64_t timestamp;
7599         int retval;
7600         int alloc_len, total_len;
7601
7602         CTL_DEBUG_PRINT(("ctl_report_timestamp\n"));
7603
7604         cdb = (struct scsi_report_timestamp *)ctsio->cdb;
7605         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7606
7607         retval = CTL_RETVAL_COMPLETE;
7608
7609         total_len = sizeof(struct scsi_report_timestamp_data);
7610         alloc_len = scsi_4btoul(cdb->length);
7611
7612         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7613
7614         ctsio->kern_sg_entries = 0;
7615
7616         if (total_len < alloc_len) {
7617                 ctsio->residual = alloc_len - total_len;
7618                 ctsio->kern_data_len = total_len;
7619                 ctsio->kern_total_len = total_len;
7620         } else {
7621                 ctsio->residual = 0;
7622                 ctsio->kern_data_len = alloc_len;
7623                 ctsio->kern_total_len = alloc_len;
7624         }
7625         ctsio->kern_data_resid = 0;
7626         ctsio->kern_rel_offset = 0;
7627
7628         data = (struct scsi_report_timestamp_data *)ctsio->kern_data_ptr;
7629         scsi_ulto2b(sizeof(*data) - 2, data->length);
7630         data->origin = RTS_ORIG_OUTSIDE;
7631         getmicrotime(&tv);
7632         timestamp = (int64_t)tv.tv_sec * 1000 + tv.tv_usec / 1000;
7633         scsi_ulto4b(timestamp >> 16, data->timestamp);
7634         scsi_ulto2b(timestamp & 0xffff, &data->timestamp[4]);
7635
7636         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7637         ctsio->be_move_done = ctl_config_move_done;
7638
7639         ctl_datamove((union ctl_io *)ctsio);
7640         return (retval);
7641 }
7642
7643 int
7644 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7645 {
7646         struct scsi_per_res_in *cdb;
7647         int alloc_len, total_len = 0;
7648         /* struct scsi_per_res_in_rsrv in_data; */
7649         struct ctl_lun *lun;
7650         struct ctl_softc *softc;
7651
7652         CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7653
7654         softc = control_softc;
7655
7656         cdb = (struct scsi_per_res_in *)ctsio->cdb;
7657
7658         alloc_len = scsi_2btoul(cdb->length);
7659
7660         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7661
7662 retry:
7663         mtx_lock(&lun->lun_lock);
7664         switch (cdb->action) {
7665         case SPRI_RK: /* read keys */
7666                 total_len = sizeof(struct scsi_per_res_in_keys) +
7667                         lun->pr_key_count *
7668                         sizeof(struct scsi_per_res_key);
7669                 break;
7670         case SPRI_RR: /* read reservation */
7671                 if (lun->flags & CTL_LUN_PR_RESERVED)
7672                         total_len = sizeof(struct scsi_per_res_in_rsrv);
7673                 else
7674                         total_len = sizeof(struct scsi_per_res_in_header);
7675                 break;
7676         case SPRI_RC: /* report capabilities */
7677                 total_len = sizeof(struct scsi_per_res_cap);
7678                 break;
7679         case SPRI_RS: /* read full status */
7680                 total_len = sizeof(struct scsi_per_res_in_header) +
7681                     (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7682                     lun->pr_key_count;
7683                 break;
7684         default:
7685                 panic("Invalid PR type %x", cdb->action);
7686         }
7687         mtx_unlock(&lun->lun_lock);
7688
7689         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7690
7691         if (total_len < alloc_len) {
7692                 ctsio->residual = alloc_len - total_len;
7693                 ctsio->kern_data_len = total_len;
7694                 ctsio->kern_total_len = total_len;
7695         } else {
7696                 ctsio->residual = 0;
7697                 ctsio->kern_data_len = alloc_len;
7698                 ctsio->kern_total_len = alloc_len;
7699         }
7700
7701         ctsio->kern_data_resid = 0;
7702         ctsio->kern_rel_offset = 0;
7703         ctsio->kern_sg_entries = 0;
7704
7705         mtx_lock(&lun->lun_lock);
7706         switch (cdb->action) {
7707         case SPRI_RK: { // read keys
7708         struct scsi_per_res_in_keys *res_keys;
7709                 int i, key_count;
7710
7711                 res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7712
7713                 /*
7714                  * We had to drop the lock to allocate our buffer, which
7715                  * leaves time for someone to come in with another
7716                  * persistent reservation.  (That is unlikely, though,
7717                  * since this should be the only persistent reservation
7718                  * command active right now.)
7719                  */
7720                 if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7721                     (lun->pr_key_count *
7722                      sizeof(struct scsi_per_res_key)))){
7723                         mtx_unlock(&lun->lun_lock);
7724                         free(ctsio->kern_data_ptr, M_CTL);
7725                         printf("%s: reservation length changed, retrying\n",
7726                                __func__);
7727                         goto retry;
7728                 }
7729
7730                 scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7731
7732                 scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7733                              lun->pr_key_count, res_keys->header.length);
7734
7735                 for (i = 0, key_count = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7736                         if (!lun->per_res[i].registered)
7737                                 continue;
7738
7739                         /*
7740                          * We used lun->pr_key_count to calculate the
7741                          * size to allocate.  If it turns out the number of
7742                          * initiators with the registered flag set is
7743                          * larger than that (i.e. they haven't been kept in
7744                          * sync), we've got a problem.
7745                          */
7746                         if (key_count >= lun->pr_key_count) {
7747 #ifdef NEEDTOPORT
7748                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
7749                                             CTL_PR_ERROR,
7750                                             csevent_LogType_Fault,
7751                                             csevent_AlertLevel_Yellow,
7752                                             csevent_FRU_ShelfController,
7753                                             csevent_FRU_Firmware,
7754                                         csevent_FRU_Unknown,
7755                                             "registered keys %d >= key "
7756                                             "count %d", key_count,
7757                                             lun->pr_key_count);
7758 #endif
7759                                 key_count++;
7760                                 continue;
7761                         }
7762                         memcpy(res_keys->keys[key_count].key,
7763                                lun->per_res[i].res_key.key,
7764                                ctl_min(sizeof(res_keys->keys[key_count].key),
7765                                sizeof(lun->per_res[i].res_key)));
7766                         key_count++;
7767                 }
7768                 break;
7769         }
7770         case SPRI_RR: { // read reservation
7771                 struct scsi_per_res_in_rsrv *res;
7772                 int tmp_len, header_only;
7773
7774                 res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7775
7776                 scsi_ulto4b(lun->PRGeneration, res->header.generation);
7777
7778                 if (lun->flags & CTL_LUN_PR_RESERVED)
7779                 {
7780                         tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7781                         scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7782                                     res->header.length);
7783                         header_only = 0;
7784                 } else {
7785                         tmp_len = sizeof(struct scsi_per_res_in_header);
7786                         scsi_ulto4b(0, res->header.length);
7787                         header_only = 1;
7788                 }
7789
7790                 /*
7791                  * We had to drop the lock to allocate our buffer, which
7792                  * leaves time for someone to come in with another
7793                  * persistent reservation.  (That is unlikely, though,
7794                  * since this should be the only persistent reservation
7795                  * command active right now.)
7796                  */
7797                 if (tmp_len != total_len) {
7798                         mtx_unlock(&lun->lun_lock);
7799                         free(ctsio->kern_data_ptr, M_CTL);
7800                         printf("%s: reservation status changed, retrying\n",
7801                                __func__);
7802                         goto retry;
7803                 }
7804
7805                 /*
7806                  * No reservation held, so we're done.
7807                  */
7808                 if (header_only != 0)
7809                         break;
7810
7811                 /*
7812                  * If the registration is an All Registrants type, the key
7813                  * is 0, since it doesn't really matter.
7814                  */
7815                 if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7816                         memcpy(res->data.reservation,
7817                                &lun->per_res[lun->pr_res_idx].res_key,
7818                                sizeof(struct scsi_per_res_key));
7819                 }
7820                 res->data.scopetype = lun->res_type;
7821                 break;
7822         }
7823         case SPRI_RC:     //report capabilities
7824         {
7825                 struct scsi_per_res_cap *res_cap;
7826                 uint16_t type_mask;
7827
7828                 res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7829                 scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7830                 res_cap->flags2 |= SPRI_TMV | SPRI_ALLOW_3;
7831                 type_mask = SPRI_TM_WR_EX_AR |
7832                             SPRI_TM_EX_AC_RO |
7833                             SPRI_TM_WR_EX_RO |
7834                             SPRI_TM_EX_AC |
7835                             SPRI_TM_WR_EX |
7836                             SPRI_TM_EX_AC_AR;
7837                 scsi_ulto2b(type_mask, res_cap->type_mask);
7838                 break;
7839         }
7840         case SPRI_RS: { // read full status
7841                 struct scsi_per_res_in_full *res_status;
7842                 struct scsi_per_res_in_full_desc *res_desc;
7843                 struct ctl_port *port;
7844                 int i, len;
7845
7846                 res_status = (struct scsi_per_res_in_full*)ctsio->kern_data_ptr;
7847
7848                 /*
7849                  * We had to drop the lock to allocate our buffer, which
7850                  * leaves time for someone to come in with another
7851                  * persistent reservation.  (That is unlikely, though,
7852                  * since this should be the only persistent reservation
7853                  * command active right now.)
7854                  */
7855                 if (total_len < (sizeof(struct scsi_per_res_in_header) +
7856                     (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7857                      lun->pr_key_count)){
7858                         mtx_unlock(&lun->lun_lock);
7859                         free(ctsio->kern_data_ptr, M_CTL);
7860                         printf("%s: reservation length changed, retrying\n",
7861                                __func__);
7862                         goto retry;
7863                 }
7864
7865                 scsi_ulto4b(lun->PRGeneration, res_status->header.generation);
7866
7867                 res_desc = &res_status->desc[0];
7868                 for (i = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7869                         if (!lun->per_res[i].registered)
7870                                 continue;
7871
7872                         memcpy(&res_desc->res_key, &lun->per_res[i].res_key.key,
7873                             sizeof(res_desc->res_key));
7874                         if ((lun->flags & CTL_LUN_PR_RESERVED) &&
7875                             (lun->pr_res_idx == i ||
7876                              lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS)) {
7877                                 res_desc->flags = SPRI_FULL_R_HOLDER;
7878                                 res_desc->scopetype = lun->res_type;
7879                         }
7880                         scsi_ulto2b(i / CTL_MAX_INIT_PER_PORT,
7881                             res_desc->rel_trgt_port_id);
7882                         len = 0;
7883                         port = softc->ctl_ports[i / CTL_MAX_INIT_PER_PORT];
7884                         if (port != NULL)
7885                                 len = ctl_create_iid(port,
7886                                     i % CTL_MAX_INIT_PER_PORT,
7887                                     res_desc->transport_id);
7888                         scsi_ulto4b(len, res_desc->additional_length);
7889                         res_desc = (struct scsi_per_res_in_full_desc *)
7890                             &res_desc->transport_id[len];
7891                 }
7892                 scsi_ulto4b((uint8_t *)res_desc - (uint8_t *)&res_status->desc[0],
7893                     res_status->header.length);
7894                 break;
7895         }
7896         default:
7897                 /*
7898                  * This is a bug, because we just checked for this above,
7899                  * and should have returned an error.
7900                  */
7901                 panic("Invalid PR type %x", cdb->action);
7902                 break; /* NOTREACHED */
7903         }
7904         mtx_unlock(&lun->lun_lock);
7905
7906         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7907         ctsio->be_move_done = ctl_config_move_done;
7908
7909         CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7910                          ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7911                          ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7912                          ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7913                          ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7914
7915         ctl_datamove((union ctl_io *)ctsio);
7916
7917         return (CTL_RETVAL_COMPLETE);
7918 }
7919
7920 /*
7921  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
7922  * it should return.
7923  */
7924 static int
7925 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
7926                 uint64_t sa_res_key, uint8_t type, uint32_t residx,
7927                 struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
7928                 struct scsi_per_res_out_parms* param)
7929 {
7930         union ctl_ha_msg persis_io;
7931         int retval, i;
7932         int isc_retval;
7933
7934         retval = 0;
7935
7936         mtx_lock(&lun->lun_lock);
7937         if (sa_res_key == 0) {
7938                 if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7939                         /* validate scope and type */
7940                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7941                              SPR_LU_SCOPE) {
7942                                 mtx_unlock(&lun->lun_lock);
7943                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7944                                                       /*sks_valid*/ 1,
7945                                                       /*command*/ 1,
7946                                                       /*field*/ 2,
7947                                                       /*bit_valid*/ 1,
7948                                                       /*bit*/ 4);
7949                                 ctl_done((union ctl_io *)ctsio);
7950                                 return (1);
7951                         }
7952
7953                         if (type>8 || type==2 || type==4 || type==0) {
7954                                 mtx_unlock(&lun->lun_lock);
7955                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7956                                                       /*sks_valid*/ 1,
7957                                                       /*command*/ 1,
7958                                                       /*field*/ 2,
7959                                                       /*bit_valid*/ 1,
7960                                                       /*bit*/ 0);
7961                                 ctl_done((union ctl_io *)ctsio);
7962                                 return (1);
7963                         }
7964
7965                         /* temporarily unregister this nexus */
7966                         lun->per_res[residx].registered = 0;
7967
7968                         /*
7969                          * Unregister everybody else and build UA for
7970                          * them
7971                          */
7972                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7973                                 if (lun->per_res[i].registered == 0)
7974                                         continue;
7975
7976                                 if (!persis_offset
7977                                  && i <CTL_MAX_INITIATORS)
7978                                         lun->pending_sense[i].ua_pending |=
7979                                                 CTL_UA_REG_PREEMPT;
7980                                 else if (persis_offset
7981                                       && i >= persis_offset)
7982                                         lun->pending_sense[i-persis_offset
7983                                                 ].ua_pending |=
7984                                                 CTL_UA_REG_PREEMPT;
7985                                 lun->per_res[i].registered = 0;
7986                                 memset(&lun->per_res[i].res_key, 0,
7987                                        sizeof(struct scsi_per_res_key));
7988                         }
7989                         lun->per_res[residx].registered = 1;
7990                         lun->pr_key_count = 1;
7991                         lun->res_type = type;
7992                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7993                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7994                                 lun->pr_res_idx = residx;
7995
7996                         /* send msg to other side */
7997                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7998                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7999                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8000                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8001                         persis_io.pr.pr_info.res_type = type;
8002                         memcpy(persis_io.pr.pr_info.sa_res_key,
8003                                param->serv_act_res_key,
8004                                sizeof(param->serv_act_res_key));
8005                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8006                              &persis_io, sizeof(persis_io), 0)) >
8007                              CTL_HA_STATUS_SUCCESS) {
8008                                 printf("CTL:Persis Out error returned "
8009                                        "from ctl_ha_msg_send %d\n",
8010                                        isc_retval);
8011                         }
8012                 } else {
8013                         /* not all registrants */
8014                         mtx_unlock(&lun->lun_lock);
8015                         free(ctsio->kern_data_ptr, M_CTL);
8016                         ctl_set_invalid_field(ctsio,
8017                                               /*sks_valid*/ 1,
8018                                               /*command*/ 0,
8019                                               /*field*/ 8,
8020                                               /*bit_valid*/ 0,
8021                                               /*bit*/ 0);
8022                         ctl_done((union ctl_io *)ctsio);
8023                         return (1);
8024                 }
8025         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8026                 || !(lun->flags & CTL_LUN_PR_RESERVED)) {
8027                 int found = 0;
8028
8029                 if (res_key == sa_res_key) {
8030                         /* special case */
8031                         /*
8032                          * The spec implies this is not good but doesn't
8033                          * say what to do. There are two choices either
8034                          * generate a res conflict or check condition
8035                          * with illegal field in parameter data. Since
8036                          * that is what is done when the sa_res_key is
8037                          * zero I'll take that approach since this has
8038                          * to do with the sa_res_key.
8039                          */
8040                         mtx_unlock(&lun->lun_lock);
8041                         free(ctsio->kern_data_ptr, M_CTL);
8042                         ctl_set_invalid_field(ctsio,
8043                                               /*sks_valid*/ 1,
8044                                               /*command*/ 0,
8045                                               /*field*/ 8,
8046                                               /*bit_valid*/ 0,
8047                                               /*bit*/ 0);
8048                         ctl_done((union ctl_io *)ctsio);
8049                         return (1);
8050                 }
8051
8052                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8053                         if (lun->per_res[i].registered
8054                          && memcmp(param->serv_act_res_key,
8055                             lun->per_res[i].res_key.key,
8056                             sizeof(struct scsi_per_res_key)) != 0)
8057                                 continue;
8058
8059                         found = 1;
8060                         lun->per_res[i].registered = 0;
8061                         memset(&lun->per_res[i].res_key, 0,
8062                                sizeof(struct scsi_per_res_key));
8063                         lun->pr_key_count--;
8064
8065                         if (!persis_offset
8066                          && i < CTL_MAX_INITIATORS)
8067                                 lun->pending_sense[i].ua_pending |=
8068                                         CTL_UA_REG_PREEMPT;
8069                         else if (persis_offset
8070                               && i >= persis_offset)
8071                                 lun->pending_sense[i-persis_offset].ua_pending|=
8072                                         CTL_UA_REG_PREEMPT;
8073                 }
8074                 if (!found) {
8075                         mtx_unlock(&lun->lun_lock);
8076                         free(ctsio->kern_data_ptr, M_CTL);
8077                         ctl_set_reservation_conflict(ctsio);
8078                         ctl_done((union ctl_io *)ctsio);
8079                         return (CTL_RETVAL_COMPLETE);
8080                 }
8081                 /* send msg to other side */
8082                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8083                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8084                 persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8085                 persis_io.pr.pr_info.residx = lun->pr_res_idx;
8086                 persis_io.pr.pr_info.res_type = type;
8087                 memcpy(persis_io.pr.pr_info.sa_res_key,
8088                        param->serv_act_res_key,
8089                        sizeof(param->serv_act_res_key));
8090                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8091                      &persis_io, sizeof(persis_io), 0)) >
8092                      CTL_HA_STATUS_SUCCESS) {
8093                         printf("CTL:Persis Out error returned from "
8094                                "ctl_ha_msg_send %d\n", isc_retval);
8095                 }
8096         } else {
8097                 /* Reserved but not all registrants */
8098                 /* sa_res_key is res holder */
8099                 if (memcmp(param->serv_act_res_key,
8100                    lun->per_res[lun->pr_res_idx].res_key.key,
8101                    sizeof(struct scsi_per_res_key)) == 0) {
8102                         /* validate scope and type */
8103                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
8104                              SPR_LU_SCOPE) {
8105                                 mtx_unlock(&lun->lun_lock);
8106                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8107                                                       /*sks_valid*/ 1,
8108                                                       /*command*/ 1,
8109                                                       /*field*/ 2,
8110                                                       /*bit_valid*/ 1,
8111                                                       /*bit*/ 4);
8112                                 ctl_done((union ctl_io *)ctsio);
8113                                 return (1);
8114                         }
8115
8116                         if (type>8 || type==2 || type==4 || type==0) {
8117                                 mtx_unlock(&lun->lun_lock);
8118                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8119                                                       /*sks_valid*/ 1,
8120                                                       /*command*/ 1,
8121                                                       /*field*/ 2,
8122                                                       /*bit_valid*/ 1,
8123                                                       /*bit*/ 0);
8124                                 ctl_done((union ctl_io *)ctsio);
8125                                 return (1);
8126                         }
8127
8128                         /*
8129                          * Do the following:
8130                          * if sa_res_key != res_key remove all
8131                          * registrants w/sa_res_key and generate UA
8132                          * for these registrants(Registrations
8133                          * Preempted) if it wasn't an exclusive
8134                          * reservation generate UA(Reservations
8135                          * Preempted) for all other registered nexuses
8136                          * if the type has changed. Establish the new
8137                          * reservation and holder. If res_key and
8138                          * sa_res_key are the same do the above
8139                          * except don't unregister the res holder.
8140                          */
8141
8142                         /*
8143                          * Temporarily unregister so it won't get
8144                          * removed or UA generated
8145                          */
8146                         lun->per_res[residx].registered = 0;
8147                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8148                                 if (lun->per_res[i].registered == 0)
8149                                         continue;
8150
8151                                 if (memcmp(param->serv_act_res_key,
8152                                     lun->per_res[i].res_key.key,
8153                                     sizeof(struct scsi_per_res_key)) == 0) {
8154                                         lun->per_res[i].registered = 0;
8155                                         memset(&lun->per_res[i].res_key,
8156                                                0,
8157                                                sizeof(struct scsi_per_res_key));
8158                                         lun->pr_key_count--;
8159
8160                                         if (!persis_offset
8161                                          && i < CTL_MAX_INITIATORS)
8162                                                 lun->pending_sense[i
8163                                                         ].ua_pending |=
8164                                                         CTL_UA_REG_PREEMPT;
8165                                         else if (persis_offset
8166                                               && i >= persis_offset)
8167                                                 lun->pending_sense[
8168                                                   i-persis_offset].ua_pending |=
8169                                                   CTL_UA_REG_PREEMPT;
8170                                 } else if (type != lun->res_type
8171                                         && (lun->res_type == SPR_TYPE_WR_EX_RO
8172                                          || lun->res_type ==SPR_TYPE_EX_AC_RO)){
8173                                                 if (!persis_offset
8174                                                  && i < CTL_MAX_INITIATORS)
8175                                                         lun->pending_sense[i
8176                                                         ].ua_pending |=
8177                                                         CTL_UA_RES_RELEASE;
8178                                                 else if (persis_offset
8179                                                       && i >= persis_offset)
8180                                                         lun->pending_sense[
8181                                                         i-persis_offset
8182                                                         ].ua_pending |=
8183                                                         CTL_UA_RES_RELEASE;
8184                                 }
8185                         }
8186                         lun->per_res[residx].registered = 1;
8187                         lun->res_type = type;
8188                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8189                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8190                                 lun->pr_res_idx = residx;
8191                         else
8192                                 lun->pr_res_idx =
8193                                         CTL_PR_ALL_REGISTRANTS;
8194
8195                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8196                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8197                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8198                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8199                         persis_io.pr.pr_info.res_type = type;
8200                         memcpy(persis_io.pr.pr_info.sa_res_key,
8201                                param->serv_act_res_key,
8202                                sizeof(param->serv_act_res_key));
8203                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8204                              &persis_io, sizeof(persis_io), 0)) >
8205                              CTL_HA_STATUS_SUCCESS) {
8206                                 printf("CTL:Persis Out error returned "
8207                                        "from ctl_ha_msg_send %d\n",
8208                                        isc_retval);
8209                         }
8210                 } else {
8211                         /*
8212                          * sa_res_key is not the res holder just
8213                          * remove registrants
8214                          */
8215                         int found=0;
8216
8217                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8218                                 if (memcmp(param->serv_act_res_key,
8219                                     lun->per_res[i].res_key.key,
8220                                     sizeof(struct scsi_per_res_key)) != 0)
8221                                         continue;
8222
8223                                 found = 1;
8224                                 lun->per_res[i].registered = 0;
8225                                 memset(&lun->per_res[i].res_key, 0,
8226                                        sizeof(struct scsi_per_res_key));
8227                                 lun->pr_key_count--;
8228
8229                                 if (!persis_offset
8230                                  && i < CTL_MAX_INITIATORS)
8231                                         lun->pending_sense[i].ua_pending |=
8232                                                 CTL_UA_REG_PREEMPT;
8233                                 else if (persis_offset
8234                                       && i >= persis_offset)
8235                                         lun->pending_sense[
8236                                                 i-persis_offset].ua_pending |=
8237                                                 CTL_UA_REG_PREEMPT;
8238                         }
8239
8240                         if (!found) {
8241                                 mtx_unlock(&lun->lun_lock);
8242                                 free(ctsio->kern_data_ptr, M_CTL);
8243                                 ctl_set_reservation_conflict(ctsio);
8244                                 ctl_done((union ctl_io *)ctsio);
8245                                 return (1);
8246                         }
8247                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8248                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8249                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8250                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8251                         persis_io.pr.pr_info.res_type = type;
8252                         memcpy(persis_io.pr.pr_info.sa_res_key,
8253                                param->serv_act_res_key,
8254                                sizeof(param->serv_act_res_key));
8255                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8256                              &persis_io, sizeof(persis_io), 0)) >
8257                              CTL_HA_STATUS_SUCCESS) {
8258                                 printf("CTL:Persis Out error returned "
8259                                        "from ctl_ha_msg_send %d\n",
8260                                 isc_retval);
8261                         }
8262                 }
8263         }
8264
8265         lun->PRGeneration++;
8266         mtx_unlock(&lun->lun_lock);
8267
8268         return (retval);
8269 }
8270
8271 static void
8272 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
8273 {
8274         int i;
8275
8276         if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8277          || lun->pr_res_idx == CTL_PR_NO_RESERVATION
8278          || memcmp(&lun->per_res[lun->pr_res_idx].res_key,
8279                    msg->pr.pr_info.sa_res_key,
8280                    sizeof(struct scsi_per_res_key)) != 0) {
8281                 uint64_t sa_res_key;
8282                 sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
8283
8284                 if (sa_res_key == 0) {
8285                         /* temporarily unregister this nexus */
8286                         lun->per_res[msg->pr.pr_info.residx].registered = 0;
8287
8288                         /*
8289                          * Unregister everybody else and build UA for
8290                          * them
8291                          */
8292                         for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8293                                 if (lun->per_res[i].registered == 0)
8294                                         continue;
8295
8296                                 if (!persis_offset
8297                                  && i < CTL_MAX_INITIATORS)
8298                                         lun->pending_sense[i].ua_pending |=
8299                                                 CTL_UA_REG_PREEMPT;
8300                                 else if (persis_offset && i >= persis_offset)
8301                                         lun->pending_sense[i -
8302                                                 persis_offset].ua_pending |=
8303                                                 CTL_UA_REG_PREEMPT;
8304                                 lun->per_res[i].registered = 0;
8305                                 memset(&lun->per_res[i].res_key, 0,
8306                                        sizeof(struct scsi_per_res_key));
8307                         }
8308
8309                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
8310                         lun->pr_key_count = 1;
8311                         lun->res_type = msg->pr.pr_info.res_type;
8312                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8313                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8314                                 lun->pr_res_idx = msg->pr.pr_info.residx;
8315                 } else {
8316                         for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8317                                 if (memcmp(msg->pr.pr_info.sa_res_key,
8318                                    lun->per_res[i].res_key.key,
8319                                    sizeof(struct scsi_per_res_key)) != 0)
8320                                         continue;
8321
8322                                 lun->per_res[i].registered = 0;
8323                                 memset(&lun->per_res[i].res_key, 0,
8324                                        sizeof(struct scsi_per_res_key));
8325                                 lun->pr_key_count--;
8326
8327                                 if (!persis_offset
8328                                  && i < persis_offset)
8329                                         lun->pending_sense[i].ua_pending |=
8330                                                 CTL_UA_REG_PREEMPT;
8331                                 else if (persis_offset
8332                                       && i >= persis_offset)
8333                                         lun->pending_sense[i -
8334                                                 persis_offset].ua_pending |=
8335                                                 CTL_UA_REG_PREEMPT;
8336                         }
8337                 }
8338         } else {
8339                 /*
8340                  * Temporarily unregister so it won't get removed
8341                  * or UA generated
8342                  */
8343                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
8344                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8345                         if (lun->per_res[i].registered == 0)
8346                                 continue;
8347
8348                         if (memcmp(msg->pr.pr_info.sa_res_key,
8349                            lun->per_res[i].res_key.key,
8350                            sizeof(struct scsi_per_res_key)) == 0) {
8351                                 lun->per_res[i].registered = 0;
8352                                 memset(&lun->per_res[i].res_key, 0,
8353                                        sizeof(struct scsi_per_res_key));
8354                                 lun->pr_key_count--;
8355                                 if (!persis_offset
8356                                  && i < CTL_MAX_INITIATORS)
8357                                         lun->pending_sense[i].ua_pending |=
8358                                                 CTL_UA_REG_PREEMPT;
8359                                 else if (persis_offset
8360                                       && i >= persis_offset)
8361                                         lun->pending_sense[i -
8362                                                 persis_offset].ua_pending |=
8363                                                 CTL_UA_REG_PREEMPT;
8364                         } else if (msg->pr.pr_info.res_type != lun->res_type
8365                                 && (lun->res_type == SPR_TYPE_WR_EX_RO
8366                                  || lun->res_type == SPR_TYPE_EX_AC_RO)) {
8367                                         if (!persis_offset
8368                                          && i < persis_offset)
8369                                                 lun->pending_sense[i
8370                                                         ].ua_pending |=
8371                                                         CTL_UA_RES_RELEASE;
8372                                         else if (persis_offset
8373                                               && i >= persis_offset)
8374                                         lun->pending_sense[i -
8375                                                 persis_offset].ua_pending |=
8376                                                 CTL_UA_RES_RELEASE;
8377                         }
8378                 }
8379                 lun->per_res[msg->pr.pr_info.residx].registered = 1;
8380                 lun->res_type = msg->pr.pr_info.res_type;
8381                 if (lun->res_type != SPR_TYPE_WR_EX_AR
8382                  && lun->res_type != SPR_TYPE_EX_AC_AR)
8383                         lun->pr_res_idx = msg->pr.pr_info.residx;
8384                 else
8385                         lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8386         }
8387         lun->PRGeneration++;
8388
8389 }
8390
8391
8392 int
8393 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
8394 {
8395         int retval;
8396         int isc_retval;
8397         u_int32_t param_len;
8398         struct scsi_per_res_out *cdb;
8399         struct ctl_lun *lun;
8400         struct scsi_per_res_out_parms* param;
8401         struct ctl_softc *softc;
8402         uint32_t residx;
8403         uint64_t res_key, sa_res_key;
8404         uint8_t type;
8405         union ctl_ha_msg persis_io;
8406         int    i;
8407
8408         CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
8409
8410         retval = CTL_RETVAL_COMPLETE;
8411
8412         softc = control_softc;
8413
8414         cdb = (struct scsi_per_res_out *)ctsio->cdb;
8415         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8416
8417         /*
8418          * We only support whole-LUN scope.  The scope & type are ignored for
8419          * register, register and ignore existing key and clear.
8420          * We sometimes ignore scope and type on preempts too!!
8421          * Verify reservation type here as well.
8422          */
8423         type = cdb->scope_type & SPR_TYPE_MASK;
8424         if ((cdb->action == SPRO_RESERVE)
8425          || (cdb->action == SPRO_RELEASE)) {
8426                 if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
8427                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8428                                               /*sks_valid*/ 1,
8429                                               /*command*/ 1,
8430                                               /*field*/ 2,
8431                                               /*bit_valid*/ 1,
8432                                               /*bit*/ 4);
8433                         ctl_done((union ctl_io *)ctsio);
8434                         return (CTL_RETVAL_COMPLETE);
8435                 }
8436
8437                 if (type>8 || type==2 || type==4 || type==0) {
8438                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8439                                               /*sks_valid*/ 1,
8440                                               /*command*/ 1,
8441                                               /*field*/ 2,
8442                                               /*bit_valid*/ 1,
8443                                               /*bit*/ 0);
8444                         ctl_done((union ctl_io *)ctsio);
8445                         return (CTL_RETVAL_COMPLETE);
8446                 }
8447         }
8448
8449         param_len = scsi_4btoul(cdb->length);
8450
8451         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
8452                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
8453                 ctsio->kern_data_len = param_len;
8454                 ctsio->kern_total_len = param_len;
8455                 ctsio->kern_data_resid = 0;
8456                 ctsio->kern_rel_offset = 0;
8457                 ctsio->kern_sg_entries = 0;
8458                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8459                 ctsio->be_move_done = ctl_config_move_done;
8460                 ctl_datamove((union ctl_io *)ctsio);
8461
8462                 return (CTL_RETVAL_COMPLETE);
8463         }
8464
8465         param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
8466
8467         residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8468         res_key = scsi_8btou64(param->res_key.key);
8469         sa_res_key = scsi_8btou64(param->serv_act_res_key);
8470
8471         /*
8472          * Validate the reservation key here except for SPRO_REG_IGNO
8473          * This must be done for all other service actions
8474          */
8475         if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
8476                 mtx_lock(&lun->lun_lock);
8477                 if (lun->per_res[residx].registered) {
8478                     if (memcmp(param->res_key.key,
8479                                lun->per_res[residx].res_key.key,
8480                                ctl_min(sizeof(param->res_key),
8481                                sizeof(lun->per_res[residx].res_key))) != 0) {
8482                                 /*
8483                                  * The current key passed in doesn't match
8484                                  * the one the initiator previously
8485                                  * registered.
8486                                  */
8487                                 mtx_unlock(&lun->lun_lock);
8488                                 free(ctsio->kern_data_ptr, M_CTL);
8489                                 ctl_set_reservation_conflict(ctsio);
8490                                 ctl_done((union ctl_io *)ctsio);
8491                                 return (CTL_RETVAL_COMPLETE);
8492                         }
8493                 } else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
8494                         /*
8495                          * We are not registered
8496                          */
8497                         mtx_unlock(&lun->lun_lock);
8498                         free(ctsio->kern_data_ptr, M_CTL);
8499                         ctl_set_reservation_conflict(ctsio);
8500                         ctl_done((union ctl_io *)ctsio);
8501                         return (CTL_RETVAL_COMPLETE);
8502                 } else if (res_key != 0) {
8503                         /*
8504                          * We are not registered and trying to register but
8505                          * the register key isn't zero.
8506                          */
8507                         mtx_unlock(&lun->lun_lock);
8508                         free(ctsio->kern_data_ptr, M_CTL);
8509                         ctl_set_reservation_conflict(ctsio);
8510                         ctl_done((union ctl_io *)ctsio);
8511                         return (CTL_RETVAL_COMPLETE);
8512                 }
8513                 mtx_unlock(&lun->lun_lock);
8514         }
8515
8516         switch (cdb->action & SPRO_ACTION_MASK) {
8517         case SPRO_REGISTER:
8518         case SPRO_REG_IGNO: {
8519
8520 #if 0
8521                 printf("Registration received\n");
8522 #endif
8523
8524                 /*
8525                  * We don't support any of these options, as we report in
8526                  * the read capabilities request (see
8527                  * ctl_persistent_reserve_in(), above).
8528                  */
8529                 if ((param->flags & SPR_SPEC_I_PT)
8530                  || (param->flags & SPR_ALL_TG_PT)
8531                  || (param->flags & SPR_APTPL)) {
8532                         int bit_ptr;
8533
8534                         if (param->flags & SPR_APTPL)
8535                                 bit_ptr = 0;
8536                         else if (param->flags & SPR_ALL_TG_PT)
8537                                 bit_ptr = 2;
8538                         else /* SPR_SPEC_I_PT */
8539                                 bit_ptr = 3;
8540
8541                         free(ctsio->kern_data_ptr, M_CTL);
8542                         ctl_set_invalid_field(ctsio,
8543                                               /*sks_valid*/ 1,
8544                                               /*command*/ 0,
8545                                               /*field*/ 20,
8546                                               /*bit_valid*/ 1,
8547                                               /*bit*/ bit_ptr);
8548                         ctl_done((union ctl_io *)ctsio);
8549                         return (CTL_RETVAL_COMPLETE);
8550                 }
8551
8552                 mtx_lock(&lun->lun_lock);
8553
8554                 /*
8555                  * The initiator wants to clear the
8556                  * key/unregister.
8557                  */
8558                 if (sa_res_key == 0) {
8559                         if ((res_key == 0
8560                           && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
8561                          || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
8562                           && !lun->per_res[residx].registered)) {
8563                                 mtx_unlock(&lun->lun_lock);
8564                                 goto done;
8565                         }
8566
8567                         lun->per_res[residx].registered = 0;
8568                         memset(&lun->per_res[residx].res_key,
8569                                0, sizeof(lun->per_res[residx].res_key));
8570                         lun->pr_key_count--;
8571
8572                         if (residx == lun->pr_res_idx) {
8573                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8574                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8575
8576                                 if ((lun->res_type == SPR_TYPE_WR_EX_RO
8577                                   || lun->res_type == SPR_TYPE_EX_AC_RO)
8578                                  && lun->pr_key_count) {
8579                                         /*
8580                                          * If the reservation is a registrants
8581                                          * only type we need to generate a UA
8582                                          * for other registered inits.  The
8583                                          * sense code should be RESERVATIONS
8584                                          * RELEASED
8585                                          */
8586
8587                                         for (i = 0; i < CTL_MAX_INITIATORS;i++){
8588                                                 if (lun->per_res[
8589                                                     i+persis_offset].registered
8590                                                     == 0)
8591                                                         continue;
8592                                                 lun->pending_sense[i
8593                                                         ].ua_pending |=
8594                                                         CTL_UA_RES_RELEASE;
8595                                         }
8596                                 }
8597                                 lun->res_type = 0;
8598                         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8599                                 if (lun->pr_key_count==0) {
8600                                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8601                                         lun->res_type = 0;
8602                                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8603                                 }
8604                         }
8605                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8606                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8607                         persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8608                         persis_io.pr.pr_info.residx = residx;
8609                         if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8610                              &persis_io, sizeof(persis_io), 0 )) >
8611                              CTL_HA_STATUS_SUCCESS) {
8612                                 printf("CTL:Persis Out error returned from "
8613                                        "ctl_ha_msg_send %d\n", isc_retval);
8614                         }
8615                 } else /* sa_res_key != 0 */ {
8616
8617                         /*
8618                          * If we aren't registered currently then increment
8619                          * the key count and set the registered flag.
8620                          */
8621                         if (!lun->per_res[residx].registered) {
8622                                 lun->pr_key_count++;
8623                                 lun->per_res[residx].registered = 1;
8624                         }
8625
8626                         memcpy(&lun->per_res[residx].res_key,
8627                                param->serv_act_res_key,
8628                                ctl_min(sizeof(param->serv_act_res_key),
8629                                sizeof(lun->per_res[residx].res_key)));
8630
8631                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8632                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8633                         persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8634                         persis_io.pr.pr_info.residx = residx;
8635                         memcpy(persis_io.pr.pr_info.sa_res_key,
8636                                param->serv_act_res_key,
8637                                sizeof(param->serv_act_res_key));
8638                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8639                              &persis_io, sizeof(persis_io), 0)) >
8640                              CTL_HA_STATUS_SUCCESS) {
8641                                 printf("CTL:Persis Out error returned from "
8642                                        "ctl_ha_msg_send %d\n", isc_retval);
8643                         }
8644                 }
8645                 lun->PRGeneration++;
8646                 mtx_unlock(&lun->lun_lock);
8647
8648                 break;
8649         }
8650         case SPRO_RESERVE:
8651 #if 0
8652                 printf("Reserve executed type %d\n", type);
8653 #endif
8654                 mtx_lock(&lun->lun_lock);
8655                 if (lun->flags & CTL_LUN_PR_RESERVED) {
8656                         /*
8657                          * if this isn't the reservation holder and it's
8658                          * not a "all registrants" type or if the type is
8659                          * different then we have a conflict
8660                          */
8661                         if ((lun->pr_res_idx != residx
8662                           && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8663                          || lun->res_type != type) {
8664                                 mtx_unlock(&lun->lun_lock);
8665                                 free(ctsio->kern_data_ptr, M_CTL);
8666                                 ctl_set_reservation_conflict(ctsio);
8667                                 ctl_done((union ctl_io *)ctsio);
8668                                 return (CTL_RETVAL_COMPLETE);
8669                         }
8670                         mtx_unlock(&lun->lun_lock);
8671                 } else /* create a reservation */ {
8672                         /*
8673                          * If it's not an "all registrants" type record
8674                          * reservation holder
8675                          */
8676                         if (type != SPR_TYPE_WR_EX_AR
8677                          && type != SPR_TYPE_EX_AC_AR)
8678                                 lun->pr_res_idx = residx; /* Res holder */
8679                         else
8680                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8681
8682                         lun->flags |= CTL_LUN_PR_RESERVED;
8683                         lun->res_type = type;
8684
8685                         mtx_unlock(&lun->lun_lock);
8686
8687                         /* send msg to other side */
8688                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8689                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8690                         persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8691                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8692                         persis_io.pr.pr_info.res_type = type;
8693                         if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8694                              &persis_io, sizeof(persis_io), 0)) >
8695                              CTL_HA_STATUS_SUCCESS) {
8696                                 printf("CTL:Persis Out error returned from "
8697                                        "ctl_ha_msg_send %d\n", isc_retval);
8698                         }
8699                 }
8700                 break;
8701
8702         case SPRO_RELEASE:
8703                 mtx_lock(&lun->lun_lock);
8704                 if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8705                         /* No reservation exists return good status */
8706                         mtx_unlock(&lun->lun_lock);
8707                         goto done;
8708                 }
8709                 /*
8710                  * Is this nexus a reservation holder?
8711                  */
8712                 if (lun->pr_res_idx != residx
8713                  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8714                         /*
8715                          * not a res holder return good status but
8716                          * do nothing
8717                          */
8718                         mtx_unlock(&lun->lun_lock);
8719                         goto done;
8720                 }
8721
8722                 if (lun->res_type != type) {
8723                         mtx_unlock(&lun->lun_lock);
8724                         free(ctsio->kern_data_ptr, M_CTL);
8725                         ctl_set_illegal_pr_release(ctsio);
8726                         ctl_done((union ctl_io *)ctsio);
8727                         return (CTL_RETVAL_COMPLETE);
8728                 }
8729
8730                 /* okay to release */
8731                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8732                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8733                 lun->res_type = 0;
8734
8735                 /*
8736                  * if this isn't an exclusive access
8737                  * res generate UA for all other
8738                  * registrants.
8739                  */
8740                 if (type != SPR_TYPE_EX_AC
8741                  && type != SPR_TYPE_WR_EX) {
8742                         /*
8743                          * temporarily unregister so we don't generate UA
8744                          */
8745                         lun->per_res[residx].registered = 0;
8746
8747                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8748                                 if (lun->per_res[i+persis_offset].registered
8749                                     == 0)
8750                                         continue;
8751                                 lun->pending_sense[i].ua_pending |=
8752                                         CTL_UA_RES_RELEASE;
8753                         }
8754
8755                         lun->per_res[residx].registered = 1;
8756                 }
8757                 mtx_unlock(&lun->lun_lock);
8758                 /* Send msg to other side */
8759                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8760                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8761                 persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8762                 if ((isc_retval=ctl_ha_msg_send( CTL_HA_CHAN_CTL, &persis_io,
8763                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8764                         printf("CTL:Persis Out error returned from "
8765                                "ctl_ha_msg_send %d\n", isc_retval);
8766                 }
8767                 break;
8768
8769         case SPRO_CLEAR:
8770                 /* send msg to other side */
8771
8772                 mtx_lock(&lun->lun_lock);
8773                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8774                 lun->res_type = 0;
8775                 lun->pr_key_count = 0;
8776                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8777
8778
8779                 memset(&lun->per_res[residx].res_key,
8780                        0, sizeof(lun->per_res[residx].res_key));
8781                 lun->per_res[residx].registered = 0;
8782
8783                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++)
8784                         if (lun->per_res[i].registered) {
8785                                 if (!persis_offset && i < CTL_MAX_INITIATORS)
8786                                         lun->pending_sense[i].ua_pending |=
8787                                                 CTL_UA_RES_PREEMPT;
8788                                 else if (persis_offset && i >= persis_offset)
8789                                         lun->pending_sense[i-persis_offset
8790                                             ].ua_pending |= CTL_UA_RES_PREEMPT;
8791
8792                                 memset(&lun->per_res[i].res_key,
8793                                        0, sizeof(struct scsi_per_res_key));
8794                                 lun->per_res[i].registered = 0;
8795                         }
8796                 lun->PRGeneration++;
8797                 mtx_unlock(&lun->lun_lock);
8798                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8799                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8800                 persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8801                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8802                      sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8803                         printf("CTL:Persis Out error returned from "
8804                                "ctl_ha_msg_send %d\n", isc_retval);
8805                 }
8806                 break;
8807
8808         case SPRO_PREEMPT: {
8809                 int nretval;
8810
8811                 nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8812                                           residx, ctsio, cdb, param);
8813                 if (nretval != 0)
8814                         return (CTL_RETVAL_COMPLETE);
8815                 break;
8816         }
8817         default:
8818                 panic("Invalid PR type %x", cdb->action);
8819         }
8820
8821 done:
8822         free(ctsio->kern_data_ptr, M_CTL);
8823         ctl_set_success(ctsio);
8824         ctl_done((union ctl_io *)ctsio);
8825
8826         return (retval);
8827 }
8828
8829 /*
8830  * This routine is for handling a message from the other SC pertaining to
8831  * persistent reserve out. All the error checking will have been done
8832  * so only perorming the action need be done here to keep the two
8833  * in sync.
8834  */
8835 static void
8836 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8837 {
8838         struct ctl_lun *lun;
8839         struct ctl_softc *softc;
8840         int i;
8841         uint32_t targ_lun;
8842
8843         softc = control_softc;
8844
8845         targ_lun = msg->hdr.nexus.targ_mapped_lun;
8846         lun = softc->ctl_luns[targ_lun];
8847         mtx_lock(&lun->lun_lock);
8848         switch(msg->pr.pr_info.action) {
8849         case CTL_PR_REG_KEY:
8850                 if (!lun->per_res[msg->pr.pr_info.residx].registered) {
8851                         lun->per_res[msg->pr.pr_info.residx].registered = 1;
8852                         lun->pr_key_count++;
8853                 }
8854                 lun->PRGeneration++;
8855                 memcpy(&lun->per_res[msg->pr.pr_info.residx].res_key,
8856                        msg->pr.pr_info.sa_res_key,
8857                        sizeof(struct scsi_per_res_key));
8858                 break;
8859
8860         case CTL_PR_UNREG_KEY:
8861                 lun->per_res[msg->pr.pr_info.residx].registered = 0;
8862                 memset(&lun->per_res[msg->pr.pr_info.residx].res_key,
8863                        0, sizeof(struct scsi_per_res_key));
8864                 lun->pr_key_count--;
8865
8866                 /* XXX Need to see if the reservation has been released */
8867                 /* if so do we need to generate UA? */
8868                 if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8869                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8870                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8871
8872                         if ((lun->res_type == SPR_TYPE_WR_EX_RO
8873                           || lun->res_type == SPR_TYPE_EX_AC_RO)
8874                          && lun->pr_key_count) {
8875                                 /*
8876                                  * If the reservation is a registrants
8877                                  * only type we need to generate a UA
8878                                  * for other registered inits.  The
8879                                  * sense code should be RESERVATIONS
8880                                  * RELEASED
8881                                  */
8882
8883                                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8884                                         if (lun->per_res[i+
8885                                             persis_offset].registered == 0)
8886                                                 continue;
8887
8888                                         lun->pending_sense[i
8889                                                 ].ua_pending |=
8890                                                 CTL_UA_RES_RELEASE;
8891                                 }
8892                         }
8893                         lun->res_type = 0;
8894                 } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8895                         if (lun->pr_key_count==0) {
8896                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8897                                 lun->res_type = 0;
8898                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8899                         }
8900                 }
8901                 lun->PRGeneration++;
8902                 break;
8903
8904         case CTL_PR_RESERVE:
8905                 lun->flags |= CTL_LUN_PR_RESERVED;
8906                 lun->res_type = msg->pr.pr_info.res_type;
8907                 lun->pr_res_idx = msg->pr.pr_info.residx;
8908
8909                 break;
8910
8911         case CTL_PR_RELEASE:
8912                 /*
8913                  * if this isn't an exclusive access res generate UA for all
8914                  * other registrants.
8915                  */
8916                 if (lun->res_type != SPR_TYPE_EX_AC
8917                  && lun->res_type != SPR_TYPE_WR_EX) {
8918                         for (i = 0; i < CTL_MAX_INITIATORS; i++)
8919                                 if (lun->per_res[i+persis_offset].registered)
8920                                         lun->pending_sense[i].ua_pending |=
8921                                                 CTL_UA_RES_RELEASE;
8922                 }
8923
8924                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8925                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8926                 lun->res_type = 0;
8927                 break;
8928
8929         case CTL_PR_PREEMPT:
8930                 ctl_pro_preempt_other(lun, msg);
8931                 break;
8932         case CTL_PR_CLEAR:
8933                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8934                 lun->res_type = 0;
8935                 lun->pr_key_count = 0;
8936                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8937
8938                 for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8939                         if (lun->per_res[i].registered == 0)
8940                                 continue;
8941                         if (!persis_offset
8942                          && i < CTL_MAX_INITIATORS)
8943                                 lun->pending_sense[i].ua_pending |=
8944                                         CTL_UA_RES_PREEMPT;
8945                         else if (persis_offset
8946                               && i >= persis_offset)
8947                                 lun->pending_sense[i-persis_offset].ua_pending|=
8948                                         CTL_UA_RES_PREEMPT;
8949                         memset(&lun->per_res[i].res_key, 0,
8950                                sizeof(struct scsi_per_res_key));
8951                         lun->per_res[i].registered = 0;
8952                 }
8953                 lun->PRGeneration++;
8954                 break;
8955         }
8956
8957         mtx_unlock(&lun->lun_lock);
8958 }
8959
8960 int
8961 ctl_read_write(struct ctl_scsiio *ctsio)
8962 {
8963         struct ctl_lun *lun;
8964         struct ctl_lba_len_flags *lbalen;
8965         uint64_t lba;
8966         uint32_t num_blocks;
8967         int fua, dpo;
8968         int retval;
8969         int isread;
8970
8971         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8972
8973         CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8974
8975         fua = 0;
8976         dpo = 0;
8977
8978         retval = CTL_RETVAL_COMPLETE;
8979
8980         isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8981               || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8982         if (lun->flags & CTL_LUN_PR_RESERVED && isread) {
8983                 uint32_t residx;
8984
8985                 /*
8986                  * XXX KDM need a lock here.
8987                  */
8988                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8989                 if ((lun->res_type == SPR_TYPE_EX_AC
8990                   && residx != lun->pr_res_idx)
8991                  || ((lun->res_type == SPR_TYPE_EX_AC_RO
8992                    || lun->res_type == SPR_TYPE_EX_AC_AR)
8993                   && !lun->per_res[residx].registered)) {
8994                         ctl_set_reservation_conflict(ctsio);
8995                         ctl_done((union ctl_io *)ctsio);
8996                         return (CTL_RETVAL_COMPLETE);
8997                 }
8998         }
8999
9000         switch (ctsio->cdb[0]) {
9001         case READ_6:
9002         case WRITE_6: {
9003                 struct scsi_rw_6 *cdb;
9004
9005                 cdb = (struct scsi_rw_6 *)ctsio->cdb;
9006
9007                 lba = scsi_3btoul(cdb->addr);
9008                 /* only 5 bits are valid in the most significant address byte */
9009                 lba &= 0x1fffff;
9010                 num_blocks = cdb->length;
9011                 /*
9012                  * This is correct according to SBC-2.
9013                  */
9014                 if (num_blocks == 0)
9015                         num_blocks = 256;
9016                 break;
9017         }
9018         case READ_10:
9019         case WRITE_10: {
9020                 struct scsi_rw_10 *cdb;
9021
9022                 cdb = (struct scsi_rw_10 *)ctsio->cdb;
9023
9024                 if (cdb->byte2 & SRW10_FUA)
9025                         fua = 1;
9026                 if (cdb->byte2 & SRW10_DPO)
9027                         dpo = 1;
9028
9029                 lba = scsi_4btoul(cdb->addr);
9030                 num_blocks = scsi_2btoul(cdb->length);
9031                 break;
9032         }
9033         case WRITE_VERIFY_10: {
9034                 struct scsi_write_verify_10 *cdb;
9035
9036                 cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
9037
9038                 /*
9039                  * XXX KDM we should do actual write verify support at some
9040                  * point.  This is obviously fake, we're just translating
9041                  * things to a write.  So we don't even bother checking the
9042                  * BYTCHK field, since we don't do any verification.  If
9043                  * the user asks for it, we'll just pretend we did it.
9044                  */
9045                 if (cdb->byte2 & SWV_DPO)
9046                         dpo = 1;
9047
9048                 lba = scsi_4btoul(cdb->addr);
9049                 num_blocks = scsi_2btoul(cdb->length);
9050                 break;
9051         }
9052         case READ_12:
9053         case WRITE_12: {
9054                 struct scsi_rw_12 *cdb;
9055
9056                 cdb = (struct scsi_rw_12 *)ctsio->cdb;
9057
9058                 if (cdb->byte2 & SRW12_FUA)
9059                         fua = 1;
9060                 if (cdb->byte2 & SRW12_DPO)
9061                         dpo = 1;
9062                 lba = scsi_4btoul(cdb->addr);
9063                 num_blocks = scsi_4btoul(cdb->length);
9064                 break;
9065         }
9066         case WRITE_VERIFY_12: {
9067                 struct scsi_write_verify_12 *cdb;
9068
9069                 cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
9070
9071                 if (cdb->byte2 & SWV_DPO)
9072                         dpo = 1;
9073
9074                 lba = scsi_4btoul(cdb->addr);
9075                 num_blocks = scsi_4btoul(cdb->length);
9076
9077                 break;
9078         }
9079         case READ_16:
9080         case WRITE_16: {
9081                 struct scsi_rw_16 *cdb;
9082
9083                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
9084
9085                 if (cdb->byte2 & SRW12_FUA)
9086                         fua = 1;
9087                 if (cdb->byte2 & SRW12_DPO)
9088                         dpo = 1;
9089
9090                 lba = scsi_8btou64(cdb->addr);
9091                 num_blocks = scsi_4btoul(cdb->length);
9092                 break;
9093         }
9094         case WRITE_VERIFY_16: {
9095                 struct scsi_write_verify_16 *cdb;
9096
9097                 cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
9098
9099                 if (cdb->byte2 & SWV_DPO)
9100                         dpo = 1;
9101
9102                 lba = scsi_8btou64(cdb->addr);
9103                 num_blocks = scsi_4btoul(cdb->length);
9104                 break;
9105         }
9106         default:
9107                 /*
9108                  * We got a command we don't support.  This shouldn't
9109                  * happen, commands should be filtered out above us.
9110                  */
9111                 ctl_set_invalid_opcode(ctsio);
9112                 ctl_done((union ctl_io *)ctsio);
9113
9114                 return (CTL_RETVAL_COMPLETE);
9115                 break; /* NOTREACHED */
9116         }
9117
9118         /*
9119          * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
9120          * interesting for us, but if RAIDCore is in write-back mode,
9121          * getting it to do write-through for a particular transaction may
9122          * not be possible.
9123          */
9124
9125         /*
9126          * The first check is to make sure we're in bounds, the second
9127          * check is to catch wrap-around problems.  If the lba + num blocks
9128          * is less than the lba, then we've wrapped around and the block
9129          * range is invalid anyway.
9130          */
9131         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9132          || ((lba + num_blocks) < lba)) {
9133                 ctl_set_lba_out_of_range(ctsio);
9134                 ctl_done((union ctl_io *)ctsio);
9135                 return (CTL_RETVAL_COMPLETE);
9136         }
9137
9138         /*
9139          * According to SBC-3, a transfer length of 0 is not an error.
9140          * Note that this cannot happen with WRITE(6) or READ(6), since 0
9141          * translates to 256 blocks for those commands.
9142          */
9143         if (num_blocks == 0) {
9144                 ctl_set_success(ctsio);
9145                 ctl_done((union ctl_io *)ctsio);
9146                 return (CTL_RETVAL_COMPLETE);
9147         }
9148
9149         lbalen = (struct ctl_lba_len_flags *)
9150             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9151         lbalen->lba = lba;
9152         lbalen->len = num_blocks;
9153         lbalen->flags = isread ? CTL_LLF_READ : CTL_LLF_WRITE;
9154
9155         ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9156         ctsio->kern_rel_offset = 0;
9157
9158         CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
9159
9160         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9161
9162         return (retval);
9163 }
9164
9165 static int
9166 ctl_cnw_cont(union ctl_io *io)
9167 {
9168         struct ctl_scsiio *ctsio;
9169         struct ctl_lun *lun;
9170         struct ctl_lba_len_flags *lbalen;
9171         int retval;
9172
9173         ctsio = &io->scsiio;
9174         ctsio->io_hdr.status = CTL_STATUS_NONE;
9175         ctsio->io_hdr.flags &= ~CTL_FLAG_IO_CONT;
9176         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9177         lbalen = (struct ctl_lba_len_flags *)
9178             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9179         lbalen->flags = CTL_LLF_WRITE;
9180
9181         CTL_DEBUG_PRINT(("ctl_cnw_cont: calling data_submit()\n"));
9182         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9183         return (retval);
9184 }
9185
9186 int
9187 ctl_cnw(struct ctl_scsiio *ctsio)
9188 {
9189         struct ctl_lun *lun;
9190         struct ctl_lba_len_flags *lbalen;
9191         uint64_t lba;
9192         uint32_t num_blocks;
9193         int fua, dpo;
9194         int retval;
9195
9196         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9197
9198         CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
9199
9200         fua = 0;
9201         dpo = 0;
9202
9203         retval = CTL_RETVAL_COMPLETE;
9204
9205         switch (ctsio->cdb[0]) {
9206         case COMPARE_AND_WRITE: {
9207                 struct scsi_compare_and_write *cdb;
9208
9209                 cdb = (struct scsi_compare_and_write *)ctsio->cdb;
9210
9211                 if (cdb->byte2 & SRW10_FUA)
9212                         fua = 1;
9213                 if (cdb->byte2 & SRW10_DPO)
9214                         dpo = 1;
9215                 lba = scsi_8btou64(cdb->addr);
9216                 num_blocks = cdb->length;
9217                 break;
9218         }
9219         default:
9220                 /*
9221                  * We got a command we don't support.  This shouldn't
9222                  * happen, commands should be filtered out above us.
9223                  */
9224                 ctl_set_invalid_opcode(ctsio);
9225                 ctl_done((union ctl_io *)ctsio);
9226
9227                 return (CTL_RETVAL_COMPLETE);
9228                 break; /* NOTREACHED */
9229         }
9230
9231         /*
9232          * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
9233          * interesting for us, but if RAIDCore is in write-back mode,
9234          * getting it to do write-through for a particular transaction may
9235          * not be possible.
9236          */
9237
9238         /*
9239          * The first check is to make sure we're in bounds, the second
9240          * check is to catch wrap-around problems.  If the lba + num blocks
9241          * is less than the lba, then we've wrapped around and the block
9242          * range is invalid anyway.
9243          */
9244         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9245          || ((lba + num_blocks) < lba)) {
9246                 ctl_set_lba_out_of_range(ctsio);
9247                 ctl_done((union ctl_io *)ctsio);
9248                 return (CTL_RETVAL_COMPLETE);
9249         }
9250
9251         /*
9252          * According to SBC-3, a transfer length of 0 is not an error.
9253          */
9254         if (num_blocks == 0) {
9255                 ctl_set_success(ctsio);
9256                 ctl_done((union ctl_io *)ctsio);
9257                 return (CTL_RETVAL_COMPLETE);
9258         }
9259
9260         ctsio->kern_total_len = 2 * num_blocks * lun->be_lun->blocksize;
9261         ctsio->kern_rel_offset = 0;
9262
9263         /*
9264          * Set the IO_CONT flag, so that if this I/O gets passed to
9265          * ctl_data_submit_done(), it'll get passed back to
9266          * ctl_ctl_cnw_cont() for further processing.
9267          */
9268         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
9269         ctsio->io_cont = ctl_cnw_cont;
9270
9271         lbalen = (struct ctl_lba_len_flags *)
9272             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9273         lbalen->lba = lba;
9274         lbalen->len = num_blocks;
9275         lbalen->flags = CTL_LLF_COMPARE;
9276
9277         CTL_DEBUG_PRINT(("ctl_cnw: calling data_submit()\n"));
9278         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9279         return (retval);
9280 }
9281
9282 int
9283 ctl_verify(struct ctl_scsiio *ctsio)
9284 {
9285         struct ctl_lun *lun;
9286         struct ctl_lba_len_flags *lbalen;
9287         uint64_t lba;
9288         uint32_t num_blocks;
9289         int bytchk, dpo;
9290         int retval;
9291
9292         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9293
9294         CTL_DEBUG_PRINT(("ctl_verify: command: %#x\n", ctsio->cdb[0]));
9295
9296         bytchk = 0;
9297         dpo = 0;
9298         retval = CTL_RETVAL_COMPLETE;
9299
9300         switch (ctsio->cdb[0]) {
9301         case VERIFY_10: {
9302                 struct scsi_verify_10 *cdb;
9303
9304                 cdb = (struct scsi_verify_10 *)ctsio->cdb;
9305                 if (cdb->byte2 & SVFY_BYTCHK)
9306                         bytchk = 1;
9307                 if (cdb->byte2 & SVFY_DPO)
9308                         dpo = 1;
9309                 lba = scsi_4btoul(cdb->addr);
9310                 num_blocks = scsi_2btoul(cdb->length);
9311                 break;
9312         }
9313         case VERIFY_12: {
9314                 struct scsi_verify_12 *cdb;
9315
9316                 cdb = (struct scsi_verify_12 *)ctsio->cdb;
9317                 if (cdb->byte2 & SVFY_BYTCHK)
9318                         bytchk = 1;
9319                 if (cdb->byte2 & SVFY_DPO)
9320                         dpo = 1;
9321                 lba = scsi_4btoul(cdb->addr);
9322                 num_blocks = scsi_4btoul(cdb->length);
9323                 break;
9324         }
9325         case VERIFY_16: {
9326                 struct scsi_rw_16 *cdb;
9327
9328                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
9329                 if (cdb->byte2 & SVFY_BYTCHK)
9330                         bytchk = 1;
9331                 if (cdb->byte2 & SVFY_DPO)
9332                         dpo = 1;
9333                 lba = scsi_8btou64(cdb->addr);
9334                 num_blocks = scsi_4btoul(cdb->length);
9335                 break;
9336         }
9337         default:
9338                 /*
9339                  * We got a command we don't support.  This shouldn't
9340                  * happen, commands should be filtered out above us.
9341                  */
9342                 ctl_set_invalid_opcode(ctsio);
9343                 ctl_done((union ctl_io *)ctsio);
9344                 return (CTL_RETVAL_COMPLETE);
9345         }
9346
9347         /*
9348          * The first check is to make sure we're in bounds, the second
9349          * check is to catch wrap-around problems.  If the lba + num blocks
9350          * is less than the lba, then we've wrapped around and the block
9351          * range is invalid anyway.
9352          */
9353         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9354          || ((lba + num_blocks) < lba)) {
9355                 ctl_set_lba_out_of_range(ctsio);
9356                 ctl_done((union ctl_io *)ctsio);
9357                 return (CTL_RETVAL_COMPLETE);
9358         }
9359
9360         /*
9361          * According to SBC-3, a transfer length of 0 is not an error.
9362          */
9363         if (num_blocks == 0) {
9364                 ctl_set_success(ctsio);
9365                 ctl_done((union ctl_io *)ctsio);
9366                 return (CTL_RETVAL_COMPLETE);
9367         }
9368
9369         lbalen = (struct ctl_lba_len_flags *)
9370             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9371         lbalen->lba = lba;
9372         lbalen->len = num_blocks;
9373         if (bytchk) {
9374                 lbalen->flags = CTL_LLF_COMPARE;
9375                 ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9376         } else {
9377                 lbalen->flags = CTL_LLF_VERIFY;
9378                 ctsio->kern_total_len = 0;
9379         }
9380         ctsio->kern_rel_offset = 0;
9381
9382         CTL_DEBUG_PRINT(("ctl_verify: calling data_submit()\n"));
9383         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9384         return (retval);
9385 }
9386
9387 int
9388 ctl_report_luns(struct ctl_scsiio *ctsio)
9389 {
9390         struct scsi_report_luns *cdb;
9391         struct scsi_report_luns_data *lun_data;
9392         struct ctl_lun *lun, *request_lun;
9393         int num_luns, retval;
9394         uint32_t alloc_len, lun_datalen;
9395         int num_filled, well_known;
9396         uint32_t initidx, targ_lun_id, lun_id;
9397
9398         retval = CTL_RETVAL_COMPLETE;
9399         well_known = 0;
9400
9401         cdb = (struct scsi_report_luns *)ctsio->cdb;
9402
9403         CTL_DEBUG_PRINT(("ctl_report_luns\n"));
9404
9405         mtx_lock(&control_softc->ctl_lock);
9406         num_luns = control_softc->num_luns;
9407         mtx_unlock(&control_softc->ctl_lock);
9408
9409         switch (cdb->select_report) {
9410         case RPL_REPORT_DEFAULT:
9411         case RPL_REPORT_ALL:
9412                 break;
9413         case RPL_REPORT_WELLKNOWN:
9414                 well_known = 1;
9415                 num_luns = 0;
9416                 break;
9417         default:
9418                 ctl_set_invalid_field(ctsio,
9419                                       /*sks_valid*/ 1,
9420                                       /*command*/ 1,
9421                                       /*field*/ 2,
9422                                       /*bit_valid*/ 0,
9423                                       /*bit*/ 0);
9424                 ctl_done((union ctl_io *)ctsio);
9425                 return (retval);
9426                 break; /* NOTREACHED */
9427         }
9428
9429         alloc_len = scsi_4btoul(cdb->length);
9430         /*
9431          * The initiator has to allocate at least 16 bytes for this request,
9432          * so he can at least get the header and the first LUN.  Otherwise
9433          * we reject the request (per SPC-3 rev 14, section 6.21).
9434          */
9435         if (alloc_len < (sizeof(struct scsi_report_luns_data) +
9436             sizeof(struct scsi_report_luns_lundata))) {
9437                 ctl_set_invalid_field(ctsio,
9438                                       /*sks_valid*/ 1,
9439                                       /*command*/ 1,
9440                                       /*field*/ 6,
9441                                       /*bit_valid*/ 0,
9442                                       /*bit*/ 0);
9443                 ctl_done((union ctl_io *)ctsio);
9444                 return (retval);
9445         }
9446
9447         request_lun = (struct ctl_lun *)
9448                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9449
9450         lun_datalen = sizeof(*lun_data) +
9451                 (num_luns * sizeof(struct scsi_report_luns_lundata));
9452
9453         ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
9454         lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
9455         ctsio->kern_sg_entries = 0;
9456
9457         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9458
9459         mtx_lock(&control_softc->ctl_lock);
9460         for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
9461                 lun_id = ctl_map_lun(ctsio->io_hdr.nexus.targ_port, targ_lun_id);
9462                 if (lun_id >= CTL_MAX_LUNS)
9463                         continue;
9464                 lun = control_softc->ctl_luns[lun_id];
9465                 if (lun == NULL)
9466                         continue;
9467
9468                 if (targ_lun_id <= 0xff) {
9469                         /*
9470                          * Peripheral addressing method, bus number 0.
9471                          */
9472                         lun_data->luns[num_filled].lundata[0] =
9473                                 RPL_LUNDATA_ATYP_PERIPH;
9474                         lun_data->luns[num_filled].lundata[1] = targ_lun_id;
9475                         num_filled++;
9476                 } else if (targ_lun_id <= 0x3fff) {
9477                         /*
9478                          * Flat addressing method.
9479                          */
9480                         lun_data->luns[num_filled].lundata[0] =
9481                                 RPL_LUNDATA_ATYP_FLAT |
9482                                 (targ_lun_id & RPL_LUNDATA_FLAT_LUN_MASK);
9483 #ifdef OLDCTLHEADERS
9484                                 (SRLD_ADDR_FLAT << SRLD_ADDR_SHIFT) |
9485                                 (targ_lun_id & SRLD_BUS_LUN_MASK);
9486 #endif
9487                         lun_data->luns[num_filled].lundata[1] =
9488 #ifdef OLDCTLHEADERS
9489                                 targ_lun_id >> SRLD_BUS_LUN_BITS;
9490 #endif
9491                                 targ_lun_id >> RPL_LUNDATA_FLAT_LUN_BITS;
9492                         num_filled++;
9493                 } else {
9494                         printf("ctl_report_luns: bogus LUN number %jd, "
9495                                "skipping\n", (intmax_t)targ_lun_id);
9496                 }
9497                 /*
9498                  * According to SPC-3, rev 14 section 6.21:
9499                  *
9500                  * "The execution of a REPORT LUNS command to any valid and
9501                  * installed logical unit shall clear the REPORTED LUNS DATA
9502                  * HAS CHANGED unit attention condition for all logical
9503                  * units of that target with respect to the requesting
9504                  * initiator. A valid and installed logical unit is one
9505                  * having a PERIPHERAL QUALIFIER of 000b in the standard
9506                  * INQUIRY data (see 6.4.2)."
9507                  *
9508                  * If request_lun is NULL, the LUN this report luns command
9509                  * was issued to is either disabled or doesn't exist. In that
9510                  * case, we shouldn't clear any pending lun change unit
9511                  * attention.
9512                  */
9513                 if (request_lun != NULL) {
9514                         mtx_lock(&lun->lun_lock);
9515                         lun->pending_sense[initidx].ua_pending &=
9516                                 ~CTL_UA_LUN_CHANGE;
9517                         mtx_unlock(&lun->lun_lock);
9518                 }
9519         }
9520         mtx_unlock(&control_softc->ctl_lock);
9521
9522         /*
9523          * It's quite possible that we've returned fewer LUNs than we allocated
9524          * space for.  Trim it.
9525          */
9526         lun_datalen = sizeof(*lun_data) +
9527                 (num_filled * sizeof(struct scsi_report_luns_lundata));
9528
9529         if (lun_datalen < alloc_len) {
9530                 ctsio->residual = alloc_len - lun_datalen;
9531                 ctsio->kern_data_len = lun_datalen;
9532                 ctsio->kern_total_len = lun_datalen;
9533         } else {
9534                 ctsio->residual = 0;
9535                 ctsio->kern_data_len = alloc_len;
9536                 ctsio->kern_total_len = alloc_len;
9537         }
9538         ctsio->kern_data_resid = 0;
9539         ctsio->kern_rel_offset = 0;
9540         ctsio->kern_sg_entries = 0;
9541
9542         /*
9543          * We set this to the actual data length, regardless of how much
9544          * space we actually have to return results.  If the user looks at
9545          * this value, he'll know whether or not he allocated enough space
9546          * and reissue the command if necessary.  We don't support well
9547          * known logical units, so if the user asks for that, return none.
9548          */
9549         scsi_ulto4b(lun_datalen - 8, lun_data->length);
9550
9551         /*
9552          * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
9553          * this request.
9554          */
9555         ctsio->scsi_status = SCSI_STATUS_OK;
9556
9557         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9558         ctsio->be_move_done = ctl_config_move_done;
9559         ctl_datamove((union ctl_io *)ctsio);
9560
9561         return (retval);
9562 }
9563
9564 int
9565 ctl_request_sense(struct ctl_scsiio *ctsio)
9566 {
9567         struct scsi_request_sense *cdb;
9568         struct scsi_sense_data *sense_ptr;
9569         struct ctl_lun *lun;
9570         uint32_t initidx;
9571         int have_error;
9572         scsi_sense_data_type sense_format;
9573
9574         cdb = (struct scsi_request_sense *)ctsio->cdb;
9575
9576         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9577
9578         CTL_DEBUG_PRINT(("ctl_request_sense\n"));
9579
9580         /*
9581          * Determine which sense format the user wants.
9582          */
9583         if (cdb->byte2 & SRS_DESC)
9584                 sense_format = SSD_TYPE_DESC;
9585         else
9586                 sense_format = SSD_TYPE_FIXED;
9587
9588         ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
9589         sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
9590         ctsio->kern_sg_entries = 0;
9591
9592         /*
9593          * struct scsi_sense_data, which is currently set to 256 bytes, is
9594          * larger than the largest allowed value for the length field in the
9595          * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
9596          */
9597         ctsio->residual = 0;
9598         ctsio->kern_data_len = cdb->length;
9599         ctsio->kern_total_len = cdb->length;
9600
9601         ctsio->kern_data_resid = 0;
9602         ctsio->kern_rel_offset = 0;
9603         ctsio->kern_sg_entries = 0;
9604
9605         /*
9606          * If we don't have a LUN, we don't have any pending sense.
9607          */
9608         if (lun == NULL)
9609                 goto no_sense;
9610
9611         have_error = 0;
9612         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9613         /*
9614          * Check for pending sense, and then for pending unit attentions.
9615          * Pending sense gets returned first, then pending unit attentions.
9616          */
9617         mtx_lock(&lun->lun_lock);
9618         if (ctl_is_set(lun->have_ca, initidx)) {
9619                 scsi_sense_data_type stored_format;
9620
9621                 /*
9622                  * Check to see which sense format was used for the stored
9623                  * sense data.
9624                  */
9625                 stored_format = scsi_sense_type(
9626                     &lun->pending_sense[initidx].sense);
9627
9628                 /*
9629                  * If the user requested a different sense format than the
9630                  * one we stored, then we need to convert it to the other
9631                  * format.  If we're going from descriptor to fixed format
9632                  * sense data, we may lose things in translation, depending
9633                  * on what options were used.
9634                  *
9635                  * If the stored format is SSD_TYPE_NONE (i.e. invalid),
9636                  * for some reason we'll just copy it out as-is.
9637                  */
9638                 if ((stored_format == SSD_TYPE_FIXED)
9639                  && (sense_format == SSD_TYPE_DESC))
9640                         ctl_sense_to_desc((struct scsi_sense_data_fixed *)
9641                             &lun->pending_sense[initidx].sense,
9642                             (struct scsi_sense_data_desc *)sense_ptr);
9643                 else if ((stored_format == SSD_TYPE_DESC)
9644                       && (sense_format == SSD_TYPE_FIXED))
9645                         ctl_sense_to_fixed((struct scsi_sense_data_desc *)
9646                             &lun->pending_sense[initidx].sense,
9647                             (struct scsi_sense_data_fixed *)sense_ptr);
9648                 else
9649                         memcpy(sense_ptr, &lun->pending_sense[initidx].sense,
9650                                ctl_min(sizeof(*sense_ptr),
9651                                sizeof(lun->pending_sense[initidx].sense)));
9652
9653                 ctl_clear_mask(lun->have_ca, initidx);
9654                 have_error = 1;
9655         } else if (lun->pending_sense[initidx].ua_pending != CTL_UA_NONE) {
9656                 ctl_ua_type ua_type;
9657
9658                 ua_type = ctl_build_ua(lun->pending_sense[initidx].ua_pending,
9659                                        sense_ptr, sense_format);
9660                 if (ua_type != CTL_UA_NONE) {
9661                         have_error = 1;
9662                         /* We're reporting this UA, so clear it */
9663                         lun->pending_sense[initidx].ua_pending &= ~ua_type;
9664                 }
9665         }
9666         mtx_unlock(&lun->lun_lock);
9667
9668         /*
9669          * We already have a pending error, return it.
9670          */
9671         if (have_error != 0) {
9672                 /*
9673                  * We report the SCSI status as OK, since the status of the
9674                  * request sense command itself is OK.
9675                  */
9676                 ctsio->scsi_status = SCSI_STATUS_OK;
9677
9678                 /*
9679                  * We report 0 for the sense length, because we aren't doing
9680                  * autosense in this case.  We're reporting sense as
9681                  * parameter data.
9682                  */
9683                 ctsio->sense_len = 0;
9684                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9685                 ctsio->be_move_done = ctl_config_move_done;
9686                 ctl_datamove((union ctl_io *)ctsio);
9687
9688                 return (CTL_RETVAL_COMPLETE);
9689         }
9690
9691 no_sense:
9692
9693         /*
9694          * No sense information to report, so we report that everything is
9695          * okay.
9696          */
9697         ctl_set_sense_data(sense_ptr,
9698                            lun,
9699                            sense_format,
9700                            /*current_error*/ 1,
9701                            /*sense_key*/ SSD_KEY_NO_SENSE,
9702                            /*asc*/ 0x00,
9703                            /*ascq*/ 0x00,
9704                            SSD_ELEM_NONE);
9705
9706         ctsio->scsi_status = SCSI_STATUS_OK;
9707
9708         /*
9709          * We report 0 for the sense length, because we aren't doing
9710          * autosense in this case.  We're reporting sense as parameter data.
9711          */
9712         ctsio->sense_len = 0;
9713         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9714         ctsio->be_move_done = ctl_config_move_done;
9715         ctl_datamove((union ctl_io *)ctsio);
9716
9717         return (CTL_RETVAL_COMPLETE);
9718 }
9719
9720 int
9721 ctl_tur(struct ctl_scsiio *ctsio)
9722 {
9723         struct ctl_lun *lun;
9724
9725         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9726
9727         CTL_DEBUG_PRINT(("ctl_tur\n"));
9728
9729         if (lun == NULL)
9730                 return (EINVAL);
9731
9732         ctsio->scsi_status = SCSI_STATUS_OK;
9733         ctsio->io_hdr.status = CTL_SUCCESS;
9734
9735         ctl_done((union ctl_io *)ctsio);
9736
9737         return (CTL_RETVAL_COMPLETE);
9738 }
9739
9740 #ifdef notyet
9741 static int
9742 ctl_cmddt_inquiry(struct ctl_scsiio *ctsio)
9743 {
9744
9745 }
9746 #endif
9747
9748 static int
9749 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
9750 {
9751         struct scsi_vpd_supported_pages *pages;
9752         int sup_page_size;
9753         struct ctl_lun *lun;
9754
9755         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9756
9757         sup_page_size = sizeof(struct scsi_vpd_supported_pages) *
9758             SCSI_EVPD_NUM_SUPPORTED_PAGES;
9759         ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9760         pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9761         ctsio->kern_sg_entries = 0;
9762
9763         if (sup_page_size < alloc_len) {
9764                 ctsio->residual = alloc_len - sup_page_size;
9765                 ctsio->kern_data_len = sup_page_size;
9766                 ctsio->kern_total_len = sup_page_size;
9767         } else {
9768                 ctsio->residual = 0;
9769                 ctsio->kern_data_len = alloc_len;
9770                 ctsio->kern_total_len = alloc_len;
9771         }
9772         ctsio->kern_data_resid = 0;
9773         ctsio->kern_rel_offset = 0;
9774         ctsio->kern_sg_entries = 0;
9775
9776         /*
9777          * The control device is always connected.  The disk device, on the
9778          * other hand, may not be online all the time.  Need to change this
9779          * to figure out whether the disk device is actually online or not.
9780          */
9781         if (lun != NULL)
9782                 pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9783                                 lun->be_lun->lun_type;
9784         else
9785                 pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9786
9787         pages->length = SCSI_EVPD_NUM_SUPPORTED_PAGES;
9788         /* Supported VPD pages */
9789         pages->page_list[0] = SVPD_SUPPORTED_PAGES;
9790         /* Serial Number */
9791         pages->page_list[1] = SVPD_UNIT_SERIAL_NUMBER;
9792         /* Device Identification */
9793         pages->page_list[2] = SVPD_DEVICE_ID;
9794         /* SCSI Ports */
9795         pages->page_list[3] = SVPD_SCSI_PORTS;
9796         /* Block limits */
9797         pages->page_list[4] = SVPD_BLOCK_LIMITS;
9798         /* Logical Block Provisioning */
9799         pages->page_list[5] = SVPD_LBP;
9800
9801         ctsio->scsi_status = SCSI_STATUS_OK;
9802
9803         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9804         ctsio->be_move_done = ctl_config_move_done;
9805         ctl_datamove((union ctl_io *)ctsio);
9806
9807         return (CTL_RETVAL_COMPLETE);
9808 }
9809
9810 static int
9811 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9812 {
9813         struct scsi_vpd_unit_serial_number *sn_ptr;
9814         struct ctl_lun *lun;
9815
9816         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9817
9818         ctsio->kern_data_ptr = malloc(sizeof(*sn_ptr), M_CTL, M_WAITOK | M_ZERO);
9819         sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9820         ctsio->kern_sg_entries = 0;
9821
9822         if (sizeof(*sn_ptr) < alloc_len) {
9823                 ctsio->residual = alloc_len - sizeof(*sn_ptr);
9824                 ctsio->kern_data_len = sizeof(*sn_ptr);
9825                 ctsio->kern_total_len = sizeof(*sn_ptr);
9826         } else {
9827                 ctsio->residual = 0;
9828                 ctsio->kern_data_len = alloc_len;
9829                 ctsio->kern_total_len = alloc_len;
9830         }
9831         ctsio->kern_data_resid = 0;
9832         ctsio->kern_rel_offset = 0;
9833         ctsio->kern_sg_entries = 0;
9834
9835         /*
9836          * The control device is always connected.  The disk device, on the
9837          * other hand, may not be online all the time.  Need to change this
9838          * to figure out whether the disk device is actually online or not.
9839          */
9840         if (lun != NULL)
9841                 sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9842                                   lun->be_lun->lun_type;
9843         else
9844                 sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9845
9846         sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9847         sn_ptr->length = ctl_min(sizeof(*sn_ptr) - 4, CTL_SN_LEN);
9848         /*
9849          * If we don't have a LUN, we just leave the serial number as
9850          * all spaces.
9851          */
9852         memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9853         if (lun != NULL) {
9854                 strncpy((char *)sn_ptr->serial_num,
9855                         (char *)lun->be_lun->serial_num, CTL_SN_LEN);
9856         }
9857         ctsio->scsi_status = SCSI_STATUS_OK;
9858
9859         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9860         ctsio->be_move_done = ctl_config_move_done;
9861         ctl_datamove((union ctl_io *)ctsio);
9862
9863         return (CTL_RETVAL_COMPLETE);
9864 }
9865
9866
9867 static int
9868 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9869 {
9870         struct scsi_vpd_device_id *devid_ptr;
9871         struct scsi_vpd_id_descriptor *desc;
9872         struct ctl_softc *ctl_softc;
9873         struct ctl_lun *lun;
9874         struct ctl_port *port;
9875         int data_len;
9876         uint8_t proto;
9877
9878         ctl_softc = control_softc;
9879
9880         port = ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)];
9881         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9882
9883         data_len = sizeof(struct scsi_vpd_device_id) +
9884             sizeof(struct scsi_vpd_id_descriptor) +
9885                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9886             sizeof(struct scsi_vpd_id_descriptor) +
9887                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9888         if (lun && lun->lun_devid)
9889                 data_len += lun->lun_devid->len;
9890         if (port->port_devid)
9891                 data_len += port->port_devid->len;
9892         if (port->target_devid)
9893                 data_len += port->target_devid->len;
9894
9895         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9896         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9897         ctsio->kern_sg_entries = 0;
9898
9899         if (data_len < alloc_len) {
9900                 ctsio->residual = alloc_len - data_len;
9901                 ctsio->kern_data_len = data_len;
9902                 ctsio->kern_total_len = data_len;
9903         } else {
9904                 ctsio->residual = 0;
9905                 ctsio->kern_data_len = alloc_len;
9906                 ctsio->kern_total_len = alloc_len;
9907         }
9908         ctsio->kern_data_resid = 0;
9909         ctsio->kern_rel_offset = 0;
9910         ctsio->kern_sg_entries = 0;
9911
9912         /*
9913          * The control device is always connected.  The disk device, on the
9914          * other hand, may not be online all the time.
9915          */
9916         if (lun != NULL)
9917                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9918                                      lun->be_lun->lun_type;
9919         else
9920                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9921         devid_ptr->page_code = SVPD_DEVICE_ID;
9922         scsi_ulto2b(data_len - 4, devid_ptr->length);
9923
9924         if (port->port_type == CTL_PORT_FC)
9925                 proto = SCSI_PROTO_FC << 4;
9926         else if (port->port_type == CTL_PORT_ISCSI)
9927                 proto = SCSI_PROTO_ISCSI << 4;
9928         else
9929                 proto = SCSI_PROTO_SPI << 4;
9930         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
9931
9932         /*
9933          * We're using a LUN association here.  i.e., this device ID is a
9934          * per-LUN identifier.
9935          */
9936         if (lun && lun->lun_devid) {
9937                 memcpy(desc, lun->lun_devid->data, lun->lun_devid->len);
9938                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
9939                     lun->lun_devid->len);
9940         }
9941
9942         /*
9943          * This is for the WWPN which is a port association.
9944          */
9945         if (port->port_devid) {
9946                 memcpy(desc, port->port_devid->data, port->port_devid->len);
9947                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
9948                     port->port_devid->len);
9949         }
9950
9951         /*
9952          * This is for the Relative Target Port(type 4h) identifier
9953          */
9954         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
9955         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
9956             SVPD_ID_TYPE_RELTARG;
9957         desc->length = 4;
9958         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port, &desc->identifier[2]);
9959         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9960             sizeof(struct scsi_vpd_id_rel_trgt_port_id));
9961
9962         /*
9963          * This is for the Target Port Group(type 5h) identifier
9964          */
9965         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
9966         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
9967             SVPD_ID_TYPE_TPORTGRP;
9968         desc->length = 4;
9969         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS + 1,
9970             &desc->identifier[2]);
9971         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9972             sizeof(struct scsi_vpd_id_trgt_port_grp_id));
9973
9974         /*
9975          * This is for the Target identifier
9976          */
9977         if (port->target_devid) {
9978                 memcpy(desc, port->target_devid->data, port->target_devid->len);
9979         }
9980
9981         ctsio->scsi_status = SCSI_STATUS_OK;
9982         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9983         ctsio->be_move_done = ctl_config_move_done;
9984         ctl_datamove((union ctl_io *)ctsio);
9985
9986         return (CTL_RETVAL_COMPLETE);
9987 }
9988
9989 static int
9990 ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len)
9991 {
9992         struct ctl_softc *softc = control_softc;
9993         struct scsi_vpd_scsi_ports *sp;
9994         struct scsi_vpd_port_designation *pd;
9995         struct scsi_vpd_port_designation_cont *pdc;
9996         struct ctl_lun *lun;
9997         struct ctl_port *port;
9998         int data_len, num_target_ports, id_len, g, pg, p;
9999         int num_target_port_groups, single;
10000
10001         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10002
10003         single = ctl_is_single;
10004         if (single)
10005                 num_target_port_groups = 1;
10006         else
10007                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
10008         num_target_ports = 0;
10009         id_len = 0;
10010         mtx_lock(&softc->ctl_lock);
10011         STAILQ_FOREACH(port, &softc->port_list, links) {
10012                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
10013                         continue;
10014                 if (ctl_map_lun_back(port->targ_port, lun->lun) >=
10015                     CTL_MAX_LUNS)
10016                         continue;
10017                 num_target_ports++;
10018                 if (port->port_devid)
10019                         id_len += port->port_devid->len;
10020         }
10021         mtx_unlock(&softc->ctl_lock);
10022
10023         data_len = sizeof(struct scsi_vpd_scsi_ports) + num_target_port_groups *
10024             num_target_ports * (sizeof(struct scsi_vpd_port_designation) +
10025              sizeof(struct scsi_vpd_port_designation_cont)) + id_len;
10026         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10027         sp = (struct scsi_vpd_scsi_ports *)ctsio->kern_data_ptr;
10028         ctsio->kern_sg_entries = 0;
10029
10030         if (data_len < alloc_len) {
10031                 ctsio->residual = alloc_len - data_len;
10032                 ctsio->kern_data_len = data_len;
10033                 ctsio->kern_total_len = data_len;
10034         } else {
10035                 ctsio->residual = 0;
10036                 ctsio->kern_data_len = alloc_len;
10037                 ctsio->kern_total_len = alloc_len;
10038         }
10039         ctsio->kern_data_resid = 0;
10040         ctsio->kern_rel_offset = 0;
10041         ctsio->kern_sg_entries = 0;
10042
10043         /*
10044          * The control device is always connected.  The disk device, on the
10045          * other hand, may not be online all the time.  Need to change this
10046          * to figure out whether the disk device is actually online or not.
10047          */
10048         if (lun != NULL)
10049                 sp->device = (SID_QUAL_LU_CONNECTED << 5) |
10050                                   lun->be_lun->lun_type;
10051         else
10052                 sp->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10053
10054         sp->page_code = SVPD_SCSI_PORTS;
10055         scsi_ulto2b(data_len - sizeof(struct scsi_vpd_scsi_ports),
10056             sp->page_length);
10057         pd = &sp->design[0];
10058
10059         mtx_lock(&softc->ctl_lock);
10060         if (softc->flags & CTL_FLAG_MASTER_SHELF)
10061                 pg = 0;
10062         else
10063                 pg = 1;
10064         for (g = 0; g < num_target_port_groups; g++) {
10065                 STAILQ_FOREACH(port, &softc->port_list, links) {
10066                         if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
10067                                 continue;
10068                         if (ctl_map_lun_back(port->targ_port, lun->lun) >=
10069                             CTL_MAX_LUNS)
10070                                 continue;
10071                         p = port->targ_port % CTL_MAX_PORTS + g * CTL_MAX_PORTS;
10072                         scsi_ulto2b(p, pd->relative_port_id);
10073                         scsi_ulto2b(0, pd->initiator_transportid_length);
10074                         pdc = (struct scsi_vpd_port_designation_cont *)
10075                             &pd->initiator_transportid[0];
10076                         if (port->port_devid && g == pg) {
10077                                 id_len = port->port_devid->len;
10078                                 scsi_ulto2b(port->port_devid->len,
10079                                     pdc->target_port_descriptors_length);
10080                                 memcpy(pdc->target_port_descriptors,
10081                                     port->port_devid->data, port->port_devid->len);
10082                         } else {
10083                                 id_len = 0;
10084                                 scsi_ulto2b(0, pdc->target_port_descriptors_length);
10085                         }
10086                         pd = (struct scsi_vpd_port_designation *)
10087                             ((uint8_t *)pdc->target_port_descriptors + id_len);
10088                 }
10089         }
10090         mtx_unlock(&softc->ctl_lock);
10091
10092         ctsio->scsi_status = SCSI_STATUS_OK;
10093         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10094         ctsio->be_move_done = ctl_config_move_done;
10095         ctl_datamove((union ctl_io *)ctsio);
10096
10097         return (CTL_RETVAL_COMPLETE);
10098 }
10099
10100 static int
10101 ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
10102 {
10103         struct scsi_vpd_block_limits *bl_ptr;
10104         struct ctl_lun *lun;
10105         int bs;
10106
10107         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10108         bs = lun->be_lun->blocksize;
10109
10110         ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
10111         bl_ptr = (struct scsi_vpd_block_limits *)ctsio->kern_data_ptr;
10112         ctsio->kern_sg_entries = 0;
10113
10114         if (sizeof(*bl_ptr) < alloc_len) {
10115                 ctsio->residual = alloc_len - sizeof(*bl_ptr);
10116                 ctsio->kern_data_len = sizeof(*bl_ptr);
10117                 ctsio->kern_total_len = sizeof(*bl_ptr);
10118         } else {
10119                 ctsio->residual = 0;
10120                 ctsio->kern_data_len = alloc_len;
10121                 ctsio->kern_total_len = alloc_len;
10122         }
10123         ctsio->kern_data_resid = 0;
10124         ctsio->kern_rel_offset = 0;
10125         ctsio->kern_sg_entries = 0;
10126
10127         /*
10128          * The control device is always connected.  The disk device, on the
10129          * other hand, may not be online all the time.  Need to change this
10130          * to figure out whether the disk device is actually online or not.
10131          */
10132         if (lun != NULL)
10133                 bl_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10134                                   lun->be_lun->lun_type;
10135         else
10136                 bl_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10137
10138         bl_ptr->page_code = SVPD_BLOCK_LIMITS;
10139         scsi_ulto2b(sizeof(*bl_ptr), bl_ptr->page_length);
10140         bl_ptr->max_cmp_write_len = 0xff;
10141         scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
10142         scsi_ulto4b(MAXPHYS / bs, bl_ptr->opt_txfer_len);
10143         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10144                 scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
10145                 scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_blk_cnt);
10146         }
10147         scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
10148
10149         ctsio->scsi_status = SCSI_STATUS_OK;
10150         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10151         ctsio->be_move_done = ctl_config_move_done;
10152         ctl_datamove((union ctl_io *)ctsio);
10153
10154         return (CTL_RETVAL_COMPLETE);
10155 }
10156
10157 static int
10158 ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
10159 {
10160         struct scsi_vpd_logical_block_prov *lbp_ptr;
10161         struct ctl_lun *lun;
10162         int bs;
10163
10164         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10165         bs = lun->be_lun->blocksize;
10166
10167         ctsio->kern_data_ptr = malloc(sizeof(*lbp_ptr), M_CTL, M_WAITOK | M_ZERO);
10168         lbp_ptr = (struct scsi_vpd_logical_block_prov *)ctsio->kern_data_ptr;
10169         ctsio->kern_sg_entries = 0;
10170
10171         if (sizeof(*lbp_ptr) < alloc_len) {
10172                 ctsio->residual = alloc_len - sizeof(*lbp_ptr);
10173                 ctsio->kern_data_len = sizeof(*lbp_ptr);
10174                 ctsio->kern_total_len = sizeof(*lbp_ptr);
10175         } else {
10176                 ctsio->residual = 0;
10177                 ctsio->kern_data_len = alloc_len;
10178                 ctsio->kern_total_len = alloc_len;
10179         }
10180         ctsio->kern_data_resid = 0;
10181         ctsio->kern_rel_offset = 0;
10182         ctsio->kern_sg_entries = 0;
10183
10184         /*
10185          * The control device is always connected.  The disk device, on the
10186          * other hand, may not be online all the time.  Need to change this
10187          * to figure out whether the disk device is actually online or not.
10188          */
10189         if (lun != NULL)
10190                 lbp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10191                                   lun->be_lun->lun_type;
10192         else
10193                 lbp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10194
10195         lbp_ptr->page_code = SVPD_LBP;
10196         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
10197                 lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 | SVPD_LBP_WS10;
10198
10199         ctsio->scsi_status = SCSI_STATUS_OK;
10200         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10201         ctsio->be_move_done = ctl_config_move_done;
10202         ctl_datamove((union ctl_io *)ctsio);
10203
10204         return (CTL_RETVAL_COMPLETE);
10205 }
10206
10207 static int
10208 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
10209 {
10210         struct scsi_inquiry *cdb;
10211         struct ctl_lun *lun;
10212         int alloc_len, retval;
10213
10214         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10215         cdb = (struct scsi_inquiry *)ctsio->cdb;
10216
10217         retval = CTL_RETVAL_COMPLETE;
10218
10219         alloc_len = scsi_2btoul(cdb->length);
10220
10221         switch (cdb->page_code) {
10222         case SVPD_SUPPORTED_PAGES:
10223                 retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
10224                 break;
10225         case SVPD_UNIT_SERIAL_NUMBER:
10226                 retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
10227                 break;
10228         case SVPD_DEVICE_ID:
10229                 retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
10230                 break;
10231         case SVPD_SCSI_PORTS:
10232                 retval = ctl_inquiry_evpd_scsi_ports(ctsio, alloc_len);
10233                 break;
10234         case SVPD_BLOCK_LIMITS:
10235                 retval = ctl_inquiry_evpd_block_limits(ctsio, alloc_len);
10236                 break;
10237         case SVPD_LBP:
10238                 retval = ctl_inquiry_evpd_lbp(ctsio, alloc_len);
10239                 break;
10240         default:
10241                 ctl_set_invalid_field(ctsio,
10242                                       /*sks_valid*/ 1,
10243                                       /*command*/ 1,
10244                                       /*field*/ 2,
10245                                       /*bit_valid*/ 0,
10246                                       /*bit*/ 0);
10247                 ctl_done((union ctl_io *)ctsio);
10248                 retval = CTL_RETVAL_COMPLETE;
10249                 break;
10250         }
10251
10252         return (retval);
10253 }
10254
10255 static int
10256 ctl_inquiry_std(struct ctl_scsiio *ctsio)
10257 {
10258         struct scsi_inquiry_data *inq_ptr;
10259         struct scsi_inquiry *cdb;
10260         struct ctl_softc *ctl_softc;
10261         struct ctl_lun *lun;
10262         char *val;
10263         uint32_t alloc_len;
10264         int is_fc;
10265
10266         ctl_softc = control_softc;
10267
10268         /*
10269          * Figure out whether we're talking to a Fibre Channel port or not.
10270          * We treat the ioctl front end, and any SCSI adapters, as packetized
10271          * SCSI front ends.
10272          */
10273         if (ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)]->port_type !=
10274             CTL_PORT_FC)
10275                 is_fc = 0;
10276         else
10277                 is_fc = 1;
10278
10279         lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10280         cdb = (struct scsi_inquiry *)ctsio->cdb;
10281         alloc_len = scsi_2btoul(cdb->length);
10282
10283         /*
10284          * We malloc the full inquiry data size here and fill it
10285          * in.  If the user only asks for less, we'll give him
10286          * that much.
10287          */
10288         ctsio->kern_data_ptr = malloc(sizeof(*inq_ptr), M_CTL, M_WAITOK | M_ZERO);
10289         inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
10290         ctsio->kern_sg_entries = 0;
10291         ctsio->kern_data_resid = 0;
10292         ctsio->kern_rel_offset = 0;
10293
10294         if (sizeof(*inq_ptr) < alloc_len) {
10295                 ctsio->residual = alloc_len - sizeof(*inq_ptr);
10296                 ctsio->kern_data_len = sizeof(*inq_ptr);
10297                 ctsio->kern_total_len = sizeof(*inq_ptr);
10298         } else {
10299                 ctsio->residual = 0;
10300                 ctsio->kern_data_len = alloc_len;
10301                 ctsio->kern_total_len = alloc_len;
10302         }
10303
10304         /*
10305          * If we have a LUN configured, report it as connected.  Otherwise,
10306          * report that it is offline or no device is supported, depending
10307          * on the value of inquiry_pq_no_lun.
10308          *
10309          * According to the spec (SPC-4 r34), the peripheral qualifier
10310          * SID_QUAL_LU_OFFLINE (001b) is used in the following scenario:
10311          *
10312          * "A peripheral device having the specified peripheral device type
10313          * is not connected to this logical unit. However, the device
10314          * server is capable of supporting the specified peripheral device
10315          * type on this logical unit."
10316          *
10317          * According to the same spec, the peripheral qualifier
10318          * SID_QUAL_BAD_LU (011b) is used in this scenario:
10319          *
10320          * "The device server is not capable of supporting a peripheral
10321          * device on this logical unit. For this peripheral qualifier the
10322          * peripheral device type shall be set to 1Fh. All other peripheral
10323          * device type values are reserved for this peripheral qualifier."
10324          *
10325          * Given the text, it would seem that we probably want to report that
10326          * the LUN is offline here.  There is no LUN connected, but we can
10327          * support a LUN at the given LUN number.
10328          *
10329          * In the real world, though, it sounds like things are a little
10330          * different:
10331          *
10332          * - Linux, when presented with a LUN with the offline peripheral
10333          *   qualifier, will create an sg driver instance for it.  So when
10334          *   you attach it to CTL, you wind up with a ton of sg driver
10335          *   instances.  (One for every LUN that Linux bothered to probe.)
10336          *   Linux does this despite the fact that it issues a REPORT LUNs
10337          *   to LUN 0 to get the inventory of supported LUNs.
10338          *
10339          * - There is other anecdotal evidence (from Emulex folks) about
10340          *   arrays that use the offline peripheral qualifier for LUNs that
10341          *   are on the "passive" path in an active/passive array.
10342          *
10343          * So the solution is provide a hopefully reasonable default
10344          * (return bad/no LUN) and allow the user to change the behavior
10345          * with a tunable/sysctl variable.
10346          */
10347         if (lun != NULL)
10348                 inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10349                                   lun->be_lun->lun_type;
10350         else if (ctl_softc->inquiry_pq_no_lun == 0)
10351                 inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10352         else
10353                 inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
10354
10355         /* RMB in byte 2 is 0 */
10356         inq_ptr->version = SCSI_REV_SPC3;
10357
10358         /*
10359          * According to SAM-3, even if a device only supports a single
10360          * level of LUN addressing, it should still set the HISUP bit:
10361          *
10362          * 4.9.1 Logical unit numbers overview
10363          *
10364          * All logical unit number formats described in this standard are
10365          * hierarchical in structure even when only a single level in that
10366          * hierarchy is used. The HISUP bit shall be set to one in the
10367          * standard INQUIRY data (see SPC-2) when any logical unit number
10368          * format described in this standard is used.  Non-hierarchical
10369          * formats are outside the scope of this standard.
10370          *
10371          * Therefore we set the HiSup bit here.
10372          *
10373          * The reponse format is 2, per SPC-3.
10374          */
10375         inq_ptr->response_format = SID_HiSup | 2;
10376
10377         inq_ptr->additional_length = sizeof(*inq_ptr) - 4;
10378         CTL_DEBUG_PRINT(("additional_length = %d\n",
10379                          inq_ptr->additional_length));
10380
10381         inq_ptr->spc3_flags = SPC3_SID_TPGS_IMPLICIT;
10382         /* 16 bit addressing */
10383         if (is_fc == 0)
10384                 inq_ptr->spc2_flags = SPC2_SID_ADDR16;
10385         /* XXX set the SID_MultiP bit here if we're actually going to
10386            respond on multiple ports */
10387         inq_ptr->spc2_flags |= SPC2_SID_MultiP;
10388
10389         /* 16 bit data bus, synchronous transfers */
10390         /* XXX these flags don't apply for FC */
10391         if (is_fc == 0)
10392                 inq_ptr->flags = SID_WBus16 | SID_Sync;
10393         /*
10394          * XXX KDM do we want to support tagged queueing on the control
10395          * device at all?
10396          */
10397         if ((lun == NULL)
10398          || (lun->be_lun->lun_type != T_PROCESSOR))
10399                 inq_ptr->flags |= SID_CmdQue;
10400         /*
10401          * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
10402          * We have 8 bytes for the vendor name, and 16 bytes for the device
10403          * name and 4 bytes for the revision.
10404          */
10405         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10406             "vendor")) == NULL) {
10407                 strcpy(inq_ptr->vendor, CTL_VENDOR);
10408         } else {
10409                 memset(inq_ptr->vendor, ' ', sizeof(inq_ptr->vendor));
10410                 strncpy(inq_ptr->vendor, val,
10411                     min(sizeof(inq_ptr->vendor), strlen(val)));
10412         }
10413         if (lun == NULL) {
10414                 strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
10415         } else if ((val = ctl_get_opt(&lun->be_lun->options, "product")) == NULL) {
10416                 switch (lun->be_lun->lun_type) {
10417                 case T_DIRECT:
10418                         strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
10419                         break;
10420                 case T_PROCESSOR:
10421                         strcpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT);
10422                         break;
10423                 default:
10424                         strcpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT);
10425                         break;
10426                 }
10427         } else {
10428                 memset(inq_ptr->product, ' ', sizeof(inq_ptr->product));
10429                 strncpy(inq_ptr->product, val,
10430                     min(sizeof(inq_ptr->product), strlen(val)));
10431         }
10432
10433         /*
10434          * XXX make this a macro somewhere so it automatically gets
10435          * incremented when we make changes.
10436          */
10437         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10438             "revision")) == NULL) {
10439                 strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
10440         } else {
10441                 memset(inq_ptr->revision, ' ', sizeof(inq_ptr->revision));
10442                 strncpy(inq_ptr->revision, val,
10443                     min(sizeof(inq_ptr->revision), strlen(val)));
10444         }
10445
10446         /*
10447          * For parallel SCSI, we support double transition and single
10448          * transition clocking.  We also support QAS (Quick Arbitration
10449          * and Selection) and Information Unit transfers on both the
10450          * control and array devices.
10451          */
10452         if (is_fc == 0)
10453                 inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
10454                                     SID_SPI_IUS;
10455
10456         /* SAM-3 */
10457         scsi_ulto2b(0x0060, inq_ptr->version1);
10458         /* SPC-3 (no version claimed) XXX should we claim a version? */
10459         scsi_ulto2b(0x0300, inq_ptr->version2);
10460         if (is_fc) {
10461                 /* FCP-2 ANSI INCITS.350:2003 */
10462                 scsi_ulto2b(0x0917, inq_ptr->version3);
10463         } else {
10464                 /* SPI-4 ANSI INCITS.362:200x */
10465                 scsi_ulto2b(0x0B56, inq_ptr->version3);
10466         }
10467
10468         if (lun == NULL) {
10469                 /* SBC-2 (no version claimed) XXX should we claim a version? */
10470                 scsi_ulto2b(0x0320, inq_ptr->version4);
10471         } else {
10472                 switch (lun->be_lun->lun_type) {
10473                 case T_DIRECT:
10474                         /*
10475                          * SBC-2 (no version claimed) XXX should we claim a
10476                          * version?
10477                          */
10478                         scsi_ulto2b(0x0320, inq_ptr->version4);
10479                         break;
10480                 case T_PROCESSOR:
10481                 default:
10482                         break;
10483                 }
10484         }
10485
10486         ctsio->scsi_status = SCSI_STATUS_OK;
10487         if (ctsio->kern_data_len > 0) {
10488                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10489                 ctsio->be_move_done = ctl_config_move_done;
10490                 ctl_datamove((union ctl_io *)ctsio);
10491         } else {
10492                 ctsio->io_hdr.status = CTL_SUCCESS;
10493                 ctl_done((union ctl_io *)ctsio);
10494         }
10495
10496         return (CTL_RETVAL_COMPLETE);
10497 }
10498
10499 int
10500 ctl_inquiry(struct ctl_scsiio *ctsio)
10501 {
10502         struct scsi_inquiry *cdb;
10503         int retval;
10504
10505         cdb = (struct scsi_inquiry *)ctsio->cdb;
10506
10507         retval = 0;
10508
10509         CTL_DEBUG_PRINT(("ctl_inquiry\n"));
10510
10511         /*
10512          * Right now, we don't support the CmdDt inquiry information.
10513          * This would be nice to support in the future.  When we do
10514          * support it, we should change this test so that it checks to make
10515          * sure SI_EVPD and SI_CMDDT aren't both set at the same time.
10516          */
10517 #ifdef notyet
10518         if (((cdb->byte2 & SI_EVPD)
10519          && (cdb->byte2 & SI_CMDDT)))
10520 #endif
10521         if (cdb->byte2 & SI_CMDDT) {
10522                 /*
10523                  * Point to the SI_CMDDT bit.  We might change this
10524                  * when we support SI_CMDDT, but since both bits would be
10525                  * "wrong", this should probably just stay as-is then.
10526                  */
10527                 ctl_set_invalid_field(ctsio,
10528                                       /*sks_valid*/ 1,
10529                                       /*command*/ 1,
10530                                       /*field*/ 1,
10531                                       /*bit_valid*/ 1,
10532                                       /*bit*/ 1);
10533                 ctl_done((union ctl_io *)ctsio);
10534                 return (CTL_RETVAL_COMPLETE);
10535         }
10536         if (cdb->byte2 & SI_EVPD)
10537                 retval = ctl_inquiry_evpd(ctsio);
10538 #ifdef notyet
10539         else if (cdb->byte2 & SI_CMDDT)
10540                 retval = ctl_inquiry_cmddt(ctsio);
10541 #endif
10542         else
10543                 retval = ctl_inquiry_std(ctsio);
10544
10545         return (retval);
10546 }
10547
10548 /*
10549  * For known CDB types, parse the LBA and length.
10550  */
10551 static int
10552 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len)
10553 {
10554         if (io->io_hdr.io_type != CTL_IO_SCSI)
10555                 return (1);
10556
10557         switch (io->scsiio.cdb[0]) {
10558         case COMPARE_AND_WRITE: {
10559                 struct scsi_compare_and_write *cdb;
10560
10561                 cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
10562
10563                 *lba = scsi_8btou64(cdb->addr);
10564                 *len = cdb->length;
10565                 break;
10566         }
10567         case READ_6:
10568         case WRITE_6: {
10569                 struct scsi_rw_6 *cdb;
10570
10571                 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
10572
10573                 *lba = scsi_3btoul(cdb->addr);
10574                 /* only 5 bits are valid in the most significant address byte */
10575                 *lba &= 0x1fffff;
10576                 *len = cdb->length;
10577                 break;
10578         }
10579         case READ_10:
10580         case WRITE_10: {
10581                 struct scsi_rw_10 *cdb;
10582
10583                 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
10584
10585                 *lba = scsi_4btoul(cdb->addr);
10586                 *len = scsi_2btoul(cdb->length);
10587                 break;
10588         }
10589         case WRITE_VERIFY_10: {
10590                 struct scsi_write_verify_10 *cdb;
10591
10592                 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
10593
10594                 *lba = scsi_4btoul(cdb->addr);
10595                 *len = scsi_2btoul(cdb->length);
10596                 break;
10597         }
10598         case READ_12:
10599         case WRITE_12: {
10600                 struct scsi_rw_12 *cdb;
10601
10602                 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
10603
10604                 *lba = scsi_4btoul(cdb->addr);
10605                 *len = scsi_4btoul(cdb->length);
10606                 break;
10607         }
10608         case WRITE_VERIFY_12: {
10609                 struct scsi_write_verify_12 *cdb;
10610
10611                 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
10612
10613                 *lba = scsi_4btoul(cdb->addr);
10614                 *len = scsi_4btoul(cdb->length);
10615                 break;
10616         }
10617         case READ_16:
10618         case WRITE_16: {
10619                 struct scsi_rw_16 *cdb;
10620
10621                 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
10622
10623                 *lba = scsi_8btou64(cdb->addr);
10624                 *len = scsi_4btoul(cdb->length);
10625                 break;
10626         }
10627         case WRITE_VERIFY_16: {
10628                 struct scsi_write_verify_16 *cdb;
10629
10630                 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
10631
10632
10633                 *lba = scsi_8btou64(cdb->addr);
10634                 *len = scsi_4btoul(cdb->length);
10635                 break;
10636         }
10637         case WRITE_SAME_10: {
10638                 struct scsi_write_same_10 *cdb;
10639
10640                 cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10641
10642                 *lba = scsi_4btoul(cdb->addr);
10643                 *len = scsi_2btoul(cdb->length);
10644                 break;
10645         }
10646         case WRITE_SAME_16: {
10647                 struct scsi_write_same_16 *cdb;
10648
10649                 cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10650
10651                 *lba = scsi_8btou64(cdb->addr);
10652                 *len = scsi_4btoul(cdb->length);
10653                 break;
10654         }
10655         case VERIFY_10: {
10656                 struct scsi_verify_10 *cdb;
10657
10658                 cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
10659
10660                 *lba = scsi_4btoul(cdb->addr);
10661                 *len = scsi_2btoul(cdb->length);
10662                 break;
10663         }
10664         case VERIFY_12: {
10665                 struct scsi_verify_12 *cdb;
10666
10667                 cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
10668
10669                 *lba = scsi_4btoul(cdb->addr);
10670                 *len = scsi_4btoul(cdb->length);
10671                 break;
10672         }
10673         case VERIFY_16: {
10674                 struct scsi_verify_16 *cdb;
10675
10676                 cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
10677
10678                 *lba = scsi_8btou64(cdb->addr);
10679                 *len = scsi_4btoul(cdb->length);
10680                 break;
10681         }
10682         default:
10683                 return (1);
10684                 break; /* NOTREACHED */
10685         }
10686
10687         return (0);
10688 }
10689
10690 static ctl_action
10691 ctl_extent_check_lba(uint64_t lba1, uint32_t len1, uint64_t lba2, uint32_t len2)
10692 {
10693         uint64_t endlba1, endlba2;
10694
10695         endlba1 = lba1 + len1 - 1;
10696         endlba2 = lba2 + len2 - 1;
10697
10698         if ((endlba1 < lba2)
10699          || (endlba2 < lba1))
10700                 return (CTL_ACTION_PASS);
10701         else
10702                 return (CTL_ACTION_BLOCK);
10703 }
10704
10705 static ctl_action
10706 ctl_extent_check(union ctl_io *io1, union ctl_io *io2)
10707 {
10708         uint64_t lba1, lba2;
10709         uint32_t len1, len2;
10710         int retval;
10711
10712         retval = ctl_get_lba_len(io1, &lba1, &len1);
10713         if (retval != 0)
10714                 return (CTL_ACTION_ERROR);
10715
10716         retval = ctl_get_lba_len(io2, &lba2, &len2);
10717         if (retval != 0)
10718                 return (CTL_ACTION_ERROR);
10719
10720         return (ctl_extent_check_lba(lba1, len1, lba2, len2));
10721 }
10722
10723 static ctl_action
10724 ctl_check_for_blockage(union ctl_io *pending_io, union ctl_io *ooa_io)
10725 {
10726         const struct ctl_cmd_entry *pending_entry, *ooa_entry;
10727         ctl_serialize_action *serialize_row;
10728
10729         /*
10730          * The initiator attempted multiple untagged commands at the same
10731          * time.  Can't do that.
10732          */
10733         if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10734          && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10735          && ((pending_io->io_hdr.nexus.targ_port ==
10736               ooa_io->io_hdr.nexus.targ_port)
10737           && (pending_io->io_hdr.nexus.initid.id ==
10738               ooa_io->io_hdr.nexus.initid.id))
10739          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
10740                 return (CTL_ACTION_OVERLAP);
10741
10742         /*
10743          * The initiator attempted to send multiple tagged commands with
10744          * the same ID.  (It's fine if different initiators have the same
10745          * tag ID.)
10746          *
10747          * Even if all of those conditions are true, we don't kill the I/O
10748          * if the command ahead of us has been aborted.  We won't end up
10749          * sending it to the FETD, and it's perfectly legal to resend a
10750          * command with the same tag number as long as the previous
10751          * instance of this tag number has been aborted somehow.
10752          */
10753         if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10754          && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10755          && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
10756          && ((pending_io->io_hdr.nexus.targ_port ==
10757               ooa_io->io_hdr.nexus.targ_port)
10758           && (pending_io->io_hdr.nexus.initid.id ==
10759               ooa_io->io_hdr.nexus.initid.id))
10760          && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
10761                 return (CTL_ACTION_OVERLAP_TAG);
10762
10763         /*
10764          * If we get a head of queue tag, SAM-3 says that we should
10765          * immediately execute it.
10766          *
10767          * What happens if this command would normally block for some other
10768          * reason?  e.g. a request sense with a head of queue tag
10769          * immediately after a write.  Normally that would block, but this
10770          * will result in its getting executed immediately...
10771          *
10772          * We currently return "pass" instead of "skip", so we'll end up
10773          * going through the rest of the queue to check for overlapped tags.
10774          *
10775          * XXX KDM check for other types of blockage first??
10776          */
10777         if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10778                 return (CTL_ACTION_PASS);
10779
10780         /*
10781          * Ordered tags have to block until all items ahead of them
10782          * have completed.  If we get called with an ordered tag, we always
10783          * block, if something else is ahead of us in the queue.
10784          */
10785         if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
10786                 return (CTL_ACTION_BLOCK);
10787
10788         /*
10789          * Simple tags get blocked until all head of queue and ordered tags
10790          * ahead of them have completed.  I'm lumping untagged commands in
10791          * with simple tags here.  XXX KDM is that the right thing to do?
10792          */
10793         if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10794           || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
10795          && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10796           || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
10797                 return (CTL_ACTION_BLOCK);
10798
10799         pending_entry = ctl_get_cmd_entry(&pending_io->scsiio);
10800         ooa_entry = ctl_get_cmd_entry(&ooa_io->scsiio);
10801
10802         serialize_row = ctl_serialize_table[ooa_entry->seridx];
10803
10804         switch (serialize_row[pending_entry->seridx]) {
10805         case CTL_SER_BLOCK:
10806                 return (CTL_ACTION_BLOCK);
10807                 break; /* NOTREACHED */
10808         case CTL_SER_EXTENT:
10809                 return (ctl_extent_check(pending_io, ooa_io));
10810                 break; /* NOTREACHED */
10811         case CTL_SER_PASS:
10812                 return (CTL_ACTION_PASS);
10813                 break; /* NOTREACHED */
10814         case CTL_SER_SKIP:
10815                 return (CTL_ACTION_SKIP);
10816                 break;
10817         default:
10818                 panic("invalid serialization value %d",
10819                       serialize_row[pending_entry->seridx]);
10820                 break; /* NOTREACHED */
10821         }
10822
10823         return (CTL_ACTION_ERROR);
10824 }
10825
10826 /*
10827  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
10828  * Assumptions:
10829  * - pending_io is generally either incoming, or on the blocked queue
10830  * - starting I/O is the I/O we want to start the check with.
10831  */
10832 static ctl_action
10833 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
10834               union ctl_io *starting_io)
10835 {
10836         union ctl_io *ooa_io;
10837         ctl_action action;
10838
10839         mtx_assert(&lun->lun_lock, MA_OWNED);
10840
10841         /*
10842          * Run back along the OOA queue, starting with the current
10843          * blocked I/O and going through every I/O before it on the
10844          * queue.  If starting_io is NULL, we'll just end up returning
10845          * CTL_ACTION_PASS.
10846          */
10847         for (ooa_io = starting_io; ooa_io != NULL;
10848              ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
10849              ooa_links)){
10850
10851                 /*
10852                  * This routine just checks to see whether
10853                  * cur_blocked is blocked by ooa_io, which is ahead
10854                  * of it in the queue.  It doesn't queue/dequeue
10855                  * cur_blocked.
10856                  */
10857                 action = ctl_check_for_blockage(pending_io, ooa_io);
10858                 switch (action) {
10859                 case CTL_ACTION_BLOCK:
10860                 case CTL_ACTION_OVERLAP:
10861                 case CTL_ACTION_OVERLAP_TAG:
10862                 case CTL_ACTION_SKIP:
10863                 case CTL_ACTION_ERROR:
10864                         return (action);
10865                         break; /* NOTREACHED */
10866                 case CTL_ACTION_PASS:
10867                         break;
10868                 default:
10869                         panic("invalid action %d", action);
10870                         break;  /* NOTREACHED */
10871                 }
10872         }
10873
10874         return (CTL_ACTION_PASS);
10875 }
10876
10877 /*
10878  * Assumptions:
10879  * - An I/O has just completed, and has been removed from the per-LUN OOA
10880  *   queue, so some items on the blocked queue may now be unblocked.
10881  */
10882 static int
10883 ctl_check_blocked(struct ctl_lun *lun)
10884 {
10885         union ctl_io *cur_blocked, *next_blocked;
10886
10887         mtx_assert(&lun->lun_lock, MA_OWNED);
10888
10889         /*
10890          * Run forward from the head of the blocked queue, checking each
10891          * entry against the I/Os prior to it on the OOA queue to see if
10892          * there is still any blockage.
10893          *
10894          * We cannot use the TAILQ_FOREACH() macro, because it can't deal
10895          * with our removing a variable on it while it is traversing the
10896          * list.
10897          */
10898         for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
10899              cur_blocked != NULL; cur_blocked = next_blocked) {
10900                 union ctl_io *prev_ooa;
10901                 ctl_action action;
10902
10903                 next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
10904                                                           blocked_links);
10905
10906                 prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
10907                                                       ctl_ooaq, ooa_links);
10908
10909                 /*
10910                  * If cur_blocked happens to be the first item in the OOA
10911                  * queue now, prev_ooa will be NULL, and the action
10912                  * returned will just be CTL_ACTION_PASS.
10913                  */
10914                 action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
10915
10916                 switch (action) {
10917                 case CTL_ACTION_BLOCK:
10918                         /* Nothing to do here, still blocked */
10919                         break;
10920                 case CTL_ACTION_OVERLAP:
10921                 case CTL_ACTION_OVERLAP_TAG:
10922                         /*
10923                          * This shouldn't happen!  In theory we've already
10924                          * checked this command for overlap...
10925                          */
10926                         break;
10927                 case CTL_ACTION_PASS:
10928                 case CTL_ACTION_SKIP: {
10929                         struct ctl_softc *softc;
10930                         const struct ctl_cmd_entry *entry;
10931                         uint32_t initidx;
10932                         int isc_retval;
10933
10934                         /*
10935                          * The skip case shouldn't happen, this transaction
10936                          * should have never made it onto the blocked queue.
10937                          */
10938                         /*
10939                          * This I/O is no longer blocked, we can remove it
10940                          * from the blocked queue.  Since this is a TAILQ
10941                          * (doubly linked list), we can do O(1) removals
10942                          * from any place on the list.
10943                          */
10944                         TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
10945                                      blocked_links);
10946                         cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10947
10948                         if (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC){
10949                                 /*
10950                                  * Need to send IO back to original side to
10951                                  * run
10952                                  */
10953                                 union ctl_ha_msg msg_info;
10954
10955                                 msg_info.hdr.original_sc =
10956                                         cur_blocked->io_hdr.original_sc;
10957                                 msg_info.hdr.serializing_sc = cur_blocked;
10958                                 msg_info.hdr.msg_type = CTL_MSG_R2R;
10959                                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10960                                      &msg_info, sizeof(msg_info), 0)) >
10961                                      CTL_HA_STATUS_SUCCESS) {
10962                                         printf("CTL:Check Blocked error from "
10963                                                "ctl_ha_msg_send %d\n",
10964                                                isc_retval);
10965                                 }
10966                                 break;
10967                         }
10968                         entry = ctl_get_cmd_entry(&cur_blocked->scsiio);
10969                         softc = control_softc;
10970
10971                         initidx = ctl_get_initindex(&cur_blocked->io_hdr.nexus);
10972
10973                         /*
10974                          * Check this I/O for LUN state changes that may
10975                          * have happened while this command was blocked.
10976                          * The LUN state may have been changed by a command
10977                          * ahead of us in the queue, so we need to re-check
10978                          * for any states that can be caused by SCSI
10979                          * commands.
10980                          */
10981                         if (ctl_scsiio_lun_check(softc, lun, entry,
10982                                                  &cur_blocked->scsiio) == 0) {
10983                                 cur_blocked->io_hdr.flags |=
10984                                                       CTL_FLAG_IS_WAS_ON_RTR;
10985                                 ctl_enqueue_rtr(cur_blocked);
10986                         } else
10987                                 ctl_done(cur_blocked);
10988                         break;
10989                 }
10990                 default:
10991                         /*
10992                          * This probably shouldn't happen -- we shouldn't
10993                          * get CTL_ACTION_ERROR, or anything else.
10994                          */
10995                         break;
10996                 }
10997         }
10998
10999         return (CTL_RETVAL_COMPLETE);
11000 }
11001
11002 /*
11003  * This routine (with one exception) checks LUN flags that can be set by
11004  * commands ahead of us in the OOA queue.  These flags have to be checked
11005  * when a command initially comes in, and when we pull a command off the
11006  * blocked queue and are preparing to execute it.  The reason we have to
11007  * check these flags for commands on the blocked queue is that the LUN
11008  * state may have been changed by a command ahead of us while we're on the
11009  * blocked queue.
11010  *
11011  * Ordering is somewhat important with these checks, so please pay
11012  * careful attention to the placement of any new checks.
11013  */
11014 static int
11015 ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
11016     const struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
11017 {
11018         int retval;
11019
11020         retval = 0;
11021
11022         mtx_assert(&lun->lun_lock, MA_OWNED);
11023
11024         /*
11025          * If this shelf is a secondary shelf controller, we have to reject
11026          * any media access commands.
11027          */
11028 #if 0
11029         /* No longer needed for HA */
11030         if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
11031          && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
11032                 ctl_set_lun_standby(ctsio);
11033                 retval = 1;
11034                 goto bailout;
11035         }
11036 #endif
11037
11038         /*
11039          * Check for a reservation conflict.  If this command isn't allowed
11040          * even on reserved LUNs, and if this initiator isn't the one who
11041          * reserved us, reject the command with a reservation conflict.
11042          */
11043         if ((lun->flags & CTL_LUN_RESERVED)
11044          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
11045                 if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
11046                  || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
11047                  || (ctsio->io_hdr.nexus.targ_target.id !=
11048                      lun->rsv_nexus.targ_target.id)) {
11049                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
11050                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
11051                         retval = 1;
11052                         goto bailout;
11053                 }
11054         }
11055
11056         if ( (lun->flags & CTL_LUN_PR_RESERVED)
11057          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV) == 0)) {
11058                 uint32_t residx;
11059
11060                 residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
11061                 /*
11062                  * if we aren't registered or it's a res holder type
11063                  * reservation and this isn't the res holder then set a
11064                  * conflict.
11065                  * NOTE: Commands which might be allowed on write exclusive
11066                  * type reservations are checked in the particular command
11067                  * for a conflict. Read and SSU are the only ones.
11068                  */
11069                 if (!lun->per_res[residx].registered
11070                  || (residx != lun->pr_res_idx && lun->res_type < 4)) {
11071                         ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
11072                         ctsio->io_hdr.status = CTL_SCSI_ERROR;
11073                         retval = 1;
11074                         goto bailout;
11075                 }
11076
11077         }
11078
11079         if ((lun->flags & CTL_LUN_OFFLINE)
11080          && ((entry->flags & CTL_CMD_FLAG_OK_ON_OFFLINE) == 0)) {
11081                 ctl_set_lun_not_ready(ctsio);
11082                 retval = 1;
11083                 goto bailout;
11084         }
11085
11086         /*
11087          * If the LUN is stopped, see if this particular command is allowed
11088          * for a stopped lun.  Otherwise, reject it with 0x04,0x02.
11089          */
11090         if ((lun->flags & CTL_LUN_STOPPED)
11091          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
11092                 /* "Logical unit not ready, initializing cmd. required" */
11093                 ctl_set_lun_stopped(ctsio);
11094                 retval = 1;
11095                 goto bailout;
11096         }
11097
11098         if ((lun->flags & CTL_LUN_INOPERABLE)
11099          && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
11100                 /* "Medium format corrupted" */
11101                 ctl_set_medium_format_corrupted(ctsio);
11102                 retval = 1;
11103                 goto bailout;
11104         }
11105
11106 bailout:
11107         return (retval);
11108
11109 }
11110
11111 static void
11112 ctl_failover_io(union ctl_io *io, int have_lock)
11113 {
11114         ctl_set_busy(&io->scsiio);
11115         ctl_done(io);
11116 }
11117
11118 static void
11119 ctl_failover(void)
11120 {
11121         struct ctl_lun *lun;
11122         struct ctl_softc *ctl_softc;
11123         union ctl_io *next_io, *pending_io;
11124         union ctl_io *io;
11125         int lun_idx;
11126         int i;
11127
11128         ctl_softc = control_softc;
11129
11130         mtx_lock(&ctl_softc->ctl_lock);
11131         /*
11132          * Remove any cmds from the other SC from the rtr queue.  These
11133          * will obviously only be for LUNs for which we're the primary.
11134          * We can't send status or get/send data for these commands.
11135          * Since they haven't been executed yet, we can just remove them.
11136          * We'll either abort them or delete them below, depending on
11137          * which HA mode we're in.
11138          */
11139 #ifdef notyet
11140         mtx_lock(&ctl_softc->queue_lock);
11141         for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->rtr_queue);
11142              io != NULL; io = next_io) {
11143                 next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
11144                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
11145                         STAILQ_REMOVE(&ctl_softc->rtr_queue, &io->io_hdr,
11146                                       ctl_io_hdr, links);
11147         }
11148         mtx_unlock(&ctl_softc->queue_lock);
11149 #endif
11150
11151         for (lun_idx=0; lun_idx < ctl_softc->num_luns; lun_idx++) {
11152                 lun = ctl_softc->ctl_luns[lun_idx];
11153                 if (lun==NULL)
11154                         continue;
11155
11156                 /*
11157                  * Processor LUNs are primary on both sides.
11158                  * XXX will this always be true?
11159                  */
11160                 if (lun->be_lun->lun_type == T_PROCESSOR)
11161                         continue;
11162
11163                 if ((lun->flags & CTL_LUN_PRIMARY_SC)
11164                  && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
11165                         printf("FAILOVER: primary lun %d\n", lun_idx);
11166                         /*
11167                          * Remove all commands from the other SC. First from the
11168                          * blocked queue then from the ooa queue. Once we have
11169                          * removed them. Call ctl_check_blocked to see if there
11170                          * is anything that can run.
11171                          */
11172                         for (io = (union ctl_io *)TAILQ_FIRST(
11173                              &lun->blocked_queue); io != NULL; io = next_io) {
11174
11175                                 next_io = (union ctl_io *)TAILQ_NEXT(
11176                                     &io->io_hdr, blocked_links);
11177
11178                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
11179                                         TAILQ_REMOVE(&lun->blocked_queue,
11180                                                      &io->io_hdr,blocked_links);
11181                                         io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
11182                                         TAILQ_REMOVE(&lun->ooa_queue,
11183                                                      &io->io_hdr, ooa_links);
11184
11185                                         ctl_free_io(io);
11186                                 }
11187                         }
11188
11189                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
11190                              io != NULL; io = next_io) {
11191
11192                                 next_io = (union ctl_io *)TAILQ_NEXT(
11193                                     &io->io_hdr, ooa_links);
11194
11195                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
11196
11197                                         TAILQ_REMOVE(&lun->ooa_queue,
11198                                                 &io->io_hdr,
11199                                                 ooa_links);
11200
11201                                         ctl_free_io(io);
11202                                 }
11203                         }
11204                         ctl_check_blocked(lun);
11205                 } else if ((lun->flags & CTL_LUN_PRIMARY_SC)
11206                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
11207
11208                         printf("FAILOVER: primary lun %d\n", lun_idx);
11209                         /*
11210                          * Abort all commands from the other SC.  We can't
11211                          * send status back for them now.  These should get
11212                          * cleaned up when they are completed or come out
11213                          * for a datamove operation.
11214                          */
11215                         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
11216                              io != NULL; io = next_io) {
11217                                 next_io = (union ctl_io *)TAILQ_NEXT(
11218                                         &io->io_hdr, ooa_links);
11219
11220                                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
11221                                         io->io_hdr.flags |= CTL_FLAG_ABORT;
11222                         }
11223                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
11224                         && (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
11225
11226                         printf("FAILOVER: secondary lun %d\n", lun_idx);
11227
11228                         lun->flags |= CTL_LUN_PRIMARY_SC;
11229
11230                         /*
11231                          * We send all I/O that was sent to this controller
11232                          * and redirected to the other side back with
11233                          * busy status, and have the initiator retry it.
11234                          * Figuring out how much data has been transferred,
11235                          * etc. and picking up where we left off would be
11236                          * very tricky.
11237                          *
11238                          * XXX KDM need to remove I/O from the blocked
11239                          * queue as well!
11240                          */
11241                         for (pending_io = (union ctl_io *)TAILQ_FIRST(
11242                              &lun->ooa_queue); pending_io != NULL;
11243                              pending_io = next_io) {
11244
11245                                 next_io =  (union ctl_io *)TAILQ_NEXT(
11246                                         &pending_io->io_hdr, ooa_links);
11247
11248                                 pending_io->io_hdr.flags &=
11249                                         ~CTL_FLAG_SENT_2OTHER_SC;
11250
11251                                 if (pending_io->io_hdr.flags &
11252                                     CTL_FLAG_IO_ACTIVE) {
11253                                         pending_io->io_hdr.flags |=
11254                                                 CTL_FLAG_FAILOVER;
11255                                 } else {
11256                                         ctl_set_busy(&pending_io->scsiio);
11257                                         ctl_done(pending_io);
11258                                 }
11259                         }
11260
11261                         /*
11262                          * Build Unit Attention
11263                          */
11264                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11265                                 lun->pending_sense[i].ua_pending |=
11266                                                      CTL_UA_ASYM_ACC_CHANGE;
11267                         }
11268                 } else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
11269                         && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
11270                         printf("FAILOVER: secondary lun %d\n", lun_idx);
11271                         /*
11272                          * if the first io on the OOA is not on the RtR queue
11273                          * add it.
11274                          */
11275                         lun->flags |= CTL_LUN_PRIMARY_SC;
11276
11277                         pending_io = (union ctl_io *)TAILQ_FIRST(
11278                             &lun->ooa_queue);
11279                         if (pending_io==NULL) {
11280                                 printf("Nothing on OOA queue\n");
11281                                 continue;
11282                         }
11283
11284                         pending_io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11285                         if ((pending_io->io_hdr.flags &
11286                              CTL_FLAG_IS_WAS_ON_RTR) == 0) {
11287                                 pending_io->io_hdr.flags |=
11288                                     CTL_FLAG_IS_WAS_ON_RTR;
11289                                 ctl_enqueue_rtr(pending_io);
11290                         }
11291 #if 0
11292                         else
11293                         {
11294                                 printf("Tag 0x%04x is running\n",
11295                                       pending_io->scsiio.tag_num);
11296                         }
11297 #endif
11298
11299                         next_io = (union ctl_io *)TAILQ_NEXT(
11300                             &pending_io->io_hdr, ooa_links);
11301                         for (pending_io=next_io; pending_io != NULL;
11302                              pending_io = next_io) {
11303                                 pending_io->io_hdr.flags &=
11304                                     ~CTL_FLAG_SENT_2OTHER_SC;
11305                                 next_io = (union ctl_io *)TAILQ_NEXT(
11306                                         &pending_io->io_hdr, ooa_links);
11307                                 if (pending_io->io_hdr.flags &
11308                                     CTL_FLAG_IS_WAS_ON_RTR) {
11309 #if 0
11310                                         printf("Tag 0x%04x is running\n",
11311                                                 pending_io->scsiio.tag_num);
11312 #endif
11313                                         continue;
11314                                 }
11315
11316                                 switch (ctl_check_ooa(lun, pending_io,
11317                                     (union ctl_io *)TAILQ_PREV(
11318                                     &pending_io->io_hdr, ctl_ooaq,
11319                                     ooa_links))) {
11320
11321                                 case CTL_ACTION_BLOCK:
11322                                         TAILQ_INSERT_TAIL(&lun->blocked_queue,
11323                                                           &pending_io->io_hdr,
11324                                                           blocked_links);
11325                                         pending_io->io_hdr.flags |=
11326                                             CTL_FLAG_BLOCKED;
11327                                         break;
11328                                 case CTL_ACTION_PASS:
11329                                 case CTL_ACTION_SKIP:
11330                                         pending_io->io_hdr.flags |=
11331                                             CTL_FLAG_IS_WAS_ON_RTR;
11332                                         ctl_enqueue_rtr(pending_io);
11333                                         break;
11334                                 case CTL_ACTION_OVERLAP:
11335                                         ctl_set_overlapped_cmd(
11336                                             (struct ctl_scsiio *)pending_io);
11337                                         ctl_done(pending_io);
11338                                         break;
11339                                 case CTL_ACTION_OVERLAP_TAG:
11340                                         ctl_set_overlapped_tag(
11341                                             (struct ctl_scsiio *)pending_io,
11342                                             pending_io->scsiio.tag_num & 0xff);
11343                                         ctl_done(pending_io);
11344                                         break;
11345                                 case CTL_ACTION_ERROR:
11346                                 default:
11347                                         ctl_set_internal_failure(
11348                                                 (struct ctl_scsiio *)pending_io,
11349                                                 0,  // sks_valid
11350                                                 0); //retry count
11351                                         ctl_done(pending_io);
11352                                         break;
11353                                 }
11354                         }
11355
11356                         /*
11357                          * Build Unit Attention
11358                          */
11359                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11360                                 lun->pending_sense[i].ua_pending |=
11361                                                      CTL_UA_ASYM_ACC_CHANGE;
11362                         }
11363                 } else {
11364                         panic("Unhandled HA mode failover, LUN flags = %#x, "
11365                               "ha_mode = #%x", lun->flags, ctl_softc->ha_mode);
11366                 }
11367         }
11368         ctl_pause_rtr = 0;
11369         mtx_unlock(&ctl_softc->ctl_lock);
11370 }
11371
11372 static int
11373 ctl_scsiio_precheck(struct ctl_softc *ctl_softc, struct ctl_scsiio *ctsio)
11374 {
11375         struct ctl_lun *lun;
11376         const struct ctl_cmd_entry *entry;
11377         uint32_t initidx, targ_lun;
11378         int retval;
11379
11380         retval = 0;
11381
11382         lun = NULL;
11383
11384         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
11385         if ((targ_lun < CTL_MAX_LUNS)
11386          && (ctl_softc->ctl_luns[targ_lun] != NULL)) {
11387                 lun = ctl_softc->ctl_luns[targ_lun];
11388                 /*
11389                  * If the LUN is invalid, pretend that it doesn't exist.
11390                  * It will go away as soon as all pending I/O has been
11391                  * completed.
11392                  */
11393                 if (lun->flags & CTL_LUN_DISABLED) {
11394                         lun = NULL;
11395                 } else {
11396                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
11397                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
11398                                 lun->be_lun;
11399                         if (lun->be_lun->lun_type == T_PROCESSOR) {
11400                                 ctsio->io_hdr.flags |= CTL_FLAG_CONTROL_DEV;
11401                         }
11402
11403                         /*
11404                          * Every I/O goes into the OOA queue for a
11405                          * particular LUN, and stays there until completion.
11406                          */
11407                         mtx_lock(&lun->lun_lock);
11408                         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr,
11409                             ooa_links);
11410                 }
11411         } else {
11412                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11413                 ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11414         }
11415
11416         /* Get command entry and return error if it is unsuppotyed. */
11417         entry = ctl_validate_command(ctsio);
11418         if (entry == NULL) {
11419                 if (lun)
11420                         mtx_unlock(&lun->lun_lock);
11421                 return (retval);
11422         }
11423
11424         ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
11425         ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
11426
11427         /*
11428          * Check to see whether we can send this command to LUNs that don't
11429          * exist.  This should pretty much only be the case for inquiry
11430          * and request sense.  Further checks, below, really require having
11431          * a LUN, so we can't really check the command anymore.  Just put
11432          * it on the rtr queue.
11433          */
11434         if (lun == NULL) {
11435                 if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) {
11436                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11437                         ctl_enqueue_rtr((union ctl_io *)ctsio);
11438                         return (retval);
11439                 }
11440
11441                 ctl_set_unsupported_lun(ctsio);
11442                 ctl_done((union ctl_io *)ctsio);
11443                 CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
11444                 return (retval);
11445         } else {
11446                 /*
11447                  * Make sure we support this particular command on this LUN.
11448                  * e.g., we don't support writes to the control LUN.
11449                  */
11450                 if (!ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
11451                         mtx_unlock(&lun->lun_lock);
11452                         ctl_set_invalid_opcode(ctsio);
11453                         ctl_done((union ctl_io *)ctsio);
11454                         return (retval);
11455                 }
11456         }
11457
11458         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11459
11460         /*
11461          * If we've got a request sense, it'll clear the contingent
11462          * allegiance condition.  Otherwise, if we have a CA condition for
11463          * this initiator, clear it, because it sent down a command other
11464          * than request sense.
11465          */
11466         if ((ctsio->cdb[0] != REQUEST_SENSE)
11467          && (ctl_is_set(lun->have_ca, initidx)))
11468                 ctl_clear_mask(lun->have_ca, initidx);
11469
11470         /*
11471          * If the command has this flag set, it handles its own unit
11472          * attention reporting, we shouldn't do anything.  Otherwise we
11473          * check for any pending unit attentions, and send them back to the
11474          * initiator.  We only do this when a command initially comes in,
11475          * not when we pull it off the blocked queue.
11476          *
11477          * According to SAM-3, section 5.3.2, the order that things get
11478          * presented back to the host is basically unit attentions caused
11479          * by some sort of reset event, busy status, reservation conflicts
11480          * or task set full, and finally any other status.
11481          *
11482          * One issue here is that some of the unit attentions we report
11483          * don't fall into the "reset" category (e.g. "reported luns data
11484          * has changed").  So reporting it here, before the reservation
11485          * check, may be technically wrong.  I guess the only thing to do
11486          * would be to check for and report the reset events here, and then
11487          * check for the other unit attention types after we check for a
11488          * reservation conflict.
11489          *
11490          * XXX KDM need to fix this
11491          */
11492         if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
11493                 ctl_ua_type ua_type;
11494
11495                 ua_type = lun->pending_sense[initidx].ua_pending;
11496                 if (ua_type != CTL_UA_NONE) {
11497                         scsi_sense_data_type sense_format;
11498
11499                         if (lun != NULL)
11500                                 sense_format = (lun->flags &
11501                                     CTL_LUN_SENSE_DESC) ? SSD_TYPE_DESC :
11502                                     SSD_TYPE_FIXED;
11503                         else
11504                                 sense_format = SSD_TYPE_FIXED;
11505
11506                         ua_type = ctl_build_ua(ua_type, &ctsio->sense_data,
11507                                                sense_format);
11508                         if (ua_type != CTL_UA_NONE) {
11509                                 ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
11510                                 ctsio->io_hdr.status = CTL_SCSI_ERROR |
11511                                                        CTL_AUTOSENSE;
11512                                 ctsio->sense_len = SSD_FULL_SIZE;
11513                                 lun->pending_sense[initidx].ua_pending &=
11514                                         ~ua_type;
11515                                 mtx_unlock(&lun->lun_lock);
11516                                 ctl_done((union ctl_io *)ctsio);
11517                                 return (retval);
11518                         }
11519                 }
11520         }
11521
11522
11523         if (ctl_scsiio_lun_check(ctl_softc, lun, entry, ctsio) != 0) {
11524                 mtx_unlock(&lun->lun_lock);
11525                 ctl_done((union ctl_io *)ctsio);
11526                 return (retval);
11527         }
11528
11529         /*
11530          * XXX CHD this is where we want to send IO to other side if
11531          * this LUN is secondary on this SC. We will need to make a copy
11532          * of the IO and flag the IO on this side as SENT_2OTHER and the flag
11533          * the copy we send as FROM_OTHER.
11534          * We also need to stuff the address of the original IO so we can
11535          * find it easily. Something similar will need be done on the other
11536          * side so when we are done we can find the copy.
11537          */
11538         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
11539                 union ctl_ha_msg msg_info;
11540                 int isc_retval;
11541
11542                 ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11543
11544                 msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
11545                 msg_info.hdr.original_sc = (union ctl_io *)ctsio;
11546 #if 0
11547                 printf("1. ctsio %p\n", ctsio);
11548 #endif
11549                 msg_info.hdr.serializing_sc = NULL;
11550                 msg_info.hdr.nexus = ctsio->io_hdr.nexus;
11551                 msg_info.scsi.tag_num = ctsio->tag_num;
11552                 msg_info.scsi.tag_type = ctsio->tag_type;
11553                 memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
11554
11555                 ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11556
11557                 if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11558                     (void *)&msg_info, sizeof(msg_info), 0)) >
11559                     CTL_HA_STATUS_SUCCESS) {
11560                         printf("CTL:precheck, ctl_ha_msg_send returned %d\n",
11561                                isc_retval);
11562                         printf("CTL:opcode is %x\n", ctsio->cdb[0]);
11563                 } else {
11564 #if 0
11565                         printf("CTL:Precheck sent msg, opcode is %x\n",opcode);
11566 #endif
11567                 }
11568
11569                 /*
11570                  * XXX KDM this I/O is off the incoming queue, but hasn't
11571                  * been inserted on any other queue.  We may need to come
11572                  * up with a holding queue while we wait for serialization
11573                  * so that we have an idea of what we're waiting for from
11574                  * the other side.
11575                  */
11576                 mtx_unlock(&lun->lun_lock);
11577                 return (retval);
11578         }
11579
11580         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
11581                               (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
11582                               ctl_ooaq, ooa_links))) {
11583         case CTL_ACTION_BLOCK:
11584                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
11585                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
11586                                   blocked_links);
11587                 mtx_unlock(&lun->lun_lock);
11588                 return (retval);
11589         case CTL_ACTION_PASS:
11590         case CTL_ACTION_SKIP:
11591                 ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11592                 mtx_unlock(&lun->lun_lock);
11593                 ctl_enqueue_rtr((union ctl_io *)ctsio);
11594                 break;
11595         case CTL_ACTION_OVERLAP:
11596                 mtx_unlock(&lun->lun_lock);
11597                 ctl_set_overlapped_cmd(ctsio);
11598                 ctl_done((union ctl_io *)ctsio);
11599                 break;
11600         case CTL_ACTION_OVERLAP_TAG:
11601                 mtx_unlock(&lun->lun_lock);
11602                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
11603                 ctl_done((union ctl_io *)ctsio);
11604                 break;
11605         case CTL_ACTION_ERROR:
11606         default:
11607                 mtx_unlock(&lun->lun_lock);
11608                 ctl_set_internal_failure(ctsio,
11609                                          /*sks_valid*/ 0,
11610                                          /*retry_count*/ 0);
11611                 ctl_done((union ctl_io *)ctsio);
11612                 break;
11613         }
11614         return (retval);
11615 }
11616
11617 const struct ctl_cmd_entry *
11618 ctl_get_cmd_entry(struct ctl_scsiio *ctsio)
11619 {
11620         const struct ctl_cmd_entry *entry;
11621         int service_action;
11622
11623         entry = &ctl_cmd_table[ctsio->cdb[0]];
11624         if (entry->flags & CTL_CMD_FLAG_SA5) {
11625                 service_action = ctsio->cdb[1] & SERVICE_ACTION_MASK;
11626                 entry = &((const struct ctl_cmd_entry *)
11627                     entry->execute)[service_action];
11628         }
11629         return (entry);
11630 }
11631
11632 const struct ctl_cmd_entry *
11633 ctl_validate_command(struct ctl_scsiio *ctsio)
11634 {
11635         const struct ctl_cmd_entry *entry;
11636         int i;
11637         uint8_t diff;
11638
11639         entry = ctl_get_cmd_entry(ctsio);
11640         if (entry->execute == NULL) {
11641                 ctl_set_invalid_opcode(ctsio);
11642                 ctl_done((union ctl_io *)ctsio);
11643                 return (NULL);
11644         }
11645         KASSERT(entry->length > 0,
11646             ("Not defined length for command 0x%02x/0x%02x",
11647              ctsio->cdb[0], ctsio->cdb[1]));
11648         for (i = 1; i < entry->length; i++) {
11649                 diff = ctsio->cdb[i] & ~entry->usage[i - 1];
11650                 if (diff == 0)
11651                         continue;
11652                 ctl_set_invalid_field(ctsio,
11653                                       /*sks_valid*/ 1,
11654                                       /*command*/ 1,
11655                                       /*field*/ i,
11656                                       /*bit_valid*/ 1,
11657                                       /*bit*/ fls(diff) - 1);
11658                 ctl_done((union ctl_io *)ctsio);
11659                 return (NULL);
11660         }
11661         return (entry);
11662 }
11663
11664 static int
11665 ctl_cmd_applicable(uint8_t lun_type, const struct ctl_cmd_entry *entry)
11666 {
11667
11668         switch (lun_type) {
11669         case T_PROCESSOR:
11670                 if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0) &&
11671                     ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
11672                         return (0);
11673                 break;
11674         case T_DIRECT:
11675                 if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0) &&
11676                     ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
11677                         return (0);
11678                 break;
11679         default:
11680                 return (0);
11681         }
11682         return (1);
11683 }
11684
11685 static int
11686 ctl_scsiio(struct ctl_scsiio *ctsio)
11687 {
11688         int retval;
11689         const struct ctl_cmd_entry *entry;
11690
11691         retval = CTL_RETVAL_COMPLETE;
11692
11693         CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
11694
11695         entry = ctl_get_cmd_entry(ctsio);
11696
11697         /*
11698          * If this I/O has been aborted, just send it straight to
11699          * ctl_done() without executing it.
11700          */
11701         if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
11702                 ctl_done((union ctl_io *)ctsio);
11703                 goto bailout;
11704         }
11705
11706         /*
11707          * All the checks should have been handled by ctl_scsiio_precheck().
11708          * We should be clear now to just execute the I/O.
11709          */
11710         retval = entry->execute(ctsio);
11711
11712 bailout:
11713         return (retval);
11714 }
11715
11716 /*
11717  * Since we only implement one target right now, a bus reset simply resets
11718  * our single target.
11719  */
11720 static int
11721 ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io)
11722 {
11723         return(ctl_target_reset(ctl_softc, io, CTL_UA_BUS_RESET));
11724 }
11725
11726 static int
11727 ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
11728                  ctl_ua_type ua_type)
11729 {
11730         struct ctl_lun *lun;
11731         int retval;
11732
11733         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11734                 union ctl_ha_msg msg_info;
11735
11736                 io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11737                 msg_info.hdr.nexus = io->io_hdr.nexus;
11738                 if (ua_type==CTL_UA_TARG_RESET)
11739                         msg_info.task.task_action = CTL_TASK_TARGET_RESET;
11740                 else
11741                         msg_info.task.task_action = CTL_TASK_BUS_RESET;
11742                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11743                 msg_info.hdr.original_sc = NULL;
11744                 msg_info.hdr.serializing_sc = NULL;
11745                 if (CTL_HA_STATUS_SUCCESS != ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11746                     (void *)&msg_info, sizeof(msg_info), 0)) {
11747                 }
11748         }
11749         retval = 0;
11750
11751         mtx_lock(&ctl_softc->ctl_lock);
11752         STAILQ_FOREACH(lun, &ctl_softc->lun_list, links)
11753                 retval += ctl_lun_reset(lun, io, ua_type);
11754         mtx_unlock(&ctl_softc->ctl_lock);
11755
11756         return (retval);
11757 }
11758
11759 /*
11760  * The LUN should always be set.  The I/O is optional, and is used to
11761  * distinguish between I/Os sent by this initiator, and by other
11762  * initiators.  We set unit attention for initiators other than this one.
11763  * SAM-3 is vague on this point.  It does say that a unit attention should
11764  * be established for other initiators when a LUN is reset (see section
11765  * 5.7.3), but it doesn't specifically say that the unit attention should
11766  * be established for this particular initiator when a LUN is reset.  Here
11767  * is the relevant text, from SAM-3 rev 8:
11768  *
11769  * 5.7.2 When a SCSI initiator port aborts its own tasks
11770  *
11771  * When a SCSI initiator port causes its own task(s) to be aborted, no
11772  * notification that the task(s) have been aborted shall be returned to
11773  * the SCSI initiator port other than the completion response for the
11774  * command or task management function action that caused the task(s) to
11775  * be aborted and notification(s) associated with related effects of the
11776  * action (e.g., a reset unit attention condition).
11777  *
11778  * XXX KDM for now, we're setting unit attention for all initiators.
11779  */
11780 static int
11781 ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
11782 {
11783         union ctl_io *xio;
11784 #if 0
11785         uint32_t initindex;
11786 #endif
11787         int i;
11788
11789         mtx_lock(&lun->lun_lock);
11790         /*
11791          * Run through the OOA queue and abort each I/O.
11792          */
11793 #if 0
11794         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
11795 #endif
11796         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11797              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11798                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
11799         }
11800
11801         /*
11802          * This version sets unit attention for every
11803          */
11804 #if 0
11805         initindex = ctl_get_initindex(&io->io_hdr.nexus);
11806         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11807                 if (initindex == i)
11808                         continue;
11809                 lun->pending_sense[i].ua_pending |= ua_type;
11810         }
11811 #endif
11812
11813         /*
11814          * A reset (any kind, really) clears reservations established with
11815          * RESERVE/RELEASE.  It does not clear reservations established
11816          * with PERSISTENT RESERVE OUT, but we don't support that at the
11817          * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
11818          * reservations made with the RESERVE/RELEASE commands, because
11819          * those commands are obsolete in SPC-3.
11820          */
11821         lun->flags &= ~CTL_LUN_RESERVED;
11822
11823         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11824                 ctl_clear_mask(lun->have_ca, i);
11825                 lun->pending_sense[i].ua_pending |= ua_type;
11826         }
11827         mtx_unlock(&lun->lun_lock);
11828
11829         return (0);
11830 }
11831
11832 static int
11833 ctl_abort_tasks_lun(struct ctl_lun *lun, uint32_t targ_port, uint32_t init_id,
11834     int other_sc)
11835 {
11836         union ctl_io *xio;
11837         int found;
11838
11839         mtx_assert(&lun->lun_lock, MA_OWNED);
11840
11841         /*
11842          * Run through the OOA queue and attempt to find the given I/O.
11843          * The target port, initiator ID, tag type and tag number have to
11844          * match the values that we got from the initiator.  If we have an
11845          * untagged command to abort, simply abort the first untagged command
11846          * we come to.  We only allow one untagged command at a time of course.
11847          */
11848         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11849              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11850
11851                 if ((targ_port == xio->io_hdr.nexus.targ_port) &&
11852                     (init_id == xio->io_hdr.nexus.initid.id)) {
11853                         xio->io_hdr.flags |= CTL_FLAG_ABORT;
11854                         found = 1;
11855                         if (!other_sc && !(lun->flags & CTL_LUN_PRIMARY_SC)) {
11856                                 union ctl_ha_msg msg_info;
11857
11858                                 msg_info.hdr.nexus = xio->io_hdr.nexus;
11859                                 msg_info.task.task_action = CTL_TASK_ABORT_TASK;
11860                                 msg_info.task.tag_num = xio->scsiio.tag_num;
11861                                 msg_info.task.tag_type = xio->scsiio.tag_type;
11862                                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11863                                 msg_info.hdr.original_sc = NULL;
11864                                 msg_info.hdr.serializing_sc = NULL;
11865                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11866                                     (void *)&msg_info, sizeof(msg_info), 0);
11867                         }
11868                 }
11869         }
11870         return (found);
11871 }
11872
11873 static int
11874 ctl_abort_task_set(union ctl_io *io)
11875 {
11876         struct ctl_softc *softc = control_softc;
11877         struct ctl_lun *lun;
11878         uint32_t targ_lun;
11879
11880         /*
11881          * Look up the LUN.
11882          */
11883         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11884         mtx_lock(&softc->ctl_lock);
11885         if ((targ_lun < CTL_MAX_LUNS) && (softc->ctl_luns[targ_lun] != NULL))
11886                 lun = softc->ctl_luns[targ_lun];
11887         else {
11888                 mtx_unlock(&softc->ctl_lock);
11889                 return (1);
11890         }
11891
11892         mtx_lock(&lun->lun_lock);
11893         mtx_unlock(&softc->ctl_lock);
11894         ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
11895             io->io_hdr.nexus.initid.id,
11896             (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
11897         mtx_unlock(&lun->lun_lock);
11898         return (0);
11899 }
11900
11901 static int
11902 ctl_i_t_nexus_reset(union ctl_io *io)
11903 {
11904         struct ctl_softc *softc = control_softc;
11905         struct ctl_lun *lun;
11906         uint32_t initindex;
11907
11908         initindex = ctl_get_initindex(&io->io_hdr.nexus);
11909         mtx_lock(&softc->ctl_lock);
11910         STAILQ_FOREACH(lun, &softc->lun_list, links) {
11911                 mtx_lock(&lun->lun_lock);
11912                 ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
11913                     io->io_hdr.nexus.initid.id,
11914                     (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
11915                 ctl_clear_mask(lun->have_ca, initindex);
11916                 lun->pending_sense[initindex].ua_pending |= CTL_UA_I_T_NEXUS_LOSS;
11917                 mtx_unlock(&lun->lun_lock);
11918         }
11919         mtx_unlock(&softc->ctl_lock);
11920         return (0);
11921 }
11922
11923 static int
11924 ctl_abort_task(union ctl_io *io)
11925 {
11926         union ctl_io *xio;
11927         struct ctl_lun *lun;
11928         struct ctl_softc *ctl_softc;
11929 #if 0
11930         struct sbuf sb;
11931         char printbuf[128];
11932 #endif
11933         int found;
11934         uint32_t targ_lun;
11935
11936         ctl_softc = control_softc;
11937         found = 0;
11938
11939         /*
11940          * Look up the LUN.
11941          */
11942         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11943         mtx_lock(&ctl_softc->ctl_lock);
11944         if ((targ_lun < CTL_MAX_LUNS)
11945          && (ctl_softc->ctl_luns[targ_lun] != NULL))
11946                 lun = ctl_softc->ctl_luns[targ_lun];
11947         else {
11948                 mtx_unlock(&ctl_softc->ctl_lock);
11949                 goto bailout;
11950         }
11951
11952 #if 0
11953         printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
11954                lun->lun, io->taskio.tag_num, io->taskio.tag_type);
11955 #endif
11956
11957         mtx_lock(&lun->lun_lock);
11958         mtx_unlock(&ctl_softc->ctl_lock);
11959         /*
11960          * Run through the OOA queue and attempt to find the given I/O.
11961          * The target port, initiator ID, tag type and tag number have to
11962          * match the values that we got from the initiator.  If we have an
11963          * untagged command to abort, simply abort the first untagged command
11964          * we come to.  We only allow one untagged command at a time of course.
11965          */
11966 #if 0
11967         TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
11968 #endif
11969         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11970              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11971 #if 0
11972                 sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
11973
11974                 sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
11975                             lun->lun, xio->scsiio.tag_num,
11976                             xio->scsiio.tag_type,
11977                             (xio->io_hdr.blocked_links.tqe_prev
11978                             == NULL) ? "" : " BLOCKED",
11979                             (xio->io_hdr.flags &
11980                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
11981                             (xio->io_hdr.flags &
11982                             CTL_FLAG_ABORT) ? " ABORT" : "",
11983                             (xio->io_hdr.flags &
11984                             CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
11985                 ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
11986                 sbuf_finish(&sb);
11987                 printf("%s\n", sbuf_data(&sb));
11988 #endif
11989
11990                 if ((xio->io_hdr.nexus.targ_port == io->io_hdr.nexus.targ_port)
11991                  && (xio->io_hdr.nexus.initid.id ==
11992                      io->io_hdr.nexus.initid.id)) {
11993                         /*
11994                          * If the abort says that the task is untagged, the
11995                          * task in the queue must be untagged.  Otherwise,
11996                          * we just check to see whether the tag numbers
11997                          * match.  This is because the QLogic firmware
11998                          * doesn't pass back the tag type in an abort
11999                          * request.
12000                          */
12001 #if 0
12002                         if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
12003                           && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
12004                          || (xio->scsiio.tag_num == io->taskio.tag_num)) {
12005 #endif
12006                         /*
12007                          * XXX KDM we've got problems with FC, because it
12008                          * doesn't send down a tag type with aborts.  So we
12009                          * can only really go by the tag number...
12010                          * This may cause problems with parallel SCSI.
12011                          * Need to figure that out!!
12012                          */
12013                         if (xio->scsiio.tag_num == io->taskio.tag_num) {
12014                                 xio->io_hdr.flags |= CTL_FLAG_ABORT;
12015                                 found = 1;
12016                                 if ((io->io_hdr.flags &
12017                                      CTL_FLAG_FROM_OTHER_SC) == 0 &&
12018                                     !(lun->flags & CTL_LUN_PRIMARY_SC)) {
12019                                         union ctl_ha_msg msg_info;
12020
12021                                         io->io_hdr.flags |=
12022                                                         CTL_FLAG_SENT_2OTHER_SC;
12023                                         msg_info.hdr.nexus = io->io_hdr.nexus;
12024                                         msg_info.task.task_action =
12025                                                 CTL_TASK_ABORT_TASK;
12026                                         msg_info.task.tag_num =
12027                                                 io->taskio.tag_num;
12028                                         msg_info.task.tag_type =
12029                                                 io->taskio.tag_type;
12030                                         msg_info.hdr.msg_type =
12031                                                 CTL_MSG_MANAGE_TASKS;
12032                                         msg_info.hdr.original_sc = NULL;
12033                                         msg_info.hdr.serializing_sc = NULL;
12034 #if 0
12035                                         printf("Sent Abort to other side\n");
12036 #endif
12037                                         if (CTL_HA_STATUS_SUCCESS !=
12038                                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12039                                                 (void *)&msg_info,
12040                                                 sizeof(msg_info), 0)) {
12041                                         }
12042                                 }
12043 #if 0
12044                                 printf("ctl_abort_task: found I/O to abort\n");
12045 #endif
12046                                 break;
12047                         }
12048                 }
12049         }
12050         mtx_unlock(&lun->lun_lock);
12051
12052 bailout:
12053
12054         if (found == 0) {
12055                 /*
12056                  * This isn't really an error.  It's entirely possible for
12057                  * the abort and command completion to cross on the wire.
12058                  * This is more of an informative/diagnostic error.
12059                  */
12060 #if 0
12061                 printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
12062                        "%d:%d:%d:%d tag %d type %d\n",
12063                        io->io_hdr.nexus.initid.id,
12064                        io->io_hdr.nexus.targ_port,
12065                        io->io_hdr.nexus.targ_target.id,
12066                        io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
12067                        io->taskio.tag_type);
12068 #endif
12069                 return (1);
12070         } else
12071                 return (0);
12072 }
12073
12074 static void
12075 ctl_run_task(union ctl_io *io)
12076 {
12077         struct ctl_softc *ctl_softc;
12078         int retval;
12079         const char *task_desc;
12080
12081         CTL_DEBUG_PRINT(("ctl_run_task\n"));
12082
12083         ctl_softc = control_softc;
12084         retval = 0;
12085
12086         KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
12087             ("ctl_run_task: Unextected io_type %d\n",
12088              io->io_hdr.io_type));
12089
12090         task_desc = ctl_scsi_task_string(&io->taskio);
12091         if (task_desc != NULL) {
12092 #ifdef NEEDTOPORT
12093                 csevent_log(CSC_CTL | CSC_SHELF_SW |
12094                             CTL_TASK_REPORT,
12095                             csevent_LogType_Trace,
12096                             csevent_Severity_Information,
12097                             csevent_AlertLevel_Green,
12098                             csevent_FRU_Firmware,
12099                             csevent_FRU_Unknown,
12100                             "CTL: received task: %s",task_desc);
12101 #endif
12102         } else {
12103 #ifdef NEEDTOPORT
12104                 csevent_log(CSC_CTL | CSC_SHELF_SW |
12105                             CTL_TASK_REPORT,
12106                             csevent_LogType_Trace,
12107                             csevent_Severity_Information,
12108                             csevent_AlertLevel_Green,
12109                             csevent_FRU_Firmware,
12110                             csevent_FRU_Unknown,
12111                             "CTL: received unknown task "
12112                             "type: %d (%#x)",
12113                             io->taskio.task_action,
12114                             io->taskio.task_action);
12115 #endif
12116         }
12117         switch (io->taskio.task_action) {
12118         case CTL_TASK_ABORT_TASK:
12119                 retval = ctl_abort_task(io);
12120                 break;
12121         case CTL_TASK_ABORT_TASK_SET:
12122                 retval = ctl_abort_task_set(io);
12123                 break;
12124         case CTL_TASK_CLEAR_ACA:
12125                 break;
12126         case CTL_TASK_CLEAR_TASK_SET:
12127                 break;
12128         case CTL_TASK_I_T_NEXUS_RESET:
12129                 retval = ctl_i_t_nexus_reset(io);
12130                 break;
12131         case CTL_TASK_LUN_RESET: {
12132                 struct ctl_lun *lun;
12133                 uint32_t targ_lun;
12134                 int retval;
12135
12136                 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12137                 mtx_lock(&ctl_softc->ctl_lock);
12138                 if ((targ_lun < CTL_MAX_LUNS)
12139                  && (ctl_softc->ctl_luns[targ_lun] != NULL))
12140                         lun = ctl_softc->ctl_luns[targ_lun];
12141                 else {
12142                         mtx_unlock(&ctl_softc->ctl_lock);
12143                         retval = 1;
12144                         break;
12145                 }
12146
12147                 if (!(io->io_hdr.flags &
12148                     CTL_FLAG_FROM_OTHER_SC)) {
12149                         union ctl_ha_msg msg_info;
12150
12151                         io->io_hdr.flags |=
12152                                 CTL_FLAG_SENT_2OTHER_SC;
12153                         msg_info.hdr.msg_type =
12154                                 CTL_MSG_MANAGE_TASKS;
12155                         msg_info.hdr.nexus = io->io_hdr.nexus;
12156                         msg_info.task.task_action =
12157                                 CTL_TASK_LUN_RESET;
12158                         msg_info.hdr.original_sc = NULL;
12159                         msg_info.hdr.serializing_sc = NULL;
12160                         if (CTL_HA_STATUS_SUCCESS !=
12161                             ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12162                             (void *)&msg_info,
12163                             sizeof(msg_info), 0)) {
12164                         }
12165                 }
12166
12167                 retval = ctl_lun_reset(lun, io,
12168                                        CTL_UA_LUN_RESET);
12169                 mtx_unlock(&ctl_softc->ctl_lock);
12170                 break;
12171         }
12172         case CTL_TASK_TARGET_RESET:
12173                 retval = ctl_target_reset(ctl_softc, io, CTL_UA_TARG_RESET);
12174                 break;
12175         case CTL_TASK_BUS_RESET:
12176                 retval = ctl_bus_reset(ctl_softc, io);
12177                 break;
12178         case CTL_TASK_PORT_LOGIN:
12179                 break;
12180         case CTL_TASK_PORT_LOGOUT:
12181                 break;
12182         default:
12183                 printf("ctl_run_task: got unknown task management event %d\n",
12184                        io->taskio.task_action);
12185                 break;
12186         }
12187         if (retval == 0)
12188                 io->io_hdr.status = CTL_SUCCESS;
12189         else
12190                 io->io_hdr.status = CTL_ERROR;
12191
12192         /*
12193          * This will queue this I/O to the done queue, but the
12194          * work thread won't be able to process it until we
12195          * return and the lock is released.
12196          */
12197         ctl_done(io);
12198 }
12199
12200 /*
12201  * For HA operation.  Handle commands that come in from the other
12202  * controller.
12203  */
12204 static void
12205 ctl_handle_isc(union ctl_io *io)
12206 {
12207         int free_io;
12208         struct ctl_lun *lun;
12209         struct ctl_softc *ctl_softc;
12210         uint32_t targ_lun;
12211
12212         ctl_softc = control_softc;
12213
12214         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12215         lun = ctl_softc->ctl_luns[targ_lun];
12216
12217         switch (io->io_hdr.msg_type) {
12218         case CTL_MSG_SERIALIZE:
12219                 free_io = ctl_serialize_other_sc_cmd(&io->scsiio);
12220                 break;
12221         case CTL_MSG_R2R: {
12222                 const struct ctl_cmd_entry *entry;
12223
12224                 /*
12225                  * This is only used in SER_ONLY mode.
12226                  */
12227                 free_io = 0;
12228                 entry = ctl_get_cmd_entry(&io->scsiio);
12229                 mtx_lock(&lun->lun_lock);
12230                 if (ctl_scsiio_lun_check(ctl_softc, lun,
12231                     entry, (struct ctl_scsiio *)io) != 0) {
12232                         mtx_unlock(&lun->lun_lock);
12233                         ctl_done(io);
12234                         break;
12235                 }
12236                 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
12237                 mtx_unlock(&lun->lun_lock);
12238                 ctl_enqueue_rtr(io);
12239                 break;
12240         }
12241         case CTL_MSG_FINISH_IO:
12242                 if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
12243                         free_io = 0;
12244                         ctl_done(io);
12245                 } else {
12246                         free_io = 1;
12247                         mtx_lock(&lun->lun_lock);
12248                         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
12249                                      ooa_links);
12250                         ctl_check_blocked(lun);
12251                         mtx_unlock(&lun->lun_lock);
12252                 }
12253                 break;
12254         case CTL_MSG_PERS_ACTION:
12255                 ctl_hndl_per_res_out_on_other_sc(
12256                         (union ctl_ha_msg *)&io->presio.pr_msg);
12257                 free_io = 1;
12258                 break;
12259         case CTL_MSG_BAD_JUJU:
12260                 free_io = 0;
12261                 ctl_done(io);
12262                 break;
12263         case CTL_MSG_DATAMOVE:
12264                 /* Only used in XFER mode */
12265                 free_io = 0;
12266                 ctl_datamove_remote(io);
12267                 break;
12268         case CTL_MSG_DATAMOVE_DONE:
12269                 /* Only used in XFER mode */
12270                 free_io = 0;
12271                 io->scsiio.be_move_done(io);
12272                 break;
12273         default:
12274                 free_io = 1;
12275                 printf("%s: Invalid message type %d\n",
12276                        __func__, io->io_hdr.msg_type);
12277                 break;
12278         }
12279         if (free_io)
12280                 ctl_free_io(io);
12281
12282 }
12283
12284
12285 /*
12286  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
12287  * there is no match.
12288  */
12289 static ctl_lun_error_pattern
12290 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
12291 {
12292         const struct ctl_cmd_entry *entry;
12293         ctl_lun_error_pattern filtered_pattern, pattern;
12294
12295         pattern = desc->error_pattern;
12296
12297         /*
12298          * XXX KDM we need more data passed into this function to match a
12299          * custom pattern, and we actually need to implement custom pattern
12300          * matching.
12301          */
12302         if (pattern & CTL_LUN_PAT_CMD)
12303                 return (CTL_LUN_PAT_CMD);
12304
12305         if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
12306                 return (CTL_LUN_PAT_ANY);
12307
12308         entry = ctl_get_cmd_entry(ctsio);
12309
12310         filtered_pattern = entry->pattern & pattern;
12311
12312         /*
12313          * If the user requested specific flags in the pattern (e.g.
12314          * CTL_LUN_PAT_RANGE), make sure the command supports all of those
12315          * flags.
12316          *
12317          * If the user did not specify any flags, it doesn't matter whether
12318          * or not the command supports the flags.
12319          */
12320         if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
12321              (pattern & ~CTL_LUN_PAT_MASK))
12322                 return (CTL_LUN_PAT_NONE);
12323
12324         /*
12325          * If the user asked for a range check, see if the requested LBA
12326          * range overlaps with this command's LBA range.
12327          */
12328         if (filtered_pattern & CTL_LUN_PAT_RANGE) {
12329                 uint64_t lba1;
12330                 uint32_t len1;
12331                 ctl_action action;
12332                 int retval;
12333
12334                 retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
12335                 if (retval != 0)
12336                         return (CTL_LUN_PAT_NONE);
12337
12338                 action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
12339                                               desc->lba_range.len);
12340                 /*
12341                  * A "pass" means that the LBA ranges don't overlap, so
12342                  * this doesn't match the user's range criteria.
12343                  */
12344                 if (action == CTL_ACTION_PASS)
12345                         return (CTL_LUN_PAT_NONE);
12346         }
12347
12348         return (filtered_pattern);
12349 }
12350
12351 static void
12352 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
12353 {
12354         struct ctl_error_desc *desc, *desc2;
12355
12356         mtx_assert(&lun->lun_lock, MA_OWNED);
12357
12358         STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
12359                 ctl_lun_error_pattern pattern;
12360                 /*
12361                  * Check to see whether this particular command matches
12362                  * the pattern in the descriptor.
12363                  */
12364                 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
12365                 if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
12366                         continue;
12367
12368                 switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
12369                 case CTL_LUN_INJ_ABORTED:
12370                         ctl_set_aborted(&io->scsiio);
12371                         break;
12372                 case CTL_LUN_INJ_MEDIUM_ERR:
12373                         ctl_set_medium_error(&io->scsiio);
12374                         break;
12375                 case CTL_LUN_INJ_UA:
12376                         /* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
12377                          * OCCURRED */
12378                         ctl_set_ua(&io->scsiio, 0x29, 0x00);
12379                         break;
12380                 case CTL_LUN_INJ_CUSTOM:
12381                         /*
12382                          * We're assuming the user knows what he is doing.
12383                          * Just copy the sense information without doing
12384                          * checks.
12385                          */
12386                         bcopy(&desc->custom_sense, &io->scsiio.sense_data,
12387                               ctl_min(sizeof(desc->custom_sense),
12388                                       sizeof(io->scsiio.sense_data)));
12389                         io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
12390                         io->scsiio.sense_len = SSD_FULL_SIZE;
12391                         io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
12392                         break;
12393                 case CTL_LUN_INJ_NONE:
12394                 default:
12395                         /*
12396                          * If this is an error injection type we don't know
12397                          * about, clear the continuous flag (if it is set)
12398                          * so it will get deleted below.
12399                          */
12400                         desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
12401                         break;
12402                 }
12403                 /*
12404                  * By default, each error injection action is a one-shot
12405                  */
12406                 if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
12407                         continue;
12408
12409                 STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
12410
12411                 free(desc, M_CTL);
12412         }
12413 }
12414
12415 #ifdef CTL_IO_DELAY
12416 static void
12417 ctl_datamove_timer_wakeup(void *arg)
12418 {
12419         union ctl_io *io;
12420
12421         io = (union ctl_io *)arg;
12422
12423         ctl_datamove(io);
12424 }
12425 #endif /* CTL_IO_DELAY */
12426
12427 void
12428 ctl_datamove(union ctl_io *io)
12429 {
12430         void (*fe_datamove)(union ctl_io *io);
12431
12432         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
12433
12434         CTL_DEBUG_PRINT(("ctl_datamove\n"));
12435
12436 #ifdef CTL_TIME_IO
12437         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12438                 char str[256];
12439                 char path_str[64];
12440                 struct sbuf sb;
12441
12442                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12443                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12444
12445                 sbuf_cat(&sb, path_str);
12446                 switch (io->io_hdr.io_type) {
12447                 case CTL_IO_SCSI:
12448                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12449                         sbuf_printf(&sb, "\n");
12450                         sbuf_cat(&sb, path_str);
12451                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12452                                     io->scsiio.tag_num, io->scsiio.tag_type);
12453                         break;
12454                 case CTL_IO_TASK:
12455                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12456                                     "Tag Type: %d\n", io->taskio.task_action,
12457                                     io->taskio.tag_num, io->taskio.tag_type);
12458                         break;
12459                 default:
12460                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12461                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12462                         break;
12463                 }
12464                 sbuf_cat(&sb, path_str);
12465                 sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
12466                             (intmax_t)time_uptime - io->io_hdr.start_time);
12467                 sbuf_finish(&sb);
12468                 printf("%s", sbuf_data(&sb));
12469         }
12470 #endif /* CTL_TIME_IO */
12471
12472 #ifdef CTL_IO_DELAY
12473         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12474                 struct ctl_lun *lun;
12475
12476                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12477
12478                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12479         } else {
12480                 struct ctl_lun *lun;
12481
12482                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12483                 if ((lun != NULL)
12484                  && (lun->delay_info.datamove_delay > 0)) {
12485                         struct callout *callout;
12486
12487                         callout = (struct callout *)&io->io_hdr.timer_bytes;
12488                         callout_init(callout, /*mpsafe*/ 1);
12489                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12490                         callout_reset(callout,
12491                                       lun->delay_info.datamove_delay * hz,
12492                                       ctl_datamove_timer_wakeup, io);
12493                         if (lun->delay_info.datamove_type ==
12494                             CTL_DELAY_TYPE_ONESHOT)
12495                                 lun->delay_info.datamove_delay = 0;
12496                         return;
12497                 }
12498         }
12499 #endif
12500
12501         /*
12502          * This command has been aborted.  Set the port status, so we fail
12503          * the data move.
12504          */
12505         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12506                 printf("ctl_datamove: tag 0x%04x on (%ju:%d:%ju:%d) aborted\n",
12507                        io->scsiio.tag_num,(uintmax_t)io->io_hdr.nexus.initid.id,
12508                        io->io_hdr.nexus.targ_port,
12509                        (uintmax_t)io->io_hdr.nexus.targ_target.id,
12510                        io->io_hdr.nexus.targ_lun);
12511                 io->io_hdr.status = CTL_CMD_ABORTED;
12512                 io->io_hdr.port_status = 31337;
12513                 /*
12514                  * Note that the backend, in this case, will get the
12515                  * callback in its context.  In other cases it may get
12516                  * called in the frontend's interrupt thread context.
12517                  */
12518                 io->scsiio.be_move_done(io);
12519                 return;
12520         }
12521
12522         /*
12523          * If we're in XFER mode and this I/O is from the other shelf
12524          * controller, we need to send the DMA to the other side to
12525          * actually transfer the data to/from the host.  In serialize only
12526          * mode the transfer happens below CTL and ctl_datamove() is only
12527          * called on the machine that originally received the I/O.
12528          */
12529         if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
12530          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12531                 union ctl_ha_msg msg;
12532                 uint32_t sg_entries_sent;
12533                 int do_sg_copy;
12534                 int i;
12535
12536                 memset(&msg, 0, sizeof(msg));
12537                 msg.hdr.msg_type = CTL_MSG_DATAMOVE;
12538                 msg.hdr.original_sc = io->io_hdr.original_sc;
12539                 msg.hdr.serializing_sc = io;
12540                 msg.hdr.nexus = io->io_hdr.nexus;
12541                 msg.dt.flags = io->io_hdr.flags;
12542                 /*
12543                  * We convert everything into a S/G list here.  We can't
12544                  * pass by reference, only by value between controllers.
12545                  * So we can't pass a pointer to the S/G list, only as many
12546                  * S/G entries as we can fit in here.  If it's possible for
12547                  * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
12548                  * then we need to break this up into multiple transfers.
12549                  */
12550                 if (io->scsiio.kern_sg_entries == 0) {
12551                         msg.dt.kern_sg_entries = 1;
12552                         /*
12553                          * If this is in cached memory, flush the cache
12554                          * before we send the DMA request to the other
12555                          * controller.  We want to do this in either the
12556                          * read or the write case.  The read case is
12557                          * straightforward.  In the write case, we want to
12558                          * make sure nothing is in the local cache that
12559                          * could overwrite the DMAed data.
12560                          */
12561                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12562                                 /*
12563                                  * XXX KDM use bus_dmamap_sync() here.
12564                                  */
12565                         }
12566
12567                         /*
12568                          * Convert to a physical address if this is a
12569                          * virtual address.
12570                          */
12571                         if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
12572                                 msg.dt.sg_list[0].addr =
12573                                         io->scsiio.kern_data_ptr;
12574                         } else {
12575                                 /*
12576                                  * XXX KDM use busdma here!
12577                                  */
12578 #if 0
12579                                 msg.dt.sg_list[0].addr = (void *)
12580                                         vtophys(io->scsiio.kern_data_ptr);
12581 #endif
12582                         }
12583
12584                         msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
12585                         do_sg_copy = 0;
12586                 } else {
12587                         struct ctl_sg_entry *sgl;
12588
12589                         do_sg_copy = 1;
12590                         msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
12591                         sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
12592                         if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12593                                 /*
12594                                  * XXX KDM use bus_dmamap_sync() here.
12595                                  */
12596                         }
12597                 }
12598
12599                 msg.dt.kern_data_len = io->scsiio.kern_data_len;
12600                 msg.dt.kern_total_len = io->scsiio.kern_total_len;
12601                 msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
12602                 msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
12603                 msg.dt.sg_sequence = 0;
12604
12605                 /*
12606                  * Loop until we've sent all of the S/G entries.  On the
12607                  * other end, we'll recompose these S/G entries into one
12608                  * contiguous list before passing it to the
12609                  */
12610                 for (sg_entries_sent = 0; sg_entries_sent <
12611                      msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
12612                         msg.dt.cur_sg_entries = ctl_min((sizeof(msg.dt.sg_list)/
12613                                 sizeof(msg.dt.sg_list[0])),
12614                                 msg.dt.kern_sg_entries - sg_entries_sent);
12615
12616                         if (do_sg_copy != 0) {
12617                                 struct ctl_sg_entry *sgl;
12618                                 int j;
12619
12620                                 sgl = (struct ctl_sg_entry *)
12621                                         io->scsiio.kern_data_ptr;
12622                                 /*
12623                                  * If this is in cached memory, flush the cache
12624                                  * before we send the DMA request to the other
12625                                  * controller.  We want to do this in either
12626                                  * the * read or the write case.  The read
12627                                  * case is straightforward.  In the write
12628                                  * case, we want to make sure nothing is
12629                                  * in the local cache that could overwrite
12630                                  * the DMAed data.
12631                                  */
12632
12633                                 for (i = sg_entries_sent, j = 0;
12634                                      i < msg.dt.cur_sg_entries; i++, j++) {
12635                                         if ((io->io_hdr.flags &
12636                                              CTL_FLAG_NO_DATASYNC) == 0) {
12637                                                 /*
12638                                                  * XXX KDM use bus_dmamap_sync()
12639                                                  */
12640                                         }
12641                                         if ((io->io_hdr.flags &
12642                                              CTL_FLAG_BUS_ADDR) == 0) {
12643                                                 /*
12644                                                  * XXX KDM use busdma.
12645                                                  */
12646 #if 0
12647                                                 msg.dt.sg_list[j].addr =(void *)
12648                                                        vtophys(sgl[i].addr);
12649 #endif
12650                                         } else {
12651                                                 msg.dt.sg_list[j].addr =
12652                                                         sgl[i].addr;
12653                                         }
12654                                         msg.dt.sg_list[j].len = sgl[i].len;
12655                                 }
12656                         }
12657
12658                         sg_entries_sent += msg.dt.cur_sg_entries;
12659                         if (sg_entries_sent >= msg.dt.kern_sg_entries)
12660                                 msg.dt.sg_last = 1;
12661                         else
12662                                 msg.dt.sg_last = 0;
12663
12664                         /*
12665                          * XXX KDM drop and reacquire the lock here?
12666                          */
12667                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12668                             sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
12669                                 /*
12670                                  * XXX do something here.
12671                                  */
12672                         }
12673
12674                         msg.dt.sent_sg_entries = sg_entries_sent;
12675                 }
12676                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12677                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER)
12678                         ctl_failover_io(io, /*have_lock*/ 0);
12679
12680         } else {
12681
12682                 /*
12683                  * Lookup the fe_datamove() function for this particular
12684                  * front end.
12685                  */
12686                 fe_datamove =
12687                     control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12688
12689                 fe_datamove(io);
12690         }
12691 }
12692
12693 static void
12694 ctl_send_datamove_done(union ctl_io *io, int have_lock)
12695 {
12696         union ctl_ha_msg msg;
12697         int isc_status;
12698
12699         memset(&msg, 0, sizeof(msg));
12700
12701         msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
12702         msg.hdr.original_sc = io;
12703         msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
12704         msg.hdr.nexus = io->io_hdr.nexus;
12705         msg.hdr.status = io->io_hdr.status;
12706         msg.scsi.tag_num = io->scsiio.tag_num;
12707         msg.scsi.tag_type = io->scsiio.tag_type;
12708         msg.scsi.scsi_status = io->scsiio.scsi_status;
12709         memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12710                sizeof(io->scsiio.sense_data));
12711         msg.scsi.sense_len = io->scsiio.sense_len;
12712         msg.scsi.sense_residual = io->scsiio.sense_residual;
12713         msg.scsi.fetd_status = io->io_hdr.port_status;
12714         msg.scsi.residual = io->scsiio.residual;
12715         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12716
12717         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12718                 ctl_failover_io(io, /*have_lock*/ have_lock);
12719                 return;
12720         }
12721
12722         isc_status = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0);
12723         if (isc_status > CTL_HA_STATUS_SUCCESS) {
12724                 /* XXX do something if this fails */
12725         }
12726
12727 }
12728
12729 /*
12730  * The DMA to the remote side is done, now we need to tell the other side
12731  * we're done so it can continue with its data movement.
12732  */
12733 static void
12734 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
12735 {
12736         union ctl_io *io;
12737
12738         io = rq->context;
12739
12740         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12741                 printf("%s: ISC DMA write failed with error %d", __func__,
12742                        rq->ret);
12743                 ctl_set_internal_failure(&io->scsiio,
12744                                          /*sks_valid*/ 1,
12745                                          /*retry_count*/ rq->ret);
12746         }
12747
12748         ctl_dt_req_free(rq);
12749
12750         /*
12751          * In this case, we had to malloc the memory locally.  Free it.
12752          */
12753         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
12754                 int i;
12755                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12756                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12757         }
12758         /*
12759          * The data is in local and remote memory, so now we need to send
12760          * status (good or back) back to the other side.
12761          */
12762         ctl_send_datamove_done(io, /*have_lock*/ 0);
12763 }
12764
12765 /*
12766  * We've moved the data from the host/controller into local memory.  Now we
12767  * need to push it over to the remote controller's memory.
12768  */
12769 static int
12770 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
12771 {
12772         int retval;
12773
12774         retval = 0;
12775
12776         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
12777                                           ctl_datamove_remote_write_cb);
12778
12779         return (retval);
12780 }
12781
12782 static void
12783 ctl_datamove_remote_write(union ctl_io *io)
12784 {
12785         int retval;
12786         void (*fe_datamove)(union ctl_io *io);
12787
12788         /*
12789          * - Get the data from the host/HBA into local memory.
12790          * - DMA memory from the local controller to the remote controller.
12791          * - Send status back to the remote controller.
12792          */
12793
12794         retval = ctl_datamove_remote_sgl_setup(io);
12795         if (retval != 0)
12796                 return;
12797
12798         /* Switch the pointer over so the FETD knows what to do */
12799         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12800
12801         /*
12802          * Use a custom move done callback, since we need to send completion
12803          * back to the other controller, not to the backend on this side.
12804          */
12805         io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
12806
12807         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12808
12809         fe_datamove(io);
12810
12811         return;
12812
12813 }
12814
12815 static int
12816 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
12817 {
12818 #if 0
12819         char str[256];
12820         char path_str[64];
12821         struct sbuf sb;
12822 #endif
12823
12824         /*
12825          * In this case, we had to malloc the memory locally.  Free it.
12826          */
12827         if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
12828                 int i;
12829                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12830                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12831         }
12832
12833 #if 0
12834         scsi_path_string(io, path_str, sizeof(path_str));
12835         sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12836         sbuf_cat(&sb, path_str);
12837         scsi_command_string(&io->scsiio, NULL, &sb);
12838         sbuf_printf(&sb, "\n");
12839         sbuf_cat(&sb, path_str);
12840         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12841                     io->scsiio.tag_num, io->scsiio.tag_type);
12842         sbuf_cat(&sb, path_str);
12843         sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
12844                     io->io_hdr.flags, io->io_hdr.status);
12845         sbuf_finish(&sb);
12846         printk("%s", sbuf_data(&sb));
12847 #endif
12848
12849
12850         /*
12851          * The read is done, now we need to send status (good or bad) back
12852          * to the other side.
12853          */
12854         ctl_send_datamove_done(io, /*have_lock*/ 0);
12855
12856         return (0);
12857 }
12858
12859 static void
12860 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
12861 {
12862         union ctl_io *io;
12863         void (*fe_datamove)(union ctl_io *io);
12864
12865         io = rq->context;
12866
12867         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12868                 printf("%s: ISC DMA read failed with error %d", __func__,
12869                        rq->ret);
12870                 ctl_set_internal_failure(&io->scsiio,
12871                                          /*sks_valid*/ 1,
12872                                          /*retry_count*/ rq->ret);
12873         }
12874
12875         ctl_dt_req_free(rq);
12876
12877         /* Switch the pointer over so the FETD knows what to do */
12878         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12879
12880         /*
12881          * Use a custom move done callback, since we need to send completion
12882          * back to the other controller, not to the backend on this side.
12883          */
12884         io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
12885
12886         /* XXX KDM add checks like the ones in ctl_datamove? */
12887
12888         fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12889
12890         fe_datamove(io);
12891 }
12892
12893 static int
12894 ctl_datamove_remote_sgl_setup(union ctl_io *io)
12895 {
12896         struct ctl_sg_entry *local_sglist, *remote_sglist;
12897         struct ctl_sg_entry *local_dma_sglist, *remote_dma_sglist;
12898         struct ctl_softc *softc;
12899         int retval;
12900         int i;
12901
12902         retval = 0;
12903         softc = control_softc;
12904
12905         local_sglist = io->io_hdr.local_sglist;
12906         local_dma_sglist = io->io_hdr.local_dma_sglist;
12907         remote_sglist = io->io_hdr.remote_sglist;
12908         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
12909
12910         if (io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) {
12911                 for (i = 0; i < io->scsiio.kern_sg_entries; i++) {
12912                         local_sglist[i].len = remote_sglist[i].len;
12913
12914                         /*
12915                          * XXX Detect the situation where the RS-level I/O
12916                          * redirector on the other side has already read the
12917                          * data off of the AOR RS on this side, and
12918                          * transferred it to remote (mirror) memory on the
12919                          * other side.  Since we already have the data in
12920                          * memory here, we just need to use it.
12921                          *
12922                          * XXX KDM this can probably be removed once we
12923                          * get the cache device code in and take the
12924                          * current AOR implementation out.
12925                          */
12926 #ifdef NEEDTOPORT
12927                         if ((remote_sglist[i].addr >=
12928                              (void *)vtophys(softc->mirr->addr))
12929                          && (remote_sglist[i].addr <
12930                              ((void *)vtophys(softc->mirr->addr) +
12931                              CacheMirrorOffset))) {
12932                                 local_sglist[i].addr = remote_sglist[i].addr -
12933                                         CacheMirrorOffset;
12934                                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
12935                                      CTL_FLAG_DATA_IN)
12936                                         io->io_hdr.flags |= CTL_FLAG_REDIR_DONE;
12937                         } else {
12938                                 local_sglist[i].addr = remote_sglist[i].addr +
12939                                         CacheMirrorOffset;
12940                         }
12941 #endif
12942 #if 0
12943                         printf("%s: local %p, remote %p, len %d\n",
12944                                __func__, local_sglist[i].addr,
12945                                remote_sglist[i].addr, local_sglist[i].len);
12946 #endif
12947                 }
12948         } else {
12949                 uint32_t len_to_go;
12950
12951                 /*
12952                  * In this case, we don't have automatically allocated
12953                  * memory for this I/O on this controller.  This typically
12954                  * happens with internal CTL I/O -- e.g. inquiry, mode
12955                  * sense, etc.  Anything coming from RAIDCore will have
12956                  * a mirror area available.
12957                  */
12958                 len_to_go = io->scsiio.kern_data_len;
12959
12960                 /*
12961                  * Clear the no datasync flag, we have to use malloced
12962                  * buffers.
12963                  */
12964                 io->io_hdr.flags &= ~CTL_FLAG_NO_DATASYNC;
12965
12966                 /*
12967                  * The difficult thing here is that the size of the various
12968                  * S/G segments may be different than the size from the
12969                  * remote controller.  That'll make it harder when DMAing
12970                  * the data back to the other side.
12971                  */
12972                 for (i = 0; (i < sizeof(io->io_hdr.remote_sglist) /
12973                      sizeof(io->io_hdr.remote_sglist[0])) &&
12974                      (len_to_go > 0); i++) {
12975                         local_sglist[i].len = ctl_min(len_to_go, 131072);
12976                         CTL_SIZE_8B(local_dma_sglist[i].len,
12977                                     local_sglist[i].len);
12978                         local_sglist[i].addr =
12979                                 malloc(local_dma_sglist[i].len, M_CTL,M_WAITOK);
12980
12981                         local_dma_sglist[i].addr = local_sglist[i].addr;
12982
12983                         if (local_sglist[i].addr == NULL) {
12984                                 int j;
12985
12986                                 printf("malloc failed for %zd bytes!",
12987                                        local_dma_sglist[i].len);
12988                                 for (j = 0; j < i; j++) {
12989                                         free(local_sglist[j].addr, M_CTL);
12990                                 }
12991                                 ctl_set_internal_failure(&io->scsiio,
12992                                                          /*sks_valid*/ 1,
12993                                                          /*retry_count*/ 4857);
12994                                 retval = 1;
12995                                 goto bailout_error;
12996
12997                         }
12998                         /* XXX KDM do we need a sync here? */
12999
13000                         len_to_go -= local_sglist[i].len;
13001                 }
13002                 /*
13003                  * Reset the number of S/G entries accordingly.  The
13004                  * original number of S/G entries is available in
13005                  * rem_sg_entries.
13006                  */
13007                 io->scsiio.kern_sg_entries = i;
13008
13009 #if 0
13010                 printf("%s: kern_sg_entries = %d\n", __func__,
13011                        io->scsiio.kern_sg_entries);
13012                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13013                         printf("%s: sg[%d] = %p, %d (DMA: %d)\n", __func__, i,
13014                                local_sglist[i].addr, local_sglist[i].len,
13015                                local_dma_sglist[i].len);
13016 #endif
13017         }
13018
13019
13020         return (retval);
13021
13022 bailout_error:
13023
13024         ctl_send_datamove_done(io, /*have_lock*/ 0);
13025
13026         return (retval);
13027 }
13028
13029 static int
13030 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
13031                          ctl_ha_dt_cb callback)
13032 {
13033         struct ctl_ha_dt_req *rq;
13034         struct ctl_sg_entry *remote_sglist, *local_sglist;
13035         struct ctl_sg_entry *remote_dma_sglist, *local_dma_sglist;
13036         uint32_t local_used, remote_used, total_used;
13037         int retval;
13038         int i, j;
13039
13040         retval = 0;
13041
13042         rq = ctl_dt_req_alloc();
13043
13044         /*
13045          * If we failed to allocate the request, and if the DMA didn't fail
13046          * anyway, set busy status.  This is just a resource allocation
13047          * failure.
13048          */
13049         if ((rq == NULL)
13050          && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
13051                 ctl_set_busy(&io->scsiio);
13052
13053         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
13054
13055                 if (rq != NULL)
13056                         ctl_dt_req_free(rq);
13057
13058                 /*
13059                  * The data move failed.  We need to return status back
13060                  * to the other controller.  No point in trying to DMA
13061                  * data to the remote controller.
13062                  */
13063
13064                 ctl_send_datamove_done(io, /*have_lock*/ 0);
13065
13066                 retval = 1;
13067
13068                 goto bailout;
13069         }
13070
13071         local_sglist = io->io_hdr.local_sglist;
13072         local_dma_sglist = io->io_hdr.local_dma_sglist;
13073         remote_sglist = io->io_hdr.remote_sglist;
13074         remote_dma_sglist = io->io_hdr.remote_dma_sglist;
13075         local_used = 0;
13076         remote_used = 0;
13077         total_used = 0;
13078
13079         if (io->io_hdr.flags & CTL_FLAG_REDIR_DONE) {
13080                 rq->ret = CTL_HA_STATUS_SUCCESS;
13081                 rq->context = io;
13082                 callback(rq);
13083                 goto bailout;
13084         }
13085
13086         /*
13087          * Pull/push the data over the wire from/to the other controller.
13088          * This takes into account the possibility that the local and
13089          * remote sglists may not be identical in terms of the size of
13090          * the elements and the number of elements.
13091          *
13092          * One fundamental assumption here is that the length allocated for
13093          * both the local and remote sglists is identical.  Otherwise, we've
13094          * essentially got a coding error of some sort.
13095          */
13096         for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
13097                 int isc_ret;
13098                 uint32_t cur_len, dma_length;
13099                 uint8_t *tmp_ptr;
13100
13101                 rq->id = CTL_HA_DATA_CTL;
13102                 rq->command = command;
13103                 rq->context = io;
13104
13105                 /*
13106                  * Both pointers should be aligned.  But it is possible
13107                  * that the allocation length is not.  They should both
13108                  * also have enough slack left over at the end, though,
13109                  * to round up to the next 8 byte boundary.
13110                  */
13111                 cur_len = ctl_min(local_sglist[i].len - local_used,
13112                                   remote_sglist[j].len - remote_used);
13113
13114                 /*
13115                  * In this case, we have a size issue and need to decrease
13116                  * the size, except in the case where we actually have less
13117                  * than 8 bytes left.  In that case, we need to increase
13118                  * the DMA length to get the last bit.
13119                  */
13120                 if ((cur_len & 0x7) != 0) {
13121                         if (cur_len > 0x7) {
13122                                 cur_len = cur_len - (cur_len & 0x7);
13123                                 dma_length = cur_len;
13124                         } else {
13125                                 CTL_SIZE_8B(dma_length, cur_len);
13126                         }
13127
13128                 } else
13129                         dma_length = cur_len;
13130
13131                 /*
13132                  * If we had to allocate memory for this I/O, instead of using
13133                  * the non-cached mirror memory, we'll need to flush the cache
13134                  * before trying to DMA to the other controller.
13135                  *
13136                  * We could end up doing this multiple times for the same
13137                  * segment if we have a larger local segment than remote
13138                  * segment.  That shouldn't be an issue.
13139                  */
13140                 if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
13141                         /*
13142                          * XXX KDM use bus_dmamap_sync() here.
13143                          */
13144                 }
13145
13146                 rq->size = dma_length;
13147
13148                 tmp_ptr = (uint8_t *)local_sglist[i].addr;
13149                 tmp_ptr += local_used;
13150
13151                 /* Use physical addresses when talking to ISC hardware */
13152                 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
13153                         /* XXX KDM use busdma */
13154 #if 0
13155                         rq->local = vtophys(tmp_ptr);
13156 #endif
13157                 } else
13158                         rq->local = tmp_ptr;
13159
13160                 tmp_ptr = (uint8_t *)remote_sglist[j].addr;
13161                 tmp_ptr += remote_used;
13162                 rq->remote = tmp_ptr;
13163
13164                 rq->callback = NULL;
13165
13166                 local_used += cur_len;
13167                 if (local_used >= local_sglist[i].len) {
13168                         i++;
13169                         local_used = 0;
13170                 }
13171
13172                 remote_used += cur_len;
13173                 if (remote_used >= remote_sglist[j].len) {
13174                         j++;
13175                         remote_used = 0;
13176                 }
13177                 total_used += cur_len;
13178
13179                 if (total_used >= io->scsiio.kern_data_len)
13180                         rq->callback = callback;
13181
13182                 if ((rq->size & 0x7) != 0) {
13183                         printf("%s: warning: size %d is not on 8b boundary\n",
13184                                __func__, rq->size);
13185                 }
13186                 if (((uintptr_t)rq->local & 0x7) != 0) {
13187                         printf("%s: warning: local %p not on 8b boundary\n",
13188                                __func__, rq->local);
13189                 }
13190                 if (((uintptr_t)rq->remote & 0x7) != 0) {
13191                         printf("%s: warning: remote %p not on 8b boundary\n",
13192                                __func__, rq->local);
13193                 }
13194 #if 0
13195                 printf("%s: %s: local %#x remote %#x size %d\n", __func__,
13196                        (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
13197                        rq->local, rq->remote, rq->size);
13198 #endif
13199
13200                 isc_ret = ctl_dt_single(rq);
13201                 if (isc_ret == CTL_HA_STATUS_WAIT)
13202                         continue;
13203
13204                 if (isc_ret == CTL_HA_STATUS_DISCONNECT) {
13205                         rq->ret = CTL_HA_STATUS_SUCCESS;
13206                 } else {
13207                         rq->ret = isc_ret;
13208                 }
13209                 callback(rq);
13210                 goto bailout;
13211         }
13212
13213 bailout:
13214         return (retval);
13215
13216 }
13217
13218 static void
13219 ctl_datamove_remote_read(union ctl_io *io)
13220 {
13221         int retval;
13222         int i;
13223
13224         /*
13225          * This will send an error to the other controller in the case of a
13226          * failure.
13227          */
13228         retval = ctl_datamove_remote_sgl_setup(io);
13229         if (retval != 0)
13230                 return;
13231
13232         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
13233                                           ctl_datamove_remote_read_cb);
13234         if ((retval != 0)
13235          && ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0)) {
13236                 /*
13237                  * Make sure we free memory if there was an error..  The
13238                  * ctl_datamove_remote_xfer() function will send the
13239                  * datamove done message, or call the callback with an
13240                  * error if there is a problem.
13241                  */
13242                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13243                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
13244         }
13245
13246         return;
13247 }
13248
13249 /*
13250  * Process a datamove request from the other controller.  This is used for
13251  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
13252  * first.  Once that is complete, the data gets DMAed into the remote
13253  * controller's memory.  For reads, we DMA from the remote controller's
13254  * memory into our memory first, and then move it out to the FETD.
13255  */
13256 static void
13257 ctl_datamove_remote(union ctl_io *io)
13258 {
13259         struct ctl_softc *softc;
13260
13261         softc = control_softc;
13262
13263         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
13264
13265         /*
13266          * Note that we look for an aborted I/O here, but don't do some of
13267          * the other checks that ctl_datamove() normally does.  We don't
13268          * need to run the task queue, because this I/O is on the ISC
13269          * queue, which is executed by the work thread after the task queue.
13270          * We don't need to run the datamove delay code, since that should
13271          * have been done if need be on the other controller.
13272          */
13273         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
13274
13275                 printf("%s: tag 0x%04x on (%d:%d:%d:%d) aborted\n", __func__,
13276                        io->scsiio.tag_num, io->io_hdr.nexus.initid.id,
13277                        io->io_hdr.nexus.targ_port,
13278                        io->io_hdr.nexus.targ_target.id,
13279                        io->io_hdr.nexus.targ_lun);
13280                 io->io_hdr.status = CTL_CMD_ABORTED;
13281                 io->io_hdr.port_status = 31338;
13282
13283                 ctl_send_datamove_done(io, /*have_lock*/ 0);
13284
13285                 return;
13286         }
13287
13288         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT) {
13289                 ctl_datamove_remote_write(io);
13290         } else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN){
13291                 ctl_datamove_remote_read(io);
13292         } else {
13293                 union ctl_ha_msg msg;
13294                 struct scsi_sense_data *sense;
13295                 uint8_t sks[3];
13296                 int retry_count;
13297
13298                 memset(&msg, 0, sizeof(msg));
13299
13300                 msg.hdr.msg_type = CTL_MSG_BAD_JUJU;
13301                 msg.hdr.status = CTL_SCSI_ERROR;
13302                 msg.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
13303
13304                 retry_count = 4243;
13305
13306                 sense = &msg.scsi.sense_data;
13307                 sks[0] = SSD_SCS_VALID;
13308                 sks[1] = (retry_count >> 8) & 0xff;
13309                 sks[2] = retry_count & 0xff;
13310
13311                 /* "Internal target failure" */
13312                 scsi_set_sense_data(sense,
13313                                     /*sense_format*/ SSD_TYPE_NONE,
13314                                     /*current_error*/ 1,
13315                                     /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
13316                                     /*asc*/ 0x44,
13317                                     /*ascq*/ 0x00,
13318                                     /*type*/ SSD_ELEM_SKS,
13319                                     /*size*/ sizeof(sks),
13320                                     /*data*/ sks,
13321                                     SSD_ELEM_NONE);
13322
13323                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
13324                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
13325                         ctl_failover_io(io, /*have_lock*/ 1);
13326                         return;
13327                 }
13328
13329                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0) >
13330                     CTL_HA_STATUS_SUCCESS) {
13331                         /* XXX KDM what to do if this fails? */
13332                 }
13333                 return;
13334         }
13335
13336 }
13337
13338 static int
13339 ctl_process_done(union ctl_io *io)
13340 {
13341         struct ctl_lun *lun;
13342         struct ctl_softc *ctl_softc;
13343         void (*fe_done)(union ctl_io *io);
13344         uint32_t targ_port = ctl_port_idx(io->io_hdr.nexus.targ_port);
13345
13346         CTL_DEBUG_PRINT(("ctl_process_done\n"));
13347
13348         fe_done =
13349             control_softc->ctl_ports[targ_port]->fe_done;
13350
13351 #ifdef CTL_TIME_IO
13352         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
13353                 char str[256];
13354                 char path_str[64];
13355                 struct sbuf sb;
13356
13357                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
13358                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
13359
13360                 sbuf_cat(&sb, path_str);
13361                 switch (io->io_hdr.io_type) {
13362                 case CTL_IO_SCSI:
13363                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
13364                         sbuf_printf(&sb, "\n");
13365                         sbuf_cat(&sb, path_str);
13366                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
13367                                     io->scsiio.tag_num, io->scsiio.tag_type);
13368                         break;
13369                 case CTL_IO_TASK:
13370                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
13371                                     "Tag Type: %d\n", io->taskio.task_action,
13372                                     io->taskio.tag_num, io->taskio.tag_type);
13373                         break;
13374                 default:
13375                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
13376                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
13377                         break;
13378                 }
13379                 sbuf_cat(&sb, path_str);
13380                 sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
13381                             (intmax_t)time_uptime - io->io_hdr.start_time);
13382                 sbuf_finish(&sb);
13383                 printf("%s", sbuf_data(&sb));
13384         }
13385 #endif /* CTL_TIME_IO */
13386
13387         switch (io->io_hdr.io_type) {
13388         case CTL_IO_SCSI:
13389                 break;
13390         case CTL_IO_TASK:
13391                 if (bootverbose || verbose > 0)
13392                         ctl_io_error_print(io, NULL);
13393                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
13394                         ctl_free_io(io);
13395                 else
13396                         fe_done(io);
13397                 return (CTL_RETVAL_COMPLETE);
13398                 break;
13399         default:
13400                 printf("ctl_process_done: invalid io type %d\n",
13401                        io->io_hdr.io_type);
13402                 panic("ctl_process_done: invalid io type %d\n",
13403                       io->io_hdr.io_type);
13404                 break; /* NOTREACHED */
13405         }
13406
13407         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13408         if (lun == NULL) {
13409                 CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
13410                                  io->io_hdr.nexus.targ_mapped_lun));
13411                 fe_done(io);
13412                 goto bailout;
13413         }
13414         ctl_softc = lun->ctl_softc;
13415
13416         mtx_lock(&lun->lun_lock);
13417
13418         /*
13419          * Check to see if we have any errors to inject here.  We only
13420          * inject errors for commands that don't already have errors set.
13421          */
13422         if ((STAILQ_FIRST(&lun->error_list) != NULL)
13423          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))
13424                 ctl_inject_error(lun, io);
13425
13426         /*
13427          * XXX KDM how do we treat commands that aren't completed
13428          * successfully?
13429          *
13430          * XXX KDM should we also track I/O latency?
13431          */
13432         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
13433             io->io_hdr.io_type == CTL_IO_SCSI) {
13434 #ifdef CTL_TIME_IO
13435                 struct bintime cur_bt;
13436 #endif
13437                 int type;
13438
13439                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13440                     CTL_FLAG_DATA_IN)
13441                         type = CTL_STATS_READ;
13442                 else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13443                     CTL_FLAG_DATA_OUT)
13444                         type = CTL_STATS_WRITE;
13445                 else
13446                         type = CTL_STATS_NO_IO;
13447
13448                 lun->stats.ports[targ_port].bytes[type] +=
13449                     io->scsiio.kern_total_len;
13450                 lun->stats.ports[targ_port].operations[type]++;
13451 #ifdef CTL_TIME_IO
13452                 bintime_add(&lun->stats.ports[targ_port].dma_time[type],
13453                    &io->io_hdr.dma_bt);
13454                 lun->stats.ports[targ_port].num_dmas[type] +=
13455                     io->io_hdr.num_dmas;
13456                 getbintime(&cur_bt);
13457                 bintime_sub(&cur_bt, &io->io_hdr.start_bt);
13458                 bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
13459 #endif
13460         }
13461
13462         /*
13463          * Remove this from the OOA queue.
13464          */
13465         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
13466
13467         /*
13468          * Run through the blocked queue on this LUN and see if anything
13469          * has become unblocked, now that this transaction is done.
13470          */
13471         ctl_check_blocked(lun);
13472
13473         /*
13474          * If the LUN has been invalidated, free it if there is nothing
13475          * left on its OOA queue.
13476          */
13477         if ((lun->flags & CTL_LUN_INVALID)
13478          && TAILQ_EMPTY(&lun->ooa_queue)) {
13479                 mtx_unlock(&lun->lun_lock);
13480                 mtx_lock(&ctl_softc->ctl_lock);
13481                 ctl_free_lun(lun);
13482                 mtx_unlock(&ctl_softc->ctl_lock);
13483         } else
13484                 mtx_unlock(&lun->lun_lock);
13485
13486         /*
13487          * If this command has been aborted, make sure we set the status
13488          * properly.  The FETD is responsible for freeing the I/O and doing
13489          * whatever it needs to do to clean up its state.
13490          */
13491         if (io->io_hdr.flags & CTL_FLAG_ABORT)
13492                 io->io_hdr.status = CTL_CMD_ABORTED;
13493
13494         /*
13495          * We print out status for every task management command.  For SCSI
13496          * commands, we filter out any unit attention errors; they happen
13497          * on every boot, and would clutter up the log.  Note:  task
13498          * management commands aren't printed here, they are printed above,
13499          * since they should never even make it down here.
13500          */
13501         switch (io->io_hdr.io_type) {
13502         case CTL_IO_SCSI: {
13503                 int error_code, sense_key, asc, ascq;
13504
13505                 sense_key = 0;
13506
13507                 if (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR)
13508                  && (io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)) {
13509                         /*
13510                          * Since this is just for printing, no need to
13511                          * show errors here.
13512                          */
13513                         scsi_extract_sense_len(&io->scsiio.sense_data,
13514                                                io->scsiio.sense_len,
13515                                                &error_code,
13516                                                &sense_key,
13517                                                &asc,
13518                                                &ascq,
13519                                                /*show_errors*/ 0);
13520                 }
13521
13522                 if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
13523                  && (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SCSI_ERROR)
13524                   || (io->scsiio.scsi_status != SCSI_STATUS_CHECK_COND)
13525                   || (sense_key != SSD_KEY_UNIT_ATTENTION))) {
13526
13527                         if ((time_uptime - ctl_softc->last_print_jiffies) <= 0){
13528                                 ctl_softc->skipped_prints++;
13529                         } else {
13530                                 uint32_t skipped_prints;
13531
13532                                 skipped_prints = ctl_softc->skipped_prints;
13533
13534                                 ctl_softc->skipped_prints = 0;
13535                                 ctl_softc->last_print_jiffies = time_uptime;
13536
13537                                 if (skipped_prints > 0) {
13538 #ifdef NEEDTOPORT
13539                                         csevent_log(CSC_CTL | CSC_SHELF_SW |
13540                                             CTL_ERROR_REPORT,
13541                                             csevent_LogType_Trace,
13542                                             csevent_Severity_Information,
13543                                             csevent_AlertLevel_Green,
13544                                             csevent_FRU_Firmware,
13545                                             csevent_FRU_Unknown,
13546                                             "High CTL error volume, %d prints "
13547                                             "skipped", skipped_prints);
13548 #endif
13549                                 }
13550                                 if (bootverbose || verbose > 0)
13551                                         ctl_io_error_print(io, NULL);
13552                         }
13553                 }
13554                 break;
13555         }
13556         case CTL_IO_TASK:
13557                 if (bootverbose || verbose > 0)
13558                         ctl_io_error_print(io, NULL);
13559                 break;
13560         default:
13561                 break;
13562         }
13563
13564         /*
13565          * Tell the FETD or the other shelf controller we're done with this
13566          * command.  Note that only SCSI commands get to this point.  Task
13567          * management commands are completed above.
13568          *
13569          * We only send status to the other controller if we're in XFER
13570          * mode.  In SER_ONLY mode, the I/O is done on the controller that
13571          * received the I/O (from CTL's perspective), and so the status is
13572          * generated there.
13573          *
13574          * XXX KDM if we hold the lock here, we could cause a deadlock
13575          * if the frontend comes back in in this context to queue
13576          * something.
13577          */
13578         if ((ctl_softc->ha_mode == CTL_HA_MODE_XFER)
13579          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
13580                 union ctl_ha_msg msg;
13581
13582                 memset(&msg, 0, sizeof(msg));
13583                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
13584                 msg.hdr.original_sc = io->io_hdr.original_sc;
13585                 msg.hdr.nexus = io->io_hdr.nexus;
13586                 msg.hdr.status = io->io_hdr.status;
13587                 msg.scsi.scsi_status = io->scsiio.scsi_status;
13588                 msg.scsi.tag_num = io->scsiio.tag_num;
13589                 msg.scsi.tag_type = io->scsiio.tag_type;
13590                 msg.scsi.sense_len = io->scsiio.sense_len;
13591                 msg.scsi.sense_residual = io->scsiio.sense_residual;
13592                 msg.scsi.residual = io->scsiio.residual;
13593                 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
13594                        sizeof(io->scsiio.sense_data));
13595                 /*
13596                  * We copy this whether or not this is an I/O-related
13597                  * command.  Otherwise, we'd have to go and check to see
13598                  * whether it's a read/write command, and it really isn't
13599                  * worth it.
13600                  */
13601                 memcpy(&msg.scsi.lbalen,
13602                        &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
13603                        sizeof(msg.scsi.lbalen));
13604
13605                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13606                                 sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
13607                         /* XXX do something here */
13608                 }
13609
13610                 ctl_free_io(io);
13611         } else
13612                 fe_done(io);
13613
13614 bailout:
13615
13616         return (CTL_RETVAL_COMPLETE);
13617 }
13618
13619 /*
13620  * Front end should call this if it doesn't do autosense.  When the request
13621  * sense comes back in from the initiator, we'll dequeue this and send it.
13622  */
13623 int
13624 ctl_queue_sense(union ctl_io *io)
13625 {
13626         struct ctl_lun *lun;
13627         struct ctl_softc *ctl_softc;
13628         uint32_t initidx, targ_lun;
13629
13630         ctl_softc = control_softc;
13631
13632         CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
13633
13634         /*
13635          * LUN lookup will likely move to the ctl_work_thread() once we
13636          * have our new queueing infrastructure (that doesn't put things on
13637          * a per-LUN queue initially).  That is so that we can handle
13638          * things like an INQUIRY to a LUN that we don't have enabled.  We
13639          * can't deal with that right now.
13640          */
13641         mtx_lock(&ctl_softc->ctl_lock);
13642
13643         /*
13644          * If we don't have a LUN for this, just toss the sense
13645          * information.
13646          */
13647         targ_lun = io->io_hdr.nexus.targ_lun;
13648         targ_lun = ctl_map_lun(io->io_hdr.nexus.targ_port, targ_lun);
13649         if ((targ_lun < CTL_MAX_LUNS)
13650          && (ctl_softc->ctl_luns[targ_lun] != NULL))
13651                 lun = ctl_softc->ctl_luns[targ_lun];
13652         else
13653                 goto bailout;
13654
13655         initidx = ctl_get_initindex(&io->io_hdr.nexus);
13656
13657         mtx_lock(&lun->lun_lock);
13658         /*
13659          * Already have CA set for this LUN...toss the sense information.
13660          */
13661         if (ctl_is_set(lun->have_ca, initidx)) {
13662                 mtx_unlock(&lun->lun_lock);
13663                 goto bailout;
13664         }
13665
13666         memcpy(&lun->pending_sense[initidx].sense, &io->scsiio.sense_data,
13667                ctl_min(sizeof(lun->pending_sense[initidx].sense),
13668                sizeof(io->scsiio.sense_data)));
13669         ctl_set_mask(lun->have_ca, initidx);
13670         mtx_unlock(&lun->lun_lock);
13671
13672 bailout:
13673         mtx_unlock(&ctl_softc->ctl_lock);
13674
13675         ctl_free_io(io);
13676
13677         return (CTL_RETVAL_COMPLETE);
13678 }
13679
13680 /*
13681  * Primary command inlet from frontend ports.  All SCSI and task I/O
13682  * requests must go through this function.
13683  */
13684 int
13685 ctl_queue(union ctl_io *io)
13686 {
13687         struct ctl_softc *ctl_softc;
13688
13689         CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
13690
13691         ctl_softc = control_softc;
13692
13693 #ifdef CTL_TIME_IO
13694         io->io_hdr.start_time = time_uptime;
13695         getbintime(&io->io_hdr.start_bt);
13696 #endif /* CTL_TIME_IO */
13697
13698         /* Map FE-specific LUN ID into global one. */
13699         io->io_hdr.nexus.targ_mapped_lun =
13700             ctl_map_lun(io->io_hdr.nexus.targ_port, io->io_hdr.nexus.targ_lun);
13701
13702         switch (io->io_hdr.io_type) {
13703         case CTL_IO_SCSI:
13704         case CTL_IO_TASK:
13705                 ctl_enqueue_incoming(io);
13706                 break;
13707         default:
13708                 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
13709                 return (EINVAL);
13710         }
13711
13712         return (CTL_RETVAL_COMPLETE);
13713 }
13714
13715 #ifdef CTL_IO_DELAY
13716 static void
13717 ctl_done_timer_wakeup(void *arg)
13718 {
13719         union ctl_io *io;
13720
13721         io = (union ctl_io *)arg;
13722         ctl_done(io);
13723 }
13724 #endif /* CTL_IO_DELAY */
13725
13726 void
13727 ctl_done(union ctl_io *io)
13728 {
13729         struct ctl_softc *ctl_softc;
13730
13731         ctl_softc = control_softc;
13732
13733         /*
13734          * Enable this to catch duplicate completion issues.
13735          */
13736 #if 0
13737         if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
13738                 printf("%s: type %d msg %d cdb %x iptl: "
13739                        "%d:%d:%d:%d tag 0x%04x "
13740                        "flag %#x status %x\n",
13741                         __func__,
13742                         io->io_hdr.io_type,
13743                         io->io_hdr.msg_type,
13744                         io->scsiio.cdb[0],
13745                         io->io_hdr.nexus.initid.id,
13746                         io->io_hdr.nexus.targ_port,
13747                         io->io_hdr.nexus.targ_target.id,
13748                         io->io_hdr.nexus.targ_lun,
13749                         (io->io_hdr.io_type ==
13750                         CTL_IO_TASK) ?
13751                         io->taskio.tag_num :
13752                         io->scsiio.tag_num,
13753                         io->io_hdr.flags,
13754                         io->io_hdr.status);
13755         } else
13756                 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
13757 #endif
13758
13759         /*
13760          * This is an internal copy of an I/O, and should not go through
13761          * the normal done processing logic.
13762          */
13763         if (io->io_hdr.flags & CTL_FLAG_INT_COPY)
13764                 return;
13765
13766         /*
13767          * We need to send a msg to the serializing shelf to finish the IO
13768          * as well.  We don't send a finish message to the other shelf if
13769          * this is a task management command.  Task management commands
13770          * aren't serialized in the OOA queue, but rather just executed on
13771          * both shelf controllers for commands that originated on that
13772          * controller.
13773          */
13774         if ((io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)
13775          && (io->io_hdr.io_type != CTL_IO_TASK)) {
13776                 union ctl_ha_msg msg_io;
13777
13778                 msg_io.hdr.msg_type = CTL_MSG_FINISH_IO;
13779                 msg_io.hdr.serializing_sc = io->io_hdr.serializing_sc;
13780                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_io,
13781                     sizeof(msg_io), 0 ) != CTL_HA_STATUS_SUCCESS) {
13782                 }
13783                 /* continue on to finish IO */
13784         }
13785 #ifdef CTL_IO_DELAY
13786         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
13787                 struct ctl_lun *lun;
13788
13789                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13790
13791                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
13792         } else {
13793                 struct ctl_lun *lun;
13794
13795                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13796
13797                 if ((lun != NULL)
13798                  && (lun->delay_info.done_delay > 0)) {
13799                         struct callout *callout;
13800
13801                         callout = (struct callout *)&io->io_hdr.timer_bytes;
13802                         callout_init(callout, /*mpsafe*/ 1);
13803                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
13804                         callout_reset(callout,
13805                                       lun->delay_info.done_delay * hz,
13806                                       ctl_done_timer_wakeup, io);
13807                         if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
13808                                 lun->delay_info.done_delay = 0;
13809                         return;
13810                 }
13811         }
13812 #endif /* CTL_IO_DELAY */
13813
13814         ctl_enqueue_done(io);
13815 }
13816
13817 int
13818 ctl_isc(struct ctl_scsiio *ctsio)
13819 {
13820         struct ctl_lun *lun;
13821         int retval;
13822
13823         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13824
13825         CTL_DEBUG_PRINT(("ctl_isc: command: %02x\n", ctsio->cdb[0]));
13826
13827         CTL_DEBUG_PRINT(("ctl_isc: calling data_submit()\n"));
13828
13829         retval = lun->backend->data_submit((union ctl_io *)ctsio);
13830
13831         return (retval);
13832 }
13833
13834
13835 static void
13836 ctl_work_thread(void *arg)
13837 {
13838         struct ctl_thread *thr = (struct ctl_thread *)arg;
13839         struct ctl_softc *softc = thr->ctl_softc;
13840         union ctl_io *io;
13841         int retval;
13842
13843         CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
13844
13845         for (;;) {
13846                 retval = 0;
13847
13848                 /*
13849                  * We handle the queues in this order:
13850                  * - ISC
13851                  * - done queue (to free up resources, unblock other commands)
13852                  * - RtR queue
13853                  * - incoming queue
13854                  *
13855                  * If those queues are empty, we break out of the loop and
13856                  * go to sleep.
13857                  */
13858                 mtx_lock(&thr->queue_lock);
13859                 io = (union ctl_io *)STAILQ_FIRST(&thr->isc_queue);
13860                 if (io != NULL) {
13861                         STAILQ_REMOVE_HEAD(&thr->isc_queue, links);
13862                         mtx_unlock(&thr->queue_lock);
13863                         ctl_handle_isc(io);
13864                         continue;
13865                 }
13866                 io = (union ctl_io *)STAILQ_FIRST(&thr->done_queue);
13867                 if (io != NULL) {
13868                         STAILQ_REMOVE_HEAD(&thr->done_queue, links);
13869                         /* clear any blocked commands, call fe_done */
13870                         mtx_unlock(&thr->queue_lock);
13871                         retval = ctl_process_done(io);
13872                         continue;
13873                 }
13874                 io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
13875                 if (io != NULL) {
13876                         STAILQ_REMOVE_HEAD(&thr->incoming_queue, links);
13877                         mtx_unlock(&thr->queue_lock);
13878                         if (io->io_hdr.io_type == CTL_IO_TASK)
13879                                 ctl_run_task(io);
13880                         else
13881                                 ctl_scsiio_precheck(softc, &io->scsiio);
13882                         continue;
13883                 }
13884                 if (!ctl_pause_rtr) {
13885                         io = (union ctl_io *)STAILQ_FIRST(&thr->rtr_queue);
13886                         if (io != NULL) {
13887                                 STAILQ_REMOVE_HEAD(&thr->rtr_queue, links);
13888                                 mtx_unlock(&thr->queue_lock);
13889                                 retval = ctl_scsiio(&io->scsiio);
13890                                 if (retval != CTL_RETVAL_COMPLETE)
13891                                         CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
13892                                 continue;
13893                         }
13894                 }
13895
13896                 /* Sleep until we have something to do. */
13897                 mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0);
13898         }
13899 }
13900
13901 static void
13902 ctl_lun_thread(void *arg)
13903 {
13904         struct ctl_softc *softc = (struct ctl_softc *)arg;
13905         struct ctl_be_lun *be_lun;
13906         int retval;
13907
13908         CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
13909
13910         for (;;) {
13911                 retval = 0;
13912                 mtx_lock(&softc->ctl_lock);
13913                 be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
13914                 if (be_lun != NULL) {
13915                         STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
13916                         mtx_unlock(&softc->ctl_lock);
13917                         ctl_create_lun(be_lun);
13918                         continue;
13919                 }
13920
13921                 /* Sleep until we have something to do. */
13922                 mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock,
13923                     PDROP | PRIBIO, "-", 0);
13924         }
13925 }
13926
13927 static void
13928 ctl_enqueue_incoming(union ctl_io *io)
13929 {
13930         struct ctl_softc *softc = control_softc;
13931         struct ctl_thread *thr;
13932         u_int idx;
13933
13934         idx = (io->io_hdr.nexus.targ_port * 127 +
13935                io->io_hdr.nexus.initid.id) % worker_threads;
13936         thr = &softc->threads[idx];
13937         mtx_lock(&thr->queue_lock);
13938         STAILQ_INSERT_TAIL(&thr->incoming_queue, &io->io_hdr, links);
13939         mtx_unlock(&thr->queue_lock);
13940         wakeup(thr);
13941 }
13942
13943 static void
13944 ctl_enqueue_rtr(union ctl_io *io)
13945 {
13946         struct ctl_softc *softc = control_softc;
13947         struct ctl_thread *thr;
13948
13949         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13950         mtx_lock(&thr->queue_lock);
13951         STAILQ_INSERT_TAIL(&thr->rtr_queue, &io->io_hdr, links);
13952         mtx_unlock(&thr->queue_lock);
13953         wakeup(thr);
13954 }
13955
13956 static void
13957 ctl_enqueue_done(union ctl_io *io)
13958 {
13959         struct ctl_softc *softc = control_softc;
13960         struct ctl_thread *thr;
13961
13962         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13963         mtx_lock(&thr->queue_lock);
13964         STAILQ_INSERT_TAIL(&thr->done_queue, &io->io_hdr, links);
13965         mtx_unlock(&thr->queue_lock);
13966         wakeup(thr);
13967 }
13968
13969 static void
13970 ctl_enqueue_isc(union ctl_io *io)
13971 {
13972         struct ctl_softc *softc = control_softc;
13973         struct ctl_thread *thr;
13974
13975         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13976         mtx_lock(&thr->queue_lock);
13977         STAILQ_INSERT_TAIL(&thr->isc_queue, &io->io_hdr, links);
13978         mtx_unlock(&thr->queue_lock);
13979         wakeup(thr);
13980 }
13981
13982 /* Initialization and failover */
13983
13984 void
13985 ctl_init_isc_msg(void)
13986 {
13987         printf("CTL: Still calling this thing\n");
13988 }
13989
13990 /*
13991  * Init component
13992  *      Initializes component into configuration defined by bootMode
13993  *      (see hasc-sv.c)
13994  *      returns hasc_Status:
13995  *              OK
13996  *              ERROR - fatal error
13997  */
13998 static ctl_ha_comp_status
13999 ctl_isc_init(struct ctl_ha_component *c)
14000 {
14001         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
14002
14003         c->status = ret;
14004         return ret;
14005 }
14006
14007 /* Start component
14008  *      Starts component in state requested. If component starts successfully,
14009  *      it must set its own state to the requestrd state
14010  *      When requested state is HASC_STATE_HA, the component may refine it
14011  *      by adding _SLAVE or _MASTER flags.
14012  *      Currently allowed state transitions are:
14013  *      UNKNOWN->HA             - initial startup
14014  *      UNKNOWN->SINGLE - initial startup when no parter detected
14015  *      HA->SINGLE              - failover
14016  * returns ctl_ha_comp_status:
14017  *              OK      - component successfully started in requested state
14018  *              FAILED  - could not start the requested state, failover may
14019  *                        be possible
14020  *              ERROR   - fatal error detected, no future startup possible
14021  */
14022 static ctl_ha_comp_status
14023 ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
14024 {
14025         ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
14026
14027         printf("%s: go\n", __func__);
14028
14029         // UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
14030         if (c->state == CTL_HA_STATE_UNKNOWN ) {
14031                 ctl_is_single = 0;
14032                 if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
14033                     != CTL_HA_STATUS_SUCCESS) {
14034                         printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
14035                         ret = CTL_HA_COMP_STATUS_ERROR;
14036                 }
14037         } else if (CTL_HA_STATE_IS_HA(c->state)
14038                 && CTL_HA_STATE_IS_SINGLE(state)){
14039                 // HA->SINGLE transition
14040                 ctl_failover();
14041                 ctl_is_single = 1;
14042         } else {
14043                 printf("ctl_isc_start:Invalid state transition %X->%X\n",
14044                        c->state, state);
14045                 ret = CTL_HA_COMP_STATUS_ERROR;
14046         }
14047         if (CTL_HA_STATE_IS_SINGLE(state))
14048                 ctl_is_single = 1;
14049
14050         c->state = state;
14051         c->status = ret;
14052         return ret;
14053 }
14054
14055 /*
14056  * Quiesce component
14057  * The component must clear any error conditions (set status to OK) and
14058  * prepare itself to another Start call
14059  * returns ctl_ha_comp_status:
14060  *      OK
14061  *      ERROR
14062  */
14063 static ctl_ha_comp_status
14064 ctl_isc_quiesce(struct ctl_ha_component *c)
14065 {
14066         int ret = CTL_HA_COMP_STATUS_OK;
14067
14068         ctl_pause_rtr = 1;
14069         c->status = ret;
14070         return ret;
14071 }
14072
14073 struct ctl_ha_component ctl_ha_component_ctlisc =
14074 {
14075         .name = "CTL ISC",
14076         .state = CTL_HA_STATE_UNKNOWN,
14077         .init = ctl_isc_init,
14078         .start = ctl_isc_start,
14079         .quiesce = ctl_isc_quiesce
14080 };
14081
14082 /*
14083  *  vim: ts=8
14084  */