sys/cam/ctl/ctl.c

   1 /*-
   2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
   3  * Copyright (c) 2012 The FreeBSD Foundation
   4  * Copyright (c) 2015 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Portions of this software were developed by Edward Tomasz Napierala
   8  * under sponsorship from the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions, and the following disclaimer,
  15  *    without modification.
  16  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  17  *    substantially similar to the "NO WARRANTY" disclaimer below
  18  *    ("Disclaimer") and any redistribution must be conditioned upon
  19  *    including a substantially similar Disclaimer requirement for further
  20  *    binary redistribution.
  21  *
  22  * NO WARRANTY
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  32  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  33  * POSSIBILITY OF SUCH DAMAGES.
  34  *
  35  * $Id$
  36  */
  37 /*
  38  * CAM Target Layer, a SCSI device emulation subsystem.
  39  *
  40  * Author: Ken Merry <ken@FreeBSD.org>
  41  */
  42
  43 #define _CTL_C
  44
  45 #include <sys/cdefs.h>
  46 __FBSDID("$FreeBSD$");
  47
  48 #include <sys/param.h>
  49 #include <sys/systm.h>
  50 #include <sys/ctype.h>
  51 #include <sys/kernel.h>
  52 #include <sys/types.h>
  53 #include <sys/kthread.h>
  54 #include <sys/bio.h>
  55 #include <sys/fcntl.h>
  56 #include <sys/lock.h>
  57 #include <sys/module.h>
  58 #include <sys/mutex.h>
  59 #include <sys/condvar.h>
  60 #include <sys/malloc.h>
  61 #include <sys/conf.h>
  62 #include <sys/ioccom.h>
  63 #include <sys/queue.h>
  64 #include <sys/sbuf.h>
  65 #include <sys/smp.h>
  66 #include <sys/endian.h>
  67 #include <sys/sysctl.h>
  68 #include <vm/uma.h>
  69
  70 #include <cam/cam.h>
  71 #include <cam/scsi/scsi_all.h>
  72 #include <cam/scsi/scsi_da.h>
  73 #include <cam/ctl/ctl_io.h>
  74 #include <cam/ctl/ctl.h>
  75 #include <cam/ctl/ctl_frontend.h>
  76 #include <cam/ctl/ctl_util.h>
  77 #include <cam/ctl/ctl_backend.h>
  78 #include <cam/ctl/ctl_ioctl.h>
  79 #include <cam/ctl/ctl_ha.h>
  80 #include <cam/ctl/ctl_private.h>
  81 #include <cam/ctl/ctl_debug.h>
  82 #include <cam/ctl/ctl_scsi_all.h>
  83 #include <cam/ctl/ctl_error.h>
  84
  85 struct ctl_softc *control_softc = NULL;
  86
  87 /*
  88  * Template mode pages.
  89  */
  90
  91 /*
  92  * Note that these are default values only.  The actual values will be
  93  * filled in when the user does a mode sense.
  94  */
  95 const static struct copan_debugconf_subpage debugconf_page_default = {
  96         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
  97         DBGCNF_SUBPAGE_CODE,            /* subpage */
  98         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
  99          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 100         DBGCNF_VERSION,                 /* page_version */
 101         {CTL_TIME_IO_DEFAULT_SECS>>8,
 102          CTL_TIME_IO_DEFAULT_SECS>>0},  /* ctl_time_io_secs */
 103 };
 104
 105 const static struct copan_debugconf_subpage debugconf_page_changeable = {
 106         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 107         DBGCNF_SUBPAGE_CODE,            /* subpage */
 108         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 109          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 110         0,                              /* page_version */
 111         {0xff,0xff},                    /* ctl_time_io_secs */
 112 };
 113
 114 const static struct scsi_da_rw_recovery_page rw_er_page_default = {
 115         /*page_code*/SMS_RW_ERROR_RECOVERY_PAGE,
 116         /*page_length*/sizeof(struct scsi_da_rw_recovery_page) - 2,
 117         /*byte3*/SMS_RWER_AWRE|SMS_RWER_ARRE,
 118         /*read_retry_count*/0,
 119         /*correction_span*/0,
 120         /*head_offset_count*/0,
 121         /*data_strobe_offset_cnt*/0,
 122         /*byte8*/SMS_RWER_LBPERE,
 123         /*write_retry_count*/0,
 124         /*reserved2*/0,
 125         /*recovery_time_limit*/{0, 0},
 126 };
 127
 128 const static struct scsi_da_rw_recovery_page rw_er_page_changeable = {
 129         /*page_code*/SMS_RW_ERROR_RECOVERY_PAGE,
 130         /*page_length*/sizeof(struct scsi_da_rw_recovery_page) - 2,
 131         /*byte3*/0,
 132         /*read_retry_count*/0,
 133         /*correction_span*/0,
 134         /*head_offset_count*/0,
 135         /*data_strobe_offset_cnt*/0,
 136         /*byte8*/0,
 137         /*write_retry_count*/0,
 138         /*reserved2*/0,
 139         /*recovery_time_limit*/{0, 0},
 140 };
 141
 142 const static struct scsi_format_page format_page_default = {
 143         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 144         /*page_length*/sizeof(struct scsi_format_page) - 2,
 145         /*tracks_per_zone*/ {0, 0},
 146         /*alt_sectors_per_zone*/ {0, 0},
 147         /*alt_tracks_per_zone*/ {0, 0},
 148         /*alt_tracks_per_lun*/ {0, 0},
 149         /*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
 150                                 CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
 151         /*bytes_per_sector*/ {0, 0},
 152         /*interleave*/ {0, 0},
 153         /*track_skew*/ {0, 0},
 154         /*cylinder_skew*/ {0, 0},
 155         /*flags*/ SFP_HSEC,
 156         /*reserved*/ {0, 0, 0}
 157 };
 158
 159 const static struct scsi_format_page format_page_changeable = {
 160         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 161         /*page_length*/sizeof(struct scsi_format_page) - 2,
 162         /*tracks_per_zone*/ {0, 0},
 163         /*alt_sectors_per_zone*/ {0, 0},
 164         /*alt_tracks_per_zone*/ {0, 0},
 165         /*alt_tracks_per_lun*/ {0, 0},
 166         /*sectors_per_track*/ {0, 0},
 167         /*bytes_per_sector*/ {0, 0},
 168         /*interleave*/ {0, 0},
 169         /*track_skew*/ {0, 0},
 170         /*cylinder_skew*/ {0, 0},
 171         /*flags*/ 0,
 172         /*reserved*/ {0, 0, 0}
 173 };
 174
 175 const static struct scsi_rigid_disk_page rigid_disk_page_default = {
 176         /*page_code*/SMS_RIGID_DISK_PAGE,
 177         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 178         /*cylinders*/ {0, 0, 0},
 179         /*heads*/ CTL_DEFAULT_HEADS,
 180         /*start_write_precomp*/ {0, 0, 0},
 181         /*start_reduced_current*/ {0, 0, 0},
 182         /*step_rate*/ {0, 0},
 183         /*landing_zone_cylinder*/ {0, 0, 0},
 184         /*rpl*/ SRDP_RPL_DISABLED,
 185         /*rotational_offset*/ 0,
 186         /*reserved1*/ 0,
 187         /*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
 188                            CTL_DEFAULT_ROTATION_RATE & 0xff},
 189         /*reserved2*/ {0, 0}
 190 };
 191
 192 const static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
 193         /*page_code*/SMS_RIGID_DISK_PAGE,
 194         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 195         /*cylinders*/ {0, 0, 0},
 196         /*heads*/ 0,
 197         /*start_write_precomp*/ {0, 0, 0},
 198         /*start_reduced_current*/ {0, 0, 0},
 199         /*step_rate*/ {0, 0},
 200         /*landing_zone_cylinder*/ {0, 0, 0},
 201         /*rpl*/ 0,
 202         /*rotational_offset*/ 0,
 203         /*reserved1*/ 0,
 204         /*rotation_rate*/ {0, 0},
 205         /*reserved2*/ {0, 0}
 206 };
 207
 208 const static struct scsi_caching_page caching_page_default = {
 209         /*page_code*/SMS_CACHING_PAGE,
 210         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 211         /*flags1*/ SCP_DISC | SCP_WCE,
 212         /*ret_priority*/ 0,
 213         /*disable_pf_transfer_len*/ {0xff, 0xff},
 214         /*min_prefetch*/ {0, 0},
 215         /*max_prefetch*/ {0xff, 0xff},
 216         /*max_pf_ceiling*/ {0xff, 0xff},
 217         /*flags2*/ 0,
 218         /*cache_segments*/ 0,
 219         /*cache_seg_size*/ {0, 0},
 220         /*reserved*/ 0,
 221         /*non_cache_seg_size*/ {0, 0, 0}
 222 };
 223
 224 const static struct scsi_caching_page caching_page_changeable = {
 225         /*page_code*/SMS_CACHING_PAGE,
 226         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 227         /*flags1*/ SCP_WCE | SCP_RCD,
 228         /*ret_priority*/ 0,
 229         /*disable_pf_transfer_len*/ {0, 0},
 230         /*min_prefetch*/ {0, 0},
 231         /*max_prefetch*/ {0, 0},
 232         /*max_pf_ceiling*/ {0, 0},
 233         /*flags2*/ 0,
 234         /*cache_segments*/ 0,
 235         /*cache_seg_size*/ {0, 0},
 236         /*reserved*/ 0,
 237         /*non_cache_seg_size*/ {0, 0, 0}
 238 };
 239
 240 const static struct scsi_control_page control_page_default = {
 241         /*page_code*/SMS_CONTROL_MODE_PAGE,
 242         /*page_length*/sizeof(struct scsi_control_page) - 2,
 243         /*rlec*/0,
 244         /*queue_flags*/SCP_QUEUE_ALG_RESTRICTED,
 245         /*eca_and_aen*/0,
 246         /*flags4*/SCP_TAS,
 247         /*aen_holdoff_period*/{0, 0},
 248         /*busy_timeout_period*/{0, 0},
 249         /*extended_selftest_completion_time*/{0, 0}
 250 };
 251
 252 const static struct scsi_control_page control_page_changeable = {
 253         /*page_code*/SMS_CONTROL_MODE_PAGE,
 254         /*page_length*/sizeof(struct scsi_control_page) - 2,
 255         /*rlec*/SCP_DSENSE,
 256         /*queue_flags*/SCP_QUEUE_ALG_MASK,
 257         /*eca_and_aen*/SCP_SWP,
 258         /*flags4*/0,
 259         /*aen_holdoff_period*/{0, 0},
 260         /*busy_timeout_period*/{0, 0},
 261         /*extended_selftest_completion_time*/{0, 0}
 262 };
 263
 264 #define CTL_CEM_LEN     (sizeof(struct scsi_control_ext_page) - 4)
 265
 266 const static struct scsi_control_ext_page control_ext_page_default = {
 267         /*page_code*/SMS_CONTROL_MODE_PAGE | SMPH_SPF,
 268         /*subpage_code*/0x01,
 269         /*page_length*/{CTL_CEM_LEN >> 8, CTL_CEM_LEN},
 270         /*flags*/0,
 271         /*prio*/0,
 272         /*max_sense*/0
 273 };
 274
 275 const static struct scsi_control_ext_page control_ext_page_changeable = {
 276         /*page_code*/SMS_CONTROL_MODE_PAGE | SMPH_SPF,
 277         /*subpage_code*/0x01,
 278         /*page_length*/{CTL_CEM_LEN >> 8, CTL_CEM_LEN},
 279         /*flags*/0,
 280         /*prio*/0,
 281         /*max_sense*/0
 282 };
 283
 284 const static struct scsi_info_exceptions_page ie_page_default = {
 285         /*page_code*/SMS_INFO_EXCEPTIONS_PAGE,
 286         /*page_length*/sizeof(struct scsi_info_exceptions_page) - 2,
 287         /*info_flags*/SIEP_FLAGS_DEXCPT,
 288         /*mrie*/0,
 289         /*interval_timer*/{0, 0, 0, 0},
 290         /*report_count*/{0, 0, 0, 0}
 291 };
 292
 293 const static struct scsi_info_exceptions_page ie_page_changeable = {
 294         /*page_code*/SMS_INFO_EXCEPTIONS_PAGE,
 295         /*page_length*/sizeof(struct scsi_info_exceptions_page) - 2,
 296         /*info_flags*/0,
 297         /*mrie*/0,
 298         /*interval_timer*/{0, 0, 0, 0},
 299         /*report_count*/{0, 0, 0, 0}
 300 };
 301
 302 #define CTL_LBPM_LEN    (sizeof(struct ctl_logical_block_provisioning_page) - 4)
 303
 304 const static struct ctl_logical_block_provisioning_page lbp_page_default = {{
 305         /*page_code*/SMS_INFO_EXCEPTIONS_PAGE | SMPH_SPF,
 306         /*subpage_code*/0x02,
 307         /*page_length*/{CTL_LBPM_LEN >> 8, CTL_LBPM_LEN},
 308         /*flags*/0,
 309         /*reserved*/{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 310         /*descr*/{}},
 311         {{/*flags*/0,
 312           /*resource*/0x01,
 313           /*reserved*/{0, 0},
 314           /*count*/{0, 0, 0, 0}},
 315          {/*flags*/0,
 316           /*resource*/0x02,
 317           /*reserved*/{0, 0},
 318           /*count*/{0, 0, 0, 0}},
 319          {/*flags*/0,
 320           /*resource*/0xf1,
 321           /*reserved*/{0, 0},
 322           /*count*/{0, 0, 0, 0}},
 323          {/*flags*/0,
 324           /*resource*/0xf2,
 325           /*reserved*/{0, 0},
 326           /*count*/{0, 0, 0, 0}}
 327         }
 328 };
 329
 330 const static struct ctl_logical_block_provisioning_page lbp_page_changeable = {{
 331         /*page_code*/SMS_INFO_EXCEPTIONS_PAGE | SMPH_SPF,
 332         /*subpage_code*/0x02,
 333         /*page_length*/{CTL_LBPM_LEN >> 8, CTL_LBPM_LEN},
 334         /*flags*/0,
 335         /*reserved*/{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 336         /*descr*/{}},
 337         {{/*flags*/0,
 338           /*resource*/0,
 339           /*reserved*/{0, 0},
 340           /*count*/{0, 0, 0, 0}},
 341          {/*flags*/0,
 342           /*resource*/0,
 343           /*reserved*/{0, 0},
 344           /*count*/{0, 0, 0, 0}},
 345          {/*flags*/0,
 346           /*resource*/0,
 347           /*reserved*/{0, 0},
 348           /*count*/{0, 0, 0, 0}},
 349          {/*flags*/0,
 350           /*resource*/0,
 351           /*reserved*/{0, 0},
 352           /*count*/{0, 0, 0, 0}}
 353         }
 354 };
 355
 356 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 357 static int worker_threads = -1;
 358 TUNABLE_INT("kern.cam.ctl.worker_threads", &worker_threads);
 359 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
 360     &worker_threads, 1, "Number of worker threads");
 361 static int ctl_debug = CTL_DEBUG_NONE;
 362 TUNABLE_INT("kern.cam.ctl.debug", &ctl_debug);
 363 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, debug, CTLFLAG_RWTUN,
 364     &ctl_debug, 0, "Enabled debug flags");
 365
 366 /*
 367  * Supported pages (0x00), Serial number (0x80), Device ID (0x83),
 368  * Extended INQUIRY Data (0x86), Mode Page Policy (0x87),
 369  * SCSI Ports (0x88), Third-party Copy (0x8F), Block limits (0xB0),
 370  * Block Device Characteristics (0xB1) and Logical Block Provisioning (0xB2)
 371  */
 372 #define SCSI_EVPD_NUM_SUPPORTED_PAGES   10
 373
 374 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 375                                   int param);
 376 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 377 static void ctl_copy_sense_data_back(union ctl_io *src, union ctl_ha_msg *dest);
 378 static int ctl_init(void);
 379 void ctl_shutdown(void);
 380 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 381 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 382 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
 383 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 384                               struct ctl_ooa *ooa_hdr,
 385                               struct ctl_ooa_entry *kern_entries);
 386 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 387                      struct thread *td);
 388 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
 389                          struct ctl_be_lun *be_lun);
 390 static int ctl_free_lun(struct ctl_lun *lun);
 391 static void ctl_create_lun(struct ctl_be_lun *be_lun);
 392 static struct ctl_port * ctl_io_port(struct ctl_io_hdr *io_hdr);
 393
 394 static int ctl_do_mode_select(union ctl_io *io);
 395 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
 396                            uint64_t res_key, uint64_t sa_res_key,
 397                            uint8_t type, uint32_t residx,
 398                            struct ctl_scsiio *ctsio,
 399                            struct scsi_per_res_out *cdb,
 400                            struct scsi_per_res_out_parms* param);
 401 static void ctl_pro_preempt_other(struct ctl_lun *lun,
 402                                   union ctl_ha_msg *msg);
 403 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
 404 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
 405 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
 406 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
 407 static int ctl_inquiry_evpd_eid(struct ctl_scsiio *ctsio, int alloc_len);
 408 static int ctl_inquiry_evpd_mpp(struct ctl_scsiio *ctsio, int alloc_len);
 409 static int ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio,
 410                                          int alloc_len);
 411 static int ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio,
 412                                          int alloc_len);
 413 static int ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len);
 414 static int ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len);
 415 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
 416 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
 417 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len);
 418 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2,
 419     bool seq);
 420 static ctl_action ctl_extent_check_seq(union ctl_io *io1, union ctl_io *io2);
 421 static ctl_action ctl_check_for_blockage(struct ctl_lun *lun,
 422     union ctl_io *pending_io, union ctl_io *ooa_io);
 423 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
 424                                 union ctl_io *starting_io);
 425 static int ctl_check_blocked(struct ctl_lun *lun);
 426 static int ctl_scsiio_lun_check(struct ctl_lun *lun,
 427                                 const struct ctl_cmd_entry *entry,
 428                                 struct ctl_scsiio *ctsio);
 429 static void ctl_failover_lun(struct ctl_lun *lun);
 430 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 431                                struct ctl_scsiio *ctsio);
 432 static int ctl_scsiio(struct ctl_scsiio *ctsio);
 433
 434 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 435 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 436                             ctl_ua_type ua_type);
 437 static int ctl_do_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 438                          ctl_ua_type ua_type);
 439 static int ctl_lun_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 440 static int ctl_abort_task(union ctl_io *io);
 441 static int ctl_abort_task_set(union ctl_io *io);
 442 static int ctl_query_task(union ctl_io *io, int task_set);
 443 static int ctl_i_t_nexus_reset(union ctl_io *io);
 444 static int ctl_query_async_event(union ctl_io *io);
 445 static void ctl_run_task(union ctl_io *io);
 446 #ifdef CTL_IO_DELAY
 447 static void ctl_datamove_timer_wakeup(void *arg);
 448 static void ctl_done_timer_wakeup(void *arg);
 449 #endif /* CTL_IO_DELAY */
 450
 451 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 452 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
 453 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
 454 static void ctl_datamove_remote_write(union ctl_io *io);
 455 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
 456 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 457 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 458 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 459                                     ctl_ha_dt_cb callback);
 460 static void ctl_datamove_remote_read(union ctl_io *io);
 461 static void ctl_datamove_remote(union ctl_io *io);
 462 static int ctl_process_done(union ctl_io *io);
 463 static void ctl_lun_thread(void *arg);
 464 static void ctl_thresh_thread(void *arg);
 465 static void ctl_work_thread(void *arg);
 466 static void ctl_enqueue_incoming(union ctl_io *io);
 467 static void ctl_enqueue_rtr(union ctl_io *io);
 468 static void ctl_enqueue_done(union ctl_io *io);
 469 static void ctl_enqueue_isc(union ctl_io *io);
 470 static const struct ctl_cmd_entry *
 471     ctl_get_cmd_entry(struct ctl_scsiio *ctsio, int *sa);
 472 static const struct ctl_cmd_entry *
 473     ctl_validate_command(struct ctl_scsiio *ctsio);
 474 static int ctl_cmd_applicable(uint8_t lun_type,
 475     const struct ctl_cmd_entry *entry);
 476
 477 static uint64_t ctl_get_prkey(struct ctl_lun *lun, uint32_t residx);
 478 static void ctl_clr_prkey(struct ctl_lun *lun, uint32_t residx);
 479 static void ctl_alloc_prkey(struct ctl_lun *lun, uint32_t residx);
 480 static void ctl_set_prkey(struct ctl_lun *lun, uint32_t residx, uint64_t key);
 481
 482 /*
 483  * Load the serialization table.  This isn't very pretty, but is probably
 484  * the easiest way to do it.
 485  */
 486 #include "ctl_ser_table.c"
 487
 488 /*
 489  * We only need to define open, close and ioctl routines for this driver.
 490  */
 491 static struct cdevsw ctl_cdevsw = {
 492         .d_version =    D_VERSION,
 493         .d_flags =      0,
 494         .d_open =       ctl_open,
 495         .d_close =      ctl_close,
 496         .d_ioctl =      ctl_ioctl,
 497         .d_name =       "ctl",
 498 };
 499
 500
 501 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 502
 503 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 504
 505 static moduledata_t ctl_moduledata = {
 506         "ctl",
 507         ctl_module_event_handler,
 508         NULL
 509 };
 510
 511 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
 512 MODULE_VERSION(ctl, 1);
 513
 514 static struct ctl_frontend ha_frontend =
 515 {
 516         .name = "ha",
 517 };
 518
 519 static void
 520 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 521                             union ctl_ha_msg *msg_info)
 522 {
 523         struct ctl_scsiio *ctsio;
 524
 525         if (msg_info->hdr.original_sc == NULL) {
 526                 printf("%s: original_sc == NULL!\n", __func__);
 527                 /* XXX KDM now what? */
 528                 return;
 529         }
 530
 531         ctsio = &msg_info->hdr.original_sc->scsiio;
 532         ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 533         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 534         ctsio->io_hdr.status = msg_info->hdr.status;
 535         ctsio->scsi_status = msg_info->scsi.scsi_status;
 536         ctsio->sense_len = msg_info->scsi.sense_len;
 537         ctsio->sense_residual = msg_info->scsi.sense_residual;
 538         ctsio->residual = msg_info->scsi.residual;
 539         memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 540                msg_info->scsi.sense_len);
 541         ctl_enqueue_isc((union ctl_io *)ctsio);
 542 }
 543
 544 static void
 545 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
 546                                 union ctl_ha_msg *msg_info)
 547 {
 548         struct ctl_scsiio *ctsio;
 549
 550         if (msg_info->hdr.serializing_sc == NULL) {
 551                 printf("%s: serializing_sc == NULL!\n", __func__);
 552                 /* XXX KDM now what? */
 553                 return;
 554         }
 555
 556         ctsio = &msg_info->hdr.serializing_sc->scsiio;
 557         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 558         ctl_enqueue_isc((union ctl_io *)ctsio);
 559 }
 560
 561 void
 562 ctl_isc_announce_lun(struct ctl_lun *lun)
 563 {
 564         struct ctl_softc *softc = lun->ctl_softc;
 565         union ctl_ha_msg *msg;
 566         struct ctl_ha_msg_lun_pr_key pr_key;
 567         int i, k;
 568
 569         if (softc->ha_link != CTL_HA_LINK_ONLINE)
 570                 return;
 571         mtx_lock(&lun->lun_lock);
 572         i = sizeof(msg->lun);
 573         if (lun->lun_devid)
 574                 i += lun->lun_devid->len;
 575         i += sizeof(pr_key) * lun->pr_key_count;
 576 alloc:
 577         mtx_unlock(&lun->lun_lock);
 578         msg = malloc(i, M_CTL, M_WAITOK);
 579         mtx_lock(&lun->lun_lock);
 580         k = sizeof(msg->lun);
 581         if (lun->lun_devid)
 582                 k += lun->lun_devid->len;
 583         k += sizeof(pr_key) * lun->pr_key_count;
 584         if (i < k) {
 585                 free(msg, M_CTL);
 586                 i = k;
 587                 goto alloc;
 588         }
 589         bzero(&msg->lun, sizeof(msg->lun));
 590         msg->hdr.msg_type = CTL_MSG_LUN_SYNC;
 591         msg->hdr.nexus.targ_lun = lun->lun;
 592         msg->hdr.nexus.targ_mapped_lun = lun->lun;
 593         msg->lun.flags = lun->flags;
 594         msg->lun.pr_generation = lun->PRGeneration;
 595         msg->lun.pr_res_idx = lun->pr_res_idx;
 596         msg->lun.pr_res_type = lun->res_type;
 597         msg->lun.pr_key_count = lun->pr_key_count;
 598         i = 0;
 599         if (lun->lun_devid) {
 600                 msg->lun.lun_devid_len = lun->lun_devid->len;
 601                 memcpy(&msg->lun.data[i], lun->lun_devid->data,
 602                     msg->lun.lun_devid_len);
 603                 i += msg->lun.lun_devid_len;
 604         }
 605         for (k = 0; k < CTL_MAX_INITIATORS; k++) {
 606                 if ((pr_key.pr_key = ctl_get_prkey(lun, k)) == 0)
 607                         continue;
 608                 pr_key.pr_iid = k;
 609                 memcpy(&msg->lun.data[i], &pr_key, sizeof(pr_key));
 610                 i += sizeof(pr_key);
 611         }
 612         mtx_unlock(&lun->lun_lock);
 613         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->port, sizeof(msg->port) + i,
 614             M_WAITOK);
 615         free(msg, M_CTL);
 616 }
 617
 618 void
 619 ctl_isc_announce_port(struct ctl_port *port)
 620 {
 621         struct ctl_softc *softc = control_softc;
 622         union ctl_ha_msg *msg;
 623         int i;
 624
 625         if (port->targ_port < softc->port_min ||
 626             port->targ_port >= softc->port_max ||
 627             softc->ha_link != CTL_HA_LINK_ONLINE)
 628                 return;
 629         i = sizeof(msg->port) + strlen(port->port_name) + 1;
 630         if (port->lun_map)
 631                 i += sizeof(uint32_t) * CTL_MAX_LUNS;
 632         if (port->port_devid)
 633                 i += port->port_devid->len;
 634         if (port->target_devid)
 635                 i += port->target_devid->len;
 636         if (port->init_devid)
 637                 i += port->init_devid->len;
 638         msg = malloc(i, M_CTL, M_WAITOK);
 639         bzero(&msg->port, sizeof(msg->port));
 640         msg->hdr.msg_type = CTL_MSG_PORT_SYNC;
 641         msg->hdr.nexus.targ_port = port->targ_port;
 642         msg->port.port_type = port->port_type;
 643         msg->port.physical_port = port->physical_port;
 644         msg->port.virtual_port = port->virtual_port;
 645         msg->port.status = port->status;
 646         i = 0;
 647         msg->port.name_len = sprintf(&msg->port.data[i],
 648             "%d:%s", softc->ha_id, port->port_name) + 1;
 649         i += msg->port.name_len;
 650         if (port->lun_map) {
 651                 msg->port.lun_map_len = sizeof(uint32_t) * CTL_MAX_LUNS;
 652                 memcpy(&msg->port.data[i], port->lun_map,
 653                     msg->port.lun_map_len);
 654                 i += msg->port.lun_map_len;
 655         }
 656         if (port->port_devid) {
 657                 msg->port.port_devid_len = port->port_devid->len;
 658                 memcpy(&msg->port.data[i], port->port_devid->data,
 659                     msg->port.port_devid_len);
 660                 i += msg->port.port_devid_len;
 661         }
 662         if (port->target_devid) {
 663                 msg->port.target_devid_len = port->target_devid->len;
 664                 memcpy(&msg->port.data[i], port->target_devid->data,
 665                     msg->port.target_devid_len);
 666                 i += msg->port.target_devid_len;
 667         }
 668         if (port->init_devid) {
 669                 msg->port.init_devid_len = port->init_devid->len;
 670                 memcpy(&msg->port.data[i], port->init_devid->data,
 671                     msg->port.init_devid_len);
 672                 i += msg->port.init_devid_len;
 673         }
 674         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->port, sizeof(msg->port) + i,
 675             M_WAITOK);
 676         free(msg, M_CTL);
 677 }
 678
 679 void
 680 ctl_isc_announce_iid(struct ctl_port *port, int iid)
 681 {
 682         struct ctl_softc *softc = control_softc;
 683         union ctl_ha_msg *msg;
 684         int i, l;
 685
 686         if (port->targ_port < softc->port_min ||
 687             port->targ_port >= softc->port_max ||
 688             softc->ha_link != CTL_HA_LINK_ONLINE)
 689                 return;
 690         mtx_lock(&softc->ctl_lock);
 691         i = sizeof(msg->iid);
 692         l = 0;
 693         if (port->wwpn_iid[iid].name)
 694                 l = strlen(port->wwpn_iid[iid].name) + 1;
 695         i += l;
 696         msg = malloc(i, M_CTL, M_NOWAIT);
 697         if (msg == NULL) {
 698                 mtx_unlock(&softc->ctl_lock);
 699                 return;
 700         }
 701         bzero(&msg->iid, sizeof(msg->iid));
 702         msg->hdr.msg_type = CTL_MSG_IID_SYNC;
 703         msg->hdr.nexus.targ_port = port->targ_port;
 704         msg->hdr.nexus.initid = iid;
 705         msg->iid.in_use = port->wwpn_iid[iid].in_use;
 706         msg->iid.name_len = l;
 707         msg->iid.wwpn = port->wwpn_iid[iid].wwpn;
 708         if (port->wwpn_iid[iid].name)
 709                 strlcpy(msg->iid.data, port->wwpn_iid[iid].name, l);
 710         mtx_unlock(&softc->ctl_lock);
 711         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->iid, i, M_NOWAIT);
 712         free(msg, M_CTL);
 713 }
 714
 715 static void
 716 ctl_isc_ha_link_up(struct ctl_softc *softc)
 717 {
 718         struct ctl_port *port;
 719         struct ctl_lun *lun;
 720         union ctl_ha_msg msg;
 721         int i;
 722
 723         /* Announce this node parameters to peer for validation. */
 724         msg.login.msg_type = CTL_MSG_LOGIN;
 725         msg.login.version = CTL_HA_VERSION;
 726         msg.login.ha_mode = softc->ha_mode;
 727         msg.login.ha_id = softc->ha_id;
 728         msg.login.max_luns = CTL_MAX_LUNS;
 729         msg.login.max_ports = CTL_MAX_PORTS;
 730         msg.login.max_init_per_port = CTL_MAX_INIT_PER_PORT;
 731         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg.login, sizeof(msg.login),
 732             M_WAITOK);
 733
 734         STAILQ_FOREACH(port, &softc->port_list, links) {
 735                 ctl_isc_announce_port(port);
 736                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
 737                         if (port->wwpn_iid[i].in_use)
 738                                 ctl_isc_announce_iid(port, i);
 739                 }
 740         }
 741         STAILQ_FOREACH(lun, &softc->lun_list, links)
 742                 ctl_isc_announce_lun(lun);
 743 }
 744
 745 static void
 746 ctl_isc_ha_link_down(struct ctl_softc *softc)
 747 {
 748         struct ctl_port *port;
 749         struct ctl_lun *lun;
 750         union ctl_io *io;
 751         int i;
 752
 753         mtx_lock(&softc->ctl_lock);
 754         STAILQ_FOREACH(lun, &softc->lun_list, links) {
 755                 mtx_lock(&lun->lun_lock);
 756                 if (lun->flags & CTL_LUN_PEER_SC_PRIMARY) {
 757                         lun->flags &= ~CTL_LUN_PEER_SC_PRIMARY;
 758                         ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
 759                 }
 760                 mtx_unlock(&lun->lun_lock);
 761
 762                 mtx_unlock(&softc->ctl_lock);
 763                 io = ctl_alloc_io(softc->othersc_pool);
 764                 mtx_lock(&softc->ctl_lock);
 765                 ctl_zero_io(io);
 766                 io->io_hdr.msg_type = CTL_MSG_FAILOVER;
 767                 io->io_hdr.nexus.targ_mapped_lun = lun->lun;
 768                 ctl_enqueue_isc(io);
 769         }
 770
 771         STAILQ_FOREACH(port, &softc->port_list, links) {
 772                 if (port->targ_port >= softc->port_min &&
 773                     port->targ_port < softc->port_max)
 774                         continue;
 775                 port->status &= ~CTL_PORT_STATUS_ONLINE;
 776                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
 777                         port->wwpn_iid[i].in_use = 0;
 778                         free(port->wwpn_iid[i].name, M_CTL);
 779                         port->wwpn_iid[i].name = NULL;
 780                 }
 781         }
 782         mtx_unlock(&softc->ctl_lock);
 783 }
 784
 785 static void
 786 ctl_isc_ua(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 787 {
 788         struct ctl_lun *lun;
 789         uint32_t iid = ctl_get_initindex(&msg->hdr.nexus);
 790
 791         mtx_lock(&softc->ctl_lock);
 792         if (msg->hdr.nexus.targ_lun < CTL_MAX_LUNS &&
 793             (lun = softc->ctl_luns[msg->hdr.nexus.targ_mapped_lun]) != NULL) {
 794                 mtx_lock(&lun->lun_lock);
 795                 mtx_unlock(&softc->ctl_lock);
 796                 if (msg->ua.ua_type == CTL_UA_THIN_PROV_THRES &&
 797                     msg->ua.ua_set)
 798                         memcpy(lun->ua_tpt_info, msg->ua.ua_info, 8);
 799                 if (msg->ua.ua_all) {
 800                         if (msg->ua.ua_set)
 801                                 ctl_est_ua_all(lun, iid, msg->ua.ua_type);
 802                         else
 803                                 ctl_clr_ua_all(lun, iid, msg->ua.ua_type);
 804                 } else {
 805                         if (msg->ua.ua_set)
 806                                 ctl_est_ua(lun, iid, msg->ua.ua_type);
 807                         else
 808                                 ctl_clr_ua(lun, iid, msg->ua.ua_type);
 809                 }
 810                 mtx_unlock(&lun->lun_lock);
 811         } else
 812                 mtx_unlock(&softc->ctl_lock);
 813 }
 814
 815 static void
 816 ctl_isc_lun_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 817 {
 818         struct ctl_lun *lun;
 819         struct ctl_ha_msg_lun_pr_key pr_key;
 820         int i, k;
 821         ctl_lun_flags oflags;
 822         uint32_t targ_lun;
 823
 824         targ_lun = msg->hdr.nexus.targ_mapped_lun;
 825         mtx_lock(&softc->ctl_lock);
 826         if ((targ_lun >= CTL_MAX_LUNS) ||
 827             ((lun = softc->ctl_luns[targ_lun]) == NULL)) {
 828                 mtx_unlock(&softc->ctl_lock);
 829                 return;
 830         }
 831         mtx_lock(&lun->lun_lock);
 832         mtx_unlock(&softc->ctl_lock);
 833         if (lun->flags & CTL_LUN_DISABLED) {
 834                 mtx_unlock(&lun->lun_lock);
 835                 return;
 836         }
 837         i = (lun->lun_devid != NULL) ? lun->lun_devid->len : 0;
 838         if (msg->lun.lun_devid_len != i || (i > 0 &&
 839             memcmp(&msg->lun.data[0], lun->lun_devid->data, i) != 0)) {
 840                 mtx_unlock(&lun->lun_lock);
 841                 printf("%s: Received conflicting HA LUN %d\n",
 842                     __func__, msg->hdr.nexus.targ_lun);
 843                 return;
 844         } else {
 845                 /* Record whether peer is primary. */
 846                 oflags = lun->flags;
 847                 if ((msg->lun.flags & CTL_LUN_PRIMARY_SC) &&
 848                     (msg->lun.flags & CTL_LUN_DISABLED) == 0)
 849                         lun->flags |= CTL_LUN_PEER_SC_PRIMARY;
 850                 else
 851                         lun->flags &= ~CTL_LUN_PEER_SC_PRIMARY;
 852                 if (oflags != lun->flags)
 853                         ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
 854
 855                 /* If peer is primary and we are not -- use data */
 856                 if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
 857                     (lun->flags & CTL_LUN_PEER_SC_PRIMARY)) {
 858                         lun->PRGeneration = msg->lun.pr_generation;
 859                         lun->pr_res_idx = msg->lun.pr_res_idx;
 860                         lun->res_type = msg->lun.pr_res_type;
 861                         lun->pr_key_count = msg->lun.pr_key_count;
 862                         for (k = 0; k < CTL_MAX_INITIATORS; k++)
 863                                 ctl_clr_prkey(lun, k);
 864                         for (k = 0; k < msg->lun.pr_key_count; k++) {
 865                                 memcpy(&pr_key, &msg->lun.data[i],
 866                                     sizeof(pr_key));
 867                                 ctl_alloc_prkey(lun, pr_key.pr_iid);
 868                                 ctl_set_prkey(lun, pr_key.pr_iid,
 869                                     pr_key.pr_key);
 870                                 i += sizeof(pr_key);
 871                         }
 872                 }
 873
 874                 mtx_unlock(&lun->lun_lock);
 875                 CTL_DEBUG_PRINT(("%s: Known LUN %d, peer is %s\n",
 876                     __func__, msg->hdr.nexus.targ_lun,
 877                     (msg->lun.flags & CTL_LUN_PRIMARY_SC) ?
 878                     "primary" : "secondary"));
 879
 880                 /* If we are primary but peer doesn't know -- notify */
 881                 if ((lun->flags & CTL_LUN_PRIMARY_SC) &&
 882                     (msg->lun.flags & CTL_LUN_PEER_SC_PRIMARY) == 0)
 883                         ctl_isc_announce_lun(lun);
 884         }
 885 }
 886
 887 static void
 888 ctl_isc_port_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 889 {
 890         struct ctl_port *port;
 891         struct ctl_lun *lun;
 892         int i, new;
 893
 894         port = softc->ctl_ports[msg->hdr.nexus.targ_port];
 895         if (port == NULL) {
 896                 CTL_DEBUG_PRINT(("%s: New port %d\n", __func__,
 897                     msg->hdr.nexus.targ_port));
 898                 new = 1;
 899                 port = malloc(sizeof(*port), M_CTL, M_WAITOK | M_ZERO);
 900                 port->frontend = &ha_frontend;
 901                 port->targ_port = msg->hdr.nexus.targ_port;
 902         } else if (port->frontend == &ha_frontend) {
 903                 CTL_DEBUG_PRINT(("%s: Updated port %d\n", __func__,
 904                     msg->hdr.nexus.targ_port));
 905                 new = 0;
 906         } else {
 907                 printf("%s: Received conflicting HA port %d\n",
 908                     __func__, msg->hdr.nexus.targ_port);
 909                 return;
 910         }
 911         port->port_type = msg->port.port_type;
 912         port->physical_port = msg->port.physical_port;
 913         port->virtual_port = msg->port.virtual_port;
 914         port->status = msg->port.status;
 915         i = 0;
 916         free(port->port_name, M_CTL);
 917         port->port_name = strndup(&msg->port.data[i], msg->port.name_len,
 918             M_CTL);
 919         i += msg->port.name_len;
 920         if (msg->port.lun_map_len != 0) {
 921                 if (port->lun_map == NULL)
 922                         port->lun_map = malloc(sizeof(uint32_t) * CTL_MAX_LUNS,
 923                             M_CTL, M_WAITOK);
 924                 memcpy(port->lun_map, &msg->port.data[i],
 925                     sizeof(uint32_t) * CTL_MAX_LUNS);
 926                 i += msg->port.lun_map_len;
 927         } else {
 928                 free(port->lun_map, M_CTL);
 929                 port->lun_map = NULL;
 930         }
 931         if (msg->port.port_devid_len != 0) {
 932                 if (port->port_devid == NULL ||
 933                     port->port_devid->len != msg->port.port_devid_len) {
 934                         free(port->port_devid, M_CTL);
 935                         port->port_devid = malloc(sizeof(struct ctl_devid) +
 936                             msg->port.port_devid_len, M_CTL, M_WAITOK);
 937                 }
 938                 memcpy(port->port_devid->data, &msg->port.data[i],
 939                     msg->port.port_devid_len);
 940                 port->port_devid->len = msg->port.port_devid_len;
 941                 i += msg->port.port_devid_len;
 942         } else {
 943                 free(port->port_devid, M_CTL);
 944                 port->port_devid = NULL;
 945         }
 946         if (msg->port.target_devid_len != 0) {
 947                 if (port->target_devid == NULL ||
 948                     port->target_devid->len != msg->port.target_devid_len) {
 949                         free(port->target_devid, M_CTL);
 950                         port->target_devid = malloc(sizeof(struct ctl_devid) +
 951                             msg->port.target_devid_len, M_CTL, M_WAITOK);
 952                 }
 953                 memcpy(port->target_devid->data, &msg->port.data[i],
 954                     msg->port.target_devid_len);
 955                 port->target_devid->len = msg->port.target_devid_len;
 956                 i += msg->port.target_devid_len;
 957         } else {
 958                 free(port->target_devid, M_CTL);
 959                 port->target_devid = NULL;
 960         }
 961         if (msg->port.init_devid_len != 0) {
 962                 if (port->init_devid == NULL ||
 963                     port->init_devid->len != msg->port.init_devid_len) {
 964                         free(port->init_devid, M_CTL);
 965                         port->init_devid = malloc(sizeof(struct ctl_devid) +
 966                             msg->port.init_devid_len, M_CTL, M_WAITOK);
 967                 }
 968                 memcpy(port->init_devid->data, &msg->port.data[i],
 969                     msg->port.init_devid_len);
 970                 port->init_devid->len = msg->port.init_devid_len;
 971                 i += msg->port.init_devid_len;
 972         } else {
 973                 free(port->init_devid, M_CTL);
 974                 port->init_devid = NULL;
 975         }
 976         if (new) {
 977                 if (ctl_port_register(port) != 0) {
 978                         printf("%s: ctl_port_register() failed with error\n",
 979                             __func__);
 980                 }
 981         }
 982         mtx_lock(&softc->ctl_lock);
 983         STAILQ_FOREACH(lun, &softc->lun_list, links) {
 984                 if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
 985                         continue;
 986                 mtx_lock(&lun->lun_lock);
 987                 ctl_est_ua_all(lun, -1, CTL_UA_INQ_CHANGE);
 988                 mtx_unlock(&lun->lun_lock);
 989         }
 990         mtx_unlock(&softc->ctl_lock);
 991 }
 992
 993 static void
 994 ctl_isc_iid_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 995 {
 996         struct ctl_port *port;
 997         int iid;
 998
 999         port = softc->ctl_ports[msg->hdr.nexus.targ_port];
1000         if (port == NULL) {
1001                 printf("%s: Received IID for unknown port %d\n",
1002                     __func__, msg->hdr.nexus.targ_port);
1003                 return;
1004         }
1005         iid = msg->hdr.nexus.initid;
1006         port->wwpn_iid[iid].in_use = msg->iid.in_use;
1007         port->wwpn_iid[iid].wwpn = msg->iid.wwpn;
1008         free(port->wwpn_iid[iid].name, M_CTL);
1009         if (msg->iid.name_len) {
1010                 port->wwpn_iid[iid].name = strndup(&msg->iid.data[0],
1011                     msg->iid.name_len, M_CTL);
1012         } else
1013                 port->wwpn_iid[iid].name = NULL;
1014 }
1015
1016 static void
1017 ctl_isc_login(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
1018 {
1019
1020         if (msg->login.version != CTL_HA_VERSION) {
1021                 printf("CTL HA peers have different versions %d != %d\n",
1022                     msg->login.version, CTL_HA_VERSION);
1023                 ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1024                 return;
1025         }
1026         if (msg->login.ha_mode != softc->ha_mode) {
1027                 printf("CTL HA peers have different ha_mode %d != %d\n",
1028                     msg->login.ha_mode, softc->ha_mode);
1029                 ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1030                 return;
1031         }
1032         if (msg->login.ha_id == softc->ha_id) {
1033                 printf("CTL HA peers have same ha_id %d\n", msg->login.ha_id);
1034                 ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1035                 return;
1036         }
1037         if (msg->login.max_luns != CTL_MAX_LUNS ||
1038             msg->login.max_ports != CTL_MAX_PORTS ||
1039             msg->login.max_init_per_port != CTL_MAX_INIT_PER_PORT) {
1040                 printf("CTL HA peers have different limits\n");
1041                 ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1042                 return;
1043         }
1044 }
1045
1046 /*
1047  * ISC (Inter Shelf Communication) event handler.  Events from the HA
1048  * subsystem come in here.
1049  */
1050 static void
1051 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
1052 {
1053         struct ctl_softc *softc;
1054         union ctl_io *io;
1055         struct ctl_prio *presio;
1056         ctl_ha_status isc_status;
1057
1058         softc = control_softc;
1059         CTL_DEBUG_PRINT(("CTL: Isc Msg event %d\n", event));
1060         if (event == CTL_HA_EVT_MSG_RECV) {
1061                 union ctl_ha_msg *msg, msgbuf;
1062
1063                 if (param > sizeof(msgbuf))
1064                         msg = malloc(param, M_CTL, M_WAITOK);
1065                 else
1066                         msg = &msgbuf;
1067                 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, msg, param,
1068                     M_WAITOK);
1069                 if (isc_status != CTL_HA_STATUS_SUCCESS) {
1070                         printf("%s: Error receiving message: %d\n",
1071                             __func__, isc_status);
1072                         if (msg != &msgbuf)
1073                                 free(msg, M_CTL);
1074                         return;
1075                 }
1076
1077                 CTL_DEBUG_PRINT(("CTL: msg_type %d\n", msg->msg_type));
1078                 switch (msg->hdr.msg_type) {
1079                 case CTL_MSG_SERIALIZE:
1080                         io = ctl_alloc_io(softc->othersc_pool);
1081                         ctl_zero_io(io);
1082                         // populate ctsio from msg
1083                         io->io_hdr.io_type = CTL_IO_SCSI;
1084                         io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
1085                         io->io_hdr.original_sc = msg->hdr.original_sc;
1086                         io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
1087                                             CTL_FLAG_IO_ACTIVE;
1088                         /*
1089                          * If we're in serialization-only mode, we don't
1090                          * want to go through full done processing.  Thus
1091                          * the COPY flag.
1092                          *
1093                          * XXX KDM add another flag that is more specific.
1094                          */
1095                         if (softc->ha_mode != CTL_HA_MODE_XFER)
1096                                 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
1097                         io->io_hdr.nexus = msg->hdr.nexus;
1098 #if 0
1099                         printf("port %u, iid %u, lun %u\n",
1100                                io->io_hdr.nexus.targ_port,
1101                                io->io_hdr.nexus.initid,
1102                                io->io_hdr.nexus.targ_lun);
1103 #endif
1104                         io->scsiio.tag_num = msg->scsi.tag_num;
1105                         io->scsiio.tag_type = msg->scsi.tag_type;
1106 #ifdef CTL_TIME_IO
1107                         io->io_hdr.start_time = time_uptime;
1108                         getbintime(&io->io_hdr.start_bt);
1109 #endif /* CTL_TIME_IO */
1110                         io->scsiio.cdb_len = msg->scsi.cdb_len;
1111                         memcpy(io->scsiio.cdb, msg->scsi.cdb,
1112                                CTL_MAX_CDBLEN);
1113                         if (softc->ha_mode == CTL_HA_MODE_XFER) {
1114                                 const struct ctl_cmd_entry *entry;
1115
1116                                 entry = ctl_get_cmd_entry(&io->scsiio, NULL);
1117                                 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
1118                                 io->io_hdr.flags |=
1119                                         entry->flags & CTL_FLAG_DATA_MASK;
1120                         }
1121                         ctl_enqueue_isc(io);
1122                         break;
1123
1124                 /* Performed on the Originating SC, XFER mode only */
1125                 case CTL_MSG_DATAMOVE: {
1126                         struct ctl_sg_entry *sgl;
1127                         int i, j;
1128
1129                         io = msg->hdr.original_sc;
1130                         if (io == NULL) {
1131                                 printf("%s: original_sc == NULL!\n", __func__);
1132                                 /* XXX KDM do something here */
1133                                 break;
1134                         }
1135                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
1136                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1137                         /*
1138                          * Keep track of this, we need to send it back over
1139                          * when the datamove is complete.
1140                          */
1141                         io->io_hdr.serializing_sc = msg->hdr.serializing_sc;
1142                         if (msg->hdr.status == CTL_SUCCESS)
1143                                 io->io_hdr.status = msg->hdr.status;
1144
1145                         if (msg->dt.sg_sequence == 0) {
1146                                 i = msg->dt.kern_sg_entries +
1147                                     msg->dt.kern_data_len /
1148                                     CTL_HA_DATAMOVE_SEGMENT + 1;
1149                                 sgl = malloc(sizeof(*sgl) * i, M_CTL,
1150                                     M_WAITOK | M_ZERO);
1151                                 io->io_hdr.remote_sglist = sgl;
1152                                 io->io_hdr.local_sglist =
1153                                     &sgl[msg->dt.kern_sg_entries];
1154
1155                                 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
1156
1157                                 io->scsiio.kern_sg_entries =
1158                                         msg->dt.kern_sg_entries;
1159                                 io->scsiio.rem_sg_entries =
1160                                         msg->dt.kern_sg_entries;
1161                                 io->scsiio.kern_data_len =
1162                                         msg->dt.kern_data_len;
1163                                 io->scsiio.kern_total_len =
1164                                         msg->dt.kern_total_len;
1165                                 io->scsiio.kern_data_resid =
1166                                         msg->dt.kern_data_resid;
1167                                 io->scsiio.kern_rel_offset =
1168                                         msg->dt.kern_rel_offset;
1169                                 io->io_hdr.flags &= ~CTL_FLAG_BUS_ADDR;
1170                                 io->io_hdr.flags |= msg->dt.flags &
1171                                     CTL_FLAG_BUS_ADDR;
1172                         } else
1173                                 sgl = (struct ctl_sg_entry *)
1174                                         io->scsiio.kern_data_ptr;
1175
1176                         for (i = msg->dt.sent_sg_entries, j = 0;
1177                              i < (msg->dt.sent_sg_entries +
1178                              msg->dt.cur_sg_entries); i++, j++) {
1179                                 sgl[i].addr = msg->dt.sg_list[j].addr;
1180                                 sgl[i].len = msg->dt.sg_list[j].len;
1181
1182 #if 0
1183                                 printf("%s: DATAMOVE: %p,%lu j=%d, i=%d\n",
1184                                     __func__, sgl[i].addr, sgl[i].len, j, i);
1185 #endif
1186                         }
1187
1188                         /*
1189                          * If this is the last piece of the I/O, we've got
1190                          * the full S/G list.  Queue processing in the thread.
1191                          * Otherwise wait for the next piece.
1192                          */
1193                         if (msg->dt.sg_last != 0)
1194                                 ctl_enqueue_isc(io);
1195                         break;
1196                 }
1197                 /* Performed on the Serializing (primary) SC, XFER mode only */
1198                 case CTL_MSG_DATAMOVE_DONE: {
1199                         if (msg->hdr.serializing_sc == NULL) {
1200                                 printf("%s: serializing_sc == NULL!\n",
1201                                        __func__);
1202                                 /* XXX KDM now what? */
1203                                 break;
1204                         }
1205                         /*
1206                          * We grab the sense information here in case
1207                          * there was a failure, so we can return status
1208                          * back to the initiator.
1209                          */
1210                         io = msg->hdr.serializing_sc;
1211                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
1212                         io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1213                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1214                         io->io_hdr.port_status = msg->scsi.fetd_status;
1215                         io->scsiio.residual = msg->scsi.residual;
1216                         if (msg->hdr.status != CTL_STATUS_NONE) {
1217                                 io->io_hdr.status = msg->hdr.status;
1218                                 io->scsiio.scsi_status = msg->scsi.scsi_status;
1219                                 io->scsiio.sense_len = msg->scsi.sense_len;
1220                                 io->scsiio.sense_residual =msg->scsi.sense_residual;
1221                                 memcpy(&io->scsiio.sense_data,
1222                                     &msg->scsi.sense_data,
1223                                     msg->scsi.sense_len);
1224                                 if (msg->hdr.status == CTL_SUCCESS)
1225                                         io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
1226                         }
1227                         ctl_enqueue_isc(io);
1228                         break;
1229                 }
1230
1231                 /* Preformed on Originating SC, SER_ONLY mode */
1232                 case CTL_MSG_R2R:
1233                         io = msg->hdr.original_sc;
1234                         if (io == NULL) {
1235                                 printf("%s: original_sc == NULL!\n",
1236                                     __func__);
1237                                 break;
1238                         }
1239                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1240                         io->io_hdr.msg_type = CTL_MSG_R2R;
1241                         io->io_hdr.serializing_sc = msg->hdr.serializing_sc;
1242                         ctl_enqueue_isc(io);
1243                         break;
1244
1245                 /*
1246                  * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
1247                  * mode.
1248                  * Performed on the Originating (i.e. secondary) SC in XFER
1249                  * mode
1250                  */
1251                 case CTL_MSG_FINISH_IO:
1252                         if (softc->ha_mode == CTL_HA_MODE_XFER)
1253                                 ctl_isc_handler_finish_xfer(softc, msg);
1254                         else
1255                                 ctl_isc_handler_finish_ser_only(softc, msg);
1256                         break;
1257
1258                 /* Preformed on Originating SC */
1259                 case CTL_MSG_BAD_JUJU:
1260                         io = msg->hdr.original_sc;
1261                         if (io == NULL) {
1262                                 printf("%s: Bad JUJU!, original_sc is NULL!\n",
1263                                        __func__);
1264                                 break;
1265                         }
1266                         ctl_copy_sense_data(msg, io);
1267                         /*
1268                          * IO should have already been cleaned up on other
1269                          * SC so clear this flag so we won't send a message
1270                          * back to finish the IO there.
1271                          */
1272                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
1273                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1274
1275                         /* io = msg->hdr.serializing_sc; */
1276                         io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
1277                         ctl_enqueue_isc(io);
1278                         break;
1279
1280                 /* Handle resets sent from the other side */
1281                 case CTL_MSG_MANAGE_TASKS: {
1282                         struct ctl_taskio *taskio;
1283                         taskio = (struct ctl_taskio *)ctl_alloc_io(
1284                             softc->othersc_pool);
1285                         ctl_zero_io((union ctl_io *)taskio);
1286                         taskio->io_hdr.io_type = CTL_IO_TASK;
1287                         taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
1288                         taskio->io_hdr.nexus = msg->hdr.nexus;
1289                         taskio->task_action = msg->task.task_action;
1290                         taskio->tag_num = msg->task.tag_num;
1291                         taskio->tag_type = msg->task.tag_type;
1292 #ifdef CTL_TIME_IO
1293                         taskio->io_hdr.start_time = time_uptime;
1294                         getbintime(&taskio->io_hdr.start_bt);
1295 #endif /* CTL_TIME_IO */
1296                         ctl_run_task((union ctl_io *)taskio);
1297                         break;
1298                 }
1299                 /* Persistent Reserve action which needs attention */
1300                 case CTL_MSG_PERS_ACTION:
1301                         presio = (struct ctl_prio *)ctl_alloc_io(
1302                             softc->othersc_pool);
1303                         ctl_zero_io((union ctl_io *)presio);
1304                         presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
1305                         presio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
1306                         presio->io_hdr.nexus = msg->hdr.nexus;
1307                         presio->pr_msg = msg->pr;
1308                         ctl_enqueue_isc((union ctl_io *)presio);
1309                         break;
1310                 case CTL_MSG_UA:
1311                         ctl_isc_ua(softc, msg, param);
1312                         break;
1313                 case CTL_MSG_PORT_SYNC:
1314                         ctl_isc_port_sync(softc, msg, param);
1315                         break;
1316                 case CTL_MSG_LUN_SYNC:
1317                         ctl_isc_lun_sync(softc, msg, param);
1318                         break;
1319                 case CTL_MSG_IID_SYNC:
1320                         ctl_isc_iid_sync(softc, msg, param);
1321                         break;
1322                 case CTL_MSG_LOGIN:
1323                         ctl_isc_login(softc, msg, param);
1324                         break;
1325                 default:
1326                         printf("Received HA message of unknown type %d\n",
1327                             msg->hdr.msg_type);
1328                         ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1329                         break;
1330                 }
1331                 if (msg != &msgbuf)
1332                         free(msg, M_CTL);
1333         } else if (event == CTL_HA_EVT_LINK_CHANGE) {
1334                 printf("CTL: HA link status changed from %d to %d\n",
1335                     softc->ha_link, param);
1336                 if (param == softc->ha_link)
1337                         return;
1338                 if (softc->ha_link == CTL_HA_LINK_ONLINE) {
1339                         softc->ha_link = param;
1340                         ctl_isc_ha_link_down(softc);
1341                 } else {
1342                         softc->ha_link = param;
1343                         if (softc->ha_link == CTL_HA_LINK_ONLINE)
1344                                 ctl_isc_ha_link_up(softc);
1345                 }
1346                 return;
1347         } else {
1348                 printf("ctl_isc_event_handler: Unknown event %d\n", event);
1349                 return;
1350         }
1351 }
1352
1353 static void
1354 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
1355 {
1356
1357         memcpy(&dest->scsiio.sense_data, &src->scsi.sense_data,
1358             src->scsi.sense_len);
1359         dest->scsiio.scsi_status = src->scsi.scsi_status;
1360         dest->scsiio.sense_len = src->scsi.sense_len;
1361         dest->io_hdr.status = src->hdr.status;
1362 }
1363
1364 static void
1365 ctl_copy_sense_data_back(union ctl_io *src, union ctl_ha_msg *dest)
1366 {
1367
1368         memcpy(&dest->scsi.sense_data, &src->scsiio.sense_data,
1369             src->scsiio.sense_len);
1370         dest->scsi.scsi_status = src->scsiio.scsi_status;
1371         dest->scsi.sense_len = src->scsiio.sense_len;
1372         dest->hdr.status = src->io_hdr.status;
1373 }
1374
1375 void
1376 ctl_est_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
1377 {
1378         struct ctl_softc *softc = lun->ctl_softc;
1379         ctl_ua_type *pu;
1380
1381         if (initidx < softc->init_min || initidx >= softc->init_max)
1382                 return;
1383         mtx_assert(&lun->lun_lock, MA_OWNED);
1384         pu = lun->pending_ua[initidx / CTL_MAX_INIT_PER_PORT];
1385         if (pu == NULL)
1386                 return;
1387         pu[initidx % CTL_MAX_INIT_PER_PORT] |= ua;
1388 }
1389
1390 void
1391 ctl_est_ua_port(struct ctl_lun *lun, int port, uint32_t except, ctl_ua_type ua)
1392 {
1393         int i;
1394
1395         mtx_assert(&lun->lun_lock, MA_OWNED);
1396         if (lun->pending_ua[port] == NULL)
1397                 return;
1398         for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1399                 if (port * CTL_MAX_INIT_PER_PORT + i == except)
1400                         continue;
1401                 lun->pending_ua[port][i] |= ua;
1402         }
1403 }
1404
1405 void
1406 ctl_est_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
1407 {
1408         struct ctl_softc *softc = lun->ctl_softc;
1409         int i;
1410
1411         mtx_assert(&lun->lun_lock, MA_OWNED);
1412         for (i = softc->port_min; i < softc->port_max; i++)
1413                 ctl_est_ua_port(lun, i, except, ua);
1414 }
1415
1416 void
1417 ctl_clr_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
1418 {
1419         struct ctl_softc *softc = lun->ctl_softc;
1420         ctl_ua_type *pu;
1421
1422         if (initidx < softc->init_min || initidx >= softc->init_max)
1423                 return;
1424         mtx_assert(&lun->lun_lock, MA_OWNED);
1425         pu = lun->pending_ua[initidx / CTL_MAX_INIT_PER_PORT];
1426         if (pu == NULL)
1427                 return;
1428         pu[initidx % CTL_MAX_INIT_PER_PORT] &= ~ua;
1429 }
1430
1431 void
1432 ctl_clr_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
1433 {
1434         struct ctl_softc *softc = lun->ctl_softc;
1435         int i, j;
1436
1437         mtx_assert(&lun->lun_lock, MA_OWNED);
1438         for (i = softc->port_min; i < softc->port_max; i++) {
1439                 if (lun->pending_ua[i] == NULL)
1440                         continue;
1441                 for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
1442                         if (i * CTL_MAX_INIT_PER_PORT + j == except)
1443                                 continue;
1444                         lun->pending_ua[i][j] &= ~ua;
1445                 }
1446         }
1447 }
1448
1449 void
1450 ctl_clr_ua_allluns(struct ctl_softc *ctl_softc, uint32_t initidx,
1451     ctl_ua_type ua_type)
1452 {
1453         struct ctl_lun *lun;
1454
1455         mtx_assert(&ctl_softc->ctl_lock, MA_OWNED);
1456         STAILQ_FOREACH(lun, &ctl_softc->lun_list, links) {
1457                 mtx_lock(&lun->lun_lock);
1458                 ctl_clr_ua(lun, initidx, ua_type);
1459                 mtx_unlock(&lun->lun_lock);
1460         }
1461 }
1462
1463 static int
1464 ctl_ha_role_sysctl(SYSCTL_HANDLER_ARGS)
1465 {
1466         struct ctl_softc *softc = (struct ctl_softc *)arg1;
1467         struct ctl_lun *lun;
1468         struct ctl_lun_req ireq;
1469         int error, value;
1470
1471         value = (softc->flags & CTL_FLAG_ACTIVE_SHELF) ? 0 : 1;
1472         error = sysctl_handle_int(oidp, &value, 0, req);
1473         if ((error != 0) || (req->newptr == NULL))
1474                 return (error);
1475
1476         mtx_lock(&softc->ctl_lock);
1477         if (value == 0)
1478                 softc->flags |= CTL_FLAG_ACTIVE_SHELF;
1479         else
1480                 softc->flags &= ~CTL_FLAG_ACTIVE_SHELF;
1481         STAILQ_FOREACH(lun, &softc->lun_list, links) {
1482                 mtx_unlock(&softc->ctl_lock);
1483                 bzero(&ireq, sizeof(ireq));
1484                 ireq.reqtype = CTL_LUNREQ_MODIFY;
1485                 ireq.reqdata.modify.lun_id = lun->lun;
1486                 lun->backend->ioctl(NULL, CTL_LUN_REQ, (caddr_t)&ireq, 0,
1487                     curthread);
1488                 if (ireq.status != CTL_LUN_OK) {
1489                         printf("%s: CTL_LUNREQ_MODIFY returned %d '%s'\n",
1490                             __func__, ireq.status, ireq.error_str);
1491                 }
1492                 mtx_lock(&softc->ctl_lock);
1493         }
1494         mtx_unlock(&softc->ctl_lock);
1495         return (0);
1496 }
1497
1498 static int
1499 ctl_init(void)
1500 {
1501         struct ctl_softc *softc;
1502         void *other_pool;
1503         int i, error, retval;
1504
1505         retval = 0;
1506         control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
1507                                M_WAITOK | M_ZERO);
1508         softc = control_softc;
1509
1510         softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
1511                               "cam/ctl");
1512
1513         softc->dev->si_drv1 = softc;
1514
1515         sysctl_ctx_init(&softc->sysctl_ctx);
1516         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
1517                 SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
1518                 CTLFLAG_RD, 0, "CAM Target Layer");
1519
1520         if (softc->sysctl_tree == NULL) {
1521                 printf("%s: unable to allocate sysctl tree\n", __func__);
1522                 destroy_dev(softc->dev);
1523                 free(control_softc, M_DEVBUF);
1524                 control_softc = NULL;
1525                 return (ENOMEM);
1526         }
1527
1528         mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
1529         softc->io_zone = uma_zcreate("CTL IO", sizeof(union ctl_io),
1530             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1531         softc->open_count = 0;
1532
1533         /*
1534          * Default to actually sending a SYNCHRONIZE CACHE command down to
1535          * the drive.
1536          */
1537         softc->flags = CTL_FLAG_REAL_SYNC;
1538
1539         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1540             OID_AUTO, "ha_mode", CTLFLAG_RDTUN, (int *)&softc->ha_mode, 0,
1541             "HA mode (0 - act/stby, 1 - serialize only, 2 - xfer)");
1542
1543         /*
1544          * In Copan's HA scheme, the "master" and "slave" roles are
1545          * figured out through the slot the controller is in.  Although it
1546          * is an active/active system, someone has to be in charge.
1547          */
1548         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1549             OID_AUTO, "ha_id", CTLFLAG_RDTUN, &softc->ha_id, 0,
1550             "HA head ID (0 - no HA)");
1551         if (softc->ha_id == 0 || softc->ha_id > NUM_TARGET_PORT_GROUPS) {
1552                 softc->flags |= CTL_FLAG_ACTIVE_SHELF;
1553                 softc->is_single = 1;
1554                 softc->port_cnt = CTL_MAX_PORTS;
1555                 softc->port_min = 0;
1556         } else {
1557                 softc->port_cnt = CTL_MAX_PORTS / NUM_TARGET_PORT_GROUPS;
1558                 softc->port_min = (softc->ha_id - 1) * softc->port_cnt;
1559         }
1560         softc->port_max = softc->port_min + softc->port_cnt;
1561         softc->init_min = softc->port_min * CTL_MAX_INIT_PER_PORT;
1562         softc->init_max = softc->port_max * CTL_MAX_INIT_PER_PORT;
1563
1564         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1565             OID_AUTO, "ha_link", CTLFLAG_RD, (int *)&softc->ha_link, 0,
1566             "HA link state (0 - offline, 1 - unknown, 2 - online)");
1567
1568         STAILQ_INIT(&softc->lun_list);
1569         STAILQ_INIT(&softc->pending_lun_queue);
1570         STAILQ_INIT(&softc->fe_list);
1571         STAILQ_INIT(&softc->port_list);
1572         STAILQ_INIT(&softc->be_list);
1573         ctl_tpc_init(softc);
1574
1575         if (ctl_pool_create(softc, "othersc", CTL_POOL_ENTRIES_OTHER_SC,
1576                             &other_pool) != 0)
1577         {
1578                 printf("ctl: can't allocate %d entry other SC pool, "
1579                        "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1580                 return (ENOMEM);
1581         }
1582         softc->othersc_pool = other_pool;
1583
1584         if (worker_threads <= 0)
1585                 worker_threads = max(1, mp_ncpus / 4);
1586         if (worker_threads > CTL_MAX_THREADS)
1587                 worker_threads = CTL_MAX_THREADS;
1588
1589         for (i = 0; i < worker_threads; i++) {
1590                 struct ctl_thread *thr = &softc->threads[i];
1591
1592                 mtx_init(&thr->queue_lock, "CTL queue mutex", NULL, MTX_DEF);
1593                 thr->ctl_softc = softc;
1594                 STAILQ_INIT(&thr->incoming_queue);
1595                 STAILQ_INIT(&thr->rtr_queue);
1596                 STAILQ_INIT(&thr->done_queue);
1597                 STAILQ_INIT(&thr->isc_queue);
1598
1599                 error = kproc_kthread_add(ctl_work_thread, thr,
1600                     &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i);
1601                 if (error != 0) {
1602                         printf("error creating CTL work thread!\n");
1603                         ctl_pool_free(other_pool);
1604                         return (error);
1605                 }
1606         }
1607         error = kproc_kthread_add(ctl_lun_thread, softc,
1608             &softc->ctl_proc, NULL, 0, 0, "ctl", "lun");
1609         if (error != 0) {
1610                 printf("error creating CTL lun thread!\n");
1611                 ctl_pool_free(other_pool);
1612                 return (error);
1613         }
1614         error = kproc_kthread_add(ctl_thresh_thread, softc,
1615             &softc->ctl_proc, NULL, 0, 0, "ctl", "thresh");
1616         if (error != 0) {
1617                 printf("error creating CTL threshold thread!\n");
1618                 ctl_pool_free(other_pool);
1619                 return (error);
1620         }
1621
1622         SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
1623             OID_AUTO, "ha_role", CTLTYPE_INT | CTLFLAG_RWTUN,
1624             softc, 0, ctl_ha_role_sysctl, "I", "HA role for this head");
1625
1626         if (softc->is_single == 0) {
1627                 ctl_frontend_register(&ha_frontend);
1628                 if (ctl_ha_msg_init(softc) != CTL_HA_STATUS_SUCCESS) {
1629                         printf("ctl_init: ctl_ha_msg_init failed.\n");
1630                         softc->is_single = 1;
1631                 } else
1632                 if (ctl_ha_msg_register(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
1633                     != CTL_HA_STATUS_SUCCESS) {
1634                         printf("ctl_init: ctl_ha_msg_register failed.\n");
1635                         softc->is_single = 1;
1636                 }
1637         }
1638         return (0);
1639 }
1640
1641 void
1642 ctl_shutdown(void)
1643 {
1644         struct ctl_softc *softc;
1645         struct ctl_lun *lun, *next_lun;
1646
1647         softc = (struct ctl_softc *)control_softc;
1648
1649         if (softc->is_single == 0) {
1650                 ctl_ha_msg_shutdown(softc);
1651                 if (ctl_ha_msg_deregister(CTL_HA_CHAN_CTL)
1652                     != CTL_HA_STATUS_SUCCESS)
1653                         printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
1654                 if (ctl_ha_msg_destroy(softc) != CTL_HA_STATUS_SUCCESS)
1655                         printf("%s: ctl_ha_msg_destroy failed.\n", __func__);
1656                 ctl_frontend_deregister(&ha_frontend);
1657         }
1658
1659         mtx_lock(&softc->ctl_lock);
1660
1661         /*
1662          * Free up each LUN.
1663          */
1664         for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1665                 next_lun = STAILQ_NEXT(lun, links);
1666                 ctl_free_lun(lun);
1667         }
1668
1669         mtx_unlock(&softc->ctl_lock);
1670
1671 #if 0
1672         ctl_shutdown_thread(softc->work_thread);
1673         mtx_destroy(&softc->queue_lock);
1674 #endif
1675
1676         ctl_tpc_shutdown(softc);
1677         uma_zdestroy(softc->io_zone);
1678         mtx_destroy(&softc->ctl_lock);
1679
1680         destroy_dev(softc->dev);
1681
1682         sysctl_ctx_free(&softc->sysctl_ctx);
1683
1684         free(control_softc, M_DEVBUF);
1685         control_softc = NULL;
1686 }
1687
1688 static int
1689 ctl_module_event_handler(module_t mod, int what, void *arg)
1690 {
1691
1692         switch (what) {
1693         case MOD_LOAD:
1694                 return (ctl_init());
1695         case MOD_UNLOAD:
1696                 return (EBUSY);
1697         default:
1698                 return (EOPNOTSUPP);
1699         }
1700 }
1701
1702 /*
1703  * XXX KDM should we do some access checks here?  Bump a reference count to
1704  * prevent a CTL module from being unloaded while someone has it open?
1705  */
1706 static int
1707 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1708 {
1709         return (0);
1710 }
1711
1712 static int
1713 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1714 {
1715         return (0);
1716 }
1717
1718 /*
1719  * Remove an initiator by port number and initiator ID.
1720  * Returns 0 for success, -1 for failure.
1721  */
1722 int
1723 ctl_remove_initiator(struct ctl_port *port, int iid)
1724 {
1725         struct ctl_softc *softc = control_softc;
1726
1727         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1728
1729         if (iid > CTL_MAX_INIT_PER_PORT) {
1730                 printf("%s: initiator ID %u > maximun %u!\n",
1731                        __func__, iid, CTL_MAX_INIT_PER_PORT);
1732                 return (-1);
1733         }
1734
1735         mtx_lock(&softc->ctl_lock);
1736         port->wwpn_iid[iid].in_use--;
1737         port->wwpn_iid[iid].last_use = time_uptime;
1738         mtx_unlock(&softc->ctl_lock);
1739         ctl_isc_announce_iid(port, iid);
1740
1741         return (0);
1742 }
1743
1744 /*
1745  * Add an initiator to the initiator map.
1746  * Returns iid for success, < 0 for failure.
1747  */
1748 int
1749 ctl_add_initiator(struct ctl_port *port, int iid, uint64_t wwpn, char *name)
1750 {
1751         struct ctl_softc *softc = control_softc;
1752         time_t best_time;
1753         int i, best;
1754
1755         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1756
1757         if (iid >= CTL_MAX_INIT_PER_PORT) {
1758                 printf("%s: WWPN %#jx initiator ID %u > maximum %u!\n",
1759                        __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1760                 free(name, M_CTL);
1761                 return (-1);
1762         }
1763
1764         mtx_lock(&softc->ctl_lock);
1765
1766         if (iid < 0 && (wwpn != 0 || name != NULL)) {
1767                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1768                         if (wwpn != 0 && wwpn == port->wwpn_iid[i].wwpn) {
1769                                 iid = i;
1770                                 break;
1771                         }
1772                         if (name != NULL && port->wwpn_iid[i].name != NULL &&
1773                             strcmp(name, port->wwpn_iid[i].name) == 0) {
1774                                 iid = i;
1775                                 break;
1776                         }
1777                 }
1778         }
1779
1780         if (iid < 0) {
1781                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1782                         if (port->wwpn_iid[i].in_use == 0 &&
1783                             port->wwpn_iid[i].wwpn == 0 &&
1784                             port->wwpn_iid[i].name == NULL) {
1785                                 iid = i;
1786                                 break;
1787                         }
1788                 }
1789         }
1790
1791         if (iid < 0) {
1792                 best = -1;
1793                 best_time = INT32_MAX;
1794                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1795                         if (port->wwpn_iid[i].in_use == 0) {
1796                                 if (port->wwpn_iid[i].last_use < best_time) {
1797                                         best = i;
1798                                         best_time = port->wwpn_iid[i].last_use;
1799                                 }
1800                         }
1801                 }
1802                 iid = best;
1803         }
1804
1805         if (iid < 0) {
1806                 mtx_unlock(&softc->ctl_lock);
1807                 free(name, M_CTL);
1808                 return (-2);
1809         }
1810
1811         if (port->wwpn_iid[iid].in_use > 0 && (wwpn != 0 || name != NULL)) {
1812                 /*
1813                  * This is not an error yet.
1814                  */
1815                 if (wwpn != 0 && wwpn == port->wwpn_iid[iid].wwpn) {
1816 #if 0
1817                         printf("%s: port %d iid %u WWPN %#jx arrived"
1818                             " again\n", __func__, port->targ_port,
1819                             iid, (uintmax_t)wwpn);
1820 #endif
1821                         goto take;
1822                 }
1823                 if (name != NULL && port->wwpn_iid[iid].name != NULL &&
1824                     strcmp(name, port->wwpn_iid[iid].name) == 0) {
1825 #if 0
1826                         printf("%s: port %d iid %u name '%s' arrived"
1827                             " again\n", __func__, port->targ_port,
1828                             iid, name);
1829 #endif
1830                         goto take;
1831                 }
1832
1833                 /*
1834                  * This is an error, but what do we do about it?  The
1835                  * driver is telling us we have a new WWPN for this
1836                  * initiator ID, so we pretty much need to use it.
1837                  */
1838                 printf("%s: port %d iid %u WWPN %#jx '%s' arrived,"
1839                     " but WWPN %#jx '%s' is still at that address\n",
1840                     __func__, port->targ_port, iid, wwpn, name,
1841                     (uintmax_t)port->wwpn_iid[iid].wwpn,
1842                     port->wwpn_iid[iid].name);
1843
1844                 /*
1845                  * XXX KDM clear have_ca and ua_pending on each LUN for
1846                  * this initiator.
1847                  */
1848         }
1849 take:
1850         free(port->wwpn_iid[iid].name, M_CTL);
1851         port->wwpn_iid[iid].name = name;
1852         port->wwpn_iid[iid].wwpn = wwpn;
1853         port->wwpn_iid[iid].in_use++;
1854         mtx_unlock(&softc->ctl_lock);
1855         ctl_isc_announce_iid(port, iid);
1856
1857         return (iid);
1858 }
1859
1860 static int
1861 ctl_create_iid(struct ctl_port *port, int iid, uint8_t *buf)
1862 {
1863         int len;
1864
1865         switch (port->port_type) {
1866         case CTL_PORT_FC:
1867         {
1868                 struct scsi_transportid_fcp *id =
1869                     (struct scsi_transportid_fcp *)buf;
1870                 if (port->wwpn_iid[iid].wwpn == 0)
1871                         return (0);
1872                 memset(id, 0, sizeof(*id));
1873                 id->format_protocol = SCSI_PROTO_FC;
1874                 scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->n_port_name);
1875                 return (sizeof(*id));
1876         }
1877         case CTL_PORT_ISCSI:
1878         {
1879                 struct scsi_transportid_iscsi_port *id =
1880                     (struct scsi_transportid_iscsi_port *)buf;
1881                 if (port->wwpn_iid[iid].name == NULL)
1882                         return (0);
1883                 memset(id, 0, 256);
1884                 id->format_protocol = SCSI_TRN_ISCSI_FORMAT_PORT |
1885                     SCSI_PROTO_ISCSI;
1886                 len = strlcpy(id->iscsi_name, port->wwpn_iid[iid].name, 252) + 1;
1887                 len = roundup2(min(len, 252), 4);
1888                 scsi_ulto2b(len, id->additional_length);
1889                 return (sizeof(*id) + len);
1890         }
1891         case CTL_PORT_SAS:
1892         {
1893                 struct scsi_transportid_sas *id =
1894                     (struct scsi_transportid_sas *)buf;
1895                 if (port->wwpn_iid[iid].wwpn == 0)
1896                         return (0);
1897                 memset(id, 0, sizeof(*id));
1898                 id->format_protocol = SCSI_PROTO_SAS;
1899                 scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->sas_address);
1900                 return (sizeof(*id));
1901         }
1902         default:
1903         {
1904                 struct scsi_transportid_spi *id =
1905                     (struct scsi_transportid_spi *)buf;
1906                 memset(id, 0, sizeof(*id));
1907                 id->format_protocol = SCSI_PROTO_SPI;
1908                 scsi_ulto2b(iid, id->scsi_addr);
1909                 scsi_ulto2b(port->targ_port, id->rel_trgt_port_id);
1910                 return (sizeof(*id));
1911         }
1912         }
1913 }
1914
1915 /*
1916  * Serialize a command that went down the "wrong" side, and so was sent to
1917  * this controller for execution.  The logic is a little different than the
1918  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1919  * sent back to the other side, but in the success case, we execute the
1920  * command on this side (XFER mode) or tell the other side to execute it
1921  * (SER_ONLY mode).
1922  */
1923 static int
1924 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
1925 {
1926         struct ctl_softc *softc;
1927         union ctl_ha_msg msg_info;
1928         struct ctl_lun *lun;
1929         const struct ctl_cmd_entry *entry;
1930         int retval = 0;
1931         uint32_t targ_lun;
1932
1933         softc = control_softc;
1934
1935         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
1936         mtx_lock(&softc->ctl_lock);
1937         if ((targ_lun < CTL_MAX_LUNS) &&
1938             ((lun = softc->ctl_luns[targ_lun]) != NULL)) {
1939                 mtx_lock(&lun->lun_lock);
1940                 mtx_unlock(&softc->ctl_lock);
1941                 /*
1942                  * If the LUN is invalid, pretend that it doesn't exist.
1943                  * It will go away as soon as all pending I/O has been
1944                  * completed.
1945                  */
1946                 if (lun->flags & CTL_LUN_DISABLED) {
1947                         mtx_unlock(&lun->lun_lock);
1948                         lun = NULL;
1949                 }
1950         } else {
1951                 mtx_unlock(&softc->ctl_lock);
1952                 lun = NULL;
1953         }
1954         if (lun == NULL) {
1955                 /*
1956                  * The other node would not send this request to us unless
1957                  * received announce that we are primary node for this LUN.
1958                  * If this LUN does not exist now, it is probably result of
1959                  * a race, so respond to initiator in the most opaque way.
1960                  */
1961                 ctl_set_busy(ctsio);
1962                 ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
1963                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1964                 msg_info.hdr.serializing_sc = NULL;
1965                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1966                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1967                     sizeof(msg_info.scsi), M_WAITOK);
1968                 return(1);
1969         }
1970
1971         entry = ctl_get_cmd_entry(ctsio, NULL);
1972         if (ctl_scsiio_lun_check(lun, entry, ctsio) != 0) {
1973                 mtx_unlock(&lun->lun_lock);
1974                 ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
1975                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1976                 msg_info.hdr.serializing_sc = NULL;
1977                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1978                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1979                     sizeof(msg_info.scsi), M_WAITOK);
1980                 return(1);
1981         }
1982
1983         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
1984         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = lun->be_lun;
1985
1986         /*
1987          * Every I/O goes into the OOA queue for a
1988          * particular LUN, and stays there until completion.
1989          */
1990 #ifdef CTL_TIME_IO
1991         if (TAILQ_EMPTY(&lun->ooa_queue))
1992                 lun->idle_time += getsbinuptime() - lun->last_busy;
1993 #endif
1994         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1995
1996         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1997                 (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1998                  ooa_links))) {
1999         case CTL_ACTION_BLOCK:
2000                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
2001                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
2002                                   blocked_links);
2003                 mtx_unlock(&lun->lun_lock);
2004                 break;
2005         case CTL_ACTION_PASS:
2006         case CTL_ACTION_SKIP:
2007                 if (softc->ha_mode == CTL_HA_MODE_XFER) {
2008                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
2009                         ctl_enqueue_rtr((union ctl_io *)ctsio);
2010                         mtx_unlock(&lun->lun_lock);
2011                 } else {
2012                         ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
2013                         mtx_unlock(&lun->lun_lock);
2014
2015                         /* send msg back to other side */
2016                         msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
2017                         msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
2018                         msg_info.hdr.msg_type = CTL_MSG_R2R;
2019                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
2020                             sizeof(msg_info.hdr), M_WAITOK);
2021                 }
2022                 break;
2023         case CTL_ACTION_OVERLAP:
2024                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
2025                 mtx_unlock(&lun->lun_lock);
2026                 retval = 1;
2027
2028                 ctl_set_overlapped_cmd(ctsio);
2029                 ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
2030                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
2031                 msg_info.hdr.serializing_sc = NULL;
2032                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
2033                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
2034                     sizeof(msg_info.scsi), M_WAITOK);
2035                 break;
2036         case CTL_ACTION_OVERLAP_TAG:
2037                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
2038                 mtx_unlock(&lun->lun_lock);
2039                 retval = 1;
2040                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num);
2041                 ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
2042                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
2043                 msg_info.hdr.serializing_sc = NULL;
2044                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
2045                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
2046                     sizeof(msg_info.scsi), M_WAITOK);
2047                 break;
2048         case CTL_ACTION_ERROR:
2049         default:
2050                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
2051                 mtx_unlock(&lun->lun_lock);
2052                 retval = 1;
2053
2054                 ctl_set_internal_failure(ctsio, /*sks_valid*/ 0,
2055                                          /*retry_count*/ 0);
2056                 ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
2057                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
2058                 msg_info.hdr.serializing_sc = NULL;
2059                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
2060                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
2061                     sizeof(msg_info.scsi), M_WAITOK);
2062                 break;
2063         }
2064         return (retval);
2065 }
2066
2067 /*
2068  * Returns 0 for success, errno for failure.
2069  */
2070 static int
2071 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
2072                    struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
2073 {
2074         union ctl_io *io;
2075         int retval;
2076
2077         retval = 0;
2078
2079         mtx_lock(&lun->lun_lock);
2080         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
2081              (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2082              ooa_links)) {
2083                 struct ctl_ooa_entry *entry;
2084
2085                 /*
2086                  * If we've got more than we can fit, just count the
2087                  * remaining entries.
2088                  */
2089                 if (*cur_fill_num >= ooa_hdr->alloc_num)
2090                         continue;
2091
2092                 entry = &kern_entries[*cur_fill_num];
2093
2094                 entry->tag_num = io->scsiio.tag_num;
2095                 entry->lun_num = lun->lun;
2096 #ifdef CTL_TIME_IO
2097                 entry->start_bt = io->io_hdr.start_bt;
2098 #endif
2099                 bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
2100                 entry->cdb_len = io->scsiio.cdb_len;
2101                 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
2102                         entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
2103
2104                 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
2105                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
2106
2107                 if (io->io_hdr.flags & CTL_FLAG_ABORT)
2108                         entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
2109
2110                 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
2111                         entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
2112
2113                 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
2114                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
2115         }
2116         mtx_unlock(&lun->lun_lock);
2117
2118         return (retval);
2119 }
2120
2121 static void *
2122 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2123                  size_t error_str_len)
2124 {
2125         void *kptr;
2126
2127         kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2128
2129         if (copyin(user_addr, kptr, len) != 0) {
2130                 snprintf(error_str, error_str_len, "Error copying %d bytes "
2131                          "from user address %p to kernel address %p", len,
2132                          user_addr, kptr);
2133                 free(kptr, M_CTL);
2134                 return (NULL);
2135         }
2136
2137         return (kptr);
2138 }
2139
2140 static void
2141 ctl_free_args(int num_args, struct ctl_be_arg *args)
2142 {
2143         int i;
2144
2145         if (args == NULL)
2146                 return;
2147
2148         for (i = 0; i < num_args; i++) {
2149                 free(args[i].kname, M_CTL);
2150                 free(args[i].kvalue, M_CTL);
2151         }
2152
2153         free(args, M_CTL);
2154 }
2155
2156 static struct ctl_be_arg *
2157 ctl_copyin_args(int num_args, struct ctl_be_arg *uargs,
2158                 char *error_str, size_t error_str_len)
2159 {
2160         struct ctl_be_arg *args;
2161         int i;
2162
2163         args = ctl_copyin_alloc(uargs, num_args * sizeof(*args),
2164                                 error_str, error_str_len);
2165
2166         if (args == NULL)
2167                 goto bailout;
2168
2169         for (i = 0; i < num_args; i++) {
2170                 args[i].kname = NULL;
2171                 args[i].kvalue = NULL;
2172         }
2173
2174         for (i = 0; i < num_args; i++) {
2175                 uint8_t *tmpptr;
2176
2177                 args[i].kname = ctl_copyin_alloc(args[i].name,
2178                         args[i].namelen, error_str, error_str_len);
2179                 if (args[i].kname == NULL)
2180                         goto bailout;
2181
2182                 if (args[i].kname[args[i].namelen - 1] != '\0') {
2183                         snprintf(error_str, error_str_len, "Argument %d "
2184                                  "name is not NUL-terminated", i);
2185                         goto bailout;
2186                 }
2187
2188                 if (args[i].flags & CTL_BEARG_RD) {
2189                         tmpptr = ctl_copyin_alloc(args[i].value,
2190                                 args[i].vallen, error_str, error_str_len);
2191                         if (tmpptr == NULL)
2192                                 goto bailout;
2193                         if ((args[i].flags & CTL_BEARG_ASCII)
2194                          && (tmpptr[args[i].vallen - 1] != '\0')) {
2195                                 snprintf(error_str, error_str_len, "Argument "
2196                                     "%d value is not NUL-terminated", i);
2197                                 goto bailout;
2198                         }
2199                         args[i].kvalue = tmpptr;
2200                 } else {
2201                         args[i].kvalue = malloc(args[i].vallen,
2202                             M_CTL, M_WAITOK | M_ZERO);
2203                 }
2204         }
2205
2206         return (args);
2207 bailout:
2208
2209         ctl_free_args(num_args, args);
2210
2211         return (NULL);
2212 }
2213
2214 static void
2215 ctl_copyout_args(int num_args, struct ctl_be_arg *args)
2216 {
2217         int i;
2218
2219         for (i = 0; i < num_args; i++) {
2220                 if (args[i].flags & CTL_BEARG_WR)
2221                         copyout(args[i].kvalue, args[i].value, args[i].vallen);
2222         }
2223 }
2224
2225 /*
2226  * Escape characters that are illegal or not recommended in XML.
2227  */
2228 int
2229 ctl_sbuf_printf_esc(struct sbuf *sb, char *str, int size)
2230 {
2231         char *end = str + size;
2232         int retval;
2233
2234         retval = 0;
2235
2236         for (; *str && str < end; str++) {
2237                 switch (*str) {
2238                 case '&':
2239                         retval = sbuf_printf(sb, "&amp;");
2240                         break;
2241                 case '>':
2242                         retval = sbuf_printf(sb, "&gt;");
2243                         break;
2244                 case '<':
2245                         retval = sbuf_printf(sb, "&lt;");
2246                         break;
2247                 default:
2248                         retval = sbuf_putc(sb, *str);
2249                         break;
2250                 }
2251
2252                 if (retval != 0)
2253                         break;
2254
2255         }
2256
2257         return (retval);
2258 }
2259
2260 static void
2261 ctl_id_sbuf(struct ctl_devid *id, struct sbuf *sb)
2262 {
2263         struct scsi_vpd_id_descriptor *desc;
2264         int i;
2265
2266         if (id == NULL || id->len < 4)
2267                 return;
2268         desc = (struct scsi_vpd_id_descriptor *)id->data;
2269         switch (desc->id_type & SVPD_ID_TYPE_MASK) {
2270         case SVPD_ID_TYPE_T10:
2271                 sbuf_printf(sb, "t10.");
2272                 break;
2273         case SVPD_ID_TYPE_EUI64:
2274                 sbuf_printf(sb, "eui.");
2275                 break;
2276         case SVPD_ID_TYPE_NAA:
2277                 sbuf_printf(sb, "naa.");
2278                 break;
2279         case SVPD_ID_TYPE_SCSI_NAME:
2280                 break;
2281         }
2282         switch (desc->proto_codeset & SVPD_ID_CODESET_MASK) {
2283         case SVPD_ID_CODESET_BINARY:
2284                 for (i = 0; i < desc->length; i++)
2285                         sbuf_printf(sb, "%02x", desc->identifier[i]);
2286                 break;
2287         case SVPD_ID_CODESET_ASCII:
2288                 sbuf_printf(sb, "%.*s", (int)desc->length,
2289                     (char *)desc->identifier);
2290                 break;
2291         case SVPD_ID_CODESET_UTF8:
2292                 sbuf_printf(sb, "%s", (char *)desc->identifier);
2293                 break;
2294         }
2295 }
2296
2297 static int
2298 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2299           struct thread *td)
2300 {
2301         struct ctl_softc *softc;
2302         struct ctl_lun *lun;
2303         int retval;
2304
2305         softc = control_softc;
2306
2307         retval = 0;
2308
2309         switch (cmd) {
2310         case CTL_IO:
2311                 retval = ctl_ioctl_io(dev, cmd, addr, flag, td);
2312                 break;
2313         case CTL_ENABLE_PORT:
2314         case CTL_DISABLE_PORT:
2315         case CTL_SET_PORT_WWNS: {
2316                 struct ctl_port *port;
2317                 struct ctl_port_entry *entry;
2318
2319                 entry = (struct ctl_port_entry *)addr;
2320
2321                 mtx_lock(&softc->ctl_lock);
2322                 STAILQ_FOREACH(port, &softc->port_list, links) {
2323                         int action, done;
2324
2325                         if (port->targ_port < softc->port_min ||
2326                             port->targ_port >= softc->port_max)
2327                                 continue;
2328
2329                         action = 0;
2330                         done = 0;
2331                         if ((entry->port_type == CTL_PORT_NONE)
2332                          && (entry->targ_port == port->targ_port)) {
2333                                 /*
2334                                  * If the user only wants to enable or
2335                                  * disable or set WWNs on a specific port,
2336                                  * do the operation and we're done.
2337                                  */
2338                                 action = 1;
2339                                 done = 1;
2340                         } else if (entry->port_type & port->port_type) {
2341                                 /*
2342                                  * Compare the user's type mask with the
2343                                  * particular frontend type to see if we
2344                                  * have a match.
2345                                  */
2346                                 action = 1;
2347                                 done = 0;
2348
2349                                 /*
2350                                  * Make sure the user isn't trying to set
2351                                  * WWNs on multiple ports at the same time.
2352                                  */
2353                                 if (cmd == CTL_SET_PORT_WWNS) {
2354                                         printf("%s: Can't set WWNs on "
2355                                                "multiple ports\n", __func__);
2356                                         retval = EINVAL;
2357                                         break;
2358                                 }
2359                         }
2360                         if (action == 0)
2361                                 continue;
2362
2363                         /*
2364                          * XXX KDM we have to drop the lock here, because
2365                          * the online/offline operations can potentially
2366                          * block.  We need to reference count the frontends
2367                          * so they can't go away,
2368                          */
2369                         if (cmd == CTL_ENABLE_PORT) {
2370                                 mtx_unlock(&softc->ctl_lock);
2371                                 ctl_port_online(port);
2372                                 mtx_lock(&softc->ctl_lock);
2373                         } else if (cmd == CTL_DISABLE_PORT) {
2374                                 mtx_unlock(&softc->ctl_lock);
2375                                 ctl_port_offline(port);
2376                                 mtx_lock(&softc->ctl_lock);
2377                         } else if (cmd == CTL_SET_PORT_WWNS) {
2378                                 ctl_port_set_wwns(port,
2379                                     (entry->flags & CTL_PORT_WWNN_VALID) ?
2380                                     1 : 0, entry->wwnn,
2381                                     (entry->flags & CTL_PORT_WWPN_VALID) ?
2382                                     1 : 0, entry->wwpn);
2383                         }
2384                         if (done != 0)
2385                                 break;
2386                 }
2387                 mtx_unlock(&softc->ctl_lock);
2388                 break;
2389         }
2390         case CTL_GET_PORT_LIST: {
2391                 struct ctl_port *port;
2392                 struct ctl_port_list *list;
2393                 int i;
2394
2395                 list = (struct ctl_port_list *)addr;
2396
2397                 if (list->alloc_len != (list->alloc_num *
2398                     sizeof(struct ctl_port_entry))) {
2399                         printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2400                                "alloc_num %u * sizeof(struct ctl_port_entry) "
2401                                "%zu\n", __func__, list->alloc_len,
2402                                list->alloc_num, sizeof(struct ctl_port_entry));
2403                         retval = EINVAL;
2404                         break;
2405                 }
2406                 list->fill_len = 0;
2407                 list->fill_num = 0;
2408                 list->dropped_num = 0;
2409                 i = 0;
2410                 mtx_lock(&softc->ctl_lock);
2411                 STAILQ_FOREACH(port, &softc->port_list, links) {
2412                         struct ctl_port_entry entry, *list_entry;
2413
2414                         if (list->fill_num >= list->alloc_num) {
2415                                 list->dropped_num++;
2416                                 continue;
2417                         }
2418
2419                         entry.port_type = port->port_type;
2420                         strlcpy(entry.port_name, port->port_name,
2421                                 sizeof(entry.port_name));
2422                         entry.targ_port = port->targ_port;
2423                         entry.physical_port = port->physical_port;
2424                         entry.virtual_port = port->virtual_port;
2425                         entry.wwnn = port->wwnn;
2426                         entry.wwpn = port->wwpn;
2427                         if (port->status & CTL_PORT_STATUS_ONLINE)
2428                                 entry.online = 1;
2429                         else
2430                                 entry.online = 0;
2431
2432                         list_entry = &list->entries[i];
2433
2434                         retval = copyout(&entry, list_entry, sizeof(entry));
2435                         if (retval != 0) {
2436                                 printf("%s: CTL_GET_PORT_LIST: copyout "
2437                                        "returned %d\n", __func__, retval);
2438                                 break;
2439                         }
2440                         i++;
2441                         list->fill_num++;
2442                         list->fill_len += sizeof(entry);
2443                 }
2444                 mtx_unlock(&softc->ctl_lock);
2445
2446                 /*
2447                  * If this is non-zero, we had a copyout fault, so there's
2448                  * probably no point in attempting to set the status inside
2449                  * the structure.
2450                  */
2451                 if (retval != 0)
2452                         break;
2453
2454                 if (list->dropped_num > 0)
2455                         list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2456                 else
2457                         list->status = CTL_PORT_LIST_OK;
2458                 break;
2459         }
2460         case CTL_DUMP_OOA: {
2461                 union ctl_io *io;
2462                 char printbuf[128];
2463                 struct sbuf sb;
2464
2465                 mtx_lock(&softc->ctl_lock);
2466                 printf("Dumping OOA queues:\n");
2467                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2468                         mtx_lock(&lun->lun_lock);
2469                         for (io = (union ctl_io *)TAILQ_FIRST(
2470                              &lun->ooa_queue); io != NULL;
2471                              io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2472                              ooa_links)) {
2473                                 sbuf_new(&sb, printbuf, sizeof(printbuf),
2474                                          SBUF_FIXEDLEN);
2475                                 sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2476                                             (intmax_t)lun->lun,
2477                                             io->scsiio.tag_num,
2478                                             (io->io_hdr.flags &
2479                                             CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2480                                             (io->io_hdr.flags &
2481                                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2482                                             (io->io_hdr.flags &
2483                                             CTL_FLAG_ABORT) ? " ABORT" : "",
2484                                             (io->io_hdr.flags &
2485                                         CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2486                                 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2487                                 sbuf_finish(&sb);
2488                                 printf("%s\n", sbuf_data(&sb));
2489                         }
2490                         mtx_unlock(&lun->lun_lock);
2491                 }
2492                 printf("OOA queues dump done\n");
2493                 mtx_unlock(&softc->ctl_lock);
2494                 break;
2495         }
2496         case CTL_GET_OOA: {
2497                 struct ctl_ooa *ooa_hdr;
2498                 struct ctl_ooa_entry *entries;
2499                 uint32_t cur_fill_num;
2500
2501                 ooa_hdr = (struct ctl_ooa *)addr;
2502
2503                 if ((ooa_hdr->alloc_len == 0)
2504                  || (ooa_hdr->alloc_num == 0)) {
2505                         printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2506                                "must be non-zero\n", __func__,
2507                                ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2508                         retval = EINVAL;
2509                         break;
2510                 }
2511
2512                 if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2513                     sizeof(struct ctl_ooa_entry))) {
2514                         printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2515                                "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2516                                __func__, ooa_hdr->alloc_len,
2517                                ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2518                         retval = EINVAL;
2519                         break;
2520                 }
2521
2522                 entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2523                 if (entries == NULL) {
2524                         printf("%s: could not allocate %d bytes for OOA "
2525                                "dump\n", __func__, ooa_hdr->alloc_len);
2526                         retval = ENOMEM;
2527                         break;
2528                 }
2529
2530                 mtx_lock(&softc->ctl_lock);
2531                 if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2532                  && ((ooa_hdr->lun_num >= CTL_MAX_LUNS)
2533                   || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2534                         mtx_unlock(&softc->ctl_lock);
2535                         free(entries, M_CTL);
2536                         printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2537                                __func__, (uintmax_t)ooa_hdr->lun_num);
2538                         retval = EINVAL;
2539                         break;
2540                 }
2541
2542                 cur_fill_num = 0;
2543
2544                 if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2545                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
2546                                 retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2547                                         ooa_hdr, entries);
2548                                 if (retval != 0)
2549                                         break;
2550                         }
2551                         if (retval != 0) {
2552                                 mtx_unlock(&softc->ctl_lock);
2553                                 free(entries, M_CTL);
2554                                 break;
2555                         }
2556                 } else {
2557                         lun = softc->ctl_luns[ooa_hdr->lun_num];
2558
2559                         retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
2560                                                     entries);
2561                 }
2562                 mtx_unlock(&softc->ctl_lock);
2563
2564                 ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2565                 ooa_hdr->fill_len = ooa_hdr->fill_num *
2566                         sizeof(struct ctl_ooa_entry);
2567                 retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2568                 if (retval != 0) {
2569                         printf("%s: error copying out %d bytes for OOA dump\n",
2570                                __func__, ooa_hdr->fill_len);
2571                 }
2572
2573                 getbintime(&ooa_hdr->cur_bt);
2574
2575                 if (cur_fill_num > ooa_hdr->alloc_num) {
2576                         ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2577                         ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2578                 } else {
2579                         ooa_hdr->dropped_num = 0;
2580                         ooa_hdr->status = CTL_OOA_OK;
2581                 }
2582
2583                 free(entries, M_CTL);
2584                 break;
2585         }
2586         case CTL_CHECK_OOA: {
2587                 union ctl_io *io;
2588                 struct ctl_ooa_info *ooa_info;
2589
2590
2591                 ooa_info = (struct ctl_ooa_info *)addr;
2592
2593                 if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2594                         ooa_info->status = CTL_OOA_INVALID_LUN;
2595                         break;
2596                 }
2597                 mtx_lock(&softc->ctl_lock);
2598                 lun = softc->ctl_luns[ooa_info->lun_id];
2599                 if (lun == NULL) {
2600                         mtx_unlock(&softc->ctl_lock);
2601                         ooa_info->status = CTL_OOA_INVALID_LUN;
2602                         break;
2603                 }
2604                 mtx_lock(&lun->lun_lock);
2605                 mtx_unlock(&softc->ctl_lock);
2606                 ooa_info->num_entries = 0;
2607                 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2608                      io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2609                      &io->io_hdr, ooa_links)) {
2610                         ooa_info->num_entries++;
2611                 }
2612                 mtx_unlock(&lun->lun_lock);
2613
2614                 ooa_info->status = CTL_OOA_SUCCESS;
2615
2616                 break;
2617         }
2618         case CTL_DELAY_IO: {
2619                 struct ctl_io_delay_info *delay_info;
2620
2621                 delay_info = (struct ctl_io_delay_info *)addr;
2622
2623 #ifdef CTL_IO_DELAY
2624                 mtx_lock(&softc->ctl_lock);
2625
2626                 if ((delay_info->lun_id >= CTL_MAX_LUNS)
2627                  || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2628                         delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2629                 } else {
2630                         lun = softc->ctl_luns[delay_info->lun_id];
2631                         mtx_lock(&lun->lun_lock);
2632
2633                         delay_info->status = CTL_DELAY_STATUS_OK;
2634
2635                         switch (delay_info->delay_type) {
2636                         case CTL_DELAY_TYPE_CONT:
2637                                 break;
2638                         case CTL_DELAY_TYPE_ONESHOT:
2639                                 break;
2640                         default:
2641                                 delay_info->status =
2642                                         CTL_DELAY_STATUS_INVALID_TYPE;
2643                                 break;
2644                         }
2645
2646                         switch (delay_info->delay_loc) {
2647                         case CTL_DELAY_LOC_DATAMOVE:
2648                                 lun->delay_info.datamove_type =
2649                                         delay_info->delay_type;
2650                                 lun->delay_info.datamove_delay =
2651                                         delay_info->delay_secs;
2652                                 break;
2653                         case CTL_DELAY_LOC_DONE:
2654                                 lun->delay_info.done_type =
2655                                         delay_info->delay_type;
2656                                 lun->delay_info.done_delay =
2657                                         delay_info->delay_secs;
2658                                 break;
2659                         default:
2660                                 delay_info->status =
2661                                         CTL_DELAY_STATUS_INVALID_LOC;
2662                                 break;
2663                         }
2664                         mtx_unlock(&lun->lun_lock);
2665                 }
2666
2667                 mtx_unlock(&softc->ctl_lock);
2668 #else
2669                 delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2670 #endif /* CTL_IO_DELAY */
2671                 break;
2672         }
2673         case CTL_REALSYNC_SET: {
2674                 int *syncstate;
2675
2676                 syncstate = (int *)addr;
2677
2678                 mtx_lock(&softc->ctl_lock);
2679                 switch (*syncstate) {
2680                 case 0:
2681                         softc->flags &= ~CTL_FLAG_REAL_SYNC;
2682                         break;
2683                 case 1:
2684                         softc->flags |= CTL_FLAG_REAL_SYNC;
2685                         break;
2686                 default:
2687                         retval = EINVAL;
2688                         break;
2689                 }
2690                 mtx_unlock(&softc->ctl_lock);
2691                 break;
2692         }
2693         case CTL_REALSYNC_GET: {
2694                 int *syncstate;
2695
2696                 syncstate = (int*)addr;
2697
2698                 mtx_lock(&softc->ctl_lock);
2699                 if (softc->flags & CTL_FLAG_REAL_SYNC)
2700                         *syncstate = 1;
2701                 else
2702                         *syncstate = 0;
2703                 mtx_unlock(&softc->ctl_lock);
2704
2705                 break;
2706         }
2707         case CTL_SETSYNC:
2708         case CTL_GETSYNC: {
2709                 struct ctl_sync_info *sync_info;
2710
2711                 sync_info = (struct ctl_sync_info *)addr;
2712
2713                 mtx_lock(&softc->ctl_lock);
2714                 lun = softc->ctl_luns[sync_info->lun_id];
2715                 if (lun == NULL) {
2716                         mtx_unlock(&softc->ctl_lock);
2717                         sync_info->status = CTL_GS_SYNC_NO_LUN;
2718                         break;
2719                 }
2720                 /*
2721                  * Get or set the sync interval.  We're not bounds checking
2722                  * in the set case, hopefully the user won't do something
2723                  * silly.
2724                  */
2725                 mtx_lock(&lun->lun_lock);
2726                 mtx_unlock(&softc->ctl_lock);
2727                 if (cmd == CTL_GETSYNC)
2728                         sync_info->sync_interval = lun->sync_interval;
2729                 else
2730                         lun->sync_interval = sync_info->sync_interval;
2731                 mtx_unlock(&lun->lun_lock);
2732
2733                 sync_info->status = CTL_GS_SYNC_OK;
2734
2735                 break;
2736         }
2737         case CTL_GETSTATS: {
2738                 struct ctl_stats *stats;
2739                 int i;
2740
2741                 stats = (struct ctl_stats *)addr;
2742
2743                 if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2744                      stats->alloc_len) {
2745                         stats->status = CTL_SS_NEED_MORE_SPACE;
2746                         stats->num_luns = softc->num_luns;
2747                         break;
2748                 }
2749                 /*
2750                  * XXX KDM no locking here.  If the LUN list changes,
2751                  * things can blow up.
2752                  */
2753                 for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2754                      i++, lun = STAILQ_NEXT(lun, links)) {
2755                         retval = copyout(&lun->stats, &stats->lun_stats[i],
2756                                          sizeof(lun->stats));
2757                         if (retval != 0)
2758                                 break;
2759                 }
2760                 stats->num_luns = softc->num_luns;
2761                 stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2762                                  softc->num_luns;
2763                 stats->status = CTL_SS_OK;
2764 #ifdef CTL_TIME_IO
2765                 stats->flags = CTL_STATS_FLAG_TIME_VALID;
2766 #else
2767                 stats->flags = CTL_STATS_FLAG_NONE;
2768 #endif
2769                 getnanouptime(&stats->timestamp);
2770                 break;
2771         }
2772         case CTL_ERROR_INJECT: {
2773                 struct ctl_error_desc *err_desc, *new_err_desc;
2774
2775                 err_desc = (struct ctl_error_desc *)addr;
2776
2777                 new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2778                                       M_WAITOK | M_ZERO);
2779                 bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2780
2781                 mtx_lock(&softc->ctl_lock);
2782                 lun = softc->ctl_luns[err_desc->lun_id];
2783                 if (lun == NULL) {
2784                         mtx_unlock(&softc->ctl_lock);
2785                         free(new_err_desc, M_CTL);
2786                         printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2787                                __func__, (uintmax_t)err_desc->lun_id);
2788                         retval = EINVAL;
2789                         break;
2790                 }
2791                 mtx_lock(&lun->lun_lock);
2792                 mtx_unlock(&softc->ctl_lock);
2793
2794                 /*
2795                  * We could do some checking here to verify the validity
2796                  * of the request, but given the complexity of error
2797                  * injection requests, the checking logic would be fairly
2798                  * complex.
2799                  *
2800                  * For now, if the request is invalid, it just won't get
2801                  * executed and might get deleted.
2802                  */
2803                 STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2804
2805                 /*
2806                  * XXX KDM check to make sure the serial number is unique,
2807                  * in case we somehow manage to wrap.  That shouldn't
2808                  * happen for a very long time, but it's the right thing to
2809                  * do.
2810                  */
2811                 new_err_desc->serial = lun->error_serial;
2812                 err_desc->serial = lun->error_serial;
2813                 lun->error_serial++;
2814
2815                 mtx_unlock(&lun->lun_lock);
2816                 break;
2817         }
2818         case CTL_ERROR_INJECT_DELETE: {
2819                 struct ctl_error_desc *delete_desc, *desc, *desc2;
2820                 int delete_done;
2821
2822                 delete_desc = (struct ctl_error_desc *)addr;
2823                 delete_done = 0;
2824
2825                 mtx_lock(&softc->ctl_lock);
2826                 lun = softc->ctl_luns[delete_desc->lun_id];
2827                 if (lun == NULL) {
2828                         mtx_unlock(&softc->ctl_lock);
2829                         printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
2830                                __func__, (uintmax_t)delete_desc->lun_id);
2831                         retval = EINVAL;
2832                         break;
2833                 }
2834                 mtx_lock(&lun->lun_lock);
2835                 mtx_unlock(&softc->ctl_lock);
2836                 STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
2837                         if (desc->serial != delete_desc->serial)
2838                                 continue;
2839
2840                         STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
2841                                       links);
2842                         free(desc, M_CTL);
2843                         delete_done = 1;
2844                 }
2845                 mtx_unlock(&lun->lun_lock);
2846                 if (delete_done == 0) {
2847                         printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
2848                                "error serial %ju on LUN %u\n", __func__,
2849                                delete_desc->serial, delete_desc->lun_id);
2850                         retval = EINVAL;
2851                         break;
2852                 }
2853                 break;
2854         }
2855         case CTL_DUMP_STRUCTS: {
2856                 int i, j, k;
2857                 struct ctl_port *port;
2858                 struct ctl_frontend *fe;
2859
2860                 mtx_lock(&softc->ctl_lock);
2861                 printf("CTL Persistent Reservation information start:\n");
2862                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2863                         lun = softc->ctl_luns[i];
2864
2865                         if ((lun == NULL)
2866                          || ((lun->flags & CTL_LUN_DISABLED) != 0))
2867                                 continue;
2868
2869                         for (j = 0; j < CTL_MAX_PORTS; j++) {
2870                                 if (lun->pr_keys[j] == NULL)
2871                                         continue;
2872                                 for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
2873                                         if (lun->pr_keys[j][k] == 0)
2874                                                 continue;
2875                                         printf("  LUN %d port %d iid %d key "
2876                                                "%#jx\n", i, j, k,
2877                                                (uintmax_t)lun->pr_keys[j][k]);
2878                                 }
2879                         }
2880                 }
2881                 printf("CTL Persistent Reservation information end\n");
2882                 printf("CTL Ports:\n");
2883                 STAILQ_FOREACH(port, &softc->port_list, links) {
2884                         printf("  Port %d '%s' Frontend '%s' Type %u pp %d vp %d WWNN "
2885                                "%#jx WWPN %#jx\n", port->targ_port, port->port_name,
2886                                port->frontend->name, port->port_type,
2887                                port->physical_port, port->virtual_port,
2888                                (uintmax_t)port->wwnn, (uintmax_t)port->wwpn);
2889                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
2890                                 if (port->wwpn_iid[j].in_use == 0 &&
2891                                     port->wwpn_iid[j].wwpn == 0 &&
2892                                     port->wwpn_iid[j].name == NULL)
2893                                         continue;
2894
2895                                 printf("    iid %u use %d WWPN %#jx '%s'\n",
2896                                     j, port->wwpn_iid[j].in_use,
2897                                     (uintmax_t)port->wwpn_iid[j].wwpn,
2898                                     port->wwpn_iid[j].name);
2899                         }
2900                 }
2901                 printf("CTL Port information end\n");
2902                 mtx_unlock(&softc->ctl_lock);
2903                 /*
2904                  * XXX KDM calling this without a lock.  We'd likely want
2905                  * to drop the lock before calling the frontend's dump
2906                  * routine anyway.
2907                  */
2908                 printf("CTL Frontends:\n");
2909                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
2910                         printf("  Frontend '%s'\n", fe->name);
2911                         if (fe->fe_dump != NULL)
2912                                 fe->fe_dump();
2913                 }
2914                 printf("CTL Frontend information end\n");
2915                 break;
2916         }
2917         case CTL_LUN_REQ: {
2918                 struct ctl_lun_req *lun_req;
2919                 struct ctl_backend_driver *backend;
2920
2921                 lun_req = (struct ctl_lun_req *)addr;
2922
2923                 backend = ctl_backend_find(lun_req->backend);
2924                 if (backend == NULL) {
2925                         lun_req->status = CTL_LUN_ERROR;
2926                         snprintf(lun_req->error_str,
2927                                  sizeof(lun_req->error_str),
2928                                  "Backend \"%s\" not found.",
2929                                  lun_req->backend);
2930                         break;
2931                 }
2932                 if (lun_req->num_be_args > 0) {
2933                         lun_req->kern_be_args = ctl_copyin_args(
2934                                 lun_req->num_be_args,
2935                                 lun_req->be_args,
2936                                 lun_req->error_str,
2937                                 sizeof(lun_req->error_str));
2938                         if (lun_req->kern_be_args == NULL) {
2939                                 lun_req->status = CTL_LUN_ERROR;
2940                                 break;
2941                         }
2942                 }
2943
2944                 retval = backend->ioctl(dev, cmd, addr, flag, td);
2945
2946                 if (lun_req->num_be_args > 0) {
2947                         ctl_copyout_args(lun_req->num_be_args,
2948                                       lun_req->kern_be_args);
2949                         ctl_free_args(lun_req->num_be_args,
2950                                       lun_req->kern_be_args);
2951                 }
2952                 break;
2953         }
2954         case CTL_LUN_LIST: {
2955                 struct sbuf *sb;
2956                 struct ctl_lun_list *list;
2957                 struct ctl_option *opt;
2958
2959                 list = (struct ctl_lun_list *)addr;
2960
2961                 /*
2962                  * Allocate a fixed length sbuf here, based on the length
2963                  * of the user's buffer.  We could allocate an auto-extending
2964                  * buffer, and then tell the user how much larger our
2965                  * amount of data is than his buffer, but that presents
2966                  * some problems:
2967                  *
2968                  * 1.  The sbuf(9) routines use a blocking malloc, and so
2969                  *     we can't hold a lock while calling them with an
2970                  *     auto-extending buffer.
2971                  *
2972                  * 2.  There is not currently a LUN reference counting
2973                  *     mechanism, outside of outstanding transactions on
2974                  *     the LUN's OOA queue.  So a LUN could go away on us
2975                  *     while we're getting the LUN number, backend-specific
2976                  *     information, etc.  Thus, given the way things
2977                  *     currently work, we need to hold the CTL lock while
2978                  *     grabbing LUN information.
2979                  *
2980                  * So, from the user's standpoint, the best thing to do is
2981                  * allocate what he thinks is a reasonable buffer length,
2982                  * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
2983                  * double the buffer length and try again.  (And repeat
2984                  * that until he succeeds.)
2985                  */
2986                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
2987                 if (sb == NULL) {
2988                         list->status = CTL_LUN_LIST_ERROR;
2989                         snprintf(list->error_str, sizeof(list->error_str),
2990                                  "Unable to allocate %d bytes for LUN list",
2991                                  list->alloc_len);
2992                         break;
2993                 }
2994
2995                 sbuf_printf(sb, "<ctllunlist>\n");
2996
2997                 mtx_lock(&softc->ctl_lock);
2998                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2999                         mtx_lock(&lun->lun_lock);
3000                         retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3001                                              (uintmax_t)lun->lun);
3002
3003                         /*
3004                          * Bail out as soon as we see that we've overfilled
3005                          * the buffer.
3006                          */
3007                         if (retval != 0)
3008                                 break;
3009
3010                         retval = sbuf_printf(sb, "\t<backend_type>%s"
3011                                              "</backend_type>\n",
3012                                              (lun->backend == NULL) ?  "none" :
3013                                              lun->backend->name);
3014
3015                         if (retval != 0)
3016                                 break;
3017
3018                         retval = sbuf_printf(sb, "\t<lun_type>%d</lun_type>\n",
3019                                              lun->be_lun->lun_type);
3020
3021                         if (retval != 0)
3022                                 break;
3023
3024                         if (lun->backend == NULL) {
3025                                 retval = sbuf_printf(sb, "</lun>\n");
3026                                 if (retval != 0)
3027                                         break;
3028                                 continue;
3029                         }
3030
3031                         retval = sbuf_printf(sb, "\t<size>%ju</size>\n",
3032                                              (lun->be_lun->maxlba > 0) ?
3033                                              lun->be_lun->maxlba + 1 : 0);
3034
3035                         if (retval != 0)
3036                                 break;
3037
3038                         retval = sbuf_printf(sb, "\t<blocksize>%u</blocksize>\n",
3039                                              lun->be_lun->blocksize);
3040
3041                         if (retval != 0)
3042                                 break;
3043
3044                         retval = sbuf_printf(sb, "\t<serial_number>");
3045
3046                         if (retval != 0)
3047                                 break;
3048
3049                         retval = ctl_sbuf_printf_esc(sb,
3050                             lun->be_lun->serial_num,
3051                             sizeof(lun->be_lun->serial_num));
3052
3053                         if (retval != 0)
3054                                 break;
3055
3056                         retval = sbuf_printf(sb, "</serial_number>\n");
3057
3058                         if (retval != 0)
3059                                 break;
3060
3061                         retval = sbuf_printf(sb, "\t<device_id>");
3062
3063                         if (retval != 0)
3064                                 break;
3065
3066                         retval = ctl_sbuf_printf_esc(sb,
3067                             lun->be_lun->device_id,
3068                             sizeof(lun->be_lun->device_id));
3069
3070                         if (retval != 0)
3071                                 break;
3072
3073                         retval = sbuf_printf(sb, "</device_id>\n");
3074
3075                         if (retval != 0)
3076                                 break;
3077
3078                         if (lun->backend->lun_info != NULL) {
3079                                 retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3080                                 if (retval != 0)
3081                                         break;
3082                         }
3083                         STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3084                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3085                                     opt->name, opt->value, opt->name);
3086                                 if (retval != 0)
3087                                         break;
3088                         }
3089
3090                         retval = sbuf_printf(sb, "</lun>\n");
3091
3092                         if (retval != 0)
3093                                 break;
3094                         mtx_unlock(&lun->lun_lock);
3095                 }
3096                 if (lun != NULL)
3097                         mtx_unlock(&lun->lun_lock);
3098                 mtx_unlock(&softc->ctl_lock);
3099
3100                 if ((retval != 0)
3101                  || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3102                         retval = 0;
3103                         sbuf_delete(sb);
3104                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3105                         snprintf(list->error_str, sizeof(list->error_str),
3106                                  "Out of space, %d bytes is too small",
3107                                  list->alloc_len);
3108                         break;
3109                 }
3110
3111                 sbuf_finish(sb);
3112
3113                 retval = copyout(sbuf_data(sb), list->lun_xml,
3114                                  sbuf_len(sb) + 1);
3115
3116                 list->fill_len = sbuf_len(sb) + 1;
3117                 list->status = CTL_LUN_LIST_OK;
3118                 sbuf_delete(sb);
3119                 break;
3120         }
3121         case CTL_ISCSI: {
3122                 struct ctl_iscsi *ci;
3123                 struct ctl_frontend *fe;
3124
3125                 ci = (struct ctl_iscsi *)addr;
3126
3127                 fe = ctl_frontend_find("iscsi");
3128                 if (fe == NULL) {
3129                         ci->status = CTL_ISCSI_ERROR;
3130                         snprintf(ci->error_str, sizeof(ci->error_str),
3131                             "Frontend \"iscsi\" not found.");
3132                         break;
3133                 }
3134
3135                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3136                 break;
3137         }
3138         case CTL_PORT_REQ: {
3139                 struct ctl_req *req;
3140                 struct ctl_frontend *fe;
3141
3142                 req = (struct ctl_req *)addr;
3143
3144                 fe = ctl_frontend_find(req->driver);
3145                 if (fe == NULL) {
3146                         req->status = CTL_LUN_ERROR;
3147                         snprintf(req->error_str, sizeof(req->error_str),
3148                             "Frontend \"%s\" not found.", req->driver);
3149                         break;
3150                 }
3151                 if (req->num_args > 0) {
3152                         req->kern_args = ctl_copyin_args(req->num_args,
3153                             req->args, req->error_str, sizeof(req->error_str));
3154                         if (req->kern_args == NULL) {
3155                                 req->status = CTL_LUN_ERROR;
3156                                 break;
3157                         }
3158                 }
3159
3160                 if (fe->ioctl)
3161                         retval = fe->ioctl(dev, cmd, addr, flag, td);
3162                 else
3163                         retval = ENODEV;
3164
3165                 if (req->num_args > 0) {
3166                         ctl_copyout_args(req->num_args, req->kern_args);
3167                         ctl_free_args(req->num_args, req->kern_args);
3168                 }
3169                 break;
3170         }
3171         case CTL_PORT_LIST: {
3172                 struct sbuf *sb;
3173                 struct ctl_port *port;
3174                 struct ctl_lun_list *list;
3175                 struct ctl_option *opt;
3176                 int j;
3177                 uint32_t plun;
3178
3179                 list = (struct ctl_lun_list *)addr;
3180
3181                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3182                 if (sb == NULL) {
3183                         list->status = CTL_LUN_LIST_ERROR;
3184                         snprintf(list->error_str, sizeof(list->error_str),
3185                                  "Unable to allocate %d bytes for LUN list",
3186                                  list->alloc_len);
3187                         break;
3188                 }
3189
3190                 sbuf_printf(sb, "<ctlportlist>\n");
3191
3192                 mtx_lock(&softc->ctl_lock);
3193                 STAILQ_FOREACH(port, &softc->port_list, links) {
3194                         retval = sbuf_printf(sb, "<targ_port id=\"%ju\">\n",
3195                                              (uintmax_t)port->targ_port);
3196
3197                         /*
3198                          * Bail out as soon as we see that we've overfilled
3199                          * the buffer.
3200                          */
3201                         if (retval != 0)
3202                                 break;
3203
3204                         retval = sbuf_printf(sb, "\t<frontend_type>%s"
3205                             "</frontend_type>\n", port->frontend->name);
3206                         if (retval != 0)
3207                                 break;
3208
3209                         retval = sbuf_printf(sb, "\t<port_type>%d</port_type>\n",
3210                                              port->port_type);
3211                         if (retval != 0)
3212                                 break;
3213
3214                         retval = sbuf_printf(sb, "\t<online>%s</online>\n",
3215                             (port->status & CTL_PORT_STATUS_ONLINE) ? "YES" : "NO");
3216                         if (retval != 0)
3217                                 break;
3218
3219                         retval = sbuf_printf(sb, "\t<port_name>%s</port_name>\n",
3220                             port->port_name);
3221                         if (retval != 0)
3222                                 break;
3223
3224                         retval = sbuf_printf(sb, "\t<physical_port>%d</physical_port>\n",
3225                             port->physical_port);
3226                         if (retval != 0)
3227                                 break;
3228
3229                         retval = sbuf_printf(sb, "\t<virtual_port>%d</virtual_port>\n",
3230                             port->virtual_port);
3231                         if (retval != 0)
3232                                 break;
3233
3234                         if (port->target_devid != NULL) {
3235                                 sbuf_printf(sb, "\t<target>");
3236                                 ctl_id_sbuf(port->target_devid, sb);
3237                                 sbuf_printf(sb, "</target>\n");
3238                         }
3239
3240                         if (port->port_devid != NULL) {
3241                                 sbuf_printf(sb, "\t<port>");
3242                                 ctl_id_sbuf(port->port_devid, sb);
3243                                 sbuf_printf(sb, "</port>\n");
3244                         }
3245
3246                         if (port->port_info != NULL) {
3247                                 retval = port->port_info(port->onoff_arg, sb);
3248                                 if (retval != 0)
3249                                         break;
3250                         }
3251                         STAILQ_FOREACH(opt, &port->options, links) {
3252                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3253                                     opt->name, opt->value, opt->name);
3254                                 if (retval != 0)
3255                                         break;
3256                         }
3257
3258                         if (port->lun_map != NULL) {
3259                                 sbuf_printf(sb, "\t<lun_map>on</lun_map>\n");
3260                                 for (j = 0; j < CTL_MAX_LUNS; j++) {
3261                                         plun = ctl_lun_map_from_port(port, j);
3262                                         if (plun >= CTL_MAX_LUNS)
3263                                                 continue;
3264                                         sbuf_printf(sb,
3265                                             "\t<lun id=\"%u\">%u</lun>\n",
3266                                             j, plun);
3267                                 }
3268                         }
3269
3270                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
3271                                 if (port->wwpn_iid[j].in_use == 0 ||
3272                                     (port->wwpn_iid[j].wwpn == 0 &&
3273                                      port->wwpn_iid[j].name == NULL))
3274                                         continue;
3275
3276                                 if (port->wwpn_iid[j].name != NULL)
3277                                         retval = sbuf_printf(sb,
3278                                             "\t<initiator id=\"%u\">%s</initiator>\n",
3279                                             j, port->wwpn_iid[j].name);
3280                                 else
3281                                         retval = sbuf_printf(sb,
3282                                             "\t<initiator id=\"%u\">naa.%08jx</initiator>\n",
3283                                             j, port->wwpn_iid[j].wwpn);
3284                                 if (retval != 0)
3285                                         break;
3286                         }
3287                         if (retval != 0)
3288                                 break;
3289
3290                         retval = sbuf_printf(sb, "</targ_port>\n");
3291                         if (retval != 0)
3292                                 break;
3293                 }
3294                 mtx_unlock(&softc->ctl_lock);
3295
3296                 if ((retval != 0)
3297                  || ((retval = sbuf_printf(sb, "</ctlportlist>\n")) != 0)) {
3298                         retval = 0;
3299                         sbuf_delete(sb);
3300                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3301                         snprintf(list->error_str, sizeof(list->error_str),
3302                                  "Out of space, %d bytes is too small",
3303                                  list->alloc_len);
3304                         break;
3305                 }
3306
3307                 sbuf_finish(sb);
3308
3309                 retval = copyout(sbuf_data(sb), list->lun_xml,
3310                                  sbuf_len(sb) + 1);
3311
3312                 list->fill_len = sbuf_len(sb) + 1;
3313                 list->status = CTL_LUN_LIST_OK;
3314                 sbuf_delete(sb);
3315                 break;
3316         }
3317         case CTL_LUN_MAP: {
3318                 struct ctl_lun_map *lm  = (struct ctl_lun_map *)addr;
3319                 struct ctl_port *port;
3320
3321                 mtx_lock(&softc->ctl_lock);
3322                 if (lm->port < softc->port_min ||
3323                     lm->port >= softc->port_max ||
3324                     (port = softc->ctl_ports[lm->port]) == NULL) {
3325                         mtx_unlock(&softc->ctl_lock);
3326                         return (ENXIO);
3327                 }
3328                 if (port->status & CTL_PORT_STATUS_ONLINE) {
3329                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
3330                                 if (ctl_lun_map_to_port(port, lun->lun) >=
3331                                     CTL_MAX_LUNS)
3332                                         continue;
3333                                 mtx_lock(&lun->lun_lock);
3334                                 ctl_est_ua_port(lun, lm->port, -1,
3335                                     CTL_UA_LUN_CHANGE);
3336                                 mtx_unlock(&lun->lun_lock);
3337                         }
3338                 }
3339                 mtx_unlock(&softc->ctl_lock); // XXX: port_enable sleeps
3340                 if (lm->plun < CTL_MAX_LUNS) {
3341                         if (lm->lun == UINT32_MAX)
3342                                 retval = ctl_lun_map_unset(port, lm->plun);
3343                         else if (lm->lun < CTL_MAX_LUNS &&
3344                             softc->ctl_luns[lm->lun] != NULL)
3345                                 retval = ctl_lun_map_set(port, lm->plun, lm->lun);
3346                         else
3347                                 return (ENXIO);
3348                 } else if (lm->plun == UINT32_MAX) {
3349                         if (lm->lun == UINT32_MAX)
3350                                 retval = ctl_lun_map_deinit(port);
3351                         else
3352                                 retval = ctl_lun_map_init(port);
3353                 } else
3354                         return (ENXIO);
3355                 if (port->status & CTL_PORT_STATUS_ONLINE)
3356                         ctl_isc_announce_port(port);
3357                 break;
3358         }
3359         default: {
3360                 /* XXX KDM should we fix this? */
3361 #if 0
3362                 struct ctl_backend_driver *backend;
3363                 unsigned int type;
3364                 int found;
3365
3366                 found = 0;
3367
3368                 /*
3369                  * We encode the backend type as the ioctl type for backend
3370                  * ioctls.  So parse it out here, and then search for a
3371                  * backend of this type.
3372                  */
3373                 type = _IOC_TYPE(cmd);
3374
3375                 STAILQ_FOREACH(backend, &softc->be_list, links) {
3376                         if (backend->type == type) {
3377                                 found = 1;
3378                                 break;
3379                         }
3380                 }
3381                 if (found == 0) {
3382                         printf("ctl: unknown ioctl command %#lx or backend "
3383                                "%d\n", cmd, type);
3384                         retval = EINVAL;
3385                         break;
3386                 }
3387                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3388 #endif
3389                 retval = ENOTTY;
3390                 break;
3391         }
3392         }
3393         return (retval);
3394 }
3395
3396 uint32_t
3397 ctl_get_initindex(struct ctl_nexus *nexus)
3398 {
3399         return (nexus->initid + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3400 }
3401
3402 int
3403 ctl_lun_map_init(struct ctl_port *port)
3404 {
3405         struct ctl_softc *softc = control_softc;
3406         struct ctl_lun *lun;
3407         uint32_t i;
3408
3409         if (port->lun_map == NULL)
3410                 port->lun_map = malloc(sizeof(uint32_t) * CTL_MAX_LUNS,
3411                     M_CTL, M_NOWAIT);
3412         if (port->lun_map == NULL)
3413                 return (ENOMEM);
3414         for (i = 0; i < CTL_MAX_LUNS; i++)
3415                 port->lun_map[i] = UINT32_MAX;
3416         if (port->status & CTL_PORT_STATUS_ONLINE) {
3417                 if (port->lun_disable != NULL) {
3418                         STAILQ_FOREACH(lun, &softc->lun_list, links)
3419                                 port->lun_disable(port->targ_lun_arg, lun->lun);
3420                 }
3421                 ctl_isc_announce_port(port);
3422         }
3423         return (0);
3424 }
3425
3426 int
3427 ctl_lun_map_deinit(struct ctl_port *port)
3428 {
3429         struct ctl_softc *softc = control_softc;
3430         struct ctl_lun *lun;
3431
3432         if (port->lun_map == NULL)
3433                 return (0);
3434         free(port->lun_map, M_CTL);
3435         port->lun_map = NULL;
3436         if (port->status & CTL_PORT_STATUS_ONLINE) {
3437                 if (port->lun_enable != NULL) {
3438                         STAILQ_FOREACH(lun, &softc->lun_list, links)
3439                                 port->lun_enable(port->targ_lun_arg, lun->lun);
3440                 }
3441                 ctl_isc_announce_port(port);
3442         }
3443         return (0);
3444 }
3445
3446 int
3447 ctl_lun_map_set(struct ctl_port *port, uint32_t plun, uint32_t glun)
3448 {
3449         int status;
3450         uint32_t old;
3451
3452         if (port->lun_map == NULL) {
3453                 status = ctl_lun_map_init(port);
3454                 if (status != 0)
3455                         return (status);
3456         }
3457         old = port->lun_map[plun];
3458         port->lun_map[plun] = glun;
3459         if ((port->status & CTL_PORT_STATUS_ONLINE) && old >= CTL_MAX_LUNS) {
3460                 if (port->lun_enable != NULL)
3461                         port->lun_enable(port->targ_lun_arg, plun);
3462                 ctl_isc_announce_port(port);
3463         }
3464         return (0);
3465 }
3466
3467 int
3468 ctl_lun_map_unset(struct ctl_port *port, uint32_t plun)
3469 {
3470         uint32_t old;
3471
3472         if (port->lun_map == NULL)
3473                 return (0);
3474         old = port->lun_map[plun];
3475         port->lun_map[plun] = UINT32_MAX;
3476         if ((port->status & CTL_PORT_STATUS_ONLINE) && old < CTL_MAX_LUNS) {
3477                 if (port->lun_disable != NULL)
3478                         port->lun_disable(port->targ_lun_arg, plun);
3479                 ctl_isc_announce_port(port);
3480         }
3481         return (0);
3482 }
3483
3484 uint32_t
3485 ctl_lun_map_from_port(struct ctl_port *port, uint32_t lun_id)
3486 {
3487
3488         if (port == NULL)
3489                 return (UINT32_MAX);
3490         if (port->lun_map == NULL || lun_id >= CTL_MAX_LUNS)
3491                 return (lun_id);
3492         return (port->lun_map[lun_id]);
3493 }
3494
3495 uint32_t
3496 ctl_lun_map_to_port(struct ctl_port *port, uint32_t lun_id)
3497 {
3498         uint32_t i;
3499
3500         if (port == NULL)
3501                 return (UINT32_MAX);
3502         if (port->lun_map == NULL)
3503                 return (lun_id);
3504         for (i = 0; i < CTL_MAX_LUNS; i++) {
3505                 if (port->lun_map[i] == lun_id)
3506                         return (i);
3507         }
3508         return (UINT32_MAX);
3509 }
3510
3511 static struct ctl_port *
3512 ctl_io_port(struct ctl_io_hdr *io_hdr)
3513 {
3514
3515         return (control_softc->ctl_ports[io_hdr->nexus.targ_port]);
3516 }
3517
3518 int
3519 ctl_ffz(uint32_t *mask, uint32_t first, uint32_t last)
3520 {
3521         int i;
3522
3523         for (i = first; i < last; i++) {
3524                 if ((mask[i / 32] & (1 << (i % 32))) == 0)
3525                         return (i);
3526         }
3527         return (-1);
3528 }
3529
3530 int
3531 ctl_set_mask(uint32_t *mask, uint32_t bit)
3532 {
3533         uint32_t chunk, piece;
3534
3535         chunk = bit >> 5;
3536         piece = bit % (sizeof(uint32_t) * 8);
3537
3538         if ((mask[chunk] & (1 << piece)) != 0)
3539                 return (-1);
3540         else
3541                 mask[chunk] |= (1 << piece);
3542
3543         return (0);
3544 }
3545
3546 int
3547 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3548 {
3549         uint32_t chunk, piece;
3550
3551         chunk = bit >> 5;
3552         piece = bit % (sizeof(uint32_t) * 8);
3553
3554         if ((mask[chunk] & (1 << piece)) == 0)
3555                 return (-1);
3556         else
3557                 mask[chunk] &= ~(1 << piece);
3558
3559         return (0);
3560 }
3561
3562 int
3563 ctl_is_set(uint32_t *mask, uint32_t bit)
3564 {
3565         uint32_t chunk, piece;
3566
3567         chunk = bit >> 5;
3568         piece = bit % (sizeof(uint32_t) * 8);
3569
3570         if ((mask[chunk] & (1 << piece)) == 0)
3571                 return (0);
3572         else
3573                 return (1);
3574 }
3575
3576 static uint64_t
3577 ctl_get_prkey(struct ctl_lun *lun, uint32_t residx)
3578 {
3579         uint64_t *t;
3580
3581         t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3582         if (t == NULL)
3583                 return (0);
3584         return (t[residx % CTL_MAX_INIT_PER_PORT]);
3585 }
3586
3587 static void
3588 ctl_clr_prkey(struct ctl_lun *lun, uint32_t residx)
3589 {
3590         uint64_t *t;
3591
3592         t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3593         if (t == NULL)
3594                 return;
3595         t[residx % CTL_MAX_INIT_PER_PORT] = 0;
3596 }
3597
3598 static void
3599 ctl_alloc_prkey(struct ctl_lun *lun, uint32_t residx)
3600 {
3601         uint64_t *p;
3602         u_int i;
3603
3604         i = residx/CTL_MAX_INIT_PER_PORT;
3605         if (lun->pr_keys[i] != NULL)
3606                 return;
3607         mtx_unlock(&lun->lun_lock);
3608         p = malloc(sizeof(uint64_t) * CTL_MAX_INIT_PER_PORT, M_CTL,
3609             M_WAITOK | M_ZERO);
3610         mtx_lock(&lun->lun_lock);
3611         if (lun->pr_keys[i] == NULL)
3612                 lun->pr_keys[i] = p;
3613         else
3614                 free(p, M_CTL);
3615 }
3616
3617 static void
3618 ctl_set_prkey(struct ctl_lun *lun, uint32_t residx, uint64_t key)
3619 {
3620         uint64_t *t;
3621
3622         t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3623         KASSERT(t != NULL, ("prkey %d is not allocated", residx));
3624         t[residx % CTL_MAX_INIT_PER_PORT] = key;
3625 }
3626
3627 /*
3628  * ctl_softc, pool_name, total_ctl_io are passed in.
3629  * npool is passed out.
3630  */
3631 int
3632 ctl_pool_create(struct ctl_softc *ctl_softc, const char *pool_name,
3633                 uint32_t total_ctl_io, void **npool)
3634 {
3635 #ifdef IO_POOLS
3636         struct ctl_io_pool *pool;
3637
3638         pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3639                                             M_NOWAIT | M_ZERO);
3640         if (pool == NULL)
3641                 return (ENOMEM);
3642
3643         snprintf(pool->name, sizeof(pool->name), "CTL IO %s", pool_name);
3644         pool->ctl_softc = ctl_softc;
3645         pool->zone = uma_zsecond_create(pool->name, NULL,
3646             NULL, NULL, NULL, ctl_softc->io_zone);
3647         /* uma_prealloc(pool->zone, total_ctl_io); */
3648
3649         *npool = pool;
3650 #else
3651         *npool = ctl_softc->io_zone;
3652 #endif
3653         return (0);
3654 }
3655
3656 void
3657 ctl_pool_free(struct ctl_io_pool *pool)
3658 {
3659
3660         if (pool == NULL)
3661                 return;
3662
3663 #ifdef IO_POOLS
3664         uma_zdestroy(pool->zone);
3665         free(pool, M_CTL);
3666 #endif
3667 }
3668
3669 union ctl_io *
3670 ctl_alloc_io(void *pool_ref)
3671 {
3672         union ctl_io *io;
3673 #ifdef IO_POOLS
3674         struct ctl_io_pool *pool = (struct ctl_io_pool *)pool_ref;
3675
3676         io = uma_zalloc(pool->zone, M_WAITOK);
3677 #else
3678         io = uma_zalloc((uma_zone_t)pool_ref, M_WAITOK);
3679 #endif
3680         if (io != NULL)
3681                 io->io_hdr.pool = pool_ref;
3682         return (io);
3683 }
3684
3685 union ctl_io *
3686 ctl_alloc_io_nowait(void *pool_ref)
3687 {
3688         union ctl_io *io;
3689 #ifdef IO_POOLS
3690         struct ctl_io_pool *pool = (struct ctl_io_pool *)pool_ref;
3691
3692         io = uma_zalloc(pool->zone, M_NOWAIT);
3693 #else
3694         io = uma_zalloc((uma_zone_t)pool_ref, M_NOWAIT);
3695 #endif
3696         if (io != NULL)
3697                 io->io_hdr.pool = pool_ref;
3698         return (io);
3699 }
3700
3701 void
3702 ctl_free_io(union ctl_io *io)
3703 {
3704 #ifdef IO_POOLS
3705         struct ctl_io_pool *pool;
3706 #endif
3707
3708         if (io == NULL)
3709                 return;
3710
3711 #ifdef IO_POOLS
3712         pool = (struct ctl_io_pool *)io->io_hdr.pool;
3713         uma_zfree(pool->zone, io);
3714 #else
3715         uma_zfree((uma_zone_t)io->io_hdr.pool, io);
3716 #endif
3717 }
3718
3719 void
3720 ctl_zero_io(union ctl_io *io)
3721 {
3722         void *pool_ref;
3723
3724         if (io == NULL)
3725                 return;
3726
3727         /*
3728          * May need to preserve linked list pointers at some point too.
3729          */
3730         pool_ref = io->io_hdr.pool;
3731         memset(io, 0, sizeof(*io));
3732         io->io_hdr.pool = pool_ref;
3733 }
3734
3735 /*
3736  * This routine is currently used for internal copies of ctl_ios that need
3737  * to persist for some reason after we've already returned status to the
3738  * FETD.  (Thus the flag set.)
3739  *
3740  * XXX XXX
3741  * Note that this makes a blind copy of all fields in the ctl_io, except
3742  * for the pool reference.  This includes any memory that has been
3743  * allocated!  That memory will no longer be valid after done has been
3744  * called, so this would be VERY DANGEROUS for command that actually does
3745  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3746  * start and stop commands, which don't transfer any data, so this is not a
3747  * problem.  If it is used for anything else, the caller would also need to
3748  * allocate data buffer space and this routine would need to be modified to
3749  * copy the data buffer(s) as well.
3750  */
3751 void
3752 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3753 {
3754         void *pool_ref;
3755
3756         if ((src == NULL)
3757          || (dest == NULL))
3758                 return;
3759
3760         /*
3761          * May need to preserve linked list pointers at some point too.
3762          */
3763         pool_ref = dest->io_hdr.pool;
3764
3765         memcpy(dest, src, MIN(sizeof(*src), sizeof(*dest)));
3766
3767         dest->io_hdr.pool = pool_ref;
3768         /*
3769          * We need to know that this is an internal copy, and doesn't need
3770          * to get passed back to the FETD that allocated it.
3771          */
3772         dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3773 }
3774
3775 int
3776 ctl_expand_number(const char *buf, uint64_t *num)
3777 {
3778         char *endptr;
3779         uint64_t number;
3780         unsigned shift;
3781
3782         number = strtoq(buf, &endptr, 0);
3783
3784         switch (tolower((unsigned char)*endptr)) {
3785         case 'e':
3786                 shift = 60;
3787                 break;
3788         case 'p':
3789                 shift = 50;
3790                 break;
3791         case 't':
3792                 shift = 40;
3793                 break;
3794         case 'g':
3795                 shift = 30;
3796                 break;
3797         case 'm':
3798                 shift = 20;
3799                 break;
3800         case 'k':
3801                 shift = 10;
3802                 break;
3803         case 'b':
3804         case '\0': /* No unit. */
3805                 *num = number;
3806                 return (0);
3807         default:
3808                 /* Unrecognized unit. */
3809                 return (-1);
3810         }
3811
3812         if ((number << shift) >> shift != number) {
3813                 /* Overflow */
3814                 return (-1);
3815         }
3816         *num = number << shift;
3817         return (0);
3818 }
3819
3820
3821 /*
3822  * This routine could be used in the future to load default and/or saved
3823  * mode page parameters for a particuar lun.
3824  */
3825 static int
3826 ctl_init_page_index(struct ctl_lun *lun)
3827 {
3828         int i;
3829         struct ctl_page_index *page_index;
3830         const char *value;
3831         uint64_t ival;
3832
3833         memcpy(&lun->mode_pages.index, page_index_template,
3834                sizeof(page_index_template));
3835
3836         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
3837
3838                 page_index = &lun->mode_pages.index[i];
3839                 /*
3840                  * If this is a disk-only mode page, there's no point in
3841                  * setting it up.  For some pages, we have to have some
3842                  * basic information about the disk in order to calculate the
3843                  * mode page data.
3844                  */
3845                 if ((lun->be_lun->lun_type != T_DIRECT)
3846                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
3847                         continue;
3848
3849                 switch (page_index->page_code & SMPH_PC_MASK) {
3850                 case SMS_RW_ERROR_RECOVERY_PAGE: {
3851                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3852                                 panic("subpage is incorrect!");
3853                         memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_CURRENT],
3854                                &rw_er_page_default,
3855                                sizeof(rw_er_page_default));
3856                         memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_CHANGEABLE],
3857                                &rw_er_page_changeable,
3858                                sizeof(rw_er_page_changeable));
3859                         memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_DEFAULT],
3860                                &rw_er_page_default,
3861                                sizeof(rw_er_page_default));
3862                         memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_SAVED],
3863                                &rw_er_page_default,
3864                                sizeof(rw_er_page_default));
3865                         page_index->page_data =
3866                                 (uint8_t *)lun->mode_pages.rw_er_page;
3867                         break;
3868                 }
3869                 case SMS_FORMAT_DEVICE_PAGE: {
3870                         struct scsi_format_page *format_page;
3871
3872                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3873                                 panic("subpage is incorrect!");
3874
3875                         /*
3876                          * Sectors per track are set above.  Bytes per
3877                          * sector need to be set here on a per-LUN basis.
3878                          */
3879                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
3880                                &format_page_default,
3881                                sizeof(format_page_default));
3882                         memcpy(&lun->mode_pages.format_page[
3883                                CTL_PAGE_CHANGEABLE], &format_page_changeable,
3884                                sizeof(format_page_changeable));
3885                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
3886                                &format_page_default,
3887                                sizeof(format_page_default));
3888                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
3889                                &format_page_default,
3890                                sizeof(format_page_default));
3891
3892                         format_page = &lun->mode_pages.format_page[
3893                                 CTL_PAGE_CURRENT];
3894                         scsi_ulto2b(lun->be_lun->blocksize,
3895                                     format_page->bytes_per_sector);
3896
3897                         format_page = &lun->mode_pages.format_page[
3898                                 CTL_PAGE_DEFAULT];
3899                         scsi_ulto2b(lun->be_lun->blocksize,
3900                                     format_page->bytes_per_sector);
3901
3902                         format_page = &lun->mode_pages.format_page[
3903                                 CTL_PAGE_SAVED];
3904                         scsi_ulto2b(lun->be_lun->blocksize,
3905                                     format_page->bytes_per_sector);
3906
3907                         page_index->page_data =
3908                                 (uint8_t *)lun->mode_pages.format_page;
3909                         break;
3910                 }
3911                 case SMS_RIGID_DISK_PAGE: {
3912                         struct scsi_rigid_disk_page *rigid_disk_page;
3913                         uint32_t sectors_per_cylinder;
3914                         uint64_t cylinders;
3915 #ifndef __XSCALE__
3916                         int shift;
3917 #endif /* !__XSCALE__ */
3918
3919                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3920                                 panic("invalid subpage value %d",
3921                                       page_index->subpage);
3922
3923                         /*
3924                          * Rotation rate and sectors per track are set
3925                          * above.  We calculate the cylinders here based on
3926                          * capacity.  Due to the number of heads and
3927                          * sectors per track we're using, smaller arrays
3928                          * may turn out to have 0 cylinders.  Linux and
3929                          * FreeBSD don't pay attention to these mode pages
3930                          * to figure out capacity, but Solaris does.  It
3931                          * seems to deal with 0 cylinders just fine, and
3932                          * works out a fake geometry based on the capacity.
3933                          */
3934                         memcpy(&lun->mode_pages.rigid_disk_page[
3935                                CTL_PAGE_DEFAULT], &rigid_disk_page_default,
3936                                sizeof(rigid_disk_page_default));
3937                         memcpy(&lun->mode_pages.rigid_disk_page[
3938                                CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
3939                                sizeof(rigid_disk_page_changeable));
3940
3941                         sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
3942                                 CTL_DEFAULT_HEADS;
3943
3944                         /*
3945                          * The divide method here will be more accurate,
3946                          * probably, but results in floating point being
3947                          * used in the kernel on i386 (__udivdi3()).  On the
3948                          * XScale, though, __udivdi3() is implemented in
3949                          * software.
3950                          *
3951                          * The shift method for cylinder calculation is
3952                          * accurate if sectors_per_cylinder is a power of
3953                          * 2.  Otherwise it might be slightly off -- you
3954                          * might have a bit of a truncation problem.
3955                          */
3956 #ifdef  __XSCALE__
3957                         cylinders = (lun->be_lun->maxlba + 1) /
3958                                 sectors_per_cylinder;
3959 #else
3960                         for (shift = 31; shift > 0; shift--) {
3961                                 if (sectors_per_cylinder & (1 << shift))
3962                                         break;
3963                         }
3964                         cylinders = (lun->be_lun->maxlba + 1) >> shift;
3965 #endif
3966
3967                         /*
3968                          * We've basically got 3 bytes, or 24 bits for the
3969                          * cylinder size in the mode page.  If we're over,
3970                          * just round down to 2^24.
3971                          */
3972                         if (cylinders > 0xffffff)
3973                                 cylinders = 0xffffff;
3974
3975                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
3976                                 CTL_PAGE_DEFAULT];
3977                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
3978
3979                         if ((value = ctl_get_opt(&lun->be_lun->options,
3980                             "rpm")) != NULL) {
3981                                 scsi_ulto2b(strtol(value, NULL, 0),
3982                                      rigid_disk_page->rotation_rate);
3983                         }
3984
3985                         memcpy(&lun->mode_pages.rigid_disk_page[CTL_PAGE_CURRENT],
3986                                &lun->mode_pages.rigid_disk_page[CTL_PAGE_DEFAULT],
3987                                sizeof(rigid_disk_page_default));
3988                         memcpy(&lun->mode_pages.rigid_disk_page[CTL_PAGE_SAVED],
3989                                &lun->mode_pages.rigid_disk_page[CTL_PAGE_DEFAULT],
3990                                sizeof(rigid_disk_page_default));
3991
3992                         page_index->page_data =
3993                                 (uint8_t *)lun->mode_pages.rigid_disk_page;
3994                         break;
3995                 }
3996                 case SMS_CACHING_PAGE: {
3997                         struct scsi_caching_page *caching_page;
3998
3999                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4000                                 panic("invalid subpage value %d",
4001                                       page_index->subpage);
4002                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
4003                                &caching_page_default,
4004                                sizeof(caching_page_default));
4005                         memcpy(&lun->mode_pages.caching_page[
4006                                CTL_PAGE_CHANGEABLE], &caching_page_changeable,
4007                                sizeof(caching_page_changeable));
4008                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4009                                &caching_page_default,
4010                                sizeof(caching_page_default));
4011                         caching_page = &lun->mode_pages.caching_page[
4012                             CTL_PAGE_SAVED];
4013                         value = ctl_get_opt(&lun->be_lun->options, "writecache");
4014                         if (value != NULL && strcmp(value, "off") == 0)
4015                                 caching_page->flags1 &= ~SCP_WCE;
4016                         value = ctl_get_opt(&lun->be_lun->options, "readcache");
4017                         if (value != NULL && strcmp(value, "off") == 0)
4018                                 caching_page->flags1 |= SCP_RCD;
4019                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
4020                                &lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4021                                sizeof(caching_page_default));
4022                         page_index->page_data =
4023                                 (uint8_t *)lun->mode_pages.caching_page;
4024                         break;
4025                 }
4026                 case SMS_CONTROL_MODE_PAGE: {
4027                         switch (page_index->subpage) {
4028                         case SMS_SUBPAGE_PAGE_0: {
4029                                 struct scsi_control_page *control_page;
4030
4031                                 memcpy(&lun->mode_pages.control_page[
4032                                     CTL_PAGE_DEFAULT],
4033                                        &control_page_default,
4034                                        sizeof(control_page_default));
4035                                 memcpy(&lun->mode_pages.control_page[
4036                                     CTL_PAGE_CHANGEABLE],
4037                                        &control_page_changeable,
4038                                        sizeof(control_page_changeable));
4039                                 memcpy(&lun->mode_pages.control_page[
4040                                     CTL_PAGE_SAVED],
4041                                        &control_page_default,
4042                                        sizeof(control_page_default));
4043                                 control_page = &lun->mode_pages.control_page[
4044                                     CTL_PAGE_SAVED];
4045                                 value = ctl_get_opt(&lun->be_lun->options,
4046                                     "reordering");
4047                                 if (value != NULL &&
4048                                     strcmp(value, "unrestricted") == 0) {
4049                                         control_page->queue_flags &=
4050                                             ~SCP_QUEUE_ALG_MASK;
4051                                         control_page->queue_flags |=
4052                                             SCP_QUEUE_ALG_UNRESTRICTED;
4053                                 }
4054                                 memcpy(&lun->mode_pages.control_page[
4055                                     CTL_PAGE_CURRENT],
4056                                        &lun->mode_pages.control_page[
4057                                     CTL_PAGE_SAVED],
4058                                        sizeof(control_page_default));
4059                                 page_index->page_data =
4060                                     (uint8_t *)lun->mode_pages.control_page;
4061                                 break;
4062                         }
4063                         case 0x01:
4064                                 memcpy(&lun->mode_pages.control_ext_page[
4065                                     CTL_PAGE_DEFAULT],
4066                                        &control_ext_page_default,
4067                                        sizeof(control_ext_page_default));
4068                                 memcpy(&lun->mode_pages.control_ext_page[
4069                                     CTL_PAGE_CHANGEABLE],
4070                                        &control_ext_page_changeable,
4071                                        sizeof(control_ext_page_changeable));
4072                                 memcpy(&lun->mode_pages.control_ext_page[
4073                                     CTL_PAGE_SAVED],
4074                                        &control_ext_page_default,
4075                                        sizeof(control_ext_page_default));
4076                                 memcpy(&lun->mode_pages.control_ext_page[
4077                                     CTL_PAGE_CURRENT],
4078                                        &lun->mode_pages.control_ext_page[
4079                                     CTL_PAGE_SAVED],
4080                                        sizeof(control_ext_page_default));
4081                                 page_index->page_data =
4082                                     (uint8_t *)lun->mode_pages.control_ext_page;
4083                                 break;
4084                         }
4085                         break;
4086                 }
4087                 case SMS_INFO_EXCEPTIONS_PAGE: {
4088                         switch (page_index->subpage) {
4089                         case SMS_SUBPAGE_PAGE_0:
4090                                 memcpy(&lun->mode_pages.ie_page[CTL_PAGE_CURRENT],
4091                                        &ie_page_default,
4092                                        sizeof(ie_page_default));
4093                                 memcpy(&lun->mode_pages.ie_page[
4094                                        CTL_PAGE_CHANGEABLE], &ie_page_changeable,
4095                                        sizeof(ie_page_changeable));
4096                                 memcpy(&lun->mode_pages.ie_page[CTL_PAGE_DEFAULT],
4097                                        &ie_page_default,
4098                                        sizeof(ie_page_default));
4099                                 memcpy(&lun->mode_pages.ie_page[CTL_PAGE_SAVED],
4100                                        &ie_page_default,
4101                                        sizeof(ie_page_default));
4102                                 page_index->page_data =
4103                                         (uint8_t *)lun->mode_pages.ie_page;
4104                                 break;
4105                         case 0x02: {
4106                                 struct ctl_logical_block_provisioning_page *page;
4107
4108                                 memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_DEFAULT],
4109                                        &lbp_page_default,
4110                                        sizeof(lbp_page_default));
4111                                 memcpy(&lun->mode_pages.lbp_page[
4112                                        CTL_PAGE_CHANGEABLE], &lbp_page_changeable,
4113                                        sizeof(lbp_page_changeable));
4114                                 memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_SAVED],
4115                                        &lbp_page_default,
4116                                        sizeof(lbp_page_default));
4117                                 page = &lun->mode_pages.lbp_page[CTL_PAGE_SAVED];
4118                                 value = ctl_get_opt(&lun->be_lun->options,
4119                                     "avail-threshold");
4120                                 if (value != NULL &&
4121                                     ctl_expand_number(value, &ival) == 0) {
4122                                         page->descr[0].flags |= SLBPPD_ENABLED |
4123                                             SLBPPD_ARMING_DEC;
4124                                         if (lun->be_lun->blocksize)
4125                                                 ival /= lun->be_lun->blocksize;
4126                                         else
4127                                                 ival /= 512;
4128                                         scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4129                                             page->descr[0].count);
4130                                 }
4131                                 value = ctl_get_opt(&lun->be_lun->options,
4132                                     "used-threshold");
4133                                 if (value != NULL &&
4134                                     ctl_expand_number(value, &ival) == 0) {
4135                                         page->descr[1].flags |= SLBPPD_ENABLED |
4136                                             SLBPPD_ARMING_INC;
4137                                         if (lun->be_lun->blocksize)
4138                                                 ival /= lun->be_lun->blocksize;
4139                                         else
4140                                                 ival /= 512;
4141                                         scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4142                                             page->descr[1].count);
4143                                 }
4144                                 value = ctl_get_opt(&lun->be_lun->options,
4145                                     "pool-avail-threshold");
4146                                 if (value != NULL &&
4147                                     ctl_expand_number(value, &ival) == 0) {
4148                                         page->descr[2].flags |= SLBPPD_ENABLED |
4149                                             SLBPPD_ARMING_DEC;
4150                                         if (lun->be_lun->blocksize)
4151                                                 ival /= lun->be_lun->blocksize;
4152                                         else
4153                                                 ival /= 512;
4154                                         scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4155                                             page->descr[2].count);
4156                                 }
4157                                 value = ctl_get_opt(&lun->be_lun->options,
4158                                     "pool-used-threshold");
4159                                 if (value != NULL &&
4160                                     ctl_expand_number(value, &ival) == 0) {
4161                                         page->descr[3].flags |= SLBPPD_ENABLED |
4162                                             SLBPPD_ARMING_INC;
4163                                         if (lun->be_lun->blocksize)
4164                                                 ival /= lun->be_lun->blocksize;
4165                                         else
4166                                                 ival /= 512;
4167                                         scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4168                                             page->descr[3].count);
4169                                 }
4170                                 memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_CURRENT],
4171                                        &lun->mode_pages.lbp_page[CTL_PAGE_SAVED],
4172                                        sizeof(lbp_page_default));
4173                                 page_index->page_data =
4174                                         (uint8_t *)lun->mode_pages.lbp_page;
4175                         }}
4176                         break;
4177                 }
4178                 case SMS_VENDOR_SPECIFIC_PAGE:{
4179                         switch (page_index->subpage) {
4180                         case DBGCNF_SUBPAGE_CODE: {
4181                                 struct copan_debugconf_subpage *current_page,
4182                                                                *saved_page;
4183
4184                                 memcpy(&lun->mode_pages.debugconf_subpage[
4185                                        CTL_PAGE_CURRENT],
4186                                        &debugconf_page_default,
4187                                        sizeof(debugconf_page_default));
4188                                 memcpy(&lun->mode_pages.debugconf_subpage[
4189                                        CTL_PAGE_CHANGEABLE],
4190                                        &debugconf_page_changeable,
4191                                        sizeof(debugconf_page_changeable));
4192                                 memcpy(&lun->mode_pages.debugconf_subpage[
4193                                        CTL_PAGE_DEFAULT],
4194                                        &debugconf_page_default,
4195                                        sizeof(debugconf_page_default));
4196                                 memcpy(&lun->mode_pages.debugconf_subpage[
4197                                        CTL_PAGE_SAVED],
4198                                        &debugconf_page_default,
4199                                        sizeof(debugconf_page_default));
4200                                 page_index->page_data =
4201                                         (uint8_t *)lun->mode_pages.debugconf_subpage;
4202
4203                                 current_page = (struct copan_debugconf_subpage *)
4204                                         (page_index->page_data +
4205                                          (page_index->page_len *
4206                                           CTL_PAGE_CURRENT));
4207                                 saved_page = (struct copan_debugconf_subpage *)
4208                                         (page_index->page_data +
4209                                          (page_index->page_len *
4210                                           CTL_PAGE_SAVED));
4211                                 break;
4212                         }
4213                         default:
4214                                 panic("invalid subpage value %d",
4215                                       page_index->subpage);
4216                                 break;
4217                         }
4218                         break;
4219                 }
4220                 default:
4221                         panic("invalid page value %d",
4222                               page_index->page_code & SMPH_PC_MASK);
4223                         break;
4224         }
4225         }
4226
4227         return (CTL_RETVAL_COMPLETE);
4228 }
4229
4230 static int
4231 ctl_init_log_page_index(struct ctl_lun *lun)
4232 {
4233         struct ctl_page_index *page_index;
4234         int i, j, k, prev;
4235
4236         memcpy(&lun->log_pages.index, log_page_index_template,
4237                sizeof(log_page_index_template));
4238
4239         prev = -1;
4240         for (i = 0, j = 0, k = 0; i < CTL_NUM_LOG_PAGES; i++) {
4241
4242                 page_index = &lun->log_pages.index[i];
4243                 /*
4244                  * If this is a disk-only mode page, there's no point in
4245                  * setting it up.  For some pages, we have to have some
4246                  * basic information about the disk in order to calculate the
4247                  * mode page data.
4248                  */
4249                 if ((lun->be_lun->lun_type != T_DIRECT)
4250                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
4251                         continue;
4252
4253                 if (page_index->page_code == SLS_LOGICAL_BLOCK_PROVISIONING &&
4254                      lun->backend->lun_attr == NULL)
4255                         continue;
4256
4257                 if (page_index->page_code != prev) {
4258                         lun->log_pages.pages_page[j] = page_index->page_code;
4259                         prev = page_index->page_code;
4260                         j++;
4261                 }
4262                 lun->log_pages.subpages_page[k*2] = page_index->page_code;
4263                 lun->log_pages.subpages_page[k*2+1] = page_index->subpage;
4264                 k++;
4265         }
4266         lun->log_pages.index[0].page_data = &lun->log_pages.pages_page[0];
4267         lun->log_pages.index[0].page_len = j;
4268         lun->log_pages.index[1].page_data = &lun->log_pages.subpages_page[0];
4269         lun->log_pages.index[1].page_len = k * 2;
4270         lun->log_pages.index[2].page_data = &lun->log_pages.lbp_page[0];
4271         lun->log_pages.index[2].page_len = 12*CTL_NUM_LBP_PARAMS;
4272         lun->log_pages.index[3].page_data = (uint8_t *)&lun->log_pages.stat_page;
4273         lun->log_pages.index[3].page_len = sizeof(lun->log_pages.stat_page);
4274
4275         return (CTL_RETVAL_COMPLETE);
4276 }
4277
4278 static int
4279 hex2bin(const char *str, uint8_t *buf, int buf_size)
4280 {
4281         int i;
4282         u_char c;
4283
4284         memset(buf, 0, buf_size);
4285         while (isspace(str[0]))
4286                 str++;
4287         if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X'))
4288                 str += 2;
4289         buf_size *= 2;
4290         for (i = 0; str[i] != 0 && i < buf_size; i++) {
4291                 c = str[i];
4292                 if (isdigit(c))
4293                         c -= '0';
4294                 else if (isalpha(c))
4295                         c -= isupper(c) ? 'A' - 10 : 'a' - 10;
4296                 else
4297                         break;
4298                 if (c >= 16)
4299                         break;
4300                 if ((i & 1) == 0)
4301                         buf[i / 2] |= (c << 4);
4302                 else
4303                         buf[i / 2] |= c;
4304         }
4305         return ((i + 1) / 2);
4306 }
4307
4308 /*
4309  * LUN allocation.
4310  *
4311  * Requirements:
4312  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4313  *   wants us to allocate the LUN and he can block.
4314  * - ctl_softc is always set
4315  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4316  *
4317  * Returns 0 for success, non-zero (errno) for failure.
4318  */
4319 static int
4320 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4321               struct ctl_be_lun *const be_lun)
4322 {
4323         struct ctl_lun *nlun, *lun;
4324         struct scsi_vpd_id_descriptor *desc;
4325         struct scsi_vpd_id_t10 *t10id;
4326         const char *eui, *naa, *scsiname, *vendor;
4327         int lun_number, i, lun_malloced;
4328         int devidlen, idlen1, idlen2 = 0, len;
4329
4330         if (be_lun == NULL)
4331                 return (EINVAL);
4332
4333         /*
4334          * We currently only support Direct Access or Processor LUN types.
4335          */
4336         switch (be_lun->lun_type) {
4337         case T_DIRECT:
4338                 break;
4339         case T_PROCESSOR:
4340                 break;
4341         case T_SEQUENTIAL:
4342         case T_CHANGER:
4343         default:
4344                 be_lun->lun_config_status(be_lun->be_lun,
4345                                           CTL_LUN_CONFIG_FAILURE);
4346                 break;
4347         }
4348         if (ctl_lun == NULL) {
4349                 lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4350                 lun_malloced = 1;
4351         } else {
4352                 lun_malloced = 0;
4353                 lun = ctl_lun;
4354         }
4355
4356         memset(lun, 0, sizeof(*lun));
4357         if (lun_malloced)
4358                 lun->flags = CTL_LUN_MALLOCED;
4359
4360         /* Generate LUN ID. */
4361         devidlen = max(CTL_DEVID_MIN_LEN,
4362             strnlen(be_lun->device_id, CTL_DEVID_LEN));
4363         idlen1 = sizeof(*t10id) + devidlen;
4364         len = sizeof(struct scsi_vpd_id_descriptor) + idlen1;
4365         scsiname = ctl_get_opt(&be_lun->options, "scsiname");
4366         if (scsiname != NULL) {
4367                 idlen2 = roundup2(strlen(scsiname) + 1, 4);
4368                 len += sizeof(struct scsi_vpd_id_descriptor) + idlen2;
4369         }
4370         eui = ctl_get_opt(&be_lun->options, "eui");
4371         if (eui != NULL) {
4372                 len += sizeof(struct scsi_vpd_id_descriptor) + 16;
4373         }
4374         naa = ctl_get_opt(&be_lun->options, "naa");
4375         if (naa != NULL) {
4376                 len += sizeof(struct scsi_vpd_id_descriptor) + 16;
4377         }
4378         lun->lun_devid = malloc(sizeof(struct ctl_devid) + len,
4379             M_CTL, M_WAITOK | M_ZERO);
4380         desc = (struct scsi_vpd_id_descriptor *)lun->lun_devid->data;
4381         desc->proto_codeset = SVPD_ID_CODESET_ASCII;
4382         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
4383         desc->length = idlen1;
4384         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
4385         memset(t10id->vendor, ' ', sizeof(t10id->vendor));
4386         if ((vendor = ctl_get_opt(&be_lun->options, "vendor")) == NULL) {
4387                 strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
4388         } else {
4389                 strncpy(t10id->vendor, vendor,
4390                     min(sizeof(t10id->vendor), strlen(vendor)));
4391         }
4392         strncpy((char *)t10id->vendor_spec_id,
4393             (char *)be_lun->device_id, devidlen);
4394         if (scsiname != NULL) {
4395                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4396                     desc->length);
4397                 desc->proto_codeset = SVPD_ID_CODESET_UTF8;
4398                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4399                     SVPD_ID_TYPE_SCSI_NAME;
4400                 desc->length = idlen2;
4401                 strlcpy(desc->identifier, scsiname, idlen2);
4402         }
4403         if (eui != NULL) {
4404                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4405                     desc->length);
4406                 desc->proto_codeset = SVPD_ID_CODESET_BINARY;
4407                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4408                     SVPD_ID_TYPE_EUI64;
4409                 desc->length = hex2bin(eui, desc->identifier, 16);
4410                 desc->length = desc->length > 12 ? 16 :
4411                     (desc->length > 8 ? 12 : 8);
4412                 len -= 16 - desc->length;
4413         }
4414         if (naa != NULL) {
4415                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4416                     desc->length);
4417                 desc->proto_codeset = SVPD_ID_CODESET_BINARY;
4418                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4419                     SVPD_ID_TYPE_NAA;
4420                 desc->length = hex2bin(naa, desc->identifier, 16);
4421                 desc->length = desc->length > 8 ? 16 : 8;
4422                 len -= 16 - desc->length;
4423         }
4424         lun->lun_devid->len = len;
4425
4426         mtx_lock(&ctl_softc->ctl_lock);
4427         /*
4428          * See if the caller requested a particular LUN number.  If so, see
4429          * if it is available.  Otherwise, allocate the first available LUN.
4430          */
4431         if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4432                 if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4433                  || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4434                         mtx_unlock(&ctl_softc->ctl_lock);
4435                         if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4436                                 printf("ctl: requested LUN ID %d is higher "
4437                                        "than CTL_MAX_LUNS - 1 (%d)\n",
4438                                        be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4439                         } else {
4440                                 /*
4441                                  * XXX KDM return an error, or just assign
4442                                  * another LUN ID in this case??
4443                                  */
4444                                 printf("ctl: requested LUN ID %d is already "
4445                                        "in use\n", be_lun->req_lun_id);
4446                         }
4447                         if (lun->flags & CTL_LUN_MALLOCED)
4448                                 free(lun, M_CTL);
4449                         be_lun->lun_config_status(be_lun->be_lun,
4450                                                   CTL_LUN_CONFIG_FAILURE);
4451                         return (ENOSPC);
4452                 }
4453                 lun_number = be_lun->req_lun_id;
4454         } else {
4455                 lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, 0, CTL_MAX_LUNS);
4456                 if (lun_number == -1) {
4457                         mtx_unlock(&ctl_softc->ctl_lock);
4458                         printf("ctl: can't allocate LUN, out of LUNs\n");
4459                         if (lun->flags & CTL_LUN_MALLOCED)
4460                                 free(lun, M_CTL);
4461                         be_lun->lun_config_status(be_lun->be_lun,
4462                                                   CTL_LUN_CONFIG_FAILURE);
4463                         return (ENOSPC);
4464                 }
4465         }
4466         ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4467
4468         mtx_init(&lun->lun_lock, "CTL LUN", NULL, MTX_DEF);
4469         lun->lun = lun_number;
4470         lun->be_lun = be_lun;
4471         /*
4472          * The processor LUN is always enabled.  Disk LUNs come on line
4473          * disabled, and must be enabled by the backend.
4474          */
4475         lun->flags |= CTL_LUN_DISABLED;
4476         lun->backend = be_lun->be;
4477         be_lun->ctl_lun = lun;
4478         be_lun->lun_id = lun_number;
4479         atomic_add_int(&be_lun->be->num_luns, 1);
4480         if (be_lun->flags & CTL_LUN_FLAG_OFFLINE)
4481                 lun->flags |= CTL_LUN_OFFLINE;
4482
4483         if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4484                 lun->flags |= CTL_LUN_STOPPED;
4485
4486         if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4487                 lun->flags |= CTL_LUN_INOPERABLE;
4488
4489         if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4490                 lun->flags |= CTL_LUN_PRIMARY_SC;
4491
4492         lun->ctl_softc = ctl_softc;
4493 #ifdef CTL_TIME_IO
4494         lun->last_busy = getsbinuptime();
4495 #endif
4496         TAILQ_INIT(&lun->ooa_queue);
4497         TAILQ_INIT(&lun->blocked_queue);
4498         STAILQ_INIT(&lun->error_list);
4499         ctl_tpc_lun_init(lun);
4500
4501         /*
4502          * Initialize the mode and log page index.
4503          */
4504         ctl_init_page_index(lun);
4505         ctl_init_log_page_index(lun);
4506
4507         /*
4508          * Now, before we insert this lun on the lun list, set the lun
4509          * inventory changed UA for all other luns.
4510          */
4511         STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4512                 mtx_lock(&nlun->lun_lock);
4513                 ctl_est_ua_all(nlun, -1, CTL_UA_LUN_CHANGE);
4514                 mtx_unlock(&nlun->lun_lock);
4515         }
4516
4517         STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4518
4519         ctl_softc->ctl_luns[lun_number] = lun;
4520
4521         ctl_softc->num_luns++;
4522
4523         /* Setup statistics gathering */
4524         lun->stats.device_type = be_lun->lun_type;
4525         lun->stats.lun_number = lun_number;
4526         if (lun->stats.device_type == T_DIRECT)
4527                 lun->stats.blocksize = be_lun->blocksize;
4528         else
4529                 lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4530         for (i = 0;i < CTL_MAX_PORTS;i++)
4531                 lun->stats.ports[i].targ_port = i;
4532
4533         mtx_unlock(&ctl_softc->ctl_lock);
4534
4535         lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4536         return (0);
4537 }
4538
4539 /*
4540  * Delete a LUN.
4541  * Assumptions:
4542  * - LUN has already been marked invalid and any pending I/O has been taken
4543  *   care of.
4544  */
4545 static int
4546 ctl_free_lun(struct ctl_lun *lun)
4547 {
4548         struct ctl_softc *softc;
4549         struct ctl_lun *nlun;
4550         int i;
4551
4552         softc = lun->ctl_softc;
4553
4554         mtx_assert(&softc->ctl_lock, MA_OWNED);
4555
4556         STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4557
4558         ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4559
4560         softc->ctl_luns[lun->lun] = NULL;
4561
4562         if (!TAILQ_EMPTY(&lun->ooa_queue))
4563                 panic("Freeing a LUN %p with outstanding I/O!!\n", lun);
4564
4565         softc->num_luns--;
4566
4567         /*
4568          * Tell the backend to free resources, if this LUN has a backend.
4569          */
4570         atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4571         lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4572
4573         ctl_tpc_lun_shutdown(lun);
4574         mtx_destroy(&lun->lun_lock);
4575         free(lun->lun_devid, M_CTL);
4576         for (i = 0; i < CTL_MAX_PORTS; i++)
4577                 free(lun->pending_ua[i], M_CTL);
4578         for (i = 0; i < CTL_MAX_PORTS; i++)
4579                 free(lun->pr_keys[i], M_CTL);
4580         free(lun->write_buffer, M_CTL);
4581         if (lun->flags & CTL_LUN_MALLOCED)
4582                 free(lun, M_CTL);
4583
4584         STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4585                 mtx_lock(&nlun->lun_lock);
4586                 ctl_est_ua_all(nlun, -1, CTL_UA_LUN_CHANGE);
4587                 mtx_unlock(&nlun->lun_lock);
4588         }
4589
4590         return (0);
4591 }
4592
4593 static void
4594 ctl_create_lun(struct ctl_be_lun *be_lun)
4595 {
4596         struct ctl_softc *softc;
4597
4598         softc = control_softc;
4599
4600         /*
4601          * ctl_alloc_lun() should handle all potential failure cases.
4602          */
4603         ctl_alloc_lun(softc, NULL, be_lun);
4604 }
4605
4606 int
4607 ctl_add_lun(struct ctl_be_lun *be_lun)
4608 {
4609         struct ctl_softc *softc = control_softc;
4610
4611         mtx_lock(&softc->ctl_lock);
4612         STAILQ_INSERT_TAIL(&softc->pending_lun_queue, be_lun, links);
4613         mtx_unlock(&softc->ctl_lock);
4614         wakeup(&softc->pending_lun_queue);
4615
4616         return (0);
4617 }
4618
4619 int
4620 ctl_enable_lun(struct ctl_be_lun *be_lun)
4621 {
4622         struct ctl_softc *softc;
4623         struct ctl_port *port, *nport;
4624         struct ctl_lun *lun;
4625         int retval;
4626
4627         lun = (struct ctl_lun *)be_lun->ctl_lun;
4628         softc = lun->ctl_softc;
4629
4630         mtx_lock(&softc->ctl_lock);
4631         mtx_lock(&lun->lun_lock);
4632         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4633                 /*
4634                  * eh?  Why did we get called if the LUN is already
4635                  * enabled?
4636                  */
4637                 mtx_unlock(&lun->lun_lock);
4638                 mtx_unlock(&softc->ctl_lock);
4639                 return (0);
4640         }
4641         lun->flags &= ~CTL_LUN_DISABLED;
4642         mtx_unlock(&lun->lun_lock);
4643
4644         for (port = STAILQ_FIRST(&softc->port_list); port != NULL; port = nport) {
4645                 nport = STAILQ_NEXT(port, links);
4646                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0 ||
4647                     port->lun_map != NULL || port->lun_enable == NULL)
4648                         continue;
4649
4650                 /*
4651                  * Drop the lock while we call the FETD's enable routine.
4652                  * This can lead to a callback into CTL (at least in the
4653                  * case of the internal initiator frontend.
4654                  */
4655                 mtx_unlock(&softc->ctl_lock);
4656                 retval = port->lun_enable(port->targ_lun_arg, lun->lun);
4657                 mtx_lock(&softc->ctl_lock);
4658                 if (retval != 0) {
4659                         printf("%s: FETD %s port %d returned error "
4660                                "%d for lun_enable on lun %jd\n",
4661                                __func__, port->port_name, port->targ_port,
4662                                retval, (intmax_t)lun->lun);
4663                 }
4664         }
4665
4666         mtx_unlock(&softc->ctl_lock);
4667         ctl_isc_announce_lun(lun);
4668
4669         return (0);
4670 }
4671
4672 int
4673 ctl_disable_lun(struct ctl_be_lun *be_lun)
4674 {
4675         struct ctl_softc *softc;
4676         struct ctl_port *port;
4677         struct ctl_lun *lun;
4678         int retval;
4679
4680         lun = (struct ctl_lun *)be_lun->ctl_lun;
4681         softc = lun->ctl_softc;
4682
4683         mtx_lock(&softc->ctl_lock);
4684         mtx_lock(&lun->lun_lock);
4685         if (lun->flags & CTL_LUN_DISABLED) {
4686                 mtx_unlock(&lun->lun_lock);
4687                 mtx_unlock(&softc->ctl_lock);
4688                 return (0);
4689         }
4690         lun->flags |= CTL_LUN_DISABLED;
4691         mtx_unlock(&lun->lun_lock);
4692
4693         STAILQ_FOREACH(port, &softc->port_list, links) {
4694                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0 ||
4695                     port->lun_map != NULL || port->lun_disable == NULL)
4696                         continue;
4697
4698                 /*
4699                  * Drop the lock before we call the frontend's disable
4700                  * routine, to avoid lock order reversals.
4701                  *
4702                  * XXX KDM what happens if the frontend list changes while
4703                  * we're traversing it?  It's unlikely, but should be handled.
4704                  */
4705                 mtx_unlock(&softc->ctl_lock);
4706                 retval = port->lun_disable(port->targ_lun_arg, lun->lun);
4707                 mtx_lock(&softc->ctl_lock);
4708                 if (retval != 0) {
4709                         printf("%s: FETD %s port %d returned error "
4710                                "%d for lun_disable on lun %jd\n",
4711                                __func__, port->port_name, port->targ_port,
4712                                retval, (intmax_t)lun->lun);
4713                 }
4714         }
4715
4716         mtx_unlock(&softc->ctl_lock);
4717         ctl_isc_announce_lun(lun);
4718
4719         return (0);
4720 }
4721
4722 int
4723 ctl_start_lun(struct ctl_be_lun *be_lun)
4724 {
4725         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4726
4727         mtx_lock(&lun->lun_lock);
4728         lun->flags &= ~CTL_LUN_STOPPED;
4729         mtx_unlock(&lun->lun_lock);
4730         return (0);
4731 }
4732
4733 int
4734 ctl_stop_lun(struct ctl_be_lun *be_lun)
4735 {
4736         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4737
4738         mtx_lock(&lun->lun_lock);
4739         lun->flags |= CTL_LUN_STOPPED;
4740         mtx_unlock(&lun->lun_lock);
4741         return (0);
4742 }
4743
4744 int
4745 ctl_lun_offline(struct ctl_be_lun *be_lun)
4746 {
4747         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4748
4749         mtx_lock(&lun->lun_lock);
4750         lun->flags |= CTL_LUN_OFFLINE;
4751         mtx_unlock(&lun->lun_lock);
4752         return (0);
4753 }
4754
4755 int
4756 ctl_lun_online(struct ctl_be_lun *be_lun)
4757 {
4758         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4759
4760         mtx_lock(&lun->lun_lock);
4761         lun->flags &= ~CTL_LUN_OFFLINE;
4762         mtx_unlock(&lun->lun_lock);
4763         return (0);
4764 }
4765
4766 int
4767 ctl_lun_primary(struct ctl_be_lun *be_lun)
4768 {
4769         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4770
4771         mtx_lock(&lun->lun_lock);
4772         lun->flags |= CTL_LUN_PRIMARY_SC;
4773         ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
4774         mtx_unlock(&lun->lun_lock);
4775         ctl_isc_announce_lun(lun);
4776         return (0);
4777 }
4778
4779 int
4780 ctl_lun_secondary(struct ctl_be_lun *be_lun)
4781 {
4782         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4783
4784         mtx_lock(&lun->lun_lock);
4785         lun->flags &= ~CTL_LUN_PRIMARY_SC;
4786         ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
4787         mtx_unlock(&lun->lun_lock);
4788         ctl_isc_announce_lun(lun);
4789         return (0);
4790 }
4791
4792 int
4793 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4794 {
4795         struct ctl_softc *softc;
4796         struct ctl_lun *lun;
4797
4798         lun = (struct ctl_lun *)be_lun->ctl_lun;
4799         softc = lun->ctl_softc;
4800
4801         mtx_lock(&lun->lun_lock);
4802
4803         /*
4804          * The LUN needs to be disabled before it can be marked invalid.
4805          */
4806         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4807                 mtx_unlock(&lun->lun_lock);
4808                 return (-1);
4809         }
4810         /*
4811          * Mark the LUN invalid.
4812          */
4813         lun->flags |= CTL_LUN_INVALID;
4814
4815         /*
4816          * If there is nothing in the OOA queue, go ahead and free the LUN.
4817          * If we have something in the OOA queue, we'll free it when the
4818          * last I/O completes.
4819          */
4820         if (TAILQ_EMPTY(&lun->ooa_queue)) {
4821                 mtx_unlock(&lun->lun_lock);
4822                 mtx_lock(&softc->ctl_lock);
4823                 ctl_free_lun(lun);
4824                 mtx_unlock(&softc->ctl_lock);
4825         } else
4826                 mtx_unlock(&lun->lun_lock);
4827
4828         return (0);
4829 }
4830
4831 int
4832 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
4833 {
4834         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4835
4836         mtx_lock(&lun->lun_lock);
4837         lun->flags |= CTL_LUN_INOPERABLE;
4838         mtx_unlock(&lun->lun_lock);
4839         return (0);
4840 }
4841
4842 int
4843 ctl_lun_operable(struct ctl_be_lun *be_lun)
4844 {
4845         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4846
4847         mtx_lock(&lun->lun_lock);
4848         lun->flags &= ~CTL_LUN_INOPERABLE;
4849         mtx_unlock(&lun->lun_lock);
4850         return (0);
4851 }
4852
4853 void
4854 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
4855 {
4856         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4857         union ctl_ha_msg msg;
4858
4859         mtx_lock(&lun->lun_lock);
4860         ctl_est_ua_all(lun, -1, CTL_UA_CAPACITY_CHANGED);
4861         mtx_unlock(&lun->lun_lock);
4862         if (lun->ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
4863                 /* Send msg to other side. */
4864                 bzero(&msg.ua, sizeof(msg.ua));
4865                 msg.hdr.msg_type = CTL_MSG_UA;
4866                 msg.hdr.nexus.initid = -1;
4867                 msg.hdr.nexus.targ_port = -1;
4868                 msg.hdr.nexus.targ_lun = lun->lun;
4869                 msg.hdr.nexus.targ_mapped_lun = lun->lun;
4870                 msg.ua.ua_all = 1;
4871                 msg.ua.ua_set = 1;
4872                 msg.ua.ua_type = CTL_UA_CAPACITY_CHANGED;
4873                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg.ua),
4874                     M_WAITOK);
4875         }
4876 }
4877
4878 /*
4879  * Backend "memory move is complete" callback for requests that never
4880  * make it down to say RAIDCore's configuration code.
4881  */
4882 int
4883 ctl_config_move_done(union ctl_io *io)
4884 {
4885         int retval;
4886
4887         CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
4888         KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
4889             ("Config I/O type isn't CTL_IO_SCSI (%d)!", io->io_hdr.io_type));
4890
4891         if ((io->io_hdr.port_status != 0) &&
4892             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
4893              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
4894                 /*
4895                  * For hardware error sense keys, the sense key
4896                  * specific value is defined to be a retry count,
4897                  * but we use it to pass back an internal FETD
4898                  * error code.  XXX KDM  Hopefully the FETD is only
4899                  * using 16 bits for an error code, since that's
4900                  * all the space we have in the sks field.
4901                  */
4902                 ctl_set_internal_failure(&io->scsiio,
4903                                          /*sks_valid*/ 1,
4904                                          /*retry_count*/
4905                                          io->io_hdr.port_status);
4906         }
4907
4908         if (ctl_debug & CTL_DEBUG_CDB_DATA)
4909                 ctl_data_print(io);
4910         if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) ||
4911             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
4912              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) ||
4913             ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
4914                 /*
4915                  * XXX KDM just assuming a single pointer here, and not a
4916                  * S/G list.  If we start using S/G lists for config data,
4917                  * we'll need to know how to clean them up here as well.
4918                  */
4919                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
4920                         free(io->scsiio.kern_data_ptr, M_CTL);
4921                 ctl_done(io);
4922                 retval = CTL_RETVAL_COMPLETE;
4923         } else {
4924                 /*
4925                  * XXX KDM now we need to continue data movement.  Some
4926                  * options:
4927                  * - call ctl_scsiio() again?  We don't do this for data
4928                  *   writes, because for those at least we know ahead of
4929                  *   time where the write will go and how long it is.  For
4930                  *   config writes, though, that information is largely
4931                  *   contained within the write itself, thus we need to
4932                  *   parse out the data again.
4933                  *
4934                  * - Call some other function once the data is in?
4935                  */
4936
4937                 /*
4938                  * XXX KDM call ctl_scsiio() again for now, and check flag
4939                  * bits to see whether we're allocated or not.
4940                  */
4941                 retval = ctl_scsiio(&io->scsiio);
4942         }
4943         return (retval);
4944 }
4945
4946 /*
4947  * This gets called by a backend driver when it is done with a
4948  * data_submit method.
4949  */
4950 void
4951 ctl_data_submit_done(union ctl_io *io)
4952 {
4953         /*
4954          * If the IO_CONT flag is set, we need to call the supplied
4955          * function to continue processing the I/O, instead of completing
4956          * the I/O just yet.
4957          *
4958          * If there is an error, though, we don't want to keep processing.
4959          * Instead, just send status back to the initiator.
4960          */
4961         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
4962             (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
4963             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
4964              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
4965                 io->scsiio.io_cont(io);
4966                 return;
4967         }
4968         ctl_done(io);
4969 }
4970
4971 /*
4972  * This gets called by a backend driver when it is done with a
4973  * configuration write.
4974  */
4975 void
4976 ctl_config_write_done(union ctl_io *io)
4977 {
4978         uint8_t *buf;
4979
4980         /*
4981          * If the IO_CONT flag is set, we need to call the supplied
4982          * function to continue processing the I/O, instead of completing
4983          * the I/O just yet.
4984          *
4985          * If there is an error, though, we don't want to keep processing.
4986          * Instead, just send status back to the initiator.
4987          */
4988         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
4989             (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
4990             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
4991              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
4992                 io->scsiio.io_cont(io);
4993                 return;
4994         }
4995         /*
4996          * Since a configuration write can be done for commands that actually
4997          * have data allocated, like write buffer, and commands that have
4998          * no data, like start/stop unit, we need to check here.
4999          */
5000         if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5001                 buf = io->scsiio.kern_data_ptr;
5002         else
5003                 buf = NULL;
5004         ctl_done(io);
5005         if (buf)
5006                 free(buf, M_CTL);
5007 }
5008
5009 void
5010 ctl_config_read_done(union ctl_io *io)
5011 {
5012         uint8_t *buf;
5013
5014         /*
5015          * If there is some error -- we are done, skip data transfer.
5016          */
5017         if ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0 ||
5018             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
5019              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
5020                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5021                         buf = io->scsiio.kern_data_ptr;
5022                 else
5023                         buf = NULL;
5024                 ctl_done(io);
5025                 if (buf)
5026                         free(buf, M_CTL);
5027                 return;
5028         }
5029
5030         /*
5031          * If the IO_CONT flag is set, we need to call the supplied
5032          * function to continue processing the I/O, instead of completing
5033          * the I/O just yet.
5034          */
5035         if (io->io_hdr.flags & CTL_FLAG_IO_CONT) {
5036                 io->scsiio.io_cont(io);
5037                 return;
5038         }
5039
5040         ctl_datamove(io);
5041 }
5042
5043 /*
5044  * SCSI release command.
5045  */
5046 int
5047 ctl_scsi_release(struct ctl_scsiio *ctsio)
5048 {
5049         int length, longid, thirdparty_id, resv_id;
5050         struct ctl_lun *lun;
5051         uint32_t residx;
5052
5053         length = 0;
5054         resv_id = 0;
5055
5056         CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
5057
5058         residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5059         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5060
5061         switch (ctsio->cdb[0]) {
5062         case RELEASE_10: {
5063                 struct scsi_release_10 *cdb;
5064
5065                 cdb = (struct scsi_release_10 *)ctsio->cdb;
5066
5067                 if (cdb->byte2 & SR10_LONGID)
5068                         longid = 1;
5069                 else
5070                         thirdparty_id = cdb->thirdparty_id;
5071
5072                 resv_id = cdb->resv_id;
5073                 length = scsi_2btoul(cdb->length);
5074                 break;
5075         }
5076         }
5077
5078
5079         /*
5080          * XXX KDM right now, we only support LUN reservation.  We don't
5081          * support 3rd party reservations, or extent reservations, which
5082          * might actually need the parameter list.  If we've gotten this
5083          * far, we've got a LUN reservation.  Anything else got kicked out
5084          * above.  So, according to SPC, ignore the length.
5085          */
5086         length = 0;
5087
5088         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5089          && (length > 0)) {
5090                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5091                 ctsio->kern_data_len = length;
5092                 ctsio->kern_total_len = length;
5093                 ctsio->kern_data_resid = 0;
5094                 ctsio->kern_rel_offset = 0;
5095                 ctsio->kern_sg_entries = 0;
5096                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5097                 ctsio->be_move_done = ctl_config_move_done;
5098                 ctl_datamove((union ctl_io *)ctsio);
5099
5100                 return (CTL_RETVAL_COMPLETE);
5101         }
5102
5103         if (length > 0)
5104                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5105
5106         mtx_lock(&lun->lun_lock);
5107
5108         /*
5109          * According to SPC, it is not an error for an intiator to attempt
5110          * to release a reservation on a LUN that isn't reserved, or that
5111          * is reserved by another initiator.  The reservation can only be
5112          * released, though, by the initiator who made it or by one of
5113          * several reset type events.
5114          */
5115         if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == residx))
5116                         lun->flags &= ~CTL_LUN_RESERVED;
5117
5118         mtx_unlock(&lun->lun_lock);
5119
5120         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5121                 free(ctsio->kern_data_ptr, M_CTL);
5122                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5123         }
5124
5125         ctl_set_success(ctsio);
5126         ctl_done((union ctl_io *)ctsio);
5127         return (CTL_RETVAL_COMPLETE);
5128 }
5129
5130 int
5131 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5132 {
5133         int extent, thirdparty, longid;
5134         int resv_id, length;
5135         uint64_t thirdparty_id;
5136         struct ctl_lun *lun;
5137         uint32_t residx;
5138
5139         extent = 0;
5140         thirdparty = 0;
5141         longid = 0;
5142         resv_id = 0;
5143         length = 0;
5144         thirdparty_id = 0;
5145
5146         CTL_DEBUG_PRINT(("ctl_reserve\n"));
5147
5148         residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5149         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5150
5151         switch (ctsio->cdb[0]) {
5152         case RESERVE_10: {
5153                 struct scsi_reserve_10 *cdb;
5154
5155                 cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5156
5157                 if (cdb->byte2 & SR10_LONGID)
5158                         longid = 1;
5159                 else
5160                         thirdparty_id = cdb->thirdparty_id;
5161
5162                 resv_id = cdb->resv_id;
5163                 length = scsi_2btoul(cdb->length);
5164                 break;
5165         }
5166         }
5167
5168         /*
5169          * XXX KDM right now, we only support LUN reservation.  We don't
5170          * support 3rd party reservations, or extent reservations, which
5171          * might actually need the parameter list.  If we've gotten this
5172          * far, we've got a LUN reservation.  Anything else got kicked out
5173          * above.  So, according to SPC, ignore the length.
5174          */
5175         length = 0;
5176
5177         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5178          && (length > 0)) {
5179                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5180                 ctsio->kern_data_len = length;
5181                 ctsio->kern_total_len = length;
5182                 ctsio->kern_data_resid = 0;
5183                 ctsio->kern_rel_offset = 0;
5184                 ctsio->kern_sg_entries = 0;
5185                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5186                 ctsio->be_move_done = ctl_config_move_done;
5187                 ctl_datamove((union ctl_io *)ctsio);
5188
5189                 return (CTL_RETVAL_COMPLETE);
5190         }
5191
5192         if (length > 0)
5193                 thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5194
5195         mtx_lock(&lun->lun_lock);
5196         if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx != residx)) {
5197                 ctl_set_reservation_conflict(ctsio);
5198                 goto bailout;
5199         }
5200
5201         lun->flags |= CTL_LUN_RESERVED;
5202         lun->res_idx = residx;
5203
5204         ctl_set_success(ctsio);
5205
5206 bailout:
5207         mtx_unlock(&lun->lun_lock);
5208
5209         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5210                 free(ctsio->kern_data_ptr, M_CTL);
5211                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5212         }
5213
5214         ctl_done((union ctl_io *)ctsio);
5215         return (CTL_RETVAL_COMPLETE);
5216 }
5217
5218 int
5219 ctl_start_stop(struct ctl_scsiio *ctsio)
5220 {
5221         struct scsi_start_stop_unit *cdb;
5222         struct ctl_lun *lun;
5223         int retval;
5224
5225         CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5226
5227         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5228         retval = 0;
5229
5230         cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5231
5232         /*
5233          * XXX KDM
5234          * We don't support the immediate bit on a stop unit.  In order to
5235          * do that, we would need to code up a way to know that a stop is
5236          * pending, and hold off any new commands until it completes, one
5237          * way or another.  Then we could accept or reject those commands
5238          * depending on its status.  We would almost need to do the reverse
5239          * of what we do below for an immediate start -- return the copy of
5240          * the ctl_io to the FETD with status to send to the host (and to
5241          * free the copy!) and then free the original I/O once the stop
5242          * actually completes.  That way, the OOA queue mechanism can work
5243          * to block commands that shouldn't proceed.  Another alternative
5244          * would be to put the copy in the queue in place of the original,
5245          * and return the original back to the caller.  That could be
5246          * slightly safer..
5247          */
5248         if ((cdb->byte2 & SSS_IMMED)
5249          && ((cdb->how & SSS_START) == 0)) {
5250                 ctl_set_invalid_field(ctsio,
5251                                       /*sks_valid*/ 1,
5252                                       /*command*/ 1,
5253                                       /*field*/ 1,
5254                                       /*bit_valid*/ 1,
5255                                       /*bit*/ 0);
5256                 ctl_done((union ctl_io *)ctsio);
5257                 return (CTL_RETVAL_COMPLETE);
5258         }
5259
5260         if ((lun->flags & CTL_LUN_PR_RESERVED)
5261          && ((cdb->how & SSS_START)==0)) {
5262                 uint32_t residx;
5263
5264                 residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5265                 if (ctl_get_prkey(lun, residx) == 0
5266                  || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5267
5268                         ctl_set_reservation_conflict(ctsio);
5269                         ctl_done((union ctl_io *)ctsio);
5270                         return (CTL_RETVAL_COMPLETE);
5271                 }
5272         }
5273
5274         /*
5275          * If there is no backend on this device, we can't start or stop
5276          * it.  In theory we shouldn't get any start/stop commands in the
5277          * first place at this level if the LUN doesn't have a backend.
5278          * That should get stopped by the command decode code.
5279          */
5280         if (lun->backend == NULL) {
5281                 ctl_set_invalid_opcode(ctsio);
5282                 ctl_done((union ctl_io *)ctsio);
5283                 return (CTL_RETVAL_COMPLETE);
5284         }
5285
5286         /*
5287          * XXX KDM Copan-specific offline behavior.
5288          * Figure out a reasonable way to port this?
5289          */
5290 #ifdef NEEDTOPORT
5291         mtx_lock(&lun->lun_lock);
5292
5293         if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5294          && (lun->flags & CTL_LUN_OFFLINE)) {
5295                 /*
5296                  * If the LUN is offline, and the on/offline bit isn't set,
5297                  * reject the start or stop.  Otherwise, let it through.
5298                  */
5299                 mtx_unlock(&lun->lun_lock);
5300                 ctl_set_lun_not_ready(ctsio);
5301                 ctl_done((union ctl_io *)ctsio);
5302         } else {
5303                 mtx_unlock(&lun->lun_lock);
5304 #endif /* NEEDTOPORT */
5305                 /*
5306                  * This could be a start or a stop when we're online,
5307                  * or a stop/offline or start/online.  A start or stop when
5308                  * we're offline is covered in the case above.
5309                  */
5310                 /*
5311                  * In the non-immediate case, we send the request to
5312                  * the backend and return status to the user when
5313                  * it is done.
5314                  *
5315                  * In the immediate case, we allocate a new ctl_io
5316                  * to hold a copy of the request, and send that to
5317                  * the backend.  We then set good status on the
5318                  * user's request and return it immediately.
5319                  */
5320                 if (cdb->byte2 & SSS_IMMED) {
5321                         union ctl_io *new_io;
5322
5323                         new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5324                         ctl_copy_io((union ctl_io *)ctsio, new_io);
5325                         retval = lun->backend->config_write(new_io);
5326                         ctl_set_success(ctsio);
5327                         ctl_done((union ctl_io *)ctsio);
5328                 } else {
5329                         retval = lun->backend->config_write(
5330                                 (union ctl_io *)ctsio);
5331                 }
5332 #ifdef NEEDTOPORT
5333         }
5334 #endif
5335         return (retval);
5336 }
5337
5338 /*
5339  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5340  * we don't really do anything with the LBA and length fields if the user
5341  * passes them in.  Instead we'll just flush out the cache for the entire
5342  * LUN.
5343  */
5344 int
5345 ctl_sync_cache(struct ctl_scsiio *ctsio)
5346 {
5347         struct ctl_lun *lun;
5348         struct ctl_softc *softc;
5349         struct ctl_lba_len_flags *lbalen;
5350         uint64_t starting_lba;
5351         uint32_t block_count;
5352         int retval;
5353         uint8_t byte2;
5354
5355         CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5356
5357         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5358         softc = lun->ctl_softc;
5359         retval = 0;
5360
5361         switch (ctsio->cdb[0]) {
5362         case SYNCHRONIZE_CACHE: {
5363                 struct scsi_sync_cache *cdb;
5364                 cdb = (struct scsi_sync_cache *)ctsio->cdb;
5365
5366                 starting_lba = scsi_4btoul(cdb->begin_lba);
5367                 block_count = scsi_2btoul(cdb->lb_count);
5368                 byte2 = cdb->byte2;
5369                 break;
5370         }
5371         case SYNCHRONIZE_CACHE_16: {
5372                 struct scsi_sync_cache_16 *cdb;
5373                 cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5374
5375                 starting_lba = scsi_8btou64(cdb->begin_lba);
5376                 block_count = scsi_4btoul(cdb->lb_count);
5377                 byte2 = cdb->byte2;
5378                 break;
5379         }
5380         default:
5381                 ctl_set_invalid_opcode(ctsio);
5382                 ctl_done((union ctl_io *)ctsio);
5383                 goto bailout;
5384                 break; /* NOTREACHED */
5385         }
5386
5387         /*
5388          * We check the LBA and length, but don't do anything with them.
5389          * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5390          * get flushed.  This check will just help satisfy anyone who wants
5391          * to see an error for an out of range LBA.
5392          */
5393         if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5394                 ctl_set_lba_out_of_range(ctsio);
5395                 ctl_done((union ctl_io *)ctsio);
5396                 goto bailout;
5397         }
5398
5399         /*
5400          * If this LUN has no backend, we can't flush the cache anyway.
5401          */
5402         if (lun->backend == NULL) {
5403                 ctl_set_invalid_opcode(ctsio);
5404                 ctl_done((union ctl_io *)ctsio);
5405                 goto bailout;
5406         }
5407
5408         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5409         lbalen->lba = starting_lba;
5410         lbalen->len = block_count;
5411         lbalen->flags = byte2;
5412
5413         /*
5414          * Check to see whether we're configured to send the SYNCHRONIZE
5415          * CACHE command directly to the back end.
5416          */
5417         mtx_lock(&lun->lun_lock);
5418         if ((softc->flags & CTL_FLAG_REAL_SYNC)
5419          && (++(lun->sync_count) >= lun->sync_interval)) {
5420                 lun->sync_count = 0;
5421                 mtx_unlock(&lun->lun_lock);
5422                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5423         } else {
5424                 mtx_unlock(&lun->lun_lock);
5425                 ctl_set_success(ctsio);
5426                 ctl_done((union ctl_io *)ctsio);
5427         }
5428
5429 bailout:
5430
5431         return (retval);
5432 }
5433
5434 int
5435 ctl_format(struct ctl_scsiio *ctsio)
5436 {
5437         struct scsi_format *cdb;
5438         struct ctl_lun *lun;
5439         int length, defect_list_len;
5440
5441         CTL_DEBUG_PRINT(("ctl_format\n"));
5442
5443         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5444
5445         cdb = (struct scsi_format *)ctsio->cdb;
5446
5447         length = 0;
5448         if (cdb->byte2 & SF_FMTDATA) {
5449                 if (cdb->byte2 & SF_LONGLIST)
5450                         length = sizeof(struct scsi_format_header_long);
5451                 else
5452                         length = sizeof(struct scsi_format_header_short);
5453         }
5454
5455         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5456          && (length > 0)) {
5457                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5458                 ctsio->kern_data_len = length;
5459                 ctsio->kern_total_len = length;
5460                 ctsio->kern_data_resid = 0;
5461                 ctsio->kern_rel_offset = 0;
5462                 ctsio->kern_sg_entries = 0;
5463                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5464                 ctsio->be_move_done = ctl_config_move_done;
5465                 ctl_datamove((union ctl_io *)ctsio);
5466
5467                 return (CTL_RETVAL_COMPLETE);
5468         }
5469
5470         defect_list_len = 0;
5471
5472         if (cdb->byte2 & SF_FMTDATA) {
5473                 if (cdb->byte2 & SF_LONGLIST) {
5474                         struct scsi_format_header_long *header;
5475
5476                         header = (struct scsi_format_header_long *)
5477                                 ctsio->kern_data_ptr;
5478
5479                         defect_list_len = scsi_4btoul(header->defect_list_len);
5480                         if (defect_list_len != 0) {
5481                                 ctl_set_invalid_field(ctsio,
5482                                                       /*sks_valid*/ 1,
5483                                                       /*command*/ 0,
5484                                                       /*field*/ 2,
5485                                                       /*bit_valid*/ 0,
5486                                                       /*bit*/ 0);
5487                                 goto bailout;
5488                         }
5489                 } else {
5490                         struct scsi_format_header_short *header;
5491
5492                         header = (struct scsi_format_header_short *)
5493                                 ctsio->kern_data_ptr;
5494
5495                         defect_list_len = scsi_2btoul(header->defect_list_len);
5496                         if (defect_list_len != 0) {
5497                                 ctl_set_invalid_field(ctsio,
5498                                                       /*sks_valid*/ 1,
5499                                                       /*command*/ 0,
5500                                                       /*field*/ 2,
5501                                                       /*bit_valid*/ 0,
5502                                                       /*bit*/ 0);
5503                                 goto bailout;
5504                         }
5505                 }
5506         }
5507
5508         /*
5509          * The format command will clear out the "Medium format corrupted"
5510          * status if set by the configuration code.  That status is really
5511          * just a way to notify the host that we have lost the media, and
5512          * get them to issue a command that will basically make them think
5513          * they're blowing away the media.
5514          */
5515         mtx_lock(&lun->lun_lock);
5516         lun->flags &= ~CTL_LUN_INOPERABLE;
5517         mtx_unlock(&lun->lun_lock);
5518
5519         ctl_set_success(ctsio);
5520 bailout:
5521
5522         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5523                 free(ctsio->kern_data_ptr, M_CTL);
5524                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5525         }
5526
5527         ctl_done((union ctl_io *)ctsio);
5528         return (CTL_RETVAL_COMPLETE);
5529 }
5530
5531 int
5532 ctl_read_buffer(struct ctl_scsiio *ctsio)
5533 {
5534         struct scsi_read_buffer *cdb;
5535         struct ctl_lun *lun;
5536         int buffer_offset, len;
5537         static uint8_t descr[4];
5538         static uint8_t echo_descr[4] = { 0 };
5539
5540         CTL_DEBUG_PRINT(("ctl_read_buffer\n"));
5541
5542         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5543         cdb = (struct scsi_read_buffer *)ctsio->cdb;
5544
5545         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA &&
5546             (cdb->byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
5547             (cdb->byte2 & RWB_MODE) != RWB_MODE_DESCR) {
5548                 ctl_set_invalid_field(ctsio,
5549                                       /*sks_valid*/ 1,
5550                                       /*command*/ 1,
5551                                       /*field*/ 1,
5552                                       /*bit_valid*/ 1,
5553                                       /*bit*/ 4);
5554                 ctl_done((union ctl_io *)ctsio);
5555                 return (CTL_RETVAL_COMPLETE);
5556         }
5557
5558         len = scsi_3btoul(cdb->length);
5559         buffer_offset = scsi_3btoul(cdb->offset);
5560
5561         if (buffer_offset + len > CTL_WRITE_BUFFER_SIZE) {
5562                 ctl_set_invalid_field(ctsio,
5563                                       /*sks_valid*/ 1,
5564                                       /*command*/ 1,
5565                                       /*field*/ 6,
5566                                       /*bit_valid*/ 0,
5567                                       /*bit*/ 0);
5568                 ctl_done((union ctl_io *)ctsio);
5569                 return (CTL_RETVAL_COMPLETE);
5570         }
5571
5572         if ((cdb->byte2 & RWB_MODE) == RWB_MODE_DESCR) {
5573                 descr[0] = 0;
5574                 scsi_ulto3b(CTL_WRITE_BUFFER_SIZE, &descr[1]);
5575                 ctsio->kern_data_ptr = descr;
5576                 len = min(len, sizeof(descr));
5577         } else if ((cdb->byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
5578                 ctsio->kern_data_ptr = echo_descr;
5579                 len = min(len, sizeof(echo_descr));
5580         } else {
5581                 if (lun->write_buffer == NULL) {
5582                         lun->write_buffer = malloc(CTL_WRITE_BUFFER_SIZE,
5583                             M_CTL, M_WAITOK);
5584                 }
5585                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5586         }
5587         ctsio->kern_data_len = len;
5588         ctsio->kern_total_len = len;
5589         ctsio->kern_data_resid = 0;
5590         ctsio->kern_rel_offset = 0;
5591         ctsio->kern_sg_entries = 0;
5592         ctl_set_success(ctsio);
5593         ctsio->be_move_done = ctl_config_move_done;
5594         ctl_datamove((union ctl_io *)ctsio);
5595         return (CTL_RETVAL_COMPLETE);
5596 }
5597
5598 int
5599 ctl_write_buffer(struct ctl_scsiio *ctsio)
5600 {
5601         struct scsi_write_buffer *cdb;
5602         struct ctl_lun *lun;
5603         int buffer_offset, len;
5604
5605         CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5606
5607         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5608         cdb = (struct scsi_write_buffer *)ctsio->cdb;
5609
5610         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5611                 ctl_set_invalid_field(ctsio,
5612                                       /*sks_valid*/ 1,
5613                                       /*command*/ 1,
5614                                       /*field*/ 1,
5615                                       /*bit_valid*/ 1,
5616                                       /*bit*/ 4);
5617                 ctl_done((union ctl_io *)ctsio);
5618                 return (CTL_RETVAL_COMPLETE);
5619         }
5620
5621         len = scsi_3btoul(cdb->length);
5622         buffer_offset = scsi_3btoul(cdb->offset);
5623
5624         if (buffer_offset + len > CTL_WRITE_BUFFER_SIZE) {
5625                 ctl_set_invalid_field(ctsio,
5626                                       /*sks_valid*/ 1,
5627                                       /*command*/ 1,
5628                                       /*field*/ 6,
5629                                       /*bit_valid*/ 0,
5630                                       /*bit*/ 0);
5631                 ctl_done((union ctl_io *)ctsio);
5632                 return (CTL_RETVAL_COMPLETE);
5633         }
5634
5635         /*
5636          * If we've got a kernel request that hasn't been malloced yet,
5637          * malloc it and tell the caller the data buffer is here.
5638          */
5639         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5640                 if (lun->write_buffer == NULL) {
5641                         lun->write_buffer = malloc(CTL_WRITE_BUFFER_SIZE,
5642                             M_CTL, M_WAITOK);
5643                 }
5644                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5645                 ctsio->kern_data_len = len;
5646                 ctsio->kern_total_len = len;
5647                 ctsio->kern_data_resid = 0;
5648                 ctsio->kern_rel_offset = 0;
5649                 ctsio->kern_sg_entries = 0;
5650                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5651                 ctsio->be_move_done = ctl_config_move_done;
5652                 ctl_datamove((union ctl_io *)ctsio);
5653
5654                 return (CTL_RETVAL_COMPLETE);
5655         }
5656
5657         ctl_set_success(ctsio);
5658         ctl_done((union ctl_io *)ctsio);
5659         return (CTL_RETVAL_COMPLETE);
5660 }
5661
5662 int
5663 ctl_write_same(struct ctl_scsiio *ctsio)
5664 {
5665         struct ctl_lun *lun;
5666         struct ctl_lba_len_flags *lbalen;
5667         uint64_t lba;
5668         uint32_t num_blocks;
5669         int len, retval;
5670         uint8_t byte2;
5671
5672         retval = CTL_RETVAL_COMPLETE;
5673
5674         CTL_DEBUG_PRINT(("ctl_write_same\n"));
5675
5676         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5677
5678         switch (ctsio->cdb[0]) {
5679         case WRITE_SAME_10: {
5680                 struct scsi_write_same_10 *cdb;
5681
5682                 cdb = (struct scsi_write_same_10 *)ctsio->cdb;
5683
5684                 lba = scsi_4btoul(cdb->addr);
5685                 num_blocks = scsi_2btoul(cdb->length);
5686                 byte2 = cdb->byte2;
5687                 break;
5688         }
5689         case WRITE_SAME_16: {
5690                 struct scsi_write_same_16 *cdb;
5691
5692                 cdb = (struct scsi_write_same_16 *)ctsio->cdb;
5693
5694                 lba = scsi_8btou64(cdb->addr);
5695                 num_blocks = scsi_4btoul(cdb->length);
5696                 byte2 = cdb->byte2;
5697                 break;
5698         }
5699         default:
5700                 /*
5701                  * We got a command we don't support.  This shouldn't
5702                  * happen, commands should be filtered out above us.
5703                  */
5704                 ctl_set_invalid_opcode(ctsio);
5705                 ctl_done((union ctl_io *)ctsio);
5706
5707                 return (CTL_RETVAL_COMPLETE);
5708                 break; /* NOTREACHED */
5709         }
5710
5711         /* NDOB and ANCHOR flags can be used only together with UNMAP */
5712         if ((byte2 & SWS_UNMAP) == 0 &&
5713             (byte2 & (SWS_NDOB | SWS_ANCHOR)) != 0) {
5714                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
5715                     /*command*/ 1, /*field*/ 1, /*bit_valid*/ 1, /*bit*/ 0);
5716                 ctl_done((union ctl_io *)ctsio);
5717                 return (CTL_RETVAL_COMPLETE);
5718         }
5719
5720         /*
5721          * The first check is to make sure we're in bounds, the second
5722          * check is to catch wrap-around problems.  If the lba + num blocks
5723          * is less than the lba, then we've wrapped around and the block
5724          * range is invalid anyway.
5725          */
5726         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5727          || ((lba + num_blocks) < lba)) {
5728                 ctl_set_lba_out_of_range(ctsio);
5729                 ctl_done((union ctl_io *)ctsio);
5730                 return (CTL_RETVAL_COMPLETE);
5731         }
5732
5733         /* Zero number of blocks means "to the last logical block" */
5734         if (num_blocks == 0) {
5735                 if ((lun->be_lun->maxlba + 1) - lba > UINT32_MAX) {
5736                         ctl_set_invalid_field(ctsio,
5737                                               /*sks_valid*/ 0,
5738                                               /*command*/ 1,
5739                                               /*field*/ 0,
5740                                               /*bit_valid*/ 0,
5741                                               /*bit*/ 0);
5742                         ctl_done((union ctl_io *)ctsio);
5743                         return (CTL_RETVAL_COMPLETE);
5744                 }
5745                 num_blocks = (lun->be_lun->maxlba + 1) - lba;
5746         }
5747
5748         len = lun->be_lun->blocksize;
5749
5750         /*
5751          * If we've got a kernel request that hasn't been malloced yet,
5752          * malloc it and tell the caller the data buffer is here.
5753          */
5754         if ((byte2 & SWS_NDOB) == 0 &&
5755             (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5756                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5757                 ctsio->kern_data_len = len;
5758                 ctsio->kern_total_len = len;
5759                 ctsio->kern_data_resid = 0;
5760                 ctsio->kern_rel_offset = 0;
5761                 ctsio->kern_sg_entries = 0;
5762                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5763                 ctsio->be_move_done = ctl_config_move_done;
5764                 ctl_datamove((union ctl_io *)ctsio);
5765
5766                 return (CTL_RETVAL_COMPLETE);
5767         }
5768
5769         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5770         lbalen->lba = lba;
5771         lbalen->len = num_blocks;
5772         lbalen->flags = byte2;
5773         retval = lun->backend->config_write((union ctl_io *)ctsio);
5774
5775         return (retval);
5776 }
5777
5778 int
5779 ctl_unmap(struct ctl_scsiio *ctsio)
5780 {
5781         struct ctl_lun *lun;
5782         struct scsi_unmap *cdb;
5783         struct ctl_ptr_len_flags *ptrlen;
5784         struct scsi_unmap_header *hdr;
5785         struct scsi_unmap_desc *buf, *end, *endnz, *range;
5786         uint64_t lba;
5787         uint32_t num_blocks;
5788         int len, retval;
5789         uint8_t byte2;
5790
5791         retval = CTL_RETVAL_COMPLETE;
5792
5793         CTL_DEBUG_PRINT(("ctl_unmap\n"));
5794
5795         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5796         cdb = (struct scsi_unmap *)ctsio->cdb;
5797
5798         len = scsi_2btoul(cdb->length);
5799         byte2 = cdb->byte2;
5800
5801         /*
5802          * If we've got a kernel request that hasn't been malloced yet,
5803          * malloc it and tell the caller the data buffer is here.
5804          */
5805         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5806                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5807                 ctsio->kern_data_len = len;
5808                 ctsio->kern_total_len = len;
5809                 ctsio->kern_data_resid = 0;
5810                 ctsio->kern_rel_offset = 0;
5811                 ctsio->kern_sg_entries = 0;
5812                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5813                 ctsio->be_move_done = ctl_config_move_done;
5814                 ctl_datamove((union ctl_io *)ctsio);
5815
5816                 return (CTL_RETVAL_COMPLETE);
5817         }
5818
5819         len = ctsio->kern_total_len - ctsio->kern_data_resid;
5820         hdr = (struct scsi_unmap_header *)ctsio->kern_data_ptr;
5821         if (len < sizeof (*hdr) ||
5822             len < (scsi_2btoul(hdr->length) + sizeof(hdr->length)) ||
5823             len < (scsi_2btoul(hdr->desc_length) + sizeof (*hdr)) ||
5824             scsi_2btoul(hdr->desc_length) % sizeof(*buf) != 0) {
5825                 ctl_set_invalid_field(ctsio,
5826                                       /*sks_valid*/ 0,
5827                                       /*command*/ 0,
5828                                       /*field*/ 0,
5829                                       /*bit_valid*/ 0,
5830                                       /*bit*/ 0);
5831                 goto done;
5832         }
5833         len = scsi_2btoul(hdr->desc_length);
5834         buf = (struct scsi_unmap_desc *)(hdr + 1);
5835         end = buf + len / sizeof(*buf);
5836
5837         endnz = buf;
5838         for (range = buf; range < end; range++) {
5839                 lba = scsi_8btou64(range->lba);
5840                 num_blocks = scsi_4btoul(range->length);
5841                 if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5842                  || ((lba + num_blocks) < lba)) {
5843                         ctl_set_lba_out_of_range(ctsio);
5844                         ctl_done((union ctl_io *)ctsio);
5845                         return (CTL_RETVAL_COMPLETE);
5846                 }
5847                 if (num_blocks != 0)
5848                         endnz = range + 1;
5849         }
5850
5851         /*
5852          * Block backend can not handle zero last range.
5853          * Filter it out and return if there is nothing left.
5854          */
5855         len = (uint8_t *)endnz - (uint8_t *)buf;
5856         if (len == 0) {
5857                 ctl_set_success(ctsio);
5858                 goto done;
5859         }
5860
5861         mtx_lock(&lun->lun_lock);
5862         ptrlen = (struct ctl_ptr_len_flags *)
5863             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5864         ptrlen->ptr = (void *)buf;
5865         ptrlen->len = len;
5866         ptrlen->flags = byte2;
5867         ctl_check_blocked(lun);
5868         mtx_unlock(&lun->lun_lock);
5869
5870         retval = lun->backend->config_write((union ctl_io *)ctsio);
5871         return (retval);
5872
5873 done:
5874         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5875                 free(ctsio->kern_data_ptr, M_CTL);
5876                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5877         }
5878         ctl_done((union ctl_io *)ctsio);
5879         return (CTL_RETVAL_COMPLETE);
5880 }
5881
5882 /*
5883  * Note that this function currently doesn't actually do anything inside
5884  * CTL to enforce things if the DQue bit is turned on.
5885  *
5886  * Also note that this function can't be used in the default case, because
5887  * the DQue bit isn't set in the changeable mask for the control mode page
5888  * anyway.  This is just here as an example for how to implement a page
5889  * handler, and a placeholder in case we want to allow the user to turn
5890  * tagged queueing on and off.
5891  *
5892  * The D_SENSE bit handling is functional, however, and will turn
5893  * descriptor sense on and off for a given LUN.
5894  */
5895 int
5896 ctl_control_page_handler(struct ctl_scsiio *ctsio,
5897                          struct ctl_page_index *page_index, uint8_t *page_ptr)
5898 {
5899         struct scsi_control_page *current_cp, *saved_cp, *user_cp;
5900         struct ctl_lun *lun;
5901         int set_ua;
5902         uint32_t initidx;
5903
5904         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5905         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5906         set_ua = 0;
5907
5908         user_cp = (struct scsi_control_page *)page_ptr;
5909         current_cp = (struct scsi_control_page *)
5910                 (page_index->page_data + (page_index->page_len *
5911                 CTL_PAGE_CURRENT));
5912         saved_cp = (struct scsi_control_page *)
5913                 (page_index->page_data + (page_index->page_len *
5914                 CTL_PAGE_SAVED));
5915
5916         mtx_lock(&lun->lun_lock);
5917         if (((current_cp->rlec & SCP_DSENSE) == 0)
5918          && ((user_cp->rlec & SCP_DSENSE) != 0)) {
5919                 /*
5920                  * Descriptor sense is currently turned off and the user
5921                  * wants to turn it on.
5922                  */
5923                 current_cp->rlec |= SCP_DSENSE;
5924                 saved_cp->rlec |= SCP_DSENSE;
5925                 lun->flags |= CTL_LUN_SENSE_DESC;
5926                 set_ua = 1;
5927         } else if (((current_cp->rlec & SCP_DSENSE) != 0)
5928                 && ((user_cp->rlec & SCP_DSENSE) == 0)) {
5929                 /*
5930                  * Descriptor sense is currently turned on, and the user
5931                  * wants to turn it off.
5932                  */
5933                 current_cp->rlec &= ~SCP_DSENSE;
5934                 saved_cp->rlec &= ~SCP_DSENSE;
5935                 lun->flags &= ~CTL_LUN_SENSE_DESC;
5936                 set_ua = 1;
5937         }
5938         if ((current_cp->queue_flags & SCP_QUEUE_ALG_MASK) !=
5939             (user_cp->queue_flags & SCP_QUEUE_ALG_MASK)) {
5940                 current_cp->queue_flags &= ~SCP_QUEUE_ALG_MASK;
5941                 current_cp->queue_flags |= user_cp->queue_flags & SCP_QUEUE_ALG_MASK;
5942                 saved_cp->queue_flags &= ~SCP_QUEUE_ALG_MASK;
5943                 saved_cp->queue_flags |= user_cp->queue_flags & SCP_QUEUE_ALG_MASK;
5944                 set_ua = 1;
5945         }
5946         if ((current_cp->eca_and_aen & SCP_SWP) !=
5947             (user_cp->eca_and_aen & SCP_SWP)) {
5948                 current_cp->eca_and_aen &= ~SCP_SWP;
5949                 current_cp->eca_and_aen |= user_cp->eca_and_aen & SCP_SWP;
5950                 saved_cp->eca_and_aen &= ~SCP_SWP;
5951                 saved_cp->eca_and_aen |= user_cp->eca_and_aen & SCP_SWP;
5952                 set_ua = 1;
5953         }
5954         if (set_ua != 0)
5955                 ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
5956         mtx_unlock(&lun->lun_lock);
5957
5958         return (0);
5959 }
5960
5961 int
5962 ctl_caching_sp_handler(struct ctl_scsiio *ctsio,
5963                      struct ctl_page_index *page_index, uint8_t *page_ptr)
5964 {
5965         struct scsi_caching_page *current_cp, *saved_cp, *user_cp;
5966         struct ctl_lun *lun;
5967         int set_ua;
5968         uint32_t initidx;
5969
5970         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5971         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5972         set_ua = 0;
5973
5974         user_cp = (struct scsi_caching_page *)page_ptr;
5975         current_cp = (struct scsi_caching_page *)
5976                 (page_index->page_data + (page_index->page_len *
5977                 CTL_PAGE_CURRENT));
5978         saved_cp = (struct scsi_caching_page *)
5979                 (page_index->page_data + (page_index->page_len *
5980                 CTL_PAGE_SAVED));
5981
5982         mtx_lock(&lun->lun_lock);
5983         if ((current_cp->flags1 & (SCP_WCE | SCP_RCD)) !=
5984             (user_cp->flags1 & (SCP_WCE | SCP_RCD))) {
5985                 current_cp->flags1 &= ~(SCP_WCE | SCP_RCD);
5986                 current_cp->flags1 |= user_cp->flags1 & (SCP_WCE | SCP_RCD);
5987                 saved_cp->flags1 &= ~(SCP_WCE | SCP_RCD);
5988                 saved_cp->flags1 |= user_cp->flags1 & (SCP_WCE | SCP_RCD);
5989                 set_ua = 1;
5990         }
5991         if (set_ua != 0)
5992                 ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
5993         mtx_unlock(&lun->lun_lock);
5994
5995         return (0);
5996 }
5997
5998 int
5999 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6000                                 struct ctl_page_index *page_index,
6001                                 uint8_t *page_ptr)
6002 {
6003         uint8_t *c;
6004         int i;
6005
6006         c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6007         ctl_time_io_secs =
6008                 (c[0] << 8) |
6009                 (c[1] << 0) |
6010                 0;
6011         CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6012         printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6013         printf("page data:");
6014         for (i=0; i<8; i++)
6015                 printf(" %.2x",page_ptr[i]);
6016         printf("\n");
6017         return (0);
6018 }
6019
6020 int
6021 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6022                                struct ctl_page_index *page_index,
6023                                int pc)
6024 {
6025         struct copan_debugconf_subpage *page;
6026
6027         page = (struct copan_debugconf_subpage *)page_index->page_data +
6028                 (page_index->page_len * pc);
6029
6030         switch (pc) {
6031         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6032         case SMS_PAGE_CTRL_DEFAULT >> 6:
6033         case SMS_PAGE_CTRL_SAVED >> 6:
6034                 /*
6035                  * We don't update the changable or default bits for this page.
6036                  */
6037                 break;
6038         case SMS_PAGE_CTRL_CURRENT >> 6:
6039                 page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6040                 page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6041                 break;
6042         default:
6043 #ifdef NEEDTOPORT
6044                 EPRINT(0, "Invalid PC %d!!", pc);
6045 #endif /* NEEDTOPORT */
6046                 break;
6047         }
6048         return (0);
6049 }
6050
6051
6052 static int
6053 ctl_do_mode_select(union ctl_io *io)
6054 {
6055         struct scsi_mode_page_header *page_header;
6056         struct ctl_page_index *page_index;
6057         struct ctl_scsiio *ctsio;
6058         int control_dev, page_len;
6059         int page_len_offset, page_len_size;
6060         union ctl_modepage_info *modepage_info;
6061         struct ctl_lun *lun;
6062         int *len_left, *len_used;
6063         int retval, i;
6064
6065         ctsio = &io->scsiio;
6066         page_index = NULL;
6067         page_len = 0;
6068         retval = CTL_RETVAL_COMPLETE;
6069
6070         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6071
6072         if (lun->be_lun->lun_type != T_DIRECT)
6073                 control_dev = 1;
6074         else
6075                 control_dev = 0;
6076
6077         modepage_info = (union ctl_modepage_info *)
6078                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6079         len_left = &modepage_info->header.len_left;
6080         len_used = &modepage_info->header.len_used;
6081
6082 do_next_page:
6083
6084         page_header = (struct scsi_mode_page_header *)
6085                 (ctsio->kern_data_ptr + *len_used);
6086
6087         if (*len_left == 0) {
6088                 free(ctsio->kern_data_ptr, M_CTL);
6089                 ctl_set_success(ctsio);
6090                 ctl_done((union ctl_io *)ctsio);
6091                 return (CTL_RETVAL_COMPLETE);
6092         } else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6093
6094                 free(ctsio->kern_data_ptr, M_CTL);
6095                 ctl_set_param_len_error(ctsio);
6096                 ctl_done((union ctl_io *)ctsio);
6097                 return (CTL_RETVAL_COMPLETE);
6098
6099         } else if ((page_header->page_code & SMPH_SPF)
6100                 && (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6101
6102                 free(ctsio->kern_data_ptr, M_CTL);
6103                 ctl_set_param_len_error(ctsio);
6104                 ctl_done((union ctl_io *)ctsio);
6105                 return (CTL_RETVAL_COMPLETE);
6106         }
6107
6108
6109         /*
6110          * XXX KDM should we do something with the block descriptor?
6111          */
6112         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6113
6114                 if ((control_dev != 0)
6115                  && (lun->mode_pages.index[i].page_flags &
6116                      CTL_PAGE_FLAG_DISK_ONLY))
6117                         continue;
6118
6119                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6120                     (page_header->page_code & SMPH_PC_MASK))
6121                         continue;
6122
6123                 /*
6124                  * If neither page has a subpage code, then we've got a
6125                  * match.
6126                  */
6127                 if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6128                  && ((page_header->page_code & SMPH_SPF) == 0)) {
6129                         page_index = &lun->mode_pages.index[i];
6130                         page_len = page_header->page_length;
6131                         break;
6132                 }
6133
6134                 /*
6135                  * If both pages have subpages, then the subpage numbers
6136                  * have to match.
6137                  */
6138                 if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6139                   && (page_header->page_code & SMPH_SPF)) {
6140                         struct scsi_mode_page_header_sp *sph;
6141
6142                         sph = (struct scsi_mode_page_header_sp *)page_header;
6143
6144                         if (lun->mode_pages.index[i].subpage ==
6145                             sph->subpage) {
6146                                 page_index = &lun->mode_pages.index[i];
6147                                 page_len = scsi_2btoul(sph->page_length);
6148                                 break;
6149                         }
6150                 }
6151         }
6152
6153         /*
6154          * If we couldn't find the page, or if we don't have a mode select
6155          * handler for it, send back an error to the user.
6156          */
6157         if ((page_index == NULL)
6158          || (page_index->select_handler == NULL)) {
6159                 ctl_set_invalid_field(ctsio,
6160                                       /*sks_valid*/ 1,
6161                                       /*command*/ 0,
6162                                       /*field*/ *len_used,
6163                                       /*bit_valid*/ 0,
6164                                       /*bit*/ 0);
6165                 free(ctsio->kern_data_ptr, M_CTL);
6166                 ctl_done((union ctl_io *)ctsio);
6167                 return (CTL_RETVAL_COMPLETE);
6168         }
6169
6170         if (page_index->page_code & SMPH_SPF) {
6171                 page_len_offset = 2;
6172                 page_len_size = 2;
6173         } else {
6174                 page_len_size = 1;
6175                 page_len_offset = 1;
6176         }
6177
6178         /*
6179          * If the length the initiator gives us isn't the one we specify in
6180          * the mode page header, or if they didn't specify enough data in
6181          * the CDB to avoid truncating this page, kick out the request.
6182          */
6183         if ((page_len != (page_index->page_len - page_len_offset -
6184                           page_len_size))
6185          || (*len_left < page_index->page_len)) {
6186
6187
6188                 ctl_set_invalid_field(ctsio,
6189                                       /*sks_valid*/ 1,
6190                                       /*command*/ 0,
6191                                       /*field*/ *len_used + page_len_offset,
6192                                       /*bit_valid*/ 0,
6193                                       /*bit*/ 0);
6194                 free(ctsio->kern_data_ptr, M_CTL);
6195                 ctl_done((union ctl_io *)ctsio);
6196                 return (CTL_RETVAL_COMPLETE);
6197         }
6198
6199         /*
6200          * Run through the mode page, checking to make sure that the bits
6201          * the user changed are actually legal for him to change.
6202          */
6203         for (i = 0; i < page_index->page_len; i++) {
6204                 uint8_t *user_byte, *change_mask, *current_byte;
6205                 int bad_bit;
6206                 int j;
6207
6208                 user_byte = (uint8_t *)page_header + i;
6209                 change_mask = page_index->page_data +
6210                               (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6211                 current_byte = page_index->page_data +
6212                                (page_index->page_len * CTL_PAGE_CURRENT) + i;
6213
6214                 /*
6215                  * Check to see whether the user set any bits in this byte
6216                  * that he is not allowed to set.
6217                  */
6218                 if ((*user_byte & ~(*change_mask)) ==
6219                     (*current_byte & ~(*change_mask)))
6220                         continue;
6221
6222                 /*
6223                  * Go through bit by bit to determine which one is illegal.
6224                  */
6225                 bad_bit = 0;
6226                 for (j = 7; j >= 0; j--) {
6227                         if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6228                             (((1 << i) & ~(*change_mask)) & *current_byte)) {
6229                                 bad_bit = i;
6230                                 break;
6231                         }
6232                 }
6233                 ctl_set_invalid_field(ctsio,
6234                                       /*sks_valid*/ 1,
6235                                       /*command*/ 0,
6236                                       /*field*/ *len_used + i,
6237                                       /*bit_valid*/ 1,
6238                                       /*bit*/ bad_bit);
6239                 free(ctsio->kern_data_ptr, M_CTL);
6240                 ctl_done((union ctl_io *)ctsio);
6241                 return (CTL_RETVAL_COMPLETE);
6242         }
6243
6244         /*
6245          * Decrement these before we call the page handler, since we may
6246          * end up getting called back one way or another before the handler
6247          * returns to this context.
6248          */
6249         *len_left -= page_index->page_len;
6250         *len_used += page_index->page_len;
6251
6252         retval = page_index->select_handler(ctsio, page_index,
6253                                             (uint8_t *)page_header);
6254
6255         /*
6256          * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6257          * wait until this queued command completes to finish processing
6258          * the mode page.  If it returns anything other than
6259          * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6260          * already set the sense information, freed the data pointer, and
6261          * completed the io for us.
6262          */
6263         if (retval != CTL_RETVAL_COMPLETE)
6264                 goto bailout_no_done;
6265
6266         /*
6267          * If the initiator sent us more than one page, parse the next one.
6268          */
6269         if (*len_left > 0)
6270                 goto do_next_page;
6271
6272         ctl_set_success(ctsio);
6273         free(ctsio->kern_data_ptr, M_CTL);
6274         ctl_done((union ctl_io *)ctsio);
6275
6276 bailout_no_done:
6277
6278         return (CTL_RETVAL_COMPLETE);
6279
6280 }
6281
6282 int
6283 ctl_mode_select(struct ctl_scsiio *ctsio)
6284 {
6285         int param_len, pf, sp;
6286         int header_size, bd_len;
6287         int len_left, len_used;
6288         struct ctl_page_index *page_index;
6289         struct ctl_lun *lun;
6290         int control_dev, page_len;
6291         union ctl_modepage_info *modepage_info;
6292         int retval;
6293
6294         pf = 0;
6295         sp = 0;
6296         page_len = 0;
6297         len_used = 0;
6298         len_left = 0;
6299         retval = 0;
6300         bd_len = 0;
6301         page_index = NULL;
6302
6303         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6304
6305         if (lun->be_lun->lun_type != T_DIRECT)
6306                 control_dev = 1;
6307         else
6308                 control_dev = 0;
6309
6310         switch (ctsio->cdb[0]) {
6311         case MODE_SELECT_6: {
6312                 struct scsi_mode_select_6 *cdb;
6313
6314                 cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6315
6316                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6317                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6318
6319                 param_len = cdb->length;
6320                 header_size = sizeof(struct scsi_mode_header_6);
6321                 break;
6322         }
6323         case MODE_SELECT_10: {
6324                 struct scsi_mode_select_10 *cdb;
6325
6326                 cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6327
6328                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6329                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6330
6331                 param_len = scsi_2btoul(cdb->length);
6332                 header_size = sizeof(struct scsi_mode_header_10);
6333                 break;
6334         }
6335         default:
6336                 ctl_set_invalid_opcode(ctsio);
6337                 ctl_done((union ctl_io *)ctsio);
6338                 return (CTL_RETVAL_COMPLETE);
6339                 break; /* NOTREACHED */
6340         }
6341
6342         /*
6343          * From SPC-3:
6344          * "A parameter list length of zero indicates that the Data-Out Buffer
6345          * shall be empty. This condition shall not be considered as an error."
6346          */
6347         if (param_len == 0) {
6348                 ctl_set_success(ctsio);
6349                 ctl_done((union ctl_io *)ctsio);
6350                 return (CTL_RETVAL_COMPLETE);
6351         }
6352
6353         /*
6354          * Since we'll hit this the first time through, prior to
6355          * allocation, we don't need to free a data buffer here.
6356          */
6357         if (param_len < header_size) {
6358                 ctl_set_param_len_error(ctsio);
6359                 ctl_done((union ctl_io *)ctsio);
6360                 return (CTL_RETVAL_COMPLETE);
6361         }
6362
6363         /*
6364          * Allocate the data buffer and grab the user's data.  In theory,
6365          * we shouldn't have to sanity check the parameter list length here
6366          * because the maximum size is 64K.  We should be able to malloc
6367          * that much without too many problems.
6368          */
6369         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6370                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6371                 ctsio->kern_data_len = param_len;
6372                 ctsio->kern_total_len = param_len;
6373                 ctsio->kern_data_resid = 0;
6374                 ctsio->kern_rel_offset = 0;
6375                 ctsio->kern_sg_entries = 0;
6376                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6377                 ctsio->be_move_done = ctl_config_move_done;
6378                 ctl_datamove((union ctl_io *)ctsio);
6379
6380                 return (CTL_RETVAL_COMPLETE);
6381         }
6382
6383         switch (ctsio->cdb[0]) {
6384         case MODE_SELECT_6: {
6385                 struct scsi_mode_header_6 *mh6;
6386
6387                 mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6388                 bd_len = mh6->blk_desc_len;
6389                 break;
6390         }
6391         case MODE_SELECT_10: {
6392                 struct scsi_mode_header_10 *mh10;
6393
6394                 mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6395                 bd_len = scsi_2btoul(mh10->blk_desc_len);
6396                 break;
6397         }
6398         default:
6399                 panic("Invalid CDB type %#x", ctsio->cdb[0]);
6400                 break;
6401         }
6402
6403         if (param_len < (header_size + bd_len)) {
6404                 free(ctsio->kern_data_ptr, M_CTL);
6405                 ctl_set_param_len_error(ctsio);
6406                 ctl_done((union ctl_io *)ctsio);
6407                 return (CTL_RETVAL_COMPLETE);
6408         }
6409
6410         /*
6411          * Set the IO_CONT flag, so that if this I/O gets passed to
6412          * ctl_config_write_done(), it'll get passed back to
6413          * ctl_do_mode_select() for further processing, or completion if
6414          * we're all done.
6415          */
6416         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6417         ctsio->io_cont = ctl_do_mode_select;
6418
6419         modepage_info = (union ctl_modepage_info *)
6420                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6421
6422         memset(modepage_info, 0, sizeof(*modepage_info));
6423
6424         len_left = param_len - header_size - bd_len;
6425         len_used = header_size + bd_len;
6426
6427         modepage_info->header.len_left = len_left;
6428         modepage_info->header.len_used = len_used;
6429
6430         return (ctl_do_mode_select((union ctl_io *)ctsio));
6431 }
6432
6433 int
6434 ctl_mode_sense(struct ctl_scsiio *ctsio)
6435 {
6436         struct ctl_lun *lun;
6437         int pc, page_code, dbd, llba, subpage;
6438         int alloc_len, page_len, header_len, total_len;
6439         struct scsi_mode_block_descr *block_desc;
6440         struct ctl_page_index *page_index;
6441         int control_dev;
6442
6443         dbd = 0;
6444         llba = 0;
6445         block_desc = NULL;
6446         page_index = NULL;
6447
6448         CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6449
6450         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6451
6452         if (lun->be_lun->lun_type != T_DIRECT)
6453                 control_dev = 1;
6454         else
6455                 control_dev = 0;
6456
6457         switch (ctsio->cdb[0]) {
6458         case MODE_SENSE_6: {
6459                 struct scsi_mode_sense_6 *cdb;
6460
6461                 cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6462
6463                 header_len = sizeof(struct scsi_mode_hdr_6);
6464                 if (cdb->byte2 & SMS_DBD)
6465                         dbd = 1;
6466                 else
6467                         header_len += sizeof(struct scsi_mode_block_descr);
6468
6469                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6470                 page_code = cdb->page & SMS_PAGE_CODE;
6471                 subpage = cdb->subpage;
6472                 alloc_len = cdb->length;
6473                 break;
6474         }
6475         case MODE_SENSE_10: {
6476                 struct scsi_mode_sense_10 *cdb;
6477
6478                 cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6479
6480                 header_len = sizeof(struct scsi_mode_hdr_10);
6481
6482                 if (cdb->byte2 & SMS_DBD)
6483                         dbd = 1;
6484                 else
6485                         header_len += sizeof(struct scsi_mode_block_descr);
6486                 if (cdb->byte2 & SMS10_LLBAA)
6487                         llba = 1;
6488                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6489                 page_code = cdb->page & SMS_PAGE_CODE;
6490                 subpage = cdb->subpage;
6491                 alloc_len = scsi_2btoul(cdb->length);
6492                 break;
6493         }
6494         default:
6495                 ctl_set_invalid_opcode(ctsio);
6496                 ctl_done((union ctl_io *)ctsio);
6497                 return (CTL_RETVAL_COMPLETE);
6498                 break; /* NOTREACHED */
6499         }
6500
6501         /*
6502          * We have to make a first pass through to calculate the size of
6503          * the pages that match the user's query.  Then we allocate enough
6504          * memory to hold it, and actually copy the data into the buffer.
6505          */
6506         switch (page_code) {
6507         case SMS_ALL_PAGES_PAGE: {
6508                 int i;
6509
6510                 page_len = 0;
6511
6512                 /*
6513                  * At the moment, values other than 0 and 0xff here are
6514                  * reserved according to SPC-3.
6515                  */
6516                 if ((subpage != SMS_SUBPAGE_PAGE_0)
6517                  && (subpage != SMS_SUBPAGE_ALL)) {
6518                         ctl_set_invalid_field(ctsio,
6519                                               /*sks_valid*/ 1,
6520                                               /*command*/ 1,
6521                                               /*field*/ 3,
6522                                               /*bit_valid*/ 0,
6523                                               /*bit*/ 0);
6524                         ctl_done((union ctl_io *)ctsio);
6525                         return (CTL_RETVAL_COMPLETE);
6526                 }
6527
6528                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6529                         if ((control_dev != 0)
6530                          && (lun->mode_pages.index[i].page_flags &
6531                              CTL_PAGE_FLAG_DISK_ONLY))
6532                                 continue;
6533
6534                         /*
6535                          * We don't use this subpage if the user didn't
6536                          * request all subpages.
6537                          */
6538                         if ((lun->mode_pages.index[i].subpage != 0)
6539                          && (subpage == SMS_SUBPAGE_PAGE_0))
6540                                 continue;
6541
6542 #if 0
6543                         printf("found page %#x len %d\n",
6544                                lun->mode_pages.index[i].page_code &
6545                                SMPH_PC_MASK,
6546                                lun->mode_pages.index[i].page_len);
6547 #endif
6548                         page_len += lun->mode_pages.index[i].page_len;
6549                 }
6550                 break;
6551         }
6552         default: {
6553                 int i;
6554
6555                 page_len = 0;
6556
6557                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6558                         /* Look for the right page code */
6559                         if ((lun->mode_pages.index[i].page_code &
6560                              SMPH_PC_MASK) != page_code)
6561                                 continue;
6562
6563                         /* Look for the right subpage or the subpage wildcard*/
6564                         if ((lun->mode_pages.index[i].subpage != subpage)
6565                          && (subpage != SMS_SUBPAGE_ALL))
6566                                 continue;
6567
6568                         /* Make sure the page is supported for this dev type */
6569                         if ((control_dev != 0)
6570                          && (lun->mode_pages.index[i].page_flags &
6571                              CTL_PAGE_FLAG_DISK_ONLY))
6572                                 continue;
6573
6574 #if 0
6575                         printf("found page %#x len %d\n",
6576                                lun->mode_pages.index[i].page_code &
6577                                SMPH_PC_MASK,
6578                                lun->mode_pages.index[i].page_len);
6579 #endif
6580
6581                         page_len += lun->mode_pages.index[i].page_len;
6582                 }
6583
6584                 if (page_len == 0) {
6585                         ctl_set_invalid_field(ctsio,
6586                                               /*sks_valid*/ 1,
6587                                               /*command*/ 1,
6588                                               /*field*/ 2,
6589                                               /*bit_valid*/ 1,
6590                                               /*bit*/ 5);
6591                         ctl_done((union ctl_io *)ctsio);
6592                         return (CTL_RETVAL_COMPLETE);
6593                 }
6594                 break;
6595         }
6596         }
6597
6598         total_len = header_len + page_len;
6599 #if 0
6600         printf("header_len = %d, page_len = %d, total_len = %d\n",
6601                header_len, page_len, total_len);
6602 #endif
6603
6604         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6605         ctsio->kern_sg_entries = 0;
6606         ctsio->kern_data_resid = 0;
6607         ctsio->kern_rel_offset = 0;
6608         if (total_len < alloc_len) {
6609                 ctsio->residual = alloc_len - total_len;
6610                 ctsio->kern_data_len = total_len;
6611                 ctsio->kern_total_len = total_len;
6612         } else {
6613                 ctsio->residual = 0;
6614                 ctsio->kern_data_len = alloc_len;
6615                 ctsio->kern_total_len = alloc_len;
6616         }
6617
6618         switch (ctsio->cdb[0]) {
6619         case MODE_SENSE_6: {
6620                 struct scsi_mode_hdr_6 *header;
6621
6622                 header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6623
6624                 header->datalen = MIN(total_len - 1, 254);
6625                 if (control_dev == 0) {
6626                         header->dev_specific = 0x10; /* DPOFUA */
6627                         if ((lun->be_lun->flags & CTL_LUN_FLAG_READONLY) ||
6628                             (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
6629                             .eca_and_aen & SCP_SWP) != 0)
6630                                     header->dev_specific |= 0x80; /* WP */
6631                 }
6632                 if (dbd)
6633                         header->block_descr_len = 0;
6634                 else
6635                         header->block_descr_len =
6636                                 sizeof(struct scsi_mode_block_descr);
6637                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6638                 break;
6639         }
6640         case MODE_SENSE_10: {
6641                 struct scsi_mode_hdr_10 *header;
6642                 int datalen;
6643
6644                 header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6645
6646                 datalen = MIN(total_len - 2, 65533);
6647                 scsi_ulto2b(datalen, header->datalen);
6648                 if (control_dev == 0) {
6649                         header->dev_specific = 0x10; /* DPOFUA */
6650                         if ((lun->be_lun->flags & CTL_LUN_FLAG_READONLY) ||
6651                             (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
6652                             .eca_and_aen & SCP_SWP) != 0)
6653                                     header->dev_specific |= 0x80; /* WP */
6654                 }
6655                 if (dbd)
6656                         scsi_ulto2b(0, header->block_descr_len);
6657                 else
6658                         scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
6659                                     header->block_descr_len);
6660                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6661                 break;
6662         }
6663         default:
6664                 panic("invalid CDB type %#x", ctsio->cdb[0]);
6665                 break; /* NOTREACHED */
6666         }
6667
6668         /*
6669          * If we've got a disk, use its blocksize in the block
6670          * descriptor.  Otherwise, just set it to 0.
6671          */
6672         if (dbd == 0) {
6673                 if (control_dev == 0)
6674                         scsi_ulto3b(lun->be_lun->blocksize,
6675                                     block_desc->block_len);
6676                 else
6677                         scsi_ulto3b(0, block_desc->block_len);
6678         }
6679
6680         switch (page_code) {
6681         case SMS_ALL_PAGES_PAGE: {
6682                 int i, data_used;
6683
6684                 data_used = header_len;
6685                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6686                         struct ctl_page_index *page_index;
6687
6688                         page_index = &lun->mode_pages.index[i];
6689
6690                         if ((control_dev != 0)
6691                          && (page_index->page_flags &
6692                             CTL_PAGE_FLAG_DISK_ONLY))
6693                                 continue;
6694
6695                         /*
6696                          * We don't use this subpage if the user didn't
6697                          * request all subpages.  We already checked (above)
6698                          * to make sure the user only specified a subpage
6699                          * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
6700                          */
6701                         if ((page_index->subpage != 0)
6702                          && (subpage == SMS_SUBPAGE_PAGE_0))
6703                                 continue;
6704
6705                         /*
6706                          * Call the handler, if it exists, to update the
6707                          * page to the latest values.
6708                          */
6709                         if (page_index->sense_handler != NULL)
6710                                 page_index->sense_handler(ctsio, page_index,pc);
6711
6712                         memcpy(ctsio->kern_data_ptr + data_used,
6713                                page_index->page_data +
6714                                (page_index->page_len * pc),
6715                                page_index->page_len);
6716                         data_used += page_index->page_len;
6717                 }
6718                 break;
6719         }
6720         default: {
6721                 int i, data_used;
6722
6723                 data_used = header_len;
6724
6725                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6726                         struct ctl_page_index *page_index;
6727
6728                         page_index = &lun->mode_pages.index[i];
6729
6730                         /* Look for the right page code */
6731                         if ((page_index->page_code & SMPH_PC_MASK) != page_code)
6732                                 continue;
6733
6734                         /* Look for the right subpage or the subpage wildcard*/
6735                         if ((page_index->subpage != subpage)
6736                          && (subpage != SMS_SUBPAGE_ALL))
6737                                 continue;
6738
6739                         /* Make sure the page is supported for this dev type */
6740                         if ((control_dev != 0)
6741                          && (page_index->page_flags &
6742                              CTL_PAGE_FLAG_DISK_ONLY))
6743                                 continue;
6744
6745                         /*
6746                          * Call the handler, if it exists, to update the
6747                          * page to the latest values.
6748                          */
6749                         if (page_index->sense_handler != NULL)
6750                                 page_index->sense_handler(ctsio, page_index,pc);
6751
6752                         memcpy(ctsio->kern_data_ptr + data_used,
6753                                page_index->page_data +
6754                                (page_index->page_len * pc),
6755                                page_index->page_len);
6756                         data_used += page_index->page_len;
6757                 }
6758                 break;
6759         }
6760         }
6761
6762         ctl_set_success(ctsio);
6763         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6764         ctsio->be_move_done = ctl_config_move_done;
6765         ctl_datamove((union ctl_io *)ctsio);
6766         return (CTL_RETVAL_COMPLETE);
6767 }
6768
6769 int
6770 ctl_lbp_log_sense_handler(struct ctl_scsiio *ctsio,
6771                                struct ctl_page_index *page_index,
6772                                int pc)
6773 {
6774         struct ctl_lun *lun;
6775         struct scsi_log_param_header *phdr;
6776         uint8_t *data;
6777         uint64_t val;
6778
6779         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6780         data = page_index->page_data;
6781
6782         if (lun->backend->lun_attr != NULL &&
6783             (val = lun->backend->lun_attr(lun->be_lun->be_lun, "blocksavail"))
6784              != UINT64_MAX) {
6785                 phdr = (struct scsi_log_param_header *)data;
6786                 scsi_ulto2b(0x0001, phdr->param_code);
6787                 phdr->param_control = SLP_LBIN | SLP_LP;
6788                 phdr->param_len = 8;
6789                 data = (uint8_t *)(phdr + 1);
6790                 scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6791                 data[4] = 0x02; /* per-pool */
6792                 data += phdr->param_len;
6793         }
6794
6795         if (lun->backend->lun_attr != NULL &&
6796             (val = lun->backend->lun_attr(lun->be_lun->be_lun, "blocksused"))
6797              != UINT64_MAX) {
6798                 phdr = (struct scsi_log_param_header *)data;
6799                 scsi_ulto2b(0x0002, phdr->param_code);
6800                 phdr->param_control = SLP_LBIN | SLP_LP;
6801                 phdr->param_len = 8;
6802                 data = (uint8_t *)(phdr + 1);
6803                 scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6804                 data[4] = 0x01; /* per-LUN */
6805                 data += phdr->param_len;
6806         }
6807
6808         if (lun->backend->lun_attr != NULL &&
6809             (val = lun->backend->lun_attr(lun->be_lun->be_lun, "poolblocksavail"))
6810              != UINT64_MAX) {
6811                 phdr = (struct scsi_log_param_header *)data;
6812                 scsi_ulto2b(0x00f1, phdr->param_code);
6813                 phdr->param_control = SLP_LBIN | SLP_LP;
6814                 phdr->param_len = 8;
6815                 data = (uint8_t *)(phdr + 1);
6816                 scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6817                 data[4] = 0x02; /* per-pool */
6818                 data += phdr->param_len;
6819         }
6820
6821         if (lun->backend->lun_attr != NULL &&
6822             (val = lun->backend->lun_attr(lun->be_lun->be_lun, "poolblocksused"))
6823              != UINT64_MAX) {
6824                 phdr = (struct scsi_log_param_header *)data;
6825                 scsi_ulto2b(0x00f2, phdr->param_code);
6826                 phdr->param_control = SLP_LBIN | SLP_LP;
6827                 phdr->param_len = 8;
6828                 data = (uint8_t *)(phdr + 1);
6829                 scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6830                 data[4] = 0x02; /* per-pool */
6831                 data += phdr->param_len;
6832         }
6833
6834         page_index->page_len = data - page_index->page_data;
6835         return (0);
6836 }
6837
6838 int
6839 ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
6840                                struct ctl_page_index *page_index,
6841                                int pc)
6842 {
6843         struct ctl_lun *lun;
6844         struct stat_page *data;
6845         uint64_t rn, wn, rb, wb;
6846         struct bintime rt, wt;
6847         int i;
6848
6849         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6850         data = (struct stat_page *)page_index->page_data;
6851
6852         scsi_ulto2b(SLP_SAP, data->sap.hdr.param_code);
6853         data->sap.hdr.param_control = SLP_LBIN;
6854         data->sap.hdr.param_len = sizeof(struct scsi_log_stat_and_perf) -
6855             sizeof(struct scsi_log_param_header);
6856         rn = wn = rb = wb = 0;
6857         bintime_clear(&rt);
6858         bintime_clear(&wt);
6859         for (i = 0; i < CTL_MAX_PORTS; i++) {
6860                 rn += lun->stats.ports[i].operations[CTL_STATS_READ];
6861                 wn += lun->stats.ports[i].operations[CTL_STATS_WRITE];
6862                 rb += lun->stats.ports[i].bytes[CTL_STATS_READ];
6863                 wb += lun->stats.ports[i].bytes[CTL_STATS_WRITE];
6864                 bintime_add(&rt, &lun->stats.ports[i].time[CTL_STATS_READ]);
6865                 bintime_add(&wt, &lun->stats.ports[i].time[CTL_STATS_WRITE]);
6866         }
6867         scsi_u64to8b(rn, data->sap.read_num);
6868         scsi_u64to8b(wn, data->sap.write_num);
6869         if (lun->stats.blocksize > 0) {
6870                 scsi_u64to8b(wb / lun->stats.blocksize,
6871                     data->sap.recvieved_lba);
6872                 scsi_u64to8b(rb / lun->stats.blocksize,
6873                     data->sap.transmitted_lba);
6874         }
6875         scsi_u64to8b((uint64_t)rt.sec * 1000 + rt.frac / (UINT64_MAX / 1000),
6876             data->sap.read_int);
6877         scsi_u64to8b((uint64_t)wt.sec * 1000 + wt.frac / (UINT64_MAX / 1000),
6878             data->sap.write_int);
6879         scsi_u64to8b(0, data->sap.weighted_num);
6880         scsi_u64to8b(0, data->sap.weighted_int);
6881         scsi_ulto2b(SLP_IT, data->it.hdr.param_code);
6882         data->it.hdr.param_control = SLP_LBIN;
6883         data->it.hdr.param_len = sizeof(struct scsi_log_idle_time) -
6884             sizeof(struct scsi_log_param_header);
6885 #ifdef CTL_TIME_IO
6886         scsi_u64to8b(lun->idle_time / SBT_1MS, data->it.idle_int);
6887 #endif
6888         scsi_ulto2b(SLP_TI, data->ti.hdr.param_code);
6889         data->it.hdr.param_control = SLP_LBIN;
6890         data->ti.hdr.param_len = sizeof(struct scsi_log_time_interval) -
6891             sizeof(struct scsi_log_param_header);
6892         scsi_ulto4b(3, data->ti.exponent);
6893         scsi_ulto4b(1, data->ti.integer);
6894
6895         page_index->page_len = sizeof(*data);
6896         return (0);
6897 }
6898
6899 int
6900 ctl_log_sense(struct ctl_scsiio *ctsio)
6901 {
6902         struct ctl_lun *lun;
6903         int i, pc, page_code, subpage;
6904         int alloc_len, total_len;
6905         struct ctl_page_index *page_index;
6906         struct scsi_log_sense *cdb;
6907         struct scsi_log_header *header;
6908
6909         CTL_DEBUG_PRINT(("ctl_log_sense\n"));
6910
6911         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6912         cdb = (struct scsi_log_sense *)ctsio->cdb;
6913         pc = (cdb->page & SLS_PAGE_CTRL_MASK) >> 6;
6914         page_code = cdb->page & SLS_PAGE_CODE;
6915         subpage = cdb->subpage;
6916         alloc_len = scsi_2btoul(cdb->length);
6917
6918         page_index = NULL;
6919         for (i = 0; i < CTL_NUM_LOG_PAGES; i++) {
6920                 page_index = &lun->log_pages.index[i];
6921
6922                 /* Look for the right page code */
6923                 if ((page_index->page_code & SL_PAGE_CODE) != page_code)
6924                         continue;
6925
6926                 /* Look for the right subpage or the subpage wildcard*/
6927                 if (page_index->subpage != subpage)
6928                         continue;
6929
6930                 break;
6931         }
6932         if (i >= CTL_NUM_LOG_PAGES) {
6933                 ctl_set_invalid_field(ctsio,
6934                                       /*sks_valid*/ 1,
6935                                       /*command*/ 1,
6936                                       /*field*/ 2,
6937                                       /*bit_valid*/ 0,
6938                                       /*bit*/ 0);
6939                 ctl_done((union ctl_io *)ctsio);
6940                 return (CTL_RETVAL_COMPLETE);
6941         }
6942
6943         total_len = sizeof(struct scsi_log_header) + page_index->page_len;
6944
6945         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6946         ctsio->kern_sg_entries = 0;
6947         ctsio->kern_data_resid = 0;
6948         ctsio->kern_rel_offset = 0;
6949         if (total_len < alloc_len) {
6950                 ctsio->residual = alloc_len - total_len;
6951                 ctsio->kern_data_len = total_len;
6952                 ctsio->kern_total_len = total_len;
6953         } else {
6954                 ctsio->residual = 0;
6955                 ctsio->kern_data_len = alloc_len;
6956                 ctsio->kern_total_len = alloc_len;
6957         }
6958
6959         header = (struct scsi_log_header *)ctsio->kern_data_ptr;
6960         header->page = page_index->page_code;
6961         if (page_index->subpage) {
6962                 header->page |= SL_SPF;
6963                 header->subpage = page_index->subpage;
6964         }
6965         scsi_ulto2b(page_index->page_len, header->datalen);
6966
6967         /*
6968          * Call the handler, if it exists, to update the
6969          * page to the latest values.
6970          */
6971         if (page_index->sense_handler != NULL)
6972                 page_index->sense_handler(ctsio, page_index, pc);
6973
6974         memcpy(header + 1, page_index->page_data, page_index->page_len);
6975
6976         ctl_set_success(ctsio);
6977         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6978         ctsio->be_move_done = ctl_config_move_done;
6979         ctl_datamove((union ctl_io *)ctsio);
6980         return (CTL_RETVAL_COMPLETE);
6981 }
6982
6983 int
6984 ctl_read_capacity(struct ctl_scsiio *ctsio)
6985 {
6986         struct scsi_read_capacity *cdb;
6987         struct scsi_read_capacity_data *data;
6988         struct ctl_lun *lun;
6989         uint32_t lba;
6990
6991         CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
6992
6993         cdb = (struct scsi_read_capacity *)ctsio->cdb;
6994
6995         lba = scsi_4btoul(cdb->addr);
6996         if (((cdb->pmi & SRC_PMI) == 0)
6997          && (lba != 0)) {
6998                 ctl_set_invalid_field(/*ctsio*/ ctsio,
6999                                       /*sks_valid*/ 1,
7000                                       /*command*/ 1,
7001                                       /*field*/ 2,
7002                                       /*bit_valid*/ 0,
7003                                       /*bit*/ 0);
7004                 ctl_done((union ctl_io *)ctsio);
7005                 return (CTL_RETVAL_COMPLETE);
7006         }
7007
7008         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7009
7010         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7011         data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
7012         ctsio->residual = 0;
7013         ctsio->kern_data_len = sizeof(*data);
7014         ctsio->kern_total_len = sizeof(*data);
7015         ctsio->kern_data_resid = 0;
7016         ctsio->kern_rel_offset = 0;
7017         ctsio->kern_sg_entries = 0;
7018
7019         /*
7020          * If the maximum LBA is greater than 0xfffffffe, the user must
7021          * issue a SERVICE ACTION IN (16) command, with the read capacity
7022          * serivce action set.
7023          */
7024         if (lun->be_lun->maxlba > 0xfffffffe)
7025                 scsi_ulto4b(0xffffffff, data->addr);
7026         else
7027                 scsi_ulto4b(lun->be_lun->maxlba, data->addr);
7028
7029         /*
7030          * XXX KDM this may not be 512 bytes...
7031          */
7032         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7033
7034         ctl_set_success(ctsio);
7035         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7036         ctsio->be_move_done = ctl_config_move_done;
7037         ctl_datamove((union ctl_io *)ctsio);
7038         return (CTL_RETVAL_COMPLETE);
7039 }
7040
7041 int
7042 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
7043 {
7044         struct scsi_read_capacity_16 *cdb;
7045         struct scsi_read_capacity_data_long *data;
7046         struct ctl_lun *lun;
7047         uint64_t lba;
7048         uint32_t alloc_len;
7049
7050         CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
7051
7052         cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
7053
7054         alloc_len = scsi_4btoul(cdb->alloc_len);
7055         lba = scsi_8btou64(cdb->addr);
7056
7057         if ((cdb->reladr & SRC16_PMI)
7058          && (lba != 0)) {
7059                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7060                                       /*sks_valid*/ 1,
7061                                       /*command*/ 1,
7062                                       /*field*/ 2,
7063                                       /*bit_valid*/ 0,
7064                                       /*bit*/ 0);
7065                 ctl_done((union ctl_io *)ctsio);
7066                 return (CTL_RETVAL_COMPLETE);
7067         }
7068
7069         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7070
7071         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7072         data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
7073
7074         if (sizeof(*data) < alloc_len) {
7075                 ctsio->residual = alloc_len - sizeof(*data);
7076                 ctsio->kern_data_len = sizeof(*data);
7077                 ctsio->kern_total_len = sizeof(*data);
7078         } else {
7079                 ctsio->residual = 0;
7080                 ctsio->kern_data_len = alloc_len;
7081                 ctsio->kern_total_len = alloc_len;
7082         }
7083         ctsio->kern_data_resid = 0;
7084         ctsio->kern_rel_offset = 0;
7085         ctsio->kern_sg_entries = 0;
7086
7087         scsi_u64to8b(lun->be_lun->maxlba, data->addr);
7088         /* XXX KDM this may not be 512 bytes... */
7089         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7090         data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
7091         scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
7092         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
7093                 data->lalba_lbp[0] |= SRC16_LBPME | SRC16_LBPRZ;
7094
7095         ctl_set_success(ctsio);
7096         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7097         ctsio->be_move_done = ctl_config_move_done;
7098         ctl_datamove((union ctl_io *)ctsio);
7099         return (CTL_RETVAL_COMPLETE);
7100 }
7101
7102 int
7103 ctl_get_lba_status(struct ctl_scsiio *ctsio)
7104 {
7105         struct scsi_get_lba_status *cdb;
7106         struct scsi_get_lba_status_data *data;
7107         struct ctl_lun *lun;
7108         struct ctl_lba_len_flags *lbalen;
7109         uint64_t lba;
7110         uint32_t alloc_len, total_len;
7111         int retval;
7112
7113         CTL_DEBUG_PRINT(("ctl_get_lba_status\n"));
7114
7115         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7116         cdb = (struct scsi_get_lba_status *)ctsio->cdb;
7117         lba = scsi_8btou64(cdb->addr);
7118         alloc_len = scsi_4btoul(cdb->alloc_len);
7119
7120         if (lba > lun->be_lun->maxlba) {
7121                 ctl_set_lba_out_of_range(ctsio);
7122                 ctl_done((union ctl_io *)ctsio);
7123                 return (CTL_RETVAL_COMPLETE);
7124         }
7125
7126         total_len = sizeof(*data) + sizeof(data->descr[0]);
7127         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7128         data = (struct scsi_get_lba_status_data *)ctsio->kern_data_ptr;
7129
7130         if (total_len < alloc_len) {
7131                 ctsio->residual = alloc_len - total_len;
7132                 ctsio->kern_data_len = total_len;
7133                 ctsio->kern_total_len = total_len;
7134         } else {
7135                 ctsio->residual = 0;
7136                 ctsio->kern_data_len = alloc_len;
7137                 ctsio->kern_total_len = alloc_len;
7138         }
7139         ctsio->kern_data_resid = 0;
7140         ctsio->kern_rel_offset = 0;
7141         ctsio->kern_sg_entries = 0;
7142
7143         /* Fill dummy data in case backend can't tell anything. */
7144         scsi_ulto4b(4 + sizeof(data->descr[0]), data->length);
7145         scsi_u64to8b(lba, data->descr[0].addr);
7146         scsi_ulto4b(MIN(UINT32_MAX, lun->be_lun->maxlba + 1 - lba),
7147             data->descr[0].length);
7148         data->descr[0].status = 0; /* Mapped or unknown. */
7149
7150         ctl_set_success(ctsio);
7151         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7152         ctsio->be_move_done = ctl_config_move_done;
7153
7154         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
7155         lbalen->lba = lba;
7156         lbalen->len = total_len;
7157         lbalen->flags = 0;
7158         retval = lun->backend->config_read((union ctl_io *)ctsio);
7159         return (CTL_RETVAL_COMPLETE);
7160 }
7161
7162 int
7163 ctl_read_defect(struct ctl_scsiio *ctsio)
7164 {
7165         struct scsi_read_defect_data_10 *ccb10;
7166         struct scsi_read_defect_data_12 *ccb12;
7167         struct scsi_read_defect_data_hdr_10 *data10;
7168         struct scsi_read_defect_data_hdr_12 *data12;
7169         uint32_t alloc_len, data_len;
7170         uint8_t format;
7171
7172         CTL_DEBUG_PRINT(("ctl_read_defect\n"));
7173
7174         if (ctsio->cdb[0] == READ_DEFECT_DATA_10) {
7175                 ccb10 = (struct scsi_read_defect_data_10 *)&ctsio->cdb;
7176                 format = ccb10->format;
7177                 alloc_len = scsi_2btoul(ccb10->alloc_length);
7178                 data_len = sizeof(*data10);
7179         } else {
7180                 ccb12 = (struct scsi_read_defect_data_12 *)&ctsio->cdb;
7181                 format = ccb12->format;
7182                 alloc_len = scsi_4btoul(ccb12->alloc_length);
7183                 data_len = sizeof(*data12);
7184         }
7185         if (alloc_len == 0) {
7186                 ctl_set_success(ctsio);
7187                 ctl_done((union ctl_io *)ctsio);
7188                 return (CTL_RETVAL_COMPLETE);
7189         }
7190
7191         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
7192         if (data_len < alloc_len) {
7193                 ctsio->residual = alloc_len - data_len;
7194                 ctsio->kern_data_len = data_len;
7195                 ctsio->kern_total_len = data_len;
7196         } else {
7197                 ctsio->residual = 0;
7198                 ctsio->kern_data_len = alloc_len;
7199                 ctsio->kern_total_len = alloc_len;
7200         }
7201         ctsio->kern_data_resid = 0;
7202         ctsio->kern_rel_offset = 0;
7203         ctsio->kern_sg_entries = 0;
7204
7205         if (ctsio->cdb[0] == READ_DEFECT_DATA_10) {
7206                 data10 = (struct scsi_read_defect_data_hdr_10 *)
7207                     ctsio->kern_data_ptr;
7208                 data10->format = format;
7209                 scsi_ulto2b(0, data10->length);
7210         } else {
7211                 data12 = (struct scsi_read_defect_data_hdr_12 *)
7212                     ctsio->kern_data_ptr;
7213                 data12->format = format;
7214                 scsi_ulto2b(0, data12->generation);
7215                 scsi_ulto4b(0, data12->length);
7216         }
7217
7218         ctl_set_success(ctsio);
7219         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7220         ctsio->be_move_done = ctl_config_move_done;
7221         ctl_datamove((union ctl_io *)ctsio);
7222         return (CTL_RETVAL_COMPLETE);
7223 }
7224
7225 int
7226 ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
7227 {
7228         struct scsi_maintenance_in *cdb;
7229         int retval;
7230         int alloc_len, ext, total_len = 0, g, pc, pg, gs, os;
7231         int num_target_port_groups, num_target_ports;
7232         struct ctl_lun *lun;
7233         struct ctl_softc *softc;
7234         struct ctl_port *port;
7235         struct scsi_target_group_data *rtg_ptr;
7236         struct scsi_target_group_data_extended *rtg_ext_ptr;
7237         struct scsi_target_port_group_descriptor *tpg_desc;
7238
7239         CTL_DEBUG_PRINT(("ctl_report_tagret_port_groups\n"));
7240
7241         cdb = (struct scsi_maintenance_in *)ctsio->cdb;
7242         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7243         softc = lun->ctl_softc;
7244
7245         retval = CTL_RETVAL_COMPLETE;
7246
7247         switch (cdb->byte2 & STG_PDF_MASK) {
7248         case STG_PDF_LENGTH:
7249                 ext = 0;
7250                 break;
7251         case STG_PDF_EXTENDED:
7252                 ext = 1;
7253                 break;
7254         default:
7255                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7256                                       /*sks_valid*/ 1,
7257                                       /*command*/ 1,
7258                                       /*field*/ 2,
7259                                       /*bit_valid*/ 1,
7260                                       /*bit*/ 5);
7261                 ctl_done((union ctl_io *)ctsio);
7262                 return(retval);
7263         }
7264
7265         if (softc->is_single)
7266                 num_target_port_groups = 1;
7267         else
7268                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
7269         num_target_ports = 0;
7270         mtx_lock(&softc->ctl_lock);
7271         STAILQ_FOREACH(port, &softc->port_list, links) {
7272                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7273                         continue;
7274                 if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
7275                         continue;
7276                 num_target_ports++;
7277         }
7278         mtx_unlock(&softc->ctl_lock);
7279
7280         if (ext)
7281                 total_len = sizeof(struct scsi_target_group_data_extended);
7282         else
7283                 total_len = sizeof(struct scsi_target_group_data);
7284         total_len += sizeof(struct scsi_target_port_group_descriptor) *
7285                 num_target_port_groups +
7286             sizeof(struct scsi_target_port_descriptor) * num_target_ports;
7287
7288         alloc_len = scsi_4btoul(cdb->length);
7289
7290         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7291
7292         ctsio->kern_sg_entries = 0;
7293
7294         if (total_len < alloc_len) {
7295                 ctsio->residual = alloc_len - total_len;
7296                 ctsio->kern_data_len = total_len;
7297                 ctsio->kern_total_len = total_len;
7298         } else {
7299                 ctsio->residual = 0;
7300                 ctsio->kern_data_len = alloc_len;
7301                 ctsio->kern_total_len = alloc_len;
7302         }
7303         ctsio->kern_data_resid = 0;
7304         ctsio->kern_rel_offset = 0;
7305
7306         if (ext) {
7307                 rtg_ext_ptr = (struct scsi_target_group_data_extended *)
7308                     ctsio->kern_data_ptr;
7309                 scsi_ulto4b(total_len - 4, rtg_ext_ptr->length);
7310                 rtg_ext_ptr->format_type = 0x10;
7311                 rtg_ext_ptr->implicit_transition_time = 0;
7312                 tpg_desc = &rtg_ext_ptr->groups[0];
7313         } else {
7314                 rtg_ptr = (struct scsi_target_group_data *)
7315                     ctsio->kern_data_ptr;
7316                 scsi_ulto4b(total_len - 4, rtg_ptr->length);
7317                 tpg_desc = &rtg_ptr->groups[0];
7318         }
7319
7320         mtx_lock(&softc->ctl_lock);
7321         pg = softc->port_min / softc->port_cnt;
7322         if (softc->ha_link == CTL_HA_LINK_OFFLINE)
7323                 gs = TPG_ASYMMETRIC_ACCESS_UNAVAILABLE;
7324         else if (softc->ha_link == CTL_HA_LINK_UNKNOWN)
7325                 gs = TPG_ASYMMETRIC_ACCESS_TRANSITIONING;
7326         else if (softc->ha_mode == CTL_HA_MODE_ACT_STBY)
7327                 gs = TPG_ASYMMETRIC_ACCESS_STANDBY;
7328         else
7329                 gs = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7330         if (lun->flags & CTL_LUN_PRIMARY_SC) {
7331                 os = gs;
7332                 gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7333         } else
7334                 os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7335         for (g = 0; g < num_target_port_groups; g++) {
7336                 tpg_desc->pref_state = (g == pg) ? gs : os;
7337                 tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP |
7338                     TPG_U_SUP | TPG_T_SUP;
7339                 scsi_ulto2b(g + 1, tpg_desc->target_port_group);
7340                 tpg_desc->status = TPG_IMPLICIT;
7341                 pc = 0;
7342                 STAILQ_FOREACH(port, &softc->port_list, links) {
7343                         if (port->targ_port < g * softc->port_cnt ||
7344                             port->targ_port >= (g + 1) * softc->port_cnt)
7345                                 continue;
7346                         if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7347                                 continue;
7348                         if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
7349                                 continue;
7350                         scsi_ulto2b(port->targ_port, tpg_desc->descriptors[pc].
7351                             relative_target_port_identifier);
7352                         pc++;
7353                 }
7354                 tpg_desc->target_port_count = pc;
7355                 tpg_desc = (struct scsi_target_port_group_descriptor *)
7356                     &tpg_desc->descriptors[pc];
7357         }
7358         mtx_unlock(&softc->ctl_lock);
7359
7360         ctl_set_success(ctsio);
7361         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7362         ctsio->be_move_done = ctl_config_move_done;
7363         ctl_datamove((union ctl_io *)ctsio);
7364         return(retval);
7365 }
7366
7367 int
7368 ctl_report_supported_opcodes(struct ctl_scsiio *ctsio)
7369 {
7370         struct ctl_lun *lun;
7371         struct scsi_report_supported_opcodes *cdb;
7372         const struct ctl_cmd_entry *entry, *sentry;
7373         struct scsi_report_supported_opcodes_all *all;
7374         struct scsi_report_supported_opcodes_descr *descr;
7375         struct scsi_report_supported_opcodes_one *one;
7376         int retval;
7377         int alloc_len, total_len;
7378         int opcode, service_action, i, j, num;
7379
7380         CTL_DEBUG_PRINT(("ctl_report_supported_opcodes\n"));
7381
7382         cdb = (struct scsi_report_supported_opcodes *)ctsio->cdb;
7383         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7384
7385         retval = CTL_RETVAL_COMPLETE;
7386
7387         opcode = cdb->requested_opcode;
7388         service_action = scsi_2btoul(cdb->requested_service_action);
7389         switch (cdb->options & RSO_OPTIONS_MASK) {
7390         case RSO_OPTIONS_ALL:
7391                 num = 0;
7392                 for (i = 0; i < 256; i++) {
7393                         entry = &ctl_cmd_table[i];
7394                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7395                                 for (j = 0; j < 32; j++) {
7396                                         sentry = &((const struct ctl_cmd_entry *)
7397                                             entry->execute)[j];
7398                                         if (ctl_cmd_applicable(
7399                                             lun->be_lun->lun_type, sentry))
7400                                                 num++;
7401                                 }
7402                         } else {
7403                                 if (ctl_cmd_applicable(lun->be_lun->lun_type,
7404                                     entry))
7405                                         num++;
7406                         }
7407                 }
7408                 total_len = sizeof(struct scsi_report_supported_opcodes_all) +
7409                     num * sizeof(struct scsi_report_supported_opcodes_descr);
7410                 break;
7411         case RSO_OPTIONS_OC:
7412                 if (ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) {
7413                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7414                                               /*sks_valid*/ 1,
7415                                               /*command*/ 1,
7416                                               /*field*/ 2,
7417                                               /*bit_valid*/ 1,
7418                                               /*bit*/ 2);
7419                         ctl_done((union ctl_io *)ctsio);
7420                         return (CTL_RETVAL_COMPLETE);
7421                 }
7422                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7423                 break;
7424         case RSO_OPTIONS_OC_SA:
7425                 if ((ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) == 0 ||
7426                     service_action >= 32) {
7427                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7428                                               /*sks_valid*/ 1,
7429                                               /*command*/ 1,
7430                                               /*field*/ 2,
7431                                               /*bit_valid*/ 1,
7432                                               /*bit*/ 2);
7433                         ctl_done((union ctl_io *)ctsio);
7434                         return (CTL_RETVAL_COMPLETE);
7435                 }
7436                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7437                 break;
7438         default:
7439                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7440                                       /*sks_valid*/ 1,
7441                                       /*command*/ 1,
7442                                       /*field*/ 2,
7443                                       /*bit_valid*/ 1,
7444                                       /*bit*/ 2);
7445                 ctl_done((union ctl_io *)ctsio);
7446                 return (CTL_RETVAL_COMPLETE);
7447         }
7448
7449         alloc_len = scsi_4btoul(cdb->length);
7450
7451         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7452
7453         ctsio->kern_sg_entries = 0;
7454
7455         if (total_len < alloc_len) {
7456                 ctsio->residual = alloc_len - total_len;
7457                 ctsio->kern_data_len = total_len;
7458                 ctsio->kern_total_len = total_len;
7459         } else {
7460                 ctsio->residual = 0;
7461                 ctsio->kern_data_len = alloc_len;
7462                 ctsio->kern_total_len = alloc_len;
7463         }
7464         ctsio->kern_data_resid = 0;
7465         ctsio->kern_rel_offset = 0;
7466
7467         switch (cdb->options & RSO_OPTIONS_MASK) {
7468         case RSO_OPTIONS_ALL:
7469                 all = (struct scsi_report_supported_opcodes_all *)
7470                     ctsio->kern_data_ptr;
7471                 num = 0;
7472                 for (i = 0; i < 256; i++) {
7473                         entry = &ctl_cmd_table[i];
7474                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7475                                 for (j = 0; j < 32; j++) {
7476                                         sentry = &((const struct ctl_cmd_entry *)
7477                                             entry->execute)[j];
7478                                         if (!ctl_cmd_applicable(
7479                                             lun->be_lun->lun_type, sentry))
7480                                                 continue;
7481                                         descr = &all->descr[num++];
7482                                         descr->opcode = i;
7483                                         scsi_ulto2b(j, descr->service_action);
7484                                         descr->flags = RSO_SERVACTV;
7485                                         scsi_ulto2b(sentry->length,
7486                                             descr->cdb_length);
7487                                 }
7488                         } else {
7489                                 if (!ctl_cmd_applicable(lun->be_lun->lun_type,
7490                                     entry))
7491                                         continue;
7492                                 descr = &all->descr[num++];
7493                                 descr->opcode = i;
7494                                 scsi_ulto2b(0, descr->service_action);
7495                                 descr->flags = 0;
7496                                 scsi_ulto2b(entry->length, descr->cdb_length);
7497                         }
7498                 }
7499                 scsi_ulto4b(
7500                     num * sizeof(struct scsi_report_supported_opcodes_descr),
7501                     all->length);
7502                 break;
7503         case RSO_OPTIONS_OC:
7504                 one = (struct scsi_report_supported_opcodes_one *)
7505                     ctsio->kern_data_ptr;
7506                 entry = &ctl_cmd_table[opcode];
7507                 goto fill_one;
7508         case RSO_OPTIONS_OC_SA:
7509                 one = (struct scsi_report_supported_opcodes_one *)
7510                     ctsio->kern_data_ptr;
7511                 entry = &ctl_cmd_table[opcode];
7512                 entry = &((const struct ctl_cmd_entry *)
7513                     entry->execute)[service_action];
7514 fill_one:
7515                 if (ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
7516                         one->support = 3;
7517                         scsi_ulto2b(entry->length, one->cdb_length);
7518                         one->cdb_usage[0] = opcode;
7519                         memcpy(&one->cdb_usage[1], entry->usage,
7520                             entry->length - 1);
7521                 } else
7522                         one->support = 1;
7523                 break;
7524         }
7525
7526         ctl_set_success(ctsio);
7527         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7528         ctsio->be_move_done = ctl_config_move_done;
7529         ctl_datamove((union ctl_io *)ctsio);
7530         return(retval);
7531 }
7532
7533 int
7534 ctl_report_supported_tmf(struct ctl_scsiio *ctsio)
7535 {
7536         struct scsi_report_supported_tmf *cdb;
7537         struct scsi_report_supported_tmf_data *data;
7538         int retval;
7539         int alloc_len, total_len;
7540
7541         CTL_DEBUG_PRINT(("ctl_report_supported_tmf\n"));
7542
7543         cdb = (struct scsi_report_supported_tmf *)ctsio->cdb;
7544
7545         retval = CTL_RETVAL_COMPLETE;
7546
7547         total_len = sizeof(struct scsi_report_supported_tmf_data);
7548         alloc_len = scsi_4btoul(cdb->length);
7549
7550         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7551
7552         ctsio->kern_sg_entries = 0;
7553
7554         if (total_len < alloc_len) {
7555                 ctsio->residual = alloc_len - total_len;
7556                 ctsio->kern_data_len = total_len;
7557                 ctsio->kern_total_len = total_len;
7558         } else {
7559                 ctsio->residual = 0;
7560                 ctsio->kern_data_len = alloc_len;
7561                 ctsio->kern_total_len = alloc_len;
7562         }
7563         ctsio->kern_data_resid = 0;
7564         ctsio->kern_rel_offset = 0;
7565
7566         data = (struct scsi_report_supported_tmf_data *)ctsio->kern_data_ptr;
7567         data->byte1 |= RST_ATS | RST_ATSS | RST_CTSS | RST_LURS | RST_QTS |
7568             RST_TRS;
7569         data->byte2 |= RST_QAES | RST_QTSS | RST_ITNRS;
7570
7571         ctl_set_success(ctsio);
7572         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7573         ctsio->be_move_done = ctl_config_move_done;
7574         ctl_datamove((union ctl_io *)ctsio);
7575         return (retval);
7576 }
7577
7578 int
7579 ctl_report_timestamp(struct ctl_scsiio *ctsio)
7580 {
7581         struct scsi_report_timestamp *cdb;
7582         struct scsi_report_timestamp_data *data;
7583         struct timeval tv;
7584         int64_t timestamp;
7585         int retval;
7586         int alloc_len, total_len;
7587
7588         CTL_DEBUG_PRINT(("ctl_report_timestamp\n"));
7589
7590         cdb = (struct scsi_report_timestamp *)ctsio->cdb;
7591
7592         retval = CTL_RETVAL_COMPLETE;
7593
7594         total_len = sizeof(struct scsi_report_timestamp_data);
7595         alloc_len = scsi_4btoul(cdb->length);
7596
7597         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7598
7599         ctsio->kern_sg_entries = 0;
7600
7601         if (total_len < alloc_len) {
7602                 ctsio->residual = alloc_len - total_len;
7603                 ctsio->kern_data_len = total_len;
7604                 ctsio->kern_total_len = total_len;
7605         } else {
7606                 ctsio->residual = 0;
7607                 ctsio->kern_data_len = alloc_len;
7608                 ctsio->kern_total_len = alloc_len;
7609         }
7610         ctsio->kern_data_resid = 0;
7611         ctsio->kern_rel_offset = 0;
7612
7613         data = (struct scsi_report_timestamp_data *)ctsio->kern_data_ptr;
7614         scsi_ulto2b(sizeof(*data) - 2, data->length);
7615         data->origin = RTS_ORIG_OUTSIDE;
7616         getmicrotime(&tv);
7617         timestamp = (int64_t)tv.tv_sec * 1000 + tv.tv_usec / 1000;
7618         scsi_ulto4b(timestamp >> 16, data->timestamp);
7619         scsi_ulto2b(timestamp & 0xffff, &data->timestamp[4]);
7620
7621         ctl_set_success(ctsio);
7622         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7623         ctsio->be_move_done = ctl_config_move_done;
7624         ctl_datamove((union ctl_io *)ctsio);
7625         return (retval);
7626 }
7627
7628 int
7629 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7630 {
7631         struct scsi_per_res_in *cdb;
7632         int alloc_len, total_len = 0;
7633         /* struct scsi_per_res_in_rsrv in_data; */
7634         struct ctl_lun *lun;
7635         struct ctl_softc *softc;
7636         uint64_t key;
7637
7638         CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7639
7640         cdb = (struct scsi_per_res_in *)ctsio->cdb;
7641
7642         alloc_len = scsi_2btoul(cdb->length);
7643
7644         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7645         softc = lun->ctl_softc;
7646
7647 retry:
7648         mtx_lock(&lun->lun_lock);
7649         switch (cdb->action) {
7650         case SPRI_RK: /* read keys */
7651                 total_len = sizeof(struct scsi_per_res_in_keys) +
7652                         lun->pr_key_count *
7653                         sizeof(struct scsi_per_res_key);
7654                 break;
7655         case SPRI_RR: /* read reservation */
7656                 if (lun->flags & CTL_LUN_PR_RESERVED)
7657                         total_len = sizeof(struct scsi_per_res_in_rsrv);
7658                 else
7659                         total_len = sizeof(struct scsi_per_res_in_header);
7660                 break;
7661         case SPRI_RC: /* report capabilities */
7662                 total_len = sizeof(struct scsi_per_res_cap);
7663                 break;
7664         case SPRI_RS: /* read full status */
7665                 total_len = sizeof(struct scsi_per_res_in_header) +
7666                     (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7667                     lun->pr_key_count;
7668                 break;
7669         default:
7670                 panic("Invalid PR type %x", cdb->action);
7671         }
7672         mtx_unlock(&lun->lun_lock);
7673
7674         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7675
7676         if (total_len < alloc_len) {
7677                 ctsio->residual = alloc_len - total_len;
7678                 ctsio->kern_data_len = total_len;
7679                 ctsio->kern_total_len = total_len;
7680         } else {
7681                 ctsio->residual = 0;
7682                 ctsio->kern_data_len = alloc_len;
7683                 ctsio->kern_total_len = alloc_len;
7684         }
7685
7686         ctsio->kern_data_resid = 0;
7687         ctsio->kern_rel_offset = 0;
7688         ctsio->kern_sg_entries = 0;
7689
7690         mtx_lock(&lun->lun_lock);
7691         switch (cdb->action) {
7692         case SPRI_RK: { // read keys
7693         struct scsi_per_res_in_keys *res_keys;
7694                 int i, key_count;
7695
7696                 res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7697
7698                 /*
7699                  * We had to drop the lock to allocate our buffer, which
7700                  * leaves time for someone to come in with another
7701                  * persistent reservation.  (That is unlikely, though,
7702                  * since this should be the only persistent reservation
7703                  * command active right now.)
7704                  */
7705                 if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7706                     (lun->pr_key_count *
7707                      sizeof(struct scsi_per_res_key)))){
7708                         mtx_unlock(&lun->lun_lock);
7709                         free(ctsio->kern_data_ptr, M_CTL);
7710                         printf("%s: reservation length changed, retrying\n",
7711                                __func__);
7712                         goto retry;
7713                 }
7714
7715                 scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7716
7717                 scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7718                              lun->pr_key_count, res_keys->header.length);
7719
7720                 for (i = 0, key_count = 0; i < CTL_MAX_INITIATORS; i++) {
7721                         if ((key = ctl_get_prkey(lun, i)) == 0)
7722                                 continue;
7723
7724                         /*
7725                          * We used lun->pr_key_count to calculate the
7726                          * size to allocate.  If it turns out the number of
7727                          * initiators with the registered flag set is
7728                          * larger than that (i.e. they haven't been kept in
7729                          * sync), we've got a problem.
7730                          */
7731                         if (key_count >= lun->pr_key_count) {
7732 #ifdef NEEDTOPORT
7733                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
7734                                             CTL_PR_ERROR,
7735                                             csevent_LogType_Fault,
7736                                             csevent_AlertLevel_Yellow,
7737                                             csevent_FRU_ShelfController,
7738                                             csevent_FRU_Firmware,
7739                                         csevent_FRU_Unknown,
7740                                             "registered keys %d >= key "
7741                                             "count %d", key_count,
7742                                             lun->pr_key_count);
7743 #endif
7744                                 key_count++;
7745                                 continue;
7746                         }
7747                         scsi_u64to8b(key, res_keys->keys[key_count].key);
7748                         key_count++;
7749                 }
7750                 break;
7751         }
7752         case SPRI_RR: { // read reservation
7753                 struct scsi_per_res_in_rsrv *res;
7754                 int tmp_len, header_only;
7755
7756                 res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7757
7758                 scsi_ulto4b(lun->PRGeneration, res->header.generation);
7759
7760                 if (lun->flags & CTL_LUN_PR_RESERVED)
7761                 {
7762                         tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7763                         scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7764                                     res->header.length);
7765                         header_only = 0;
7766                 } else {
7767                         tmp_len = sizeof(struct scsi_per_res_in_header);
7768                         scsi_ulto4b(0, res->header.length);
7769                         header_only = 1;
7770                 }
7771
7772                 /*
7773                  * We had to drop the lock to allocate our buffer, which
7774                  * leaves time for someone to come in with another
7775                  * persistent reservation.  (That is unlikely, though,
7776                  * since this should be the only persistent reservation
7777                  * command active right now.)
7778                  */
7779                 if (tmp_len != total_len) {
7780                         mtx_unlock(&lun->lun_lock);
7781                         free(ctsio->kern_data_ptr, M_CTL);
7782                         printf("%s: reservation status changed, retrying\n",
7783                                __func__);
7784                         goto retry;
7785                 }
7786
7787                 /*
7788                  * No reservation held, so we're done.
7789                  */
7790                 if (header_only != 0)
7791                         break;
7792
7793                 /*
7794                  * If the registration is an All Registrants type, the key
7795                  * is 0, since it doesn't really matter.
7796                  */
7797                 if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7798                         scsi_u64to8b(ctl_get_prkey(lun, lun->pr_res_idx),
7799                             res->data.reservation);
7800                 }
7801                 res->data.scopetype = lun->res_type;
7802                 break;
7803         }
7804         case SPRI_RC:     //report capabilities
7805         {
7806                 struct scsi_per_res_cap *res_cap;
7807                 uint16_t type_mask;
7808
7809                 res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7810                 scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7811                 res_cap->flags2 |= SPRI_TMV | SPRI_ALLOW_5;
7812                 type_mask = SPRI_TM_WR_EX_AR |
7813                             SPRI_TM_EX_AC_RO |
7814                             SPRI_TM_WR_EX_RO |
7815                             SPRI_TM_EX_AC |
7816                             SPRI_TM_WR_EX |
7817                             SPRI_TM_EX_AC_AR;
7818                 scsi_ulto2b(type_mask, res_cap->type_mask);
7819                 break;
7820         }
7821         case SPRI_RS: { // read full status
7822                 struct scsi_per_res_in_full *res_status;
7823                 struct scsi_per_res_in_full_desc *res_desc;
7824                 struct ctl_port *port;
7825                 int i, len;
7826
7827                 res_status = (struct scsi_per_res_in_full*)ctsio->kern_data_ptr;
7828
7829                 /*
7830                  * We had to drop the lock to allocate our buffer, which
7831                  * leaves time for someone to come in with another
7832                  * persistent reservation.  (That is unlikely, though,
7833                  * since this should be the only persistent reservation
7834                  * command active right now.)
7835                  */
7836                 if (total_len < (sizeof(struct scsi_per_res_in_header) +
7837                     (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7838                      lun->pr_key_count)){
7839                         mtx_unlock(&lun->lun_lock);
7840                         free(ctsio->kern_data_ptr, M_CTL);
7841                         printf("%s: reservation length changed, retrying\n",
7842                                __func__);
7843                         goto retry;
7844                 }
7845
7846                 scsi_ulto4b(lun->PRGeneration, res_status->header.generation);
7847
7848                 res_desc = &res_status->desc[0];
7849                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
7850                         if ((key = ctl_get_prkey(lun, i)) == 0)
7851                                 continue;
7852
7853                         scsi_u64to8b(key, res_desc->res_key.key);
7854                         if ((lun->flags & CTL_LUN_PR_RESERVED) &&
7855                             (lun->pr_res_idx == i ||
7856                              lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS)) {
7857                                 res_desc->flags = SPRI_FULL_R_HOLDER;
7858                                 res_desc->scopetype = lun->res_type;
7859                         }
7860                         scsi_ulto2b(i / CTL_MAX_INIT_PER_PORT,
7861                             res_desc->rel_trgt_port_id);
7862                         len = 0;
7863                         port = softc->ctl_ports[i / CTL_MAX_INIT_PER_PORT];
7864                         if (port != NULL)
7865                                 len = ctl_create_iid(port,
7866                                     i % CTL_MAX_INIT_PER_PORT,
7867                                     res_desc->transport_id);
7868                         scsi_ulto4b(len, res_desc->additional_length);
7869                         res_desc = (struct scsi_per_res_in_full_desc *)
7870                             &res_desc->transport_id[len];
7871                 }
7872                 scsi_ulto4b((uint8_t *)res_desc - (uint8_t *)&res_status->desc[0],
7873                     res_status->header.length);
7874                 break;
7875         }
7876         default:
7877                 /*
7878                  * This is a bug, because we just checked for this above,
7879                  * and should have returned an error.
7880                  */
7881                 panic("Invalid PR type %x", cdb->action);
7882                 break; /* NOTREACHED */
7883         }
7884         mtx_unlock(&lun->lun_lock);
7885
7886         ctl_set_success(ctsio);
7887         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7888         ctsio->be_move_done = ctl_config_move_done;
7889         ctl_datamove((union ctl_io *)ctsio);
7890         return (CTL_RETVAL_COMPLETE);
7891 }
7892
7893 /*
7894  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
7895  * it should return.
7896  */
7897 static int
7898 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
7899                 uint64_t sa_res_key, uint8_t type, uint32_t residx,
7900                 struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
7901                 struct scsi_per_res_out_parms* param)
7902 {
7903         union ctl_ha_msg persis_io;
7904         int i;
7905
7906         mtx_lock(&lun->lun_lock);
7907         if (sa_res_key == 0) {
7908                 if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7909                         /* validate scope and type */
7910                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7911                              SPR_LU_SCOPE) {
7912                                 mtx_unlock(&lun->lun_lock);
7913                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7914                                                       /*sks_valid*/ 1,
7915                                                       /*command*/ 1,
7916                                                       /*field*/ 2,
7917                                                       /*bit_valid*/ 1,
7918                                                       /*bit*/ 4);
7919                                 ctl_done((union ctl_io *)ctsio);
7920                                 return (1);
7921                         }
7922
7923                         if (type>8 || type==2 || type==4 || type==0) {
7924                                 mtx_unlock(&lun->lun_lock);
7925                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7926                                                       /*sks_valid*/ 1,
7927                                                       /*command*/ 1,
7928                                                       /*field*/ 2,
7929                                                       /*bit_valid*/ 1,
7930                                                       /*bit*/ 0);
7931                                 ctl_done((union ctl_io *)ctsio);
7932                                 return (1);
7933                         }
7934
7935                         /*
7936                          * Unregister everybody else and build UA for
7937                          * them
7938                          */
7939                         for(i = 0; i < CTL_MAX_INITIATORS; i++) {
7940                                 if (i == residx || ctl_get_prkey(lun, i) == 0)
7941                                         continue;
7942
7943                                 ctl_clr_prkey(lun, i);
7944                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
7945                         }
7946                         lun->pr_key_count = 1;
7947                         lun->res_type = type;
7948                         if (lun->res_type != SPR_TYPE_WR_EX_AR
7949                          && lun->res_type != SPR_TYPE_EX_AC_AR)
7950                                 lun->pr_res_idx = residx;
7951                         lun->PRGeneration++;
7952                         mtx_unlock(&lun->lun_lock);
7953
7954                         /* send msg to other side */
7955                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7956                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7957                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7958                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
7959                         persis_io.pr.pr_info.res_type = type;
7960                         memcpy(persis_io.pr.pr_info.sa_res_key,
7961                                param->serv_act_res_key,
7962                                sizeof(param->serv_act_res_key));
7963                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
7964                             sizeof(persis_io.pr), M_WAITOK);
7965                 } else {
7966                         /* not all registrants */
7967                         mtx_unlock(&lun->lun_lock);
7968                         free(ctsio->kern_data_ptr, M_CTL);
7969                         ctl_set_invalid_field(ctsio,
7970                                               /*sks_valid*/ 1,
7971                                               /*command*/ 0,
7972                                               /*field*/ 8,
7973                                               /*bit_valid*/ 0,
7974                                               /*bit*/ 0);
7975                         ctl_done((union ctl_io *)ctsio);
7976                         return (1);
7977                 }
7978         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7979                 || !(lun->flags & CTL_LUN_PR_RESERVED)) {
7980                 int found = 0;
7981
7982                 if (res_key == sa_res_key) {
7983                         /* special case */
7984                         /*
7985                          * The spec implies this is not good but doesn't
7986                          * say what to do. There are two choices either
7987                          * generate a res conflict or check condition
7988                          * with illegal field in parameter data. Since
7989                          * that is what is done when the sa_res_key is
7990                          * zero I'll take that approach since this has
7991                          * to do with the sa_res_key.
7992                          */
7993                         mtx_unlock(&lun->lun_lock);
7994                         free(ctsio->kern_data_ptr, M_CTL);
7995                         ctl_set_invalid_field(ctsio,
7996                                               /*sks_valid*/ 1,
7997                                               /*command*/ 0,
7998                                               /*field*/ 8,
7999                                               /*bit_valid*/ 0,
8000                                               /*bit*/ 0);
8001                         ctl_done((union ctl_io *)ctsio);
8002                         return (1);
8003                 }
8004
8005                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8006                         if (ctl_get_prkey(lun, i) != sa_res_key)
8007                                 continue;
8008
8009                         found = 1;
8010                         ctl_clr_prkey(lun, i);
8011                         lun->pr_key_count--;
8012                         ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8013                 }
8014                 if (!found) {
8015                         mtx_unlock(&lun->lun_lock);
8016                         free(ctsio->kern_data_ptr, M_CTL);
8017                         ctl_set_reservation_conflict(ctsio);
8018                         ctl_done((union ctl_io *)ctsio);
8019                         return (CTL_RETVAL_COMPLETE);
8020                 }
8021                 lun->PRGeneration++;
8022                 mtx_unlock(&lun->lun_lock);
8023
8024                 /* send msg to other side */
8025                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8026                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8027                 persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8028                 persis_io.pr.pr_info.residx = lun->pr_res_idx;
8029                 persis_io.pr.pr_info.res_type = type;
8030                 memcpy(persis_io.pr.pr_info.sa_res_key,
8031                        param->serv_act_res_key,
8032                        sizeof(param->serv_act_res_key));
8033                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8034                     sizeof(persis_io.pr), M_WAITOK);
8035         } else {
8036                 /* Reserved but not all registrants */
8037                 /* sa_res_key is res holder */
8038                 if (sa_res_key == ctl_get_prkey(lun, lun->pr_res_idx)) {
8039                         /* validate scope and type */
8040                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
8041                              SPR_LU_SCOPE) {
8042                                 mtx_unlock(&lun->lun_lock);
8043                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8044                                                       /*sks_valid*/ 1,
8045                                                       /*command*/ 1,
8046                                                       /*field*/ 2,
8047                                                       /*bit_valid*/ 1,
8048                                                       /*bit*/ 4);
8049                                 ctl_done((union ctl_io *)ctsio);
8050                                 return (1);
8051                         }
8052
8053                         if (type>8 || type==2 || type==4 || type==0) {
8054                                 mtx_unlock(&lun->lun_lock);
8055                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8056                                                       /*sks_valid*/ 1,
8057                                                       /*command*/ 1,
8058                                                       /*field*/ 2,
8059                                                       /*bit_valid*/ 1,
8060                                                       /*bit*/ 0);
8061                                 ctl_done((union ctl_io *)ctsio);
8062                                 return (1);
8063                         }
8064
8065                         /*
8066                          * Do the following:
8067                          * if sa_res_key != res_key remove all
8068                          * registrants w/sa_res_key and generate UA
8069                          * for these registrants(Registrations
8070                          * Preempted) if it wasn't an exclusive
8071                          * reservation generate UA(Reservations
8072                          * Preempted) for all other registered nexuses
8073                          * if the type has changed. Establish the new
8074                          * reservation and holder. If res_key and
8075                          * sa_res_key are the same do the above
8076                          * except don't unregister the res holder.
8077                          */
8078
8079                         for(i = 0; i < CTL_MAX_INITIATORS; i++) {
8080                                 if (i == residx || ctl_get_prkey(lun, i) == 0)
8081                                         continue;
8082
8083                                 if (sa_res_key == ctl_get_prkey(lun, i)) {
8084                                         ctl_clr_prkey(lun, i);
8085                                         lun->pr_key_count--;
8086                                         ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8087                                 } else if (type != lun->res_type
8088                                         && (lun->res_type == SPR_TYPE_WR_EX_RO
8089                                          || lun->res_type ==SPR_TYPE_EX_AC_RO)){
8090                                         ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8091                                 }
8092                         }
8093                         lun->res_type = type;
8094                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8095                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8096                                 lun->pr_res_idx = residx;
8097                         else
8098                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8099                         lun->PRGeneration++;
8100                         mtx_unlock(&lun->lun_lock);
8101
8102                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8103                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8104                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8105                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8106                         persis_io.pr.pr_info.res_type = type;
8107                         memcpy(persis_io.pr.pr_info.sa_res_key,
8108                                param->serv_act_res_key,
8109                                sizeof(param->serv_act_res_key));
8110                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8111                             sizeof(persis_io.pr), M_WAITOK);
8112                 } else {
8113                         /*
8114                          * sa_res_key is not the res holder just
8115                          * remove registrants
8116                          */
8117                         int found=0;
8118
8119                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8120                                 if (sa_res_key != ctl_get_prkey(lun, i))
8121                                         continue;
8122
8123                                 found = 1;
8124                                 ctl_clr_prkey(lun, i);
8125                                 lun->pr_key_count--;
8126                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8127                         }
8128
8129                         if (!found) {
8130                                 mtx_unlock(&lun->lun_lock);
8131                                 free(ctsio->kern_data_ptr, M_CTL);
8132                                 ctl_set_reservation_conflict(ctsio);
8133                                 ctl_done((union ctl_io *)ctsio);
8134                                 return (1);
8135                         }
8136                         lun->PRGeneration++;
8137                         mtx_unlock(&lun->lun_lock);
8138
8139                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8140                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8141                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8142                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8143                         persis_io.pr.pr_info.res_type = type;
8144                         memcpy(persis_io.pr.pr_info.sa_res_key,
8145                                param->serv_act_res_key,
8146                                sizeof(param->serv_act_res_key));
8147                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8148                             sizeof(persis_io.pr), M_WAITOK);
8149                 }
8150         }
8151         return (0);
8152 }
8153
8154 static void
8155 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
8156 {
8157         uint64_t sa_res_key;
8158         int i;
8159
8160         sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
8161
8162         if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8163          || lun->pr_res_idx == CTL_PR_NO_RESERVATION
8164          || sa_res_key != ctl_get_prkey(lun, lun->pr_res_idx)) {
8165                 if (sa_res_key == 0) {
8166                         /*
8167                          * Unregister everybody else and build UA for
8168                          * them
8169                          */
8170                         for(i = 0; i < CTL_MAX_INITIATORS; i++) {
8171                                 if (i == msg->pr.pr_info.residx ||
8172                                     ctl_get_prkey(lun, i) == 0)
8173                                         continue;
8174
8175                                 ctl_clr_prkey(lun, i);
8176                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8177                         }
8178
8179                         lun->pr_key_count = 1;
8180                         lun->res_type = msg->pr.pr_info.res_type;
8181                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8182                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8183                                 lun->pr_res_idx = msg->pr.pr_info.residx;
8184                 } else {
8185                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8186                                 if (sa_res_key == ctl_get_prkey(lun, i))
8187                                         continue;
8188
8189                                 ctl_clr_prkey(lun, i);
8190                                 lun->pr_key_count--;
8191                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8192                         }
8193                 }
8194         } else {
8195                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8196                         if (i == msg->pr.pr_info.residx ||
8197                             ctl_get_prkey(lun, i) == 0)
8198                                 continue;
8199
8200                         if (sa_res_key == ctl_get_prkey(lun, i)) {
8201                                 ctl_clr_prkey(lun, i);
8202                                 lun->pr_key_count--;
8203                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8204                         } else if (msg->pr.pr_info.res_type != lun->res_type
8205                                 && (lun->res_type == SPR_TYPE_WR_EX_RO
8206                                  || lun->res_type == SPR_TYPE_EX_AC_RO)) {
8207                                 ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8208                         }
8209                 }
8210                 lun->res_type = msg->pr.pr_info.res_type;
8211                 if (lun->res_type != SPR_TYPE_WR_EX_AR
8212                  && lun->res_type != SPR_TYPE_EX_AC_AR)
8213                         lun->pr_res_idx = msg->pr.pr_info.residx;
8214                 else
8215                         lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8216         }
8217         lun->PRGeneration++;
8218
8219 }
8220
8221
8222 int
8223 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
8224 {
8225         int retval;
8226         u_int32_t param_len;
8227         struct scsi_per_res_out *cdb;
8228         struct ctl_lun *lun;
8229         struct scsi_per_res_out_parms* param;
8230         struct ctl_softc *softc;
8231         uint32_t residx;
8232         uint64_t res_key, sa_res_key, key;
8233         uint8_t type;
8234         union ctl_ha_msg persis_io;
8235         int    i;
8236
8237         CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
8238
8239         retval = CTL_RETVAL_COMPLETE;
8240
8241         cdb = (struct scsi_per_res_out *)ctsio->cdb;
8242         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8243         softc = lun->ctl_softc;
8244
8245         /*
8246          * We only support whole-LUN scope.  The scope & type are ignored for
8247          * register, register and ignore existing key and clear.
8248          * We sometimes ignore scope and type on preempts too!!
8249          * Verify reservation type here as well.
8250          */
8251         type = cdb->scope_type & SPR_TYPE_MASK;
8252         if ((cdb->action == SPRO_RESERVE)
8253          || (cdb->action == SPRO_RELEASE)) {
8254                 if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
8255                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8256                                               /*sks_valid*/ 1,
8257                                               /*command*/ 1,
8258                                               /*field*/ 2,
8259                                               /*bit_valid*/ 1,
8260                                               /*bit*/ 4);
8261                         ctl_done((union ctl_io *)ctsio);
8262                         return (CTL_RETVAL_COMPLETE);
8263                 }
8264
8265                 if (type>8 || type==2 || type==4 || type==0) {
8266                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8267                                               /*sks_valid*/ 1,
8268                                               /*command*/ 1,
8269                                               /*field*/ 2,
8270                                               /*bit_valid*/ 1,
8271                                               /*bit*/ 0);
8272                         ctl_done((union ctl_io *)ctsio);
8273                         return (CTL_RETVAL_COMPLETE);
8274                 }
8275         }
8276
8277         param_len = scsi_4btoul(cdb->length);
8278
8279         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
8280                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
8281                 ctsio->kern_data_len = param_len;
8282                 ctsio->kern_total_len = param_len;
8283                 ctsio->kern_data_resid = 0;
8284                 ctsio->kern_rel_offset = 0;
8285                 ctsio->kern_sg_entries = 0;
8286                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8287                 ctsio->be_move_done = ctl_config_move_done;
8288                 ctl_datamove((union ctl_io *)ctsio);
8289
8290                 return (CTL_RETVAL_COMPLETE);
8291         }
8292
8293         param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
8294
8295         residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
8296         res_key = scsi_8btou64(param->res_key.key);
8297         sa_res_key = scsi_8btou64(param->serv_act_res_key);
8298
8299         /*
8300          * Validate the reservation key here except for SPRO_REG_IGNO
8301          * This must be done for all other service actions
8302          */
8303         if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
8304                 mtx_lock(&lun->lun_lock);
8305                 if ((key = ctl_get_prkey(lun, residx)) != 0) {
8306                         if (res_key != key) {
8307                                 /*
8308                                  * The current key passed in doesn't match
8309                                  * the one the initiator previously
8310                                  * registered.
8311                                  */
8312                                 mtx_unlock(&lun->lun_lock);
8313                                 free(ctsio->kern_data_ptr, M_CTL);
8314                                 ctl_set_reservation_conflict(ctsio);
8315                                 ctl_done((union ctl_io *)ctsio);
8316                                 return (CTL_RETVAL_COMPLETE);
8317                         }
8318                 } else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
8319                         /*
8320                          * We are not registered
8321                          */
8322                         mtx_unlock(&lun->lun_lock);
8323                         free(ctsio->kern_data_ptr, M_CTL);
8324                         ctl_set_reservation_conflict(ctsio);
8325                         ctl_done((union ctl_io *)ctsio);
8326                         return (CTL_RETVAL_COMPLETE);
8327                 } else if (res_key != 0) {
8328                         /*
8329                          * We are not registered and trying to register but
8330                          * the register key isn't zero.
8331                          */
8332                         mtx_unlock(&lun->lun_lock);
8333                         free(ctsio->kern_data_ptr, M_CTL);
8334                         ctl_set_reservation_conflict(ctsio);
8335                         ctl_done((union ctl_io *)ctsio);
8336                         return (CTL_RETVAL_COMPLETE);
8337                 }
8338                 mtx_unlock(&lun->lun_lock);
8339         }
8340
8341         switch (cdb->action & SPRO_ACTION_MASK) {
8342         case SPRO_REGISTER:
8343         case SPRO_REG_IGNO: {
8344
8345 #if 0
8346                 printf("Registration received\n");
8347 #endif
8348
8349                 /*
8350                  * We don't support any of these options, as we report in
8351                  * the read capabilities request (see
8352                  * ctl_persistent_reserve_in(), above).
8353                  */
8354                 if ((param->flags & SPR_SPEC_I_PT)
8355                  || (param->flags & SPR_ALL_TG_PT)
8356                  || (param->flags & SPR_APTPL)) {
8357                         int bit_ptr;
8358
8359                         if (param->flags & SPR_APTPL)
8360                                 bit_ptr = 0;
8361                         else if (param->flags & SPR_ALL_TG_PT)
8362                                 bit_ptr = 2;
8363                         else /* SPR_SPEC_I_PT */
8364                                 bit_ptr = 3;
8365
8366                         free(ctsio->kern_data_ptr, M_CTL);
8367                         ctl_set_invalid_field(ctsio,
8368                                               /*sks_valid*/ 1,
8369                                               /*command*/ 0,
8370                                               /*field*/ 20,
8371                                               /*bit_valid*/ 1,
8372                                               /*bit*/ bit_ptr);
8373                         ctl_done((union ctl_io *)ctsio);
8374                         return (CTL_RETVAL_COMPLETE);
8375                 }
8376
8377                 mtx_lock(&lun->lun_lock);
8378
8379                 /*
8380                  * The initiator wants to clear the
8381                  * key/unregister.
8382                  */
8383                 if (sa_res_key == 0) {
8384                         if ((res_key == 0
8385                           && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
8386                          || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
8387                           && ctl_get_prkey(lun, residx) == 0)) {
8388                                 mtx_unlock(&lun->lun_lock);
8389                                 goto done;
8390                         }
8391
8392                         ctl_clr_prkey(lun, residx);
8393                         lun->pr_key_count--;
8394
8395                         if (residx == lun->pr_res_idx) {
8396                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8397                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8398
8399                                 if ((lun->res_type == SPR_TYPE_WR_EX_RO
8400                                   || lun->res_type == SPR_TYPE_EX_AC_RO)
8401                                  && lun->pr_key_count) {
8402                                         /*
8403                                          * If the reservation is a registrants
8404                                          * only type we need to generate a UA
8405                                          * for other registered inits.  The
8406                                          * sense code should be RESERVATIONS
8407                                          * RELEASED
8408                                          */
8409
8410                                         for (i = softc->init_min; i < softc->init_max; i++){
8411                                                 if (ctl_get_prkey(lun, i) == 0)
8412                                                         continue;
8413                                                 ctl_est_ua(lun, i,
8414                                                     CTL_UA_RES_RELEASE);
8415                                         }
8416                                 }
8417                                 lun->res_type = 0;
8418                         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8419                                 if (lun->pr_key_count==0) {
8420                                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8421                                         lun->res_type = 0;
8422                                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8423                                 }
8424                         }
8425                         lun->PRGeneration++;
8426                         mtx_unlock(&lun->lun_lock);
8427
8428                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8429                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8430                         persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8431                         persis_io.pr.pr_info.residx = residx;
8432                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8433                             sizeof(persis_io.pr), M_WAITOK);
8434                 } else /* sa_res_key != 0 */ {
8435
8436                         /*
8437                          * If we aren't registered currently then increment
8438                          * the key count and set the registered flag.
8439                          */
8440                         ctl_alloc_prkey(lun, residx);
8441                         if (ctl_get_prkey(lun, residx) == 0)
8442                                 lun->pr_key_count++;
8443                         ctl_set_prkey(lun, residx, sa_res_key);
8444                         lun->PRGeneration++;
8445                         mtx_unlock(&lun->lun_lock);
8446
8447                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8448                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8449                         persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8450                         persis_io.pr.pr_info.residx = residx;
8451                         memcpy(persis_io.pr.pr_info.sa_res_key,
8452                                param->serv_act_res_key,
8453                                sizeof(param->serv_act_res_key));
8454                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8455                             sizeof(persis_io.pr), M_WAITOK);
8456                 }
8457
8458                 break;
8459         }
8460         case SPRO_RESERVE:
8461 #if 0
8462                 printf("Reserve executed type %d\n", type);
8463 #endif
8464                 mtx_lock(&lun->lun_lock);
8465                 if (lun->flags & CTL_LUN_PR_RESERVED) {
8466                         /*
8467                          * if this isn't the reservation holder and it's
8468                          * not a "all registrants" type or if the type is
8469                          * different then we have a conflict
8470                          */
8471                         if ((lun->pr_res_idx != residx
8472                           && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8473                          || lun->res_type != type) {
8474                                 mtx_unlock(&lun->lun_lock);
8475                                 free(ctsio->kern_data_ptr, M_CTL);
8476                                 ctl_set_reservation_conflict(ctsio);
8477                                 ctl_done((union ctl_io *)ctsio);
8478                                 return (CTL_RETVAL_COMPLETE);
8479                         }
8480                         mtx_unlock(&lun->lun_lock);
8481                 } else /* create a reservation */ {
8482                         /*
8483                          * If it's not an "all registrants" type record
8484                          * reservation holder
8485                          */
8486                         if (type != SPR_TYPE_WR_EX_AR
8487                          && type != SPR_TYPE_EX_AC_AR)
8488                                 lun->pr_res_idx = residx; /* Res holder */
8489                         else
8490                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8491
8492                         lun->flags |= CTL_LUN_PR_RESERVED;
8493                         lun->res_type = type;
8494
8495                         mtx_unlock(&lun->lun_lock);
8496
8497                         /* send msg to other side */
8498                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8499                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8500                         persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8501                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8502                         persis_io.pr.pr_info.res_type = type;
8503                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8504                             sizeof(persis_io.pr), M_WAITOK);
8505                 }
8506                 break;
8507
8508         case SPRO_RELEASE:
8509                 mtx_lock(&lun->lun_lock);
8510                 if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8511                         /* No reservation exists return good status */
8512                         mtx_unlock(&lun->lun_lock);
8513                         goto done;
8514                 }
8515                 /*
8516                  * Is this nexus a reservation holder?
8517                  */
8518                 if (lun->pr_res_idx != residx
8519                  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8520                         /*
8521                          * not a res holder return good status but
8522                          * do nothing
8523                          */
8524                         mtx_unlock(&lun->lun_lock);
8525                         goto done;
8526                 }
8527
8528                 if (lun->res_type != type) {
8529                         mtx_unlock(&lun->lun_lock);
8530                         free(ctsio->kern_data_ptr, M_CTL);
8531                         ctl_set_illegal_pr_release(ctsio);
8532                         ctl_done((union ctl_io *)ctsio);
8533                         return (CTL_RETVAL_COMPLETE);
8534                 }
8535
8536                 /* okay to release */
8537                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8538                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8539                 lun->res_type = 0;
8540
8541                 /*
8542                  * if this isn't an exclusive access
8543                  * res generate UA for all other
8544                  * registrants.
8545                  */
8546                 if (type != SPR_TYPE_EX_AC
8547                  && type != SPR_TYPE_WR_EX) {
8548                         for (i = softc->init_min; i < softc->init_max; i++) {
8549                                 if (i == residx || ctl_get_prkey(lun, i) == 0)
8550                                         continue;
8551                                 ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8552                         }
8553                 }
8554                 mtx_unlock(&lun->lun_lock);
8555
8556                 /* Send msg to other side */
8557                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8558                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8559                 persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8560                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8561                      sizeof(persis_io.pr), M_WAITOK);
8562                 break;
8563
8564         case SPRO_CLEAR:
8565                 /* send msg to other side */
8566
8567                 mtx_lock(&lun->lun_lock);
8568                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8569                 lun->res_type = 0;
8570                 lun->pr_key_count = 0;
8571                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8572
8573                 ctl_clr_prkey(lun, residx);
8574                 for (i = 0; i < CTL_MAX_INITIATORS; i++)
8575                         if (ctl_get_prkey(lun, i) != 0) {
8576                                 ctl_clr_prkey(lun, i);
8577                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8578                         }
8579                 lun->PRGeneration++;
8580                 mtx_unlock(&lun->lun_lock);
8581
8582                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8583                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8584                 persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8585                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8586                      sizeof(persis_io.pr), M_WAITOK);
8587                 break;
8588
8589         case SPRO_PREEMPT:
8590         case SPRO_PRE_ABO: {
8591                 int nretval;
8592
8593                 nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8594                                           residx, ctsio, cdb, param);
8595                 if (nretval != 0)
8596                         return (CTL_RETVAL_COMPLETE);
8597                 break;
8598         }
8599         default:
8600                 panic("Invalid PR type %x", cdb->action);
8601         }
8602
8603 done:
8604         free(ctsio->kern_data_ptr, M_CTL);
8605         ctl_set_success(ctsio);
8606         ctl_done((union ctl_io *)ctsio);
8607
8608         return (retval);
8609 }
8610
8611 /*
8612  * This routine is for handling a message from the other SC pertaining to
8613  * persistent reserve out. All the error checking will have been done
8614  * so only perorming the action need be done here to keep the two
8615  * in sync.
8616  */
8617 static void
8618 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8619 {
8620         struct ctl_lun *lun;
8621         struct ctl_softc *softc;
8622         int i;
8623         uint32_t residx, targ_lun;
8624
8625         softc = control_softc;
8626         targ_lun = msg->hdr.nexus.targ_mapped_lun;
8627         mtx_lock(&softc->ctl_lock);
8628         if ((targ_lun >= CTL_MAX_LUNS) ||
8629             ((lun = softc->ctl_luns[targ_lun]) == NULL)) {
8630                 mtx_unlock(&softc->ctl_lock);
8631                 return;
8632         }
8633         mtx_lock(&lun->lun_lock);
8634         mtx_unlock(&softc->ctl_lock);
8635         if (lun->flags & CTL_LUN_DISABLED) {
8636                 mtx_unlock(&lun->lun_lock);
8637                 return;
8638         }
8639         residx = ctl_get_initindex(&msg->hdr.nexus);
8640         switch(msg->pr.pr_info.action) {
8641         case CTL_PR_REG_KEY:
8642                 ctl_alloc_prkey(lun, msg->pr.pr_info.residx);
8643                 if (ctl_get_prkey(lun, msg->pr.pr_info.residx) == 0)
8644                         lun->pr_key_count++;
8645                 ctl_set_prkey(lun, msg->pr.pr_info.residx,
8646                     scsi_8btou64(msg->pr.pr_info.sa_res_key));
8647                 lun->PRGeneration++;
8648                 break;
8649
8650         case CTL_PR_UNREG_KEY:
8651                 ctl_clr_prkey(lun, msg->pr.pr_info.residx);
8652                 lun->pr_key_count--;
8653
8654                 /* XXX Need to see if the reservation has been released */
8655                 /* if so do we need to generate UA? */
8656                 if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8657                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8658                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8659
8660                         if ((lun->res_type == SPR_TYPE_WR_EX_RO
8661                           || lun->res_type == SPR_TYPE_EX_AC_RO)
8662                          && lun->pr_key_count) {
8663                                 /*
8664                                  * If the reservation is a registrants
8665                                  * only type we need to generate a UA
8666                                  * for other registered inits.  The
8667                                  * sense code should be RESERVATIONS
8668                                  * RELEASED
8669                                  */
8670
8671                                 for (i = softc->init_min; i < softc->init_max; i++) {
8672                                         if (ctl_get_prkey(lun, i) == 0)
8673                                                 continue;
8674
8675                                         ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8676                                 }
8677                         }
8678                         lun->res_type = 0;
8679                 } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8680                         if (lun->pr_key_count==0) {
8681                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8682                                 lun->res_type = 0;
8683                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8684                         }
8685                 }
8686                 lun->PRGeneration++;
8687                 break;
8688
8689         case CTL_PR_RESERVE:
8690                 lun->flags |= CTL_LUN_PR_RESERVED;
8691                 lun->res_type = msg->pr.pr_info.res_type;
8692                 lun->pr_res_idx = msg->pr.pr_info.residx;
8693
8694                 break;
8695
8696         case CTL_PR_RELEASE:
8697                 /*
8698                  * if this isn't an exclusive access res generate UA for all
8699                  * other registrants.
8700                  */
8701                 if (lun->res_type != SPR_TYPE_EX_AC
8702                  && lun->res_type != SPR_TYPE_WR_EX) {
8703                         for (i = softc->init_min; i < softc->init_max; i++)
8704                                 if (i == residx || ctl_get_prkey(lun, i) == 0)
8705                                         continue;
8706                                 ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8707                 }
8708
8709                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8710                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8711                 lun->res_type = 0;
8712                 break;
8713
8714         case CTL_PR_PREEMPT:
8715                 ctl_pro_preempt_other(lun, msg);
8716                 break;
8717         case CTL_PR_CLEAR:
8718                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8719                 lun->res_type = 0;
8720                 lun->pr_key_count = 0;
8721                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8722
8723                 for (i=0; i < CTL_MAX_INITIATORS; i++) {
8724                         if (ctl_get_prkey(lun, i) == 0)
8725                                 continue;
8726                         ctl_clr_prkey(lun, i);
8727                         ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8728                 }
8729                 lun->PRGeneration++;
8730                 break;
8731         }
8732
8733         mtx_unlock(&lun->lun_lock);
8734 }
8735
8736 int
8737 ctl_read_write(struct ctl_scsiio *ctsio)
8738 {
8739         struct ctl_lun *lun;
8740         struct ctl_lba_len_flags *lbalen;
8741         uint64_t lba;
8742         uint32_t num_blocks;
8743         int flags, retval;
8744         int isread;
8745
8746         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8747
8748         CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8749
8750         flags = 0;
8751         retval = CTL_RETVAL_COMPLETE;
8752
8753         isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8754               || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8755         switch (ctsio->cdb[0]) {
8756         case READ_6:
8757         case WRITE_6: {
8758                 struct scsi_rw_6 *cdb;
8759
8760                 cdb = (struct scsi_rw_6 *)ctsio->cdb;
8761
8762                 lba = scsi_3btoul(cdb->addr);
8763                 /* only 5 bits are valid in the most significant address byte */
8764                 lba &= 0x1fffff;
8765                 num_blocks = cdb->length;
8766                 /*
8767                  * This is correct according to SBC-2.
8768                  */
8769                 if (num_blocks == 0)
8770                         num_blocks = 256;
8771                 break;
8772         }
8773         case READ_10:
8774         case WRITE_10: {
8775                 struct scsi_rw_10 *cdb;
8776
8777                 cdb = (struct scsi_rw_10 *)ctsio->cdb;
8778                 if (cdb->byte2 & SRW10_FUA)
8779                         flags |= CTL_LLF_FUA;
8780                 if (cdb->byte2 & SRW10_DPO)
8781                         flags |= CTL_LLF_DPO;
8782                 lba = scsi_4btoul(cdb->addr);
8783                 num_blocks = scsi_2btoul(cdb->length);
8784                 break;
8785         }
8786         case WRITE_VERIFY_10: {
8787                 struct scsi_write_verify_10 *cdb;
8788
8789                 cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
8790                 flags |= CTL_LLF_FUA;
8791                 if (cdb->byte2 & SWV_DPO)
8792                         flags |= CTL_LLF_DPO;
8793                 lba = scsi_4btoul(cdb->addr);
8794                 num_blocks = scsi_2btoul(cdb->length);
8795                 break;
8796         }
8797         case READ_12:
8798         case WRITE_12: {
8799                 struct scsi_rw_12 *cdb;
8800
8801                 cdb = (struct scsi_rw_12 *)ctsio->cdb;
8802                 if (cdb->byte2 & SRW12_FUA)
8803                         flags |= CTL_LLF_FUA;
8804                 if (cdb->byte2 & SRW12_DPO)
8805                         flags |= CTL_LLF_DPO;
8806                 lba = scsi_4btoul(cdb->addr);
8807                 num_blocks = scsi_4btoul(cdb->length);
8808                 break;
8809         }
8810         case WRITE_VERIFY_12: {
8811                 struct scsi_write_verify_12 *cdb;
8812
8813                 cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
8814                 flags |= CTL_LLF_FUA;
8815                 if (cdb->byte2 & SWV_DPO)
8816                         flags |= CTL_LLF_DPO;
8817                 lba = scsi_4btoul(cdb->addr);
8818                 num_blocks = scsi_4btoul(cdb->length);
8819                 break;
8820         }
8821         case READ_16:
8822         case WRITE_16: {
8823                 struct scsi_rw_16 *cdb;
8824
8825                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
8826                 if (cdb->byte2 & SRW12_FUA)
8827                         flags |= CTL_LLF_FUA;
8828                 if (cdb->byte2 & SRW12_DPO)
8829                         flags |= CTL_LLF_DPO;
8830                 lba = scsi_8btou64(cdb->addr);
8831                 num_blocks = scsi_4btoul(cdb->length);
8832                 break;
8833         }
8834         case WRITE_ATOMIC_16: {
8835                 struct scsi_rw_16 *cdb;
8836
8837                 if (lun->be_lun->atomicblock == 0) {
8838                         ctl_set_invalid_opcode(ctsio);
8839                         ctl_done((union ctl_io *)ctsio);
8840                         return (CTL_RETVAL_COMPLETE);
8841                 }
8842
8843                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
8844                 if (cdb->byte2 & SRW12_FUA)
8845                         flags |= CTL_LLF_FUA;
8846                 if (cdb->byte2 & SRW12_DPO)
8847                         flags |= CTL_LLF_DPO;
8848                 lba = scsi_8btou64(cdb->addr);
8849                 num_blocks = scsi_4btoul(cdb->length);
8850                 if (num_blocks > lun->be_lun->atomicblock) {
8851                         ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
8852                             /*command*/ 1, /*field*/ 12, /*bit_valid*/ 0,
8853                             /*bit*/ 0);
8854                         ctl_done((union ctl_io *)ctsio);
8855                         return (CTL_RETVAL_COMPLETE);
8856                 }
8857                 break;
8858         }
8859         case WRITE_VERIFY_16: {
8860                 struct scsi_write_verify_16 *cdb;
8861
8862                 cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
8863                 flags |= CTL_LLF_FUA;
8864                 if (cdb->byte2 & SWV_DPO)
8865                         flags |= CTL_LLF_DPO;
8866                 lba = scsi_8btou64(cdb->addr);
8867                 num_blocks = scsi_4btoul(cdb->length);
8868                 break;
8869         }
8870         default:
8871                 /*
8872                  * We got a command we don't support.  This shouldn't
8873                  * happen, commands should be filtered out above us.
8874                  */
8875                 ctl_set_invalid_opcode(ctsio);
8876                 ctl_done((union ctl_io *)ctsio);
8877
8878                 return (CTL_RETVAL_COMPLETE);
8879                 break; /* NOTREACHED */
8880         }
8881
8882         /*
8883          * The first check is to make sure we're in bounds, the second
8884          * check is to catch wrap-around problems.  If the lba + num blocks
8885          * is less than the lba, then we've wrapped around and the block
8886          * range is invalid anyway.
8887          */
8888         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
8889          || ((lba + num_blocks) < lba)) {
8890                 ctl_set_lba_out_of_range(ctsio);
8891                 ctl_done((union ctl_io *)ctsio);
8892                 return (CTL_RETVAL_COMPLETE);
8893         }
8894
8895         /*
8896          * According to SBC-3, a transfer length of 0 is not an error.
8897          * Note that this cannot happen with WRITE(6) or READ(6), since 0
8898          * translates to 256 blocks for those commands.
8899          */
8900         if (num_blocks == 0) {
8901                 ctl_set_success(ctsio);
8902                 ctl_done((union ctl_io *)ctsio);
8903                 return (CTL_RETVAL_COMPLETE);
8904         }
8905
8906         /* Set FUA and/or DPO if caches are disabled. */
8907         if (isread) {
8908                 if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
8909                     SCP_RCD) != 0)
8910                         flags |= CTL_LLF_FUA | CTL_LLF_DPO;
8911         } else {
8912                 if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
8913                     SCP_WCE) == 0)
8914                         flags |= CTL_LLF_FUA;
8915         }
8916
8917         lbalen = (struct ctl_lba_len_flags *)
8918             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
8919         lbalen->lba = lba;
8920         lbalen->len = num_blocks;
8921         lbalen->flags = (isread ? CTL_LLF_READ : CTL_LLF_WRITE) | flags;
8922
8923         ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
8924         ctsio->kern_rel_offset = 0;
8925
8926         CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
8927
8928         retval = lun->backend->data_submit((union ctl_io *)ctsio);
8929
8930         return (retval);
8931 }
8932
8933 static int
8934 ctl_cnw_cont(union ctl_io *io)
8935 {
8936         struct ctl_scsiio *ctsio;
8937         struct ctl_lun *lun;
8938         struct ctl_lba_len_flags *lbalen;
8939         int retval;
8940
8941         ctsio = &io->scsiio;
8942         ctsio->io_hdr.status = CTL_STATUS_NONE;
8943         ctsio->io_hdr.flags &= ~CTL_FLAG_IO_CONT;
8944         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8945         lbalen = (struct ctl_lba_len_flags *)
8946             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
8947         lbalen->flags &= ~CTL_LLF_COMPARE;
8948         lbalen->flags |= CTL_LLF_WRITE;
8949
8950         CTL_DEBUG_PRINT(("ctl_cnw_cont: calling data_submit()\n"));
8951         retval = lun->backend->data_submit((union ctl_io *)ctsio);
8952         return (retval);
8953 }
8954
8955 int
8956 ctl_cnw(struct ctl_scsiio *ctsio)
8957 {
8958         struct ctl_lun *lun;
8959         struct ctl_lba_len_flags *lbalen;
8960         uint64_t lba;
8961         uint32_t num_blocks;
8962         int flags, retval;
8963
8964         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8965
8966         CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
8967
8968         flags = 0;
8969         retval = CTL_RETVAL_COMPLETE;
8970
8971         switch (ctsio->cdb[0]) {
8972         case COMPARE_AND_WRITE: {
8973                 struct scsi_compare_and_write *cdb;
8974
8975                 cdb = (struct scsi_compare_and_write *)ctsio->cdb;
8976                 if (cdb->byte2 & SRW10_FUA)
8977                         flags |= CTL_LLF_FUA;
8978                 if (cdb->byte2 & SRW10_DPO)
8979                         flags |= CTL_LLF_DPO;
8980                 lba = scsi_8btou64(cdb->addr);
8981                 num_blocks = cdb->length;
8982                 break;
8983         }
8984         default:
8985                 /*
8986                  * We got a command we don't support.  This shouldn't
8987                  * happen, commands should be filtered out above us.
8988                  */
8989                 ctl_set_invalid_opcode(ctsio);
8990                 ctl_done((union ctl_io *)ctsio);
8991
8992                 return (CTL_RETVAL_COMPLETE);
8993                 break; /* NOTREACHED */
8994         }
8995
8996         /*
8997          * The first check is to make sure we're in bounds, the second
8998          * check is to catch wrap-around problems.  If the lba + num blocks
8999          * is less than the lba, then we've wrapped around and the block
9000          * range is invalid anyway.
9001          */
9002         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9003          || ((lba + num_blocks) < lba)) {
9004                 ctl_set_lba_out_of_range(ctsio);
9005                 ctl_done((union ctl_io *)ctsio);
9006                 return (CTL_RETVAL_COMPLETE);
9007         }
9008
9009         /*
9010          * According to SBC-3, a transfer length of 0 is not an error.
9011          */
9012         if (num_blocks == 0) {
9013                 ctl_set_success(ctsio);
9014                 ctl_done((union ctl_io *)ctsio);
9015                 return (CTL_RETVAL_COMPLETE);
9016         }
9017
9018         /* Set FUA if write cache is disabled. */
9019         if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
9020             SCP_WCE) == 0)
9021                 flags |= CTL_LLF_FUA;
9022
9023         ctsio->kern_total_len = 2 * num_blocks * lun->be_lun->blocksize;
9024         ctsio->kern_rel_offset = 0;
9025
9026         /*
9027          * Set the IO_CONT flag, so that if this I/O gets passed to
9028          * ctl_data_submit_done(), it'll get passed back to
9029          * ctl_ctl_cnw_cont() for further processing.
9030          */
9031         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
9032         ctsio->io_cont = ctl_cnw_cont;
9033
9034         lbalen = (struct ctl_lba_len_flags *)
9035             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9036         lbalen->lba = lba;
9037         lbalen->len = num_blocks;
9038         lbalen->flags = CTL_LLF_COMPARE | flags;
9039
9040         CTL_DEBUG_PRINT(("ctl_cnw: calling data_submit()\n"));
9041         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9042         return (retval);
9043 }
9044
9045 int
9046 ctl_verify(struct ctl_scsiio *ctsio)
9047 {
9048         struct ctl_lun *lun;
9049         struct ctl_lba_len_flags *lbalen;
9050         uint64_t lba;
9051         uint32_t num_blocks;
9052         int bytchk, flags;
9053         int retval;
9054
9055         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9056
9057         CTL_DEBUG_PRINT(("ctl_verify: command: %#x\n", ctsio->cdb[0]));
9058
9059         bytchk = 0;
9060         flags = CTL_LLF_FUA;
9061         retval = CTL_RETVAL_COMPLETE;
9062
9063         switch (ctsio->cdb[0]) {
9064         case VERIFY_10: {
9065                 struct scsi_verify_10 *cdb;
9066
9067                 cdb = (struct scsi_verify_10 *)ctsio->cdb;
9068                 if (cdb->byte2 & SVFY_BYTCHK)
9069                         bytchk = 1;
9070                 if (cdb->byte2 & SVFY_DPO)
9071                         flags |= CTL_LLF_DPO;
9072                 lba = scsi_4btoul(cdb->addr);
9073                 num_blocks = scsi_2btoul(cdb->length);
9074                 break;
9075         }
9076         case VERIFY_12: {
9077                 struct scsi_verify_12 *cdb;
9078
9079                 cdb = (struct scsi_verify_12 *)ctsio->cdb;
9080                 if (cdb->byte2 & SVFY_BYTCHK)
9081                         bytchk = 1;
9082                 if (cdb->byte2 & SVFY_DPO)
9083                         flags |= CTL_LLF_DPO;
9084                 lba = scsi_4btoul(cdb->addr);
9085                 num_blocks = scsi_4btoul(cdb->length);
9086                 break;
9087         }
9088         case VERIFY_16: {
9089                 struct scsi_rw_16 *cdb;
9090
9091                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
9092                 if (cdb->byte2 & SVFY_BYTCHK)
9093                         bytchk = 1;
9094                 if (cdb->byte2 & SVFY_DPO)
9095                         flags |= CTL_LLF_DPO;
9096                 lba = scsi_8btou64(cdb->addr);
9097                 num_blocks = scsi_4btoul(cdb->length);
9098                 break;
9099         }
9100         default:
9101                 /*
9102                  * We got a command we don't support.  This shouldn't
9103                  * happen, commands should be filtered out above us.
9104                  */
9105                 ctl_set_invalid_opcode(ctsio);
9106                 ctl_done((union ctl_io *)ctsio);
9107                 return (CTL_RETVAL_COMPLETE);
9108         }
9109
9110         /*
9111          * The first check is to make sure we're in bounds, the second
9112          * check is to catch wrap-around problems.  If the lba + num blocks
9113          * is less than the lba, then we've wrapped around and the block
9114          * range is invalid anyway.
9115          */
9116         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9117          || ((lba + num_blocks) < lba)) {
9118                 ctl_set_lba_out_of_range(ctsio);
9119                 ctl_done((union ctl_io *)ctsio);
9120                 return (CTL_RETVAL_COMPLETE);
9121         }
9122
9123         /*
9124          * According to SBC-3, a transfer length of 0 is not an error.
9125          */
9126         if (num_blocks == 0) {
9127                 ctl_set_success(ctsio);
9128                 ctl_done((union ctl_io *)ctsio);
9129                 return (CTL_RETVAL_COMPLETE);
9130         }
9131
9132         lbalen = (struct ctl_lba_len_flags *)
9133             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9134         lbalen->lba = lba;
9135         lbalen->len = num_blocks;
9136         if (bytchk) {
9137                 lbalen->flags = CTL_LLF_COMPARE | flags;
9138                 ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9139         } else {
9140                 lbalen->flags = CTL_LLF_VERIFY | flags;
9141                 ctsio->kern_total_len = 0;
9142         }
9143         ctsio->kern_rel_offset = 0;
9144
9145         CTL_DEBUG_PRINT(("ctl_verify: calling data_submit()\n"));
9146         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9147         return (retval);
9148 }
9149
9150 int
9151 ctl_report_luns(struct ctl_scsiio *ctsio)
9152 {
9153         struct ctl_softc *softc = control_softc;
9154         struct scsi_report_luns *cdb;
9155         struct scsi_report_luns_data *lun_data;
9156         struct ctl_lun *lun, *request_lun;
9157         struct ctl_port *port;
9158         int num_luns, retval;
9159         uint32_t alloc_len, lun_datalen;
9160         int num_filled, well_known;
9161         uint32_t initidx, targ_lun_id, lun_id;
9162
9163         retval = CTL_RETVAL_COMPLETE;
9164         well_known = 0;
9165
9166         cdb = (struct scsi_report_luns *)ctsio->cdb;
9167         port = ctl_io_port(&ctsio->io_hdr);
9168
9169         CTL_DEBUG_PRINT(("ctl_report_luns\n"));
9170
9171         mtx_lock(&softc->ctl_lock);
9172         num_luns = 0;
9173         for (targ_lun_id = 0; targ_lun_id < CTL_MAX_LUNS; targ_lun_id++) {
9174                 if (ctl_lun_map_from_port(port, targ_lun_id) < CTL_MAX_LUNS)
9175                         num_luns++;
9176         }
9177         mtx_unlock(&softc->ctl_lock);
9178
9179         switch (cdb->select_report) {
9180         case RPL_REPORT_DEFAULT:
9181         case RPL_REPORT_ALL:
9182                 break;
9183         case RPL_REPORT_WELLKNOWN:
9184                 well_known = 1;
9185                 num_luns = 0;
9186                 break;
9187         default:
9188                 ctl_set_invalid_field(ctsio,
9189                                       /*sks_valid*/ 1,
9190                                       /*command*/ 1,
9191                                       /*field*/ 2,
9192                                       /*bit_valid*/ 0,
9193                                       /*bit*/ 0);
9194                 ctl_done((union ctl_io *)ctsio);
9195                 return (retval);
9196                 break; /* NOTREACHED */
9197         }
9198
9199         alloc_len = scsi_4btoul(cdb->length);
9200         /*
9201          * The initiator has to allocate at least 16 bytes for this request,
9202          * so he can at least get the header and the first LUN.  Otherwise
9203          * we reject the request (per SPC-3 rev 14, section 6.21).
9204          */
9205         if (alloc_len < (sizeof(struct scsi_report_luns_data) +
9206             sizeof(struct scsi_report_luns_lundata))) {
9207                 ctl_set_invalid_field(ctsio,
9208                                       /*sks_valid*/ 1,
9209                                       /*command*/ 1,
9210                                       /*field*/ 6,
9211                                       /*bit_valid*/ 0,
9212                                       /*bit*/ 0);
9213                 ctl_done((union ctl_io *)ctsio);
9214                 return (retval);
9215         }
9216
9217         request_lun = (struct ctl_lun *)
9218                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9219
9220         lun_datalen = sizeof(*lun_data) +
9221                 (num_luns * sizeof(struct scsi_report_luns_lundata));
9222
9223         ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
9224         lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
9225         ctsio->kern_sg_entries = 0;
9226
9227         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9228
9229         mtx_lock(&softc->ctl_lock);
9230         for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
9231                 lun_id = ctl_lun_map_from_port(port, targ_lun_id);
9232                 if (lun_id >= CTL_MAX_LUNS)
9233                         continue;
9234                 lun = softc->ctl_luns[lun_id];
9235                 if (lun == NULL)
9236                         continue;
9237
9238                 if (targ_lun_id <= 0xff) {
9239                         /*
9240                          * Peripheral addressing method, bus number 0.
9241                          */
9242                         lun_data->luns[num_filled].lundata[0] =
9243                                 RPL_LUNDATA_ATYP_PERIPH;
9244                         lun_data->luns[num_filled].lundata[1] = targ_lun_id;
9245                         num_filled++;
9246                 } else if (targ_lun_id <= 0x3fff) {
9247                         /*
9248                          * Flat addressing method.
9249                          */
9250                         lun_data->luns[num_filled].lundata[0] =
9251                                 RPL_LUNDATA_ATYP_FLAT | (targ_lun_id >> 8);
9252                         lun_data->luns[num_filled].lundata[1] =
9253                                 (targ_lun_id & 0xff);
9254                         num_filled++;
9255                 } else if (targ_lun_id <= 0xffffff) {
9256                         /*
9257                          * Extended flat addressing method.
9258                          */
9259                         lun_data->luns[num_filled].lundata[0] =
9260                             RPL_LUNDATA_ATYP_EXTLUN | 0x12;
9261                         scsi_ulto3b(targ_lun_id,
9262                             &lun_data->luns[num_filled].lundata[1]);
9263                         num_filled++;
9264                 } else {
9265                         printf("ctl_report_luns: bogus LUN number %jd, "
9266                                "skipping\n", (intmax_t)targ_lun_id);
9267                 }
9268                 /*
9269                  * According to SPC-3, rev 14 section 6.21:
9270                  *
9271                  * "The execution of a REPORT LUNS command to any valid and
9272                  * installed logical unit shall clear the REPORTED LUNS DATA
9273                  * HAS CHANGED unit attention condition for all logical
9274                  * units of that target with respect to the requesting
9275                  * initiator. A valid and installed logical unit is one
9276                  * having a PERIPHERAL QUALIFIER of 000b in the standard
9277                  * INQUIRY data (see 6.4.2)."
9278                  *
9279                  * If request_lun is NULL, the LUN this report luns command
9280                  * was issued to is either disabled or doesn't exist. In that
9281                  * case, we shouldn't clear any pending lun change unit
9282                  * attention.
9283                  */
9284                 if (request_lun != NULL) {
9285                         mtx_lock(&lun->lun_lock);
9286                         ctl_clr_ua(lun, initidx, CTL_UA_LUN_CHANGE);
9287                         mtx_unlock(&lun->lun_lock);
9288                 }
9289         }
9290         mtx_unlock(&softc->ctl_lock);
9291
9292         /*
9293          * It's quite possible that we've returned fewer LUNs than we allocated
9294          * space for.  Trim it.
9295          */
9296         lun_datalen = sizeof(*lun_data) +
9297                 (num_filled * sizeof(struct scsi_report_luns_lundata));
9298
9299         if (lun_datalen < alloc_len) {
9300                 ctsio->residual = alloc_len - lun_datalen;
9301                 ctsio->kern_data_len = lun_datalen;
9302                 ctsio->kern_total_len = lun_datalen;
9303         } else {
9304                 ctsio->residual = 0;
9305                 ctsio->kern_data_len = alloc_len;
9306                 ctsio->kern_total_len = alloc_len;
9307         }
9308         ctsio->kern_data_resid = 0;
9309         ctsio->kern_rel_offset = 0;
9310         ctsio->kern_sg_entries = 0;
9311
9312         /*
9313          * We set this to the actual data length, regardless of how much
9314          * space we actually have to return results.  If the user looks at
9315          * this value, he'll know whether or not he allocated enough space
9316          * and reissue the command if necessary.  We don't support well
9317          * known logical units, so if the user asks for that, return none.
9318          */
9319         scsi_ulto4b(lun_datalen - 8, lun_data->length);
9320
9321         /*
9322          * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
9323          * this request.
9324          */
9325         ctl_set_success(ctsio);
9326         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9327         ctsio->be_move_done = ctl_config_move_done;
9328         ctl_datamove((union ctl_io *)ctsio);
9329         return (retval);
9330 }
9331
9332 int
9333 ctl_request_sense(struct ctl_scsiio *ctsio)
9334 {
9335         struct scsi_request_sense *cdb;
9336         struct scsi_sense_data *sense_ptr;
9337         struct ctl_softc *ctl_softc;
9338         struct ctl_lun *lun;
9339         uint32_t initidx;
9340         int have_error;
9341         scsi_sense_data_type sense_format;
9342         ctl_ua_type ua_type;
9343
9344         cdb = (struct scsi_request_sense *)ctsio->cdb;
9345
9346         ctl_softc = control_softc;
9347         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9348
9349         CTL_DEBUG_PRINT(("ctl_request_sense\n"));
9350
9351         /*
9352          * Determine which sense format the user wants.
9353          */
9354         if (cdb->byte2 & SRS_DESC)
9355                 sense_format = SSD_TYPE_DESC;
9356         else
9357                 sense_format = SSD_TYPE_FIXED;
9358
9359         ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
9360         sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
9361         ctsio->kern_sg_entries = 0;
9362
9363         /*
9364          * struct scsi_sense_data, which is currently set to 256 bytes, is
9365          * larger than the largest allowed value for the length field in the
9366          * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
9367          */
9368         ctsio->residual = 0;
9369         ctsio->kern_data_len = cdb->length;
9370         ctsio->kern_total_len = cdb->length;
9371
9372         ctsio->kern_data_resid = 0;
9373         ctsio->kern_rel_offset = 0;
9374         ctsio->kern_sg_entries = 0;
9375
9376         /*
9377          * If we don't have a LUN, we don't have any pending sense.
9378          */
9379         if (lun == NULL)
9380                 goto no_sense;
9381
9382         have_error = 0;
9383         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9384         /*
9385          * Check for pending sense, and then for pending unit attentions.
9386          * Pending sense gets returned first, then pending unit attentions.
9387          */
9388         mtx_lock(&lun->lun_lock);
9389 #ifdef CTL_WITH_CA
9390         if (ctl_is_set(lun->have_ca, initidx)) {
9391                 scsi_sense_data_type stored_format;
9392
9393                 /*
9394                  * Check to see which sense format was used for the stored
9395                  * sense data.
9396                  */
9397                 stored_format = scsi_sense_type(&lun->pending_sense[initidx]);
9398
9399                 /*
9400                  * If the user requested a different sense format than the
9401                  * one we stored, then we need to convert it to the other
9402                  * format.  If we're going from descriptor to fixed format
9403                  * sense data, we may lose things in translation, depending
9404                  * on what options were used.
9405                  *
9406                  * If the stored format is SSD_TYPE_NONE (i.e. invalid),
9407                  * for some reason we'll just copy it out as-is.
9408                  */
9409                 if ((stored_format == SSD_TYPE_FIXED)
9410                  && (sense_format == SSD_TYPE_DESC))
9411                         ctl_sense_to_desc((struct scsi_sense_data_fixed *)
9412                             &lun->pending_sense[initidx],
9413                             (struct scsi_sense_data_desc *)sense_ptr);
9414                 else if ((stored_format == SSD_TYPE_DESC)
9415                       && (sense_format == SSD_TYPE_FIXED))
9416                         ctl_sense_to_fixed((struct scsi_sense_data_desc *)
9417                             &lun->pending_sense[initidx],
9418                             (struct scsi_sense_data_fixed *)sense_ptr);
9419                 else
9420                         memcpy(sense_ptr, &lun->pending_sense[initidx],
9421                                MIN(sizeof(*sense_ptr),
9422                                sizeof(lun->pending_sense[initidx])));
9423
9424                 ctl_clear_mask(lun->have_ca, initidx);
9425                 have_error = 1;
9426         } else
9427 #endif
9428         {
9429                 ua_type = ctl_build_ua(lun, initidx, sense_ptr, sense_format);
9430                 if (ua_type != CTL_UA_NONE)
9431                         have_error = 1;
9432                 if (ua_type == CTL_UA_LUN_CHANGE) {
9433                         mtx_unlock(&lun->lun_lock);
9434                         mtx_lock(&ctl_softc->ctl_lock);
9435                         ctl_clr_ua_allluns(ctl_softc, initidx, ua_type);
9436                         mtx_unlock(&ctl_softc->ctl_lock);
9437                         mtx_lock(&lun->lun_lock);
9438                 }
9439
9440         }
9441         mtx_unlock(&lun->lun_lock);
9442
9443         /*
9444          * We already have a pending error, return it.
9445          */
9446         if (have_error != 0) {
9447                 /*
9448                  * We report the SCSI status as OK, since the status of the
9449                  * request sense command itself is OK.
9450                  * We report 0 for the sense length, because we aren't doing
9451                  * autosense in this case.  We're reporting sense as
9452                  * parameter data.
9453                  */
9454                 ctl_set_success(ctsio);
9455                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9456                 ctsio->be_move_done = ctl_config_move_done;
9457                 ctl_datamove((union ctl_io *)ctsio);
9458                 return (CTL_RETVAL_COMPLETE);
9459         }
9460
9461 no_sense:
9462
9463         /*
9464          * No sense information to report, so we report that everything is
9465          * okay.
9466          */
9467         ctl_set_sense_data(sense_ptr,
9468                            lun,
9469                            sense_format,
9470                            /*current_error*/ 1,
9471                            /*sense_key*/ SSD_KEY_NO_SENSE,
9472                            /*asc*/ 0x00,
9473                            /*ascq*/ 0x00,
9474                            SSD_ELEM_NONE);
9475
9476         /*
9477          * We report 0 for the sense length, because we aren't doing
9478          * autosense in this case.  We're reporting sense as parameter data.
9479          */
9480         ctl_set_success(ctsio);
9481         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9482         ctsio->be_move_done = ctl_config_move_done;
9483         ctl_datamove((union ctl_io *)ctsio);
9484         return (CTL_RETVAL_COMPLETE);
9485 }
9486
9487 int
9488 ctl_tur(struct ctl_scsiio *ctsio)
9489 {
9490
9491         CTL_DEBUG_PRINT(("ctl_tur\n"));
9492
9493         ctl_set_success(ctsio);
9494         ctl_done((union ctl_io *)ctsio);
9495
9496         return (CTL_RETVAL_COMPLETE);
9497 }
9498
9499 /*
9500  * SCSI VPD page 0x00, the Supported VPD Pages page.
9501  */
9502 static int
9503 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
9504 {
9505         struct scsi_vpd_supported_pages *pages;
9506         int sup_page_size;
9507         struct ctl_lun *lun;
9508         int p;
9509
9510         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9511
9512         sup_page_size = sizeof(struct scsi_vpd_supported_pages) *
9513             SCSI_EVPD_NUM_SUPPORTED_PAGES;
9514         ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9515         pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9516         ctsio->kern_sg_entries = 0;
9517
9518         if (sup_page_size < alloc_len) {
9519                 ctsio->residual = alloc_len - sup_page_size;
9520                 ctsio->kern_data_len = sup_page_size;
9521                 ctsio->kern_total_len = sup_page_size;
9522         } else {
9523                 ctsio->residual = 0;
9524                 ctsio->kern_data_len = alloc_len;
9525                 ctsio->kern_total_len = alloc_len;
9526         }
9527         ctsio->kern_data_resid = 0;
9528         ctsio->kern_rel_offset = 0;
9529         ctsio->kern_sg_entries = 0;
9530
9531         /*
9532          * The control device is always connected.  The disk device, on the
9533          * other hand, may not be online all the time.  Need to change this
9534          * to figure out whether the disk device is actually online or not.
9535          */
9536         if (lun != NULL)
9537                 pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9538                                 lun->be_lun->lun_type;
9539         else
9540                 pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9541
9542         p = 0;
9543         /* Supported VPD pages */
9544         pages->page_list[p++] = SVPD_SUPPORTED_PAGES;
9545         /* Serial Number */
9546         pages->page_list[p++] = SVPD_UNIT_SERIAL_NUMBER;
9547         /* Device Identification */
9548         pages->page_list[p++] = SVPD_DEVICE_ID;
9549         /* Extended INQUIRY Data */
9550         pages->page_list[p++] = SVPD_EXTENDED_INQUIRY_DATA;
9551         /* Mode Page Policy */
9552         pages->page_list[p++] = SVPD_MODE_PAGE_POLICY;
9553         /* SCSI Ports */
9554         pages->page_list[p++] = SVPD_SCSI_PORTS;
9555         /* Third-party Copy */
9556         pages->page_list[p++] = SVPD_SCSI_TPC;
9557         if (lun != NULL && lun->be_lun->lun_type == T_DIRECT) {
9558                 /* Block limits */
9559                 pages->page_list[p++] = SVPD_BLOCK_LIMITS;
9560                 /* Block Device Characteristics */
9561                 pages->page_list[p++] = SVPD_BDC;
9562                 /* Logical Block Provisioning */
9563                 pages->page_list[p++] = SVPD_LBP;
9564         }
9565         pages->length = p;
9566
9567         ctl_set_success(ctsio);
9568         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9569         ctsio->be_move_done = ctl_config_move_done;
9570         ctl_datamove((union ctl_io *)ctsio);
9571         return (CTL_RETVAL_COMPLETE);
9572 }
9573
9574 /*
9575  * SCSI VPD page 0x80, the Unit Serial Number page.
9576  */
9577 static int
9578 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9579 {
9580         struct scsi_vpd_unit_serial_number *sn_ptr;
9581         struct ctl_lun *lun;
9582         int data_len;
9583
9584         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9585
9586         data_len = 4 + CTL_SN_LEN;
9587         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9588         sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9589         if (data_len < alloc_len) {
9590                 ctsio->residual = alloc_len - data_len;
9591                 ctsio->kern_data_len = data_len;
9592                 ctsio->kern_total_len = data_len;
9593         } else {
9594                 ctsio->residual = 0;
9595                 ctsio->kern_data_len = alloc_len;
9596                 ctsio->kern_total_len = alloc_len;
9597         }
9598         ctsio->kern_data_resid = 0;
9599         ctsio->kern_rel_offset = 0;
9600         ctsio->kern_sg_entries = 0;
9601
9602         /*
9603          * The control device is always connected.  The disk device, on the
9604          * other hand, may not be online all the time.  Need to change this
9605          * to figure out whether the disk device is actually online or not.
9606          */
9607         if (lun != NULL)
9608                 sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9609                                   lun->be_lun->lun_type;
9610         else
9611                 sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9612
9613         sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9614         sn_ptr->length = CTL_SN_LEN;
9615         /*
9616          * If we don't have a LUN, we just leave the serial number as
9617          * all spaces.
9618          */
9619         if (lun != NULL) {
9620                 strncpy((char *)sn_ptr->serial_num,
9621                         (char *)lun->be_lun->serial_num, CTL_SN_LEN);
9622         } else
9623                 memset(sn_ptr->serial_num, 0x20, CTL_SN_LEN);
9624
9625         ctl_set_success(ctsio);
9626         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9627         ctsio->be_move_done = ctl_config_move_done;
9628         ctl_datamove((union ctl_io *)ctsio);
9629         return (CTL_RETVAL_COMPLETE);
9630 }
9631
9632
9633 /*
9634  * SCSI VPD page 0x86, the Extended INQUIRY Data page.
9635  */
9636 static int
9637 ctl_inquiry_evpd_eid(struct ctl_scsiio *ctsio, int alloc_len)
9638 {
9639         struct scsi_vpd_extended_inquiry_data *eid_ptr;
9640         struct ctl_lun *lun;
9641         int data_len;
9642
9643         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9644
9645         data_len = sizeof(struct scsi_vpd_extended_inquiry_data);
9646         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9647         eid_ptr = (struct scsi_vpd_extended_inquiry_data *)ctsio->kern_data_ptr;
9648         ctsio->kern_sg_entries = 0;
9649
9650         if (data_len < alloc_len) {
9651                 ctsio->residual = alloc_len - data_len;
9652                 ctsio->kern_data_len = data_len;
9653                 ctsio->kern_total_len = data_len;
9654         } else {
9655                 ctsio->residual = 0;
9656                 ctsio->kern_data_len = alloc_len;
9657                 ctsio->kern_total_len = alloc_len;
9658         }
9659         ctsio->kern_data_resid = 0;
9660         ctsio->kern_rel_offset = 0;
9661         ctsio->kern_sg_entries = 0;
9662
9663         /*
9664          * The control device is always connected.  The disk device, on the
9665          * other hand, may not be online all the time.
9666          */
9667         if (lun != NULL)
9668                 eid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9669                                      lun->be_lun->lun_type;
9670         else
9671                 eid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9672         eid_ptr->page_code = SVPD_EXTENDED_INQUIRY_DATA;
9673         scsi_ulto2b(data_len - 4, eid_ptr->page_length);
9674         /*
9675          * We support head of queue, ordered and simple tags.
9676          */
9677         eid_ptr->flags2 = SVPD_EID_HEADSUP | SVPD_EID_ORDSUP | SVPD_EID_SIMPSUP;
9678         /*
9679          * Volatile cache supported.
9680          */
9681         eid_ptr->flags3 = SVPD_EID_V_SUP;
9682
9683         /*
9684          * This means that we clear the REPORTED LUNS DATA HAS CHANGED unit
9685          * attention for a particular IT nexus on all LUNs once we report
9686          * it to that nexus once.  This bit is required as of SPC-4.
9687          */
9688         eid_ptr->flags4 = SVPD_EID_LUICLT;
9689
9690         /*
9691          * XXX KDM in order to correctly answer this, we would need
9692          * information from the SIM to determine how much sense data it
9693          * can send.  So this would really be a path inquiry field, most
9694          * likely.  This can be set to a maximum of 252 according to SPC-4,
9695          * but the hardware may or may not be able to support that much.
9696          * 0 just means that the maximum sense data length is not reported.
9697          */
9698         eid_ptr->max_sense_length = 0;
9699
9700         ctl_set_success(ctsio);
9701         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9702         ctsio->be_move_done = ctl_config_move_done;
9703         ctl_datamove((union ctl_io *)ctsio);
9704         return (CTL_RETVAL_COMPLETE);
9705 }
9706
9707 static int
9708 ctl_inquiry_evpd_mpp(struct ctl_scsiio *ctsio, int alloc_len)
9709 {
9710         struct scsi_vpd_mode_page_policy *mpp_ptr;
9711         struct ctl_lun *lun;
9712         int data_len;
9713
9714         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9715
9716         data_len = sizeof(struct scsi_vpd_mode_page_policy) +
9717             sizeof(struct scsi_vpd_mode_page_policy_descr);
9718
9719         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9720         mpp_ptr = (struct scsi_vpd_mode_page_policy *)ctsio->kern_data_ptr;
9721         ctsio->kern_sg_entries = 0;
9722
9723         if (data_len < alloc_len) {
9724                 ctsio->residual = alloc_len - data_len;
9725                 ctsio->kern_data_len = data_len;
9726                 ctsio->kern_total_len = data_len;
9727         } else {
9728                 ctsio->residual = 0;
9729                 ctsio->kern_data_len = alloc_len;
9730                 ctsio->kern_total_len = alloc_len;
9731         }
9732         ctsio->kern_data_resid = 0;
9733         ctsio->kern_rel_offset = 0;
9734         ctsio->kern_sg_entries = 0;
9735
9736         /*
9737          * The control device is always connected.  The disk device, on the
9738          * other hand, may not be online all the time.
9739          */
9740         if (lun != NULL)
9741                 mpp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9742                                      lun->be_lun->lun_type;
9743         else
9744                 mpp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9745         mpp_ptr->page_code = SVPD_MODE_PAGE_POLICY;
9746         scsi_ulto2b(data_len - 4, mpp_ptr->page_length);
9747         mpp_ptr->descr[0].page_code = 0x3f;
9748         mpp_ptr->descr[0].subpage_code = 0xff;
9749         mpp_ptr->descr[0].policy = SVPD_MPP_SHARED;
9750
9751         ctl_set_success(ctsio);
9752         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9753         ctsio->be_move_done = ctl_config_move_done;
9754         ctl_datamove((union ctl_io *)ctsio);
9755         return (CTL_RETVAL_COMPLETE);
9756 }
9757
9758 /*
9759  * SCSI VPD page 0x83, the Device Identification page.
9760  */
9761 static int
9762 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9763 {
9764         struct scsi_vpd_device_id *devid_ptr;
9765         struct scsi_vpd_id_descriptor *desc;
9766         struct ctl_softc *softc;
9767         struct ctl_lun *lun;
9768         struct ctl_port *port;
9769         int data_len;
9770         uint8_t proto;
9771
9772         softc = control_softc;
9773
9774         port = ctl_io_port(&ctsio->io_hdr);
9775         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9776
9777         data_len = sizeof(struct scsi_vpd_device_id) +
9778             sizeof(struct scsi_vpd_id_descriptor) +
9779                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9780             sizeof(struct scsi_vpd_id_descriptor) +
9781                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9782         if (lun && lun->lun_devid)
9783                 data_len += lun->lun_devid->len;
9784         if (port && port->port_devid)
9785                 data_len += port->port_devid->len;
9786         if (port && port->target_devid)
9787                 data_len += port->target_devid->len;
9788
9789         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9790         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9791         ctsio->kern_sg_entries = 0;
9792
9793         if (data_len < alloc_len) {
9794                 ctsio->residual = alloc_len - data_len;
9795                 ctsio->kern_data_len = data_len;
9796                 ctsio->kern_total_len = data_len;
9797         } else {
9798                 ctsio->residual = 0;
9799                 ctsio->kern_data_len = alloc_len;
9800                 ctsio->kern_total_len = alloc_len;
9801         }
9802         ctsio->kern_data_resid = 0;
9803         ctsio->kern_rel_offset = 0;
9804         ctsio->kern_sg_entries = 0;
9805
9806         /*
9807          * The control device is always connected.  The disk device, on the
9808          * other hand, may not be online all the time.
9809          */
9810         if (lun != NULL)
9811                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9812                                      lun->be_lun->lun_type;
9813         else
9814                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9815         devid_ptr->page_code = SVPD_DEVICE_ID;
9816         scsi_ulto2b(data_len - 4, devid_ptr->length);
9817
9818         if (port && port->port_type == CTL_PORT_FC)
9819                 proto = SCSI_PROTO_FC << 4;
9820         else if (port && port->port_type == CTL_PORT_ISCSI)
9821                 proto = SCSI_PROTO_ISCSI << 4;
9822         else
9823                 proto = SCSI_PROTO_SPI << 4;
9824         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
9825
9826         /*
9827          * We're using a LUN association here.  i.e., this device ID is a
9828          * per-LUN identifier.
9829          */
9830         if (lun && lun->lun_devid) {
9831                 memcpy(desc, lun->lun_devid->data, lun->lun_devid->len);
9832                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
9833                     lun->lun_devid->len);
9834         }
9835
9836         /*
9837          * This is for the WWPN which is a port association.
9838          */
9839         if (port && port->port_devid) {
9840                 memcpy(desc, port->port_devid->data, port->port_devid->len);
9841                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
9842                     port->port_devid->len);
9843         }
9844
9845         /*
9846          * This is for the Relative Target Port(type 4h) identifier
9847          */
9848         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
9849         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
9850             SVPD_ID_TYPE_RELTARG;
9851         desc->length = 4;
9852         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port, &desc->identifier[2]);
9853         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9854             sizeof(struct scsi_vpd_id_rel_trgt_port_id));
9855
9856         /*
9857          * This is for the Target Port Group(type 5h) identifier
9858          */
9859         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
9860         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
9861             SVPD_ID_TYPE_TPORTGRP;
9862         desc->length = 4;
9863         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port / softc->port_cnt + 1,
9864             &desc->identifier[2]);
9865         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9866             sizeof(struct scsi_vpd_id_trgt_port_grp_id));
9867
9868         /*
9869          * This is for the Target identifier
9870          */
9871         if (port && port->target_devid) {
9872                 memcpy(desc, port->target_devid->data, port->target_devid->len);
9873         }
9874
9875         ctl_set_success(ctsio);
9876         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9877         ctsio->be_move_done = ctl_config_move_done;
9878         ctl_datamove((union ctl_io *)ctsio);
9879         return (CTL_RETVAL_COMPLETE);
9880 }
9881
9882 static int
9883 ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len)
9884 {
9885         struct ctl_softc *softc = control_softc;
9886         struct scsi_vpd_scsi_ports *sp;
9887         struct scsi_vpd_port_designation *pd;
9888         struct scsi_vpd_port_designation_cont *pdc;
9889         struct ctl_lun *lun;
9890         struct ctl_port *port;
9891         int data_len, num_target_ports, iid_len, id_len;
9892
9893         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9894
9895         num_target_ports = 0;
9896         iid_len = 0;
9897         id_len = 0;
9898         mtx_lock(&softc->ctl_lock);
9899         STAILQ_FOREACH(port, &softc->port_list, links) {
9900                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
9901                         continue;
9902                 if (lun != NULL &&
9903                     ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
9904                         continue;
9905                 num_target_ports++;
9906                 if (port->init_devid)
9907                         iid_len += port->init_devid->len;
9908                 if (port->port_devid)
9909                         id_len += port->port_devid->len;
9910         }
9911         mtx_unlock(&softc->ctl_lock);
9912
9913         data_len = sizeof(struct scsi_vpd_scsi_ports) +
9914             num_target_ports * (sizeof(struct scsi_vpd_port_designation) +
9915              sizeof(struct scsi_vpd_port_designation_cont)) + iid_len + id_len;
9916         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9917         sp = (struct scsi_vpd_scsi_ports *)ctsio->kern_data_ptr;
9918         ctsio->kern_sg_entries = 0;
9919
9920         if (data_len < alloc_len) {
9921                 ctsio->residual = alloc_len - data_len;
9922                 ctsio->kern_data_len = data_len;
9923                 ctsio->kern_total_len = data_len;
9924         } else {
9925                 ctsio->residual = 0;
9926                 ctsio->kern_data_len = alloc_len;
9927                 ctsio->kern_total_len = alloc_len;
9928         }
9929         ctsio->kern_data_resid = 0;
9930         ctsio->kern_rel_offset = 0;
9931         ctsio->kern_sg_entries = 0;
9932
9933         /*
9934          * The control device is always connected.  The disk device, on the
9935          * other hand, may not be online all the time.  Need to change this
9936          * to figure out whether the disk device is actually online or not.
9937          */
9938         if (lun != NULL)
9939                 sp->device = (SID_QUAL_LU_CONNECTED << 5) |
9940                                   lun->be_lun->lun_type;
9941         else
9942                 sp->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9943
9944         sp->page_code = SVPD_SCSI_PORTS;
9945         scsi_ulto2b(data_len - sizeof(struct scsi_vpd_scsi_ports),
9946             sp->page_length);
9947         pd = &sp->design[0];
9948
9949         mtx_lock(&softc->ctl_lock);
9950         STAILQ_FOREACH(port, &softc->port_list, links) {
9951                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
9952                         continue;
9953                 if (lun != NULL &&
9954                     ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
9955                         continue;
9956                 scsi_ulto2b(port->targ_port, pd->relative_port_id);
9957                 if (port->init_devid) {
9958                         iid_len = port->init_devid->len;
9959                         memcpy(pd->initiator_transportid,
9960                             port->init_devid->data, port->init_devid->len);
9961                 } else
9962                         iid_len = 0;
9963                 scsi_ulto2b(iid_len, pd->initiator_transportid_length);
9964                 pdc = (struct scsi_vpd_port_designation_cont *)
9965                     (&pd->initiator_transportid[iid_len]);
9966                 if (port->port_devid) {
9967                         id_len = port->port_devid->len;
9968                         memcpy(pdc->target_port_descriptors,
9969                             port->port_devid->data, port->port_devid->len);
9970                 } else
9971                         id_len = 0;
9972                 scsi_ulto2b(id_len, pdc->target_port_descriptors_length);
9973                 pd = (struct scsi_vpd_port_designation *)
9974                     ((uint8_t *)pdc->target_port_descriptors + id_len);
9975         }
9976         mtx_unlock(&softc->ctl_lock);
9977
9978         ctl_set_success(ctsio);
9979         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9980         ctsio->be_move_done = ctl_config_move_done;
9981         ctl_datamove((union ctl_io *)ctsio);
9982         return (CTL_RETVAL_COMPLETE);
9983 }
9984
9985 static int
9986 ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
9987 {
9988         struct scsi_vpd_block_limits *bl_ptr;
9989         struct ctl_lun *lun;
9990         int bs;
9991
9992         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9993
9994         ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
9995         bl_ptr = (struct scsi_vpd_block_limits *)ctsio->kern_data_ptr;
9996         ctsio->kern_sg_entries = 0;
9997
9998         if (sizeof(*bl_ptr) < alloc_len) {
9999                 ctsio->residual = alloc_len - sizeof(*bl_ptr);
10000                 ctsio->kern_data_len = sizeof(*bl_ptr);
10001                 ctsio->kern_total_len = sizeof(*bl_ptr);
10002         } else {
10003                 ctsio->residual = 0;
10004                 ctsio->kern_data_len = alloc_len;
10005                 ctsio->kern_total_len = alloc_len;
10006         }
10007         ctsio->kern_data_resid = 0;
10008         ctsio->kern_rel_offset = 0;
10009         ctsio->kern_sg_entries = 0;
10010
10011         /*
10012          * The control device is always connected.  The disk device, on the
10013          * other hand, may not be online all the time.  Need to change this
10014          * to figure out whether the disk device is actually online or not.
10015          */
10016         if (lun != NULL)
10017                 bl_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10018                                   lun->be_lun->lun_type;
10019         else
10020                 bl_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10021
10022         bl_ptr->page_code = SVPD_BLOCK_LIMITS;
10023         scsi_ulto2b(sizeof(*bl_ptr) - 4, bl_ptr->page_length);
10024         bl_ptr->max_cmp_write_len = 0xff;
10025         scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
10026         if (lun != NULL) {
10027                 bs = lun->be_lun->blocksize;
10028                 scsi_ulto4b(lun->be_lun->opttxferlen, bl_ptr->opt_txfer_len);
10029                 if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10030                         scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
10031                         scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_blk_cnt);
10032                         if (lun->be_lun->ublockexp != 0) {
10033                                 scsi_ulto4b((1 << lun->be_lun->ublockexp),
10034                                     bl_ptr->opt_unmap_grain);
10035                                 scsi_ulto4b(0x80000000 | lun->be_lun->ublockoff,
10036                                     bl_ptr->unmap_grain_align);
10037                         }
10038                 }
10039                 scsi_ulto4b(lun->be_lun->atomicblock,
10040                     bl_ptr->max_atomic_transfer_length);
10041                 scsi_ulto4b(0, bl_ptr->atomic_alignment);
10042                 scsi_ulto4b(0, bl_ptr->atomic_transfer_length_granularity);
10043         }
10044         scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
10045
10046         ctl_set_success(ctsio);
10047         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10048         ctsio->be_move_done = ctl_config_move_done;
10049         ctl_datamove((union ctl_io *)ctsio);
10050         return (CTL_RETVAL_COMPLETE);
10051 }
10052
10053 static int
10054 ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len)
10055 {
10056         struct scsi_vpd_block_device_characteristics *bdc_ptr;
10057         struct ctl_lun *lun;
10058         const char *value;
10059         u_int i;
10060
10061         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10062
10063         ctsio->kern_data_ptr = malloc(sizeof(*bdc_ptr), M_CTL, M_WAITOK | M_ZERO);
10064         bdc_ptr = (struct scsi_vpd_block_device_characteristics *)ctsio->kern_data_ptr;
10065         ctsio->kern_sg_entries = 0;
10066
10067         if (sizeof(*bdc_ptr) < alloc_len) {
10068                 ctsio->residual = alloc_len - sizeof(*bdc_ptr);
10069                 ctsio->kern_data_len = sizeof(*bdc_ptr);
10070                 ctsio->kern_total_len = sizeof(*bdc_ptr);
10071         } else {
10072                 ctsio->residual = 0;
10073                 ctsio->kern_data_len = alloc_len;
10074                 ctsio->kern_total_len = alloc_len;
10075         }
10076         ctsio->kern_data_resid = 0;
10077         ctsio->kern_rel_offset = 0;
10078         ctsio->kern_sg_entries = 0;
10079
10080         /*
10081          * The control device is always connected.  The disk device, on the
10082          * other hand, may not be online all the time.  Need to change this
10083          * to figure out whether the disk device is actually online or not.
10084          */
10085         if (lun != NULL)
10086                 bdc_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10087                                   lun->be_lun->lun_type;
10088         else
10089                 bdc_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10090         bdc_ptr->page_code = SVPD_BDC;
10091         scsi_ulto2b(sizeof(*bdc_ptr) - 4, bdc_ptr->page_length);
10092         if (lun != NULL &&
10093             (value = ctl_get_opt(&lun->be_lun->options, "rpm")) != NULL)
10094                 i = strtol(value, NULL, 0);
10095         else
10096                 i = CTL_DEFAULT_ROTATION_RATE;
10097         scsi_ulto2b(i, bdc_ptr->medium_rotation_rate);
10098         if (lun != NULL &&
10099             (value = ctl_get_opt(&lun->be_lun->options, "formfactor")) != NULL)
10100                 i = strtol(value, NULL, 0);
10101         else
10102                 i = 0;
10103         bdc_ptr->wab_wac_ff = (i & 0x0f);
10104         bdc_ptr->flags = SVPD_FUAB | SVPD_VBULS;
10105
10106         ctl_set_success(ctsio);
10107         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10108         ctsio->be_move_done = ctl_config_move_done;
10109         ctl_datamove((union ctl_io *)ctsio);
10110         return (CTL_RETVAL_COMPLETE);
10111 }
10112
10113 static int
10114 ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
10115 {
10116         struct scsi_vpd_logical_block_prov *lbp_ptr;
10117         struct ctl_lun *lun;
10118
10119         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10120
10121         ctsio->kern_data_ptr = malloc(sizeof(*lbp_ptr), M_CTL, M_WAITOK | M_ZERO);
10122         lbp_ptr = (struct scsi_vpd_logical_block_prov *)ctsio->kern_data_ptr;
10123         ctsio->kern_sg_entries = 0;
10124
10125         if (sizeof(*lbp_ptr) < alloc_len) {
10126                 ctsio->residual = alloc_len - sizeof(*lbp_ptr);
10127                 ctsio->kern_data_len = sizeof(*lbp_ptr);
10128                 ctsio->kern_total_len = sizeof(*lbp_ptr);
10129         } else {
10130                 ctsio->residual = 0;
10131                 ctsio->kern_data_len = alloc_len;
10132                 ctsio->kern_total_len = alloc_len;
10133         }
10134         ctsio->kern_data_resid = 0;
10135         ctsio->kern_rel_offset = 0;
10136         ctsio->kern_sg_entries = 0;
10137
10138         /*
10139          * The control device is always connected.  The disk device, on the
10140          * other hand, may not be online all the time.  Need to change this
10141          * to figure out whether the disk device is actually online or not.
10142          */
10143         if (lun != NULL)
10144                 lbp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10145                                   lun->be_lun->lun_type;
10146         else
10147                 lbp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10148
10149         lbp_ptr->page_code = SVPD_LBP;
10150         scsi_ulto2b(sizeof(*lbp_ptr) - 4, lbp_ptr->page_length);
10151         lbp_ptr->threshold_exponent = CTL_LBP_EXPONENT;
10152         if (lun != NULL && lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10153                 lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 |
10154                     SVPD_LBP_WS10 | SVPD_LBP_RZ | SVPD_LBP_ANC_SUP;
10155                 lbp_ptr->prov_type = SVPD_LBP_THIN;
10156         }
10157
10158         ctl_set_success(ctsio);
10159         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10160         ctsio->be_move_done = ctl_config_move_done;
10161         ctl_datamove((union ctl_io *)ctsio);
10162         return (CTL_RETVAL_COMPLETE);
10163 }
10164
10165 /*
10166  * INQUIRY with the EVPD bit set.
10167  */
10168 static int
10169 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
10170 {
10171         struct ctl_lun *lun;
10172         struct scsi_inquiry *cdb;
10173         int alloc_len, retval;
10174
10175         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10176         cdb = (struct scsi_inquiry *)ctsio->cdb;
10177         alloc_len = scsi_2btoul(cdb->length);
10178
10179         switch (cdb->page_code) {
10180         case SVPD_SUPPORTED_PAGES:
10181                 retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
10182                 break;
10183         case SVPD_UNIT_SERIAL_NUMBER:
10184                 retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
10185                 break;
10186         case SVPD_DEVICE_ID:
10187                 retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
10188                 break;
10189         case SVPD_EXTENDED_INQUIRY_DATA:
10190                 retval = ctl_inquiry_evpd_eid(ctsio, alloc_len);
10191                 break;
10192         case SVPD_MODE_PAGE_POLICY:
10193                 retval = ctl_inquiry_evpd_mpp(ctsio, alloc_len);
10194                 break;
10195         case SVPD_SCSI_PORTS:
10196                 retval = ctl_inquiry_evpd_scsi_ports(ctsio, alloc_len);
10197                 break;
10198         case SVPD_SCSI_TPC:
10199                 retval = ctl_inquiry_evpd_tpc(ctsio, alloc_len);
10200                 break;
10201         case SVPD_BLOCK_LIMITS:
10202                 if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10203                         goto err;
10204                 retval = ctl_inquiry_evpd_block_limits(ctsio, alloc_len);
10205                 break;
10206         case SVPD_BDC:
10207                 if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10208                         goto err;
10209                 retval = ctl_inquiry_evpd_bdc(ctsio, alloc_len);
10210                 break;
10211         case SVPD_LBP:
10212                 if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10213                         goto err;
10214                 retval = ctl_inquiry_evpd_lbp(ctsio, alloc_len);
10215                 break;
10216         default:
10217 err:
10218                 ctl_set_invalid_field(ctsio,
10219                                       /*sks_valid*/ 1,
10220                                       /*command*/ 1,
10221                                       /*field*/ 2,
10222                                       /*bit_valid*/ 0,
10223                                       /*bit*/ 0);
10224                 ctl_done((union ctl_io *)ctsio);
10225                 retval = CTL_RETVAL_COMPLETE;
10226                 break;
10227         }
10228
10229         return (retval);
10230 }
10231
10232 /*
10233  * Standard INQUIRY data.
10234  */
10235 static int
10236 ctl_inquiry_std(struct ctl_scsiio *ctsio)
10237 {
10238         struct scsi_inquiry_data *inq_ptr;
10239         struct scsi_inquiry *cdb;
10240         struct ctl_softc *softc;
10241         struct ctl_port *port;
10242         struct ctl_lun *lun;
10243         char *val;
10244         uint32_t alloc_len, data_len;
10245         ctl_port_type port_type;
10246
10247         softc = control_softc;
10248
10249         /*
10250          * Figure out whether we're talking to a Fibre Channel port or not.
10251          * We treat the ioctl front end, and any SCSI adapters, as packetized
10252          * SCSI front ends.
10253          */
10254         port = ctl_io_port(&ctsio->io_hdr);
10255         if (port != NULL)
10256                 port_type = port->port_type;
10257         else
10258                 port_type = CTL_PORT_SCSI;
10259         if (port_type == CTL_PORT_IOCTL || port_type == CTL_PORT_INTERNAL)
10260                 port_type = CTL_PORT_SCSI;
10261
10262         lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10263         cdb = (struct scsi_inquiry *)ctsio->cdb;
10264         alloc_len = scsi_2btoul(cdb->length);
10265
10266         /*
10267          * We malloc the full inquiry data size here and fill it
10268          * in.  If the user only asks for less, we'll give him
10269          * that much.
10270          */
10271         data_len = offsetof(struct scsi_inquiry_data, vendor_specific1);
10272         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10273         inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
10274         ctsio->kern_sg_entries = 0;
10275         ctsio->kern_data_resid = 0;
10276         ctsio->kern_rel_offset = 0;
10277
10278         if (data_len < alloc_len) {
10279                 ctsio->residual = alloc_len - data_len;
10280                 ctsio->kern_data_len = data_len;
10281                 ctsio->kern_total_len = data_len;
10282         } else {
10283                 ctsio->residual = 0;
10284                 ctsio->kern_data_len = alloc_len;
10285                 ctsio->kern_total_len = alloc_len;
10286         }
10287
10288         if (lun != NULL) {
10289                 if ((lun->flags & CTL_LUN_PRIMARY_SC) ||
10290                     softc->ha_link >= CTL_HA_LINK_UNKNOWN) {
10291                         inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10292                             lun->be_lun->lun_type;
10293                 } else {
10294                         inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) |
10295                             lun->be_lun->lun_type;
10296                 }
10297         } else
10298                 inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
10299
10300         /* RMB in byte 2 is 0 */
10301         inq_ptr->version = SCSI_REV_SPC4;
10302
10303         /*
10304          * According to SAM-3, even if a device only supports a single
10305          * level of LUN addressing, it should still set the HISUP bit:
10306          *
10307          * 4.9.1 Logical unit numbers overview
10308          *
10309          * All logical unit number formats described in this standard are
10310          * hierarchical in structure even when only a single level in that
10311          * hierarchy is used. The HISUP bit shall be set to one in the
10312          * standard INQUIRY data (see SPC-2) when any logical unit number
10313          * format described in this standard is used.  Non-hierarchical
10314          * formats are outside the scope of this standard.
10315          *
10316          * Therefore we set the HiSup bit here.
10317          *
10318          * The reponse format is 2, per SPC-3.
10319          */
10320         inq_ptr->response_format = SID_HiSup | 2;
10321
10322         inq_ptr->additional_length = data_len -
10323             (offsetof(struct scsi_inquiry_data, additional_length) + 1);
10324         CTL_DEBUG_PRINT(("additional_length = %d\n",
10325                          inq_ptr->additional_length));
10326
10327         inq_ptr->spc3_flags = SPC3_SID_3PC | SPC3_SID_TPGS_IMPLICIT;
10328         /* 16 bit addressing */
10329         if (port_type == CTL_PORT_SCSI)
10330                 inq_ptr->spc2_flags = SPC2_SID_ADDR16;
10331         /* XXX set the SID_MultiP bit here if we're actually going to
10332            respond on multiple ports */
10333         inq_ptr->spc2_flags |= SPC2_SID_MultiP;
10334
10335         /* 16 bit data bus, synchronous transfers */
10336         if (port_type == CTL_PORT_SCSI)
10337                 inq_ptr->flags = SID_WBus16 | SID_Sync;
10338         /*
10339          * XXX KDM do we want to support tagged queueing on the control
10340          * device at all?
10341          */
10342         if ((lun == NULL)
10343          || (lun->be_lun->lun_type != T_PROCESSOR))
10344                 inq_ptr->flags |= SID_CmdQue;
10345         /*
10346          * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
10347          * We have 8 bytes for the vendor name, and 16 bytes for the device
10348          * name and 4 bytes for the revision.
10349          */
10350         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10351             "vendor")) == NULL) {
10352                 strncpy(inq_ptr->vendor, CTL_VENDOR, sizeof(inq_ptr->vendor));
10353         } else {
10354                 memset(inq_ptr->vendor, ' ', sizeof(inq_ptr->vendor));
10355                 strncpy(inq_ptr->vendor, val,
10356                     min(sizeof(inq_ptr->vendor), strlen(val)));
10357         }
10358         if (lun == NULL) {
10359                 strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
10360                     sizeof(inq_ptr->product));
10361         } else if ((val = ctl_get_opt(&lun->be_lun->options, "product")) == NULL) {
10362                 switch (lun->be_lun->lun_type) {
10363                 case T_DIRECT:
10364                         strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
10365                             sizeof(inq_ptr->product));
10366                         break;
10367                 case T_PROCESSOR:
10368                         strncpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT,
10369                             sizeof(inq_ptr->product));
10370                         break;
10371                 default:
10372                         strncpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT,
10373                             sizeof(inq_ptr->product));
10374                         break;
10375                 }
10376         } else {
10377                 memset(inq_ptr->product, ' ', sizeof(inq_ptr->product));
10378                 strncpy(inq_ptr->product, val,
10379                     min(sizeof(inq_ptr->product), strlen(val)));
10380         }
10381
10382         /*
10383          * XXX make this a macro somewhere so it automatically gets
10384          * incremented when we make changes.
10385          */
10386         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10387             "revision")) == NULL) {
10388                 strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
10389         } else {
10390                 memset(inq_ptr->revision, ' ', sizeof(inq_ptr->revision));
10391                 strncpy(inq_ptr->revision, val,
10392                     min(sizeof(inq_ptr->revision), strlen(val)));
10393         }
10394
10395         /*
10396          * For parallel SCSI, we support double transition and single
10397          * transition clocking.  We also support QAS (Quick Arbitration
10398          * and Selection) and Information Unit transfers on both the
10399          * control and array devices.
10400          */
10401         if (port_type == CTL_PORT_SCSI)
10402                 inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
10403                                     SID_SPI_IUS;
10404
10405         /* SAM-5 (no version claimed) */
10406         scsi_ulto2b(0x00A0, inq_ptr->version1);
10407         /* SPC-4 (no version claimed) */
10408         scsi_ulto2b(0x0460, inq_ptr->version2);
10409         if (port_type == CTL_PORT_FC) {
10410                 /* FCP-2 ANSI INCITS.350:2003 */
10411                 scsi_ulto2b(0x0917, inq_ptr->version3);
10412         } else if (port_type == CTL_PORT_SCSI) {
10413                 /* SPI-4 ANSI INCITS.362:200x */
10414                 scsi_ulto2b(0x0B56, inq_ptr->version3);
10415         } else if (port_type == CTL_PORT_ISCSI) {
10416                 /* iSCSI (no version claimed) */
10417                 scsi_ulto2b(0x0960, inq_ptr->version3);
10418         } else if (port_type == CTL_PORT_SAS) {
10419                 /* SAS (no version claimed) */
10420                 scsi_ulto2b(0x0BE0, inq_ptr->version3);
10421         }
10422
10423         if (lun == NULL) {
10424                 /* SBC-4 (no version claimed) */
10425                 scsi_ulto2b(0x0600, inq_ptr->version4);
10426         } else {
10427                 switch (lun->be_lun->lun_type) {
10428                 case T_DIRECT:
10429                         /* SBC-4 (no version claimed) */
10430                         scsi_ulto2b(0x0600, inq_ptr->version4);
10431                         break;
10432                 case T_PROCESSOR:
10433                 default:
10434                         break;
10435                 }
10436         }
10437
10438         ctl_set_success(ctsio);
10439         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10440         ctsio->be_move_done = ctl_config_move_done;
10441         ctl_datamove((union ctl_io *)ctsio);
10442         return (CTL_RETVAL_COMPLETE);
10443 }
10444
10445 int
10446 ctl_inquiry(struct ctl_scsiio *ctsio)
10447 {
10448         struct scsi_inquiry *cdb;
10449         int retval;
10450
10451         CTL_DEBUG_PRINT(("ctl_inquiry\n"));
10452
10453         cdb = (struct scsi_inquiry *)ctsio->cdb;
10454         if (cdb->byte2 & SI_EVPD)
10455                 retval = ctl_inquiry_evpd(ctsio);
10456         else if (cdb->page_code == 0)
10457                 retval = ctl_inquiry_std(ctsio);
10458         else {
10459                 ctl_set_invalid_field(ctsio,
10460                                       /*sks_valid*/ 1,
10461                                       /*command*/ 1,
10462                                       /*field*/ 2,
10463                                       /*bit_valid*/ 0,
10464                                       /*bit*/ 0);
10465                 ctl_done((union ctl_io *)ctsio);
10466                 return (CTL_RETVAL_COMPLETE);
10467         }
10468
10469         return (retval);
10470 }
10471
10472 /*
10473  * For known CDB types, parse the LBA and length.
10474  */
10475 static int
10476 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len)
10477 {
10478         if (io->io_hdr.io_type != CTL_IO_SCSI)
10479                 return (1);
10480
10481         switch (io->scsiio.cdb[0]) {
10482         case COMPARE_AND_WRITE: {
10483                 struct scsi_compare_and_write *cdb;
10484
10485                 cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
10486
10487                 *lba = scsi_8btou64(cdb->addr);
10488                 *len = cdb->length;
10489                 break;
10490         }
10491         case READ_6:
10492         case WRITE_6: {
10493                 struct scsi_rw_6 *cdb;
10494
10495                 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
10496
10497                 *lba = scsi_3btoul(cdb->addr);
10498                 /* only 5 bits are valid in the most significant address byte */
10499                 *lba &= 0x1fffff;
10500                 *len = cdb->length;
10501                 break;
10502         }
10503         case READ_10:
10504         case WRITE_10: {
10505                 struct scsi_rw_10 *cdb;
10506
10507                 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
10508
10509                 *lba = scsi_4btoul(cdb->addr);
10510                 *len = scsi_2btoul(cdb->length);
10511                 break;
10512         }
10513         case WRITE_VERIFY_10: {
10514                 struct scsi_write_verify_10 *cdb;
10515
10516                 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
10517
10518                 *lba = scsi_4btoul(cdb->addr);
10519                 *len = scsi_2btoul(cdb->length);
10520                 break;
10521         }
10522         case READ_12:
10523         case WRITE_12: {
10524                 struct scsi_rw_12 *cdb;
10525
10526                 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
10527
10528                 *lba = scsi_4btoul(cdb->addr);
10529                 *len = scsi_4btoul(cdb->length);
10530                 break;
10531         }
10532         case WRITE_VERIFY_12: {
10533                 struct scsi_write_verify_12 *cdb;
10534
10535                 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
10536
10537                 *lba = scsi_4btoul(cdb->addr);
10538                 *len = scsi_4btoul(cdb->length);
10539                 break;
10540         }
10541         case READ_16:
10542         case WRITE_16:
10543         case WRITE_ATOMIC_16: {
10544                 struct scsi_rw_16 *cdb;
10545
10546                 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
10547
10548                 *lba = scsi_8btou64(cdb->addr);
10549                 *len = scsi_4btoul(cdb->length);
10550                 break;
10551         }
10552         case WRITE_VERIFY_16: {
10553                 struct scsi_write_verify_16 *cdb;
10554
10555                 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
10556
10557                 *lba = scsi_8btou64(cdb->addr);
10558                 *len = scsi_4btoul(cdb->length);
10559                 break;
10560         }
10561         case WRITE_SAME_10: {
10562                 struct scsi_write_same_10 *cdb;
10563
10564                 cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10565
10566                 *lba = scsi_4btoul(cdb->addr);
10567                 *len = scsi_2btoul(cdb->length);
10568                 break;
10569         }
10570         case WRITE_SAME_16: {
10571                 struct scsi_write_same_16 *cdb;
10572
10573                 cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10574
10575                 *lba = scsi_8btou64(cdb->addr);
10576                 *len = scsi_4btoul(cdb->length);
10577                 break;
10578         }
10579         case VERIFY_10: {
10580                 struct scsi_verify_10 *cdb;
10581
10582                 cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
10583
10584                 *lba = scsi_4btoul(cdb->addr);
10585                 *len = scsi_2btoul(cdb->length);
10586                 break;
10587         }
10588         case VERIFY_12: {
10589                 struct scsi_verify_12 *cdb;
10590
10591                 cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
10592
10593                 *lba = scsi_4btoul(cdb->addr);
10594                 *len = scsi_4btoul(cdb->length);
10595                 break;
10596         }
10597         case VERIFY_16: {
10598                 struct scsi_verify_16 *cdb;
10599
10600                 cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
10601
10602                 *lba = scsi_8btou64(cdb->addr);
10603                 *len = scsi_4btoul(cdb->length);
10604                 break;
10605         }
10606         case UNMAP: {
10607                 *lba = 0;
10608                 *len = UINT64_MAX;
10609                 break;
10610         }
10611         case SERVICE_ACTION_IN: {       /* GET LBA STATUS */
10612                 struct scsi_get_lba_status *cdb;
10613
10614                 cdb = (struct scsi_get_lba_status *)io->scsiio.cdb;
10615                 *lba = scsi_8btou64(cdb->addr);
10616                 *len = UINT32_MAX;
10617                 break;
10618         }
10619         default:
10620                 return (1);
10621                 break; /* NOTREACHED */
10622         }
10623
10624         return (0);
10625 }
10626
10627 static ctl_action
10628 ctl_extent_check_lba(uint64_t lba1, uint64_t len1, uint64_t lba2, uint64_t len2,
10629     bool seq)
10630 {
10631         uint64_t endlba1, endlba2;
10632
10633         endlba1 = lba1 + len1 - (seq ? 0 : 1);
10634         endlba2 = lba2 + len2 - 1;
10635
10636         if ((endlba1 < lba2) || (endlba2 < lba1))
10637                 return (CTL_ACTION_PASS);
10638         else
10639                 return (CTL_ACTION_BLOCK);
10640 }
10641
10642 static int
10643 ctl_extent_check_unmap(union ctl_io *io, uint64_t lba2, uint64_t len2)
10644 {
10645         struct ctl_ptr_len_flags *ptrlen;
10646         struct scsi_unmap_desc *buf, *end, *range;
10647         uint64_t lba;
10648         uint32_t len;
10649
10650         /* If not UNMAP -- go other way. */
10651         if (io->io_hdr.io_type != CTL_IO_SCSI ||
10652             io->scsiio.cdb[0] != UNMAP)
10653                 return (CTL_ACTION_ERROR);
10654
10655         /* If UNMAP without data -- block and wait for data. */
10656         ptrlen = (struct ctl_ptr_len_flags *)
10657             &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
10658         if ((io->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0 ||
10659             ptrlen->ptr == NULL)
10660                 return (CTL_ACTION_BLOCK);
10661
10662         /* UNMAP with data -- check for collision. */
10663         buf = (struct scsi_unmap_desc *)ptrlen->ptr;
10664         end = buf + ptrlen->len / sizeof(*buf);
10665         for (range = buf; range < end; range++) {
10666                 lba = scsi_8btou64(range->lba);
10667                 len = scsi_4btoul(range->length);
10668                 if ((lba < lba2 + len2) && (lba + len > lba2))
10669                         return (CTL_ACTION_BLOCK);
10670         }
10671         return (CTL_ACTION_PASS);
10672 }
10673
10674 static ctl_action
10675 ctl_extent_check(union ctl_io *io1, union ctl_io *io2, bool seq)
10676 {
10677         uint64_t lba1, lba2;
10678         uint64_t len1, len2;
10679         int retval;
10680
10681         if (ctl_get_lba_len(io2, &lba2, &len2) != 0)
10682                 return (CTL_ACTION_ERROR);
10683
10684         retval = ctl_extent_check_unmap(io1, lba2, len2);
10685         if (retval != CTL_ACTION_ERROR)
10686                 return (retval);
10687
10688         if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
10689                 return (CTL_ACTION_ERROR);
10690
10691         if (io1->io_hdr.flags & CTL_FLAG_SERSEQ_DONE)
10692                 seq = FALSE;
10693         return (ctl_extent_check_lba(lba1, len1, lba2, len2, seq));
10694 }
10695
10696 static ctl_action
10697 ctl_extent_check_seq(union ctl_io *io1, union ctl_io *io2)
10698 {
10699         uint64_t lba1, lba2;
10700         uint64_t len1, len2;
10701
10702         if (io1->io_hdr.flags & CTL_FLAG_SERSEQ_DONE)
10703                 return (CTL_ACTION_PASS);
10704         if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
10705                 return (CTL_ACTION_ERROR);
10706         if (ctl_get_lba_len(io2, &lba2, &len2) != 0)
10707                 return (CTL_ACTION_ERROR);
10708
10709         if (lba1 + len1 == lba2)
10710                 return (CTL_ACTION_BLOCK);
10711         return (CTL_ACTION_PASS);
10712 }
10713
10714 static ctl_action
10715 ctl_check_for_blockage(struct ctl_lun *lun, union ctl_io *pending_io,
10716     union ctl_io *ooa_io)
10717 {
10718         const struct ctl_cmd_entry *pending_entry, *ooa_entry;
10719         ctl_serialize_action *serialize_row;
10720
10721         /*
10722          * The initiator attempted multiple untagged commands at the same
10723          * time.  Can't do that.
10724          */
10725         if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10726          && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10727          && ((pending_io->io_hdr.nexus.targ_port ==
10728               ooa_io->io_hdr.nexus.targ_port)
10729           && (pending_io->io_hdr.nexus.initid ==
10730               ooa_io->io_hdr.nexus.initid))
10731          && ((ooa_io->io_hdr.flags & (CTL_FLAG_ABORT |
10732               CTL_FLAG_STATUS_SENT)) == 0))
10733                 return (CTL_ACTION_OVERLAP);
10734
10735         /*
10736          * The initiator attempted to send multiple tagged commands with
10737          * the same ID.  (It's fine if different initiators have the same
10738          * tag ID.)
10739          *
10740          * Even if all of those conditions are true, we don't kill the I/O
10741          * if the command ahead of us has been aborted.  We won't end up
10742          * sending it to the FETD, and it's perfectly legal to resend a
10743          * command with the same tag number as long as the previous
10744          * instance of this tag number has been aborted somehow.
10745          */
10746         if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10747          && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10748          && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
10749          && ((pending_io->io_hdr.nexus.targ_port ==
10750               ooa_io->io_hdr.nexus.targ_port)
10751           && (pending_io->io_hdr.nexus.initid ==
10752               ooa_io->io_hdr.nexus.initid))
10753          && ((ooa_io->io_hdr.flags & (CTL_FLAG_ABORT |
10754               CTL_FLAG_STATUS_SENT)) == 0))
10755                 return (CTL_ACTION_OVERLAP_TAG);
10756
10757         /*
10758          * If we get a head of queue tag, SAM-3 says that we should
10759          * immediately execute it.
10760          *
10761          * What happens if this command would normally block for some other
10762          * reason?  e.g. a request sense with a head of queue tag
10763          * immediately after a write.  Normally that would block, but this
10764          * will result in its getting executed immediately...
10765          *
10766          * We currently return "pass" instead of "skip", so we'll end up
10767          * going through the rest of the queue to check for overlapped tags.
10768          *
10769          * XXX KDM check for other types of blockage first??
10770          */
10771         if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10772                 return (CTL_ACTION_PASS);
10773
10774         /*
10775          * Ordered tags have to block until all items ahead of them
10776          * have completed.  If we get called with an ordered tag, we always
10777          * block, if something else is ahead of us in the queue.
10778          */
10779         if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
10780                 return (CTL_ACTION_BLOCK);
10781
10782         /*
10783          * Simple tags get blocked until all head of queue and ordered tags
10784          * ahead of them have completed.  I'm lumping untagged commands in
10785          * with simple tags here.  XXX KDM is that the right thing to do?
10786          */
10787         if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10788           || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
10789          && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10790           || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
10791                 return (CTL_ACTION_BLOCK);
10792
10793         pending_entry = ctl_get_cmd_entry(&pending_io->scsiio, NULL);
10794         ooa_entry = ctl_get_cmd_entry(&ooa_io->scsiio, NULL);
10795
10796         serialize_row = ctl_serialize_table[ooa_entry->seridx];
10797
10798         switch (serialize_row[pending_entry->seridx]) {
10799         case CTL_SER_BLOCK:
10800                 return (CTL_ACTION_BLOCK);
10801         case CTL_SER_EXTENT:
10802                 return (ctl_extent_check(ooa_io, pending_io,
10803                     (lun->be_lun && lun->be_lun->serseq == CTL_LUN_SERSEQ_ON)));
10804         case CTL_SER_EXTENTOPT:
10805                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT].queue_flags
10806                     & SCP_QUEUE_ALG_MASK) != SCP_QUEUE_ALG_UNRESTRICTED)
10807                         return (ctl_extent_check(ooa_io, pending_io,
10808                             (lun->be_lun &&
10809                              lun->be_lun->serseq == CTL_LUN_SERSEQ_ON)));
10810                 return (CTL_ACTION_PASS);
10811         case CTL_SER_EXTENTSEQ:
10812                 if (lun->be_lun && lun->be_lun->serseq != CTL_LUN_SERSEQ_OFF)
10813                         return (ctl_extent_check_seq(ooa_io, pending_io));
10814                 return (CTL_ACTION_PASS);
10815         case CTL_SER_PASS:
10816                 return (CTL_ACTION_PASS);
10817         case CTL_SER_BLOCKOPT:
10818                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT].queue_flags
10819                     & SCP_QUEUE_ALG_MASK) != SCP_QUEUE_ALG_UNRESTRICTED)
10820                         return (CTL_ACTION_BLOCK);
10821                 return (CTL_ACTION_PASS);
10822         case CTL_SER_SKIP:
10823                 return (CTL_ACTION_SKIP);
10824         default:
10825                 panic("invalid serialization value %d",
10826                       serialize_row[pending_entry->seridx]);
10827         }
10828
10829         return (CTL_ACTION_ERROR);
10830 }
10831
10832 /*
10833  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
10834  * Assumptions:
10835  * - pending_io is generally either incoming, or on the blocked queue
10836  * - starting I/O is the I/O we want to start the check with.
10837  */
10838 static ctl_action
10839 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
10840               union ctl_io *starting_io)
10841 {
10842         union ctl_io *ooa_io;
10843         ctl_action action;
10844
10845         mtx_assert(&lun->lun_lock, MA_OWNED);
10846
10847         /*
10848          * Run back along the OOA queue, starting with the current
10849          * blocked I/O and going through every I/O before it on the
10850          * queue.  If starting_io is NULL, we'll just end up returning
10851          * CTL_ACTION_PASS.
10852          */
10853         for (ooa_io = starting_io; ooa_io != NULL;
10854              ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
10855              ooa_links)){
10856
10857                 /*
10858                  * This routine just checks to see whether
10859                  * cur_blocked is blocked by ooa_io, which is ahead
10860                  * of it in the queue.  It doesn't queue/dequeue
10861                  * cur_blocked.
10862                  */
10863                 action = ctl_check_for_blockage(lun, pending_io, ooa_io);
10864                 switch (action) {
10865                 case CTL_ACTION_BLOCK:
10866                 case CTL_ACTION_OVERLAP:
10867                 case CTL_ACTION_OVERLAP_TAG:
10868                 case CTL_ACTION_SKIP:
10869                 case CTL_ACTION_ERROR:
10870                         return (action);
10871                         break; /* NOTREACHED */
10872                 case CTL_ACTION_PASS:
10873                         break;
10874                 default:
10875                         panic("invalid action %d", action);
10876                         break;  /* NOTREACHED */
10877                 }
10878         }
10879
10880         return (CTL_ACTION_PASS);
10881 }
10882
10883 /*
10884  * Assumptions:
10885  * - An I/O has just completed, and has been removed from the per-LUN OOA
10886  *   queue, so some items on the blocked queue may now be unblocked.
10887  */
10888 static int
10889 ctl_check_blocked(struct ctl_lun *lun)
10890 {
10891         struct ctl_softc *softc = lun->ctl_softc;
10892         union ctl_io *cur_blocked, *next_blocked;
10893
10894         mtx_assert(&lun->lun_lock, MA_OWNED);
10895
10896         /*
10897          * Run forward from the head of the blocked queue, checking each
10898          * entry against the I/Os prior to it on the OOA queue to see if
10899          * there is still any blockage.
10900          *
10901          * We cannot use the TAILQ_FOREACH() macro, because it can't deal
10902          * with our removing a variable on it while it is traversing the
10903          * list.
10904          */
10905         for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
10906              cur_blocked != NULL; cur_blocked = next_blocked) {
10907                 union ctl_io *prev_ooa;
10908                 ctl_action action;
10909
10910                 next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
10911                                                           blocked_links);
10912
10913                 prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
10914                                                       ctl_ooaq, ooa_links);
10915
10916                 /*
10917                  * If cur_blocked happens to be the first item in the OOA
10918                  * queue now, prev_ooa will be NULL, and the action
10919                  * returned will just be CTL_ACTION_PASS.
10920                  */
10921                 action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
10922
10923                 switch (action) {
10924                 case CTL_ACTION_BLOCK:
10925                         /* Nothing to do here, still blocked */
10926                         break;
10927                 case CTL_ACTION_OVERLAP:
10928                 case CTL_ACTION_OVERLAP_TAG:
10929                         /*
10930                          * This shouldn't happen!  In theory we've already
10931                          * checked this command for overlap...
10932                          */
10933                         break;
10934                 case CTL_ACTION_PASS:
10935                 case CTL_ACTION_SKIP: {
10936                         const struct ctl_cmd_entry *entry;
10937
10938                         /*
10939                          * The skip case shouldn't happen, this transaction
10940                          * should have never made it onto the blocked queue.
10941                          */
10942                         /*
10943                          * This I/O is no longer blocked, we can remove it
10944                          * from the blocked queue.  Since this is a TAILQ
10945                          * (doubly linked list), we can do O(1) removals
10946                          * from any place on the list.
10947                          */
10948                         TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
10949                                      blocked_links);
10950                         cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10951
10952                         if ((softc->ha_mode != CTL_HA_MODE_XFER) &&
10953                             (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)){
10954                                 /*
10955                                  * Need to send IO back to original side to
10956                                  * run
10957                                  */
10958                                 union ctl_ha_msg msg_info;
10959
10960                                 cur_blocked->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
10961                                 msg_info.hdr.original_sc =
10962                                         cur_blocked->io_hdr.original_sc;
10963                                 msg_info.hdr.serializing_sc = cur_blocked;
10964                                 msg_info.hdr.msg_type = CTL_MSG_R2R;
10965                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
10966                                     sizeof(msg_info.hdr), M_NOWAIT);
10967                                 break;
10968                         }
10969                         entry = ctl_get_cmd_entry(&cur_blocked->scsiio, NULL);
10970
10971                         /*
10972                          * Check this I/O for LUN state changes that may
10973                          * have happened while this command was blocked.
10974                          * The LUN state may have been changed by a command
10975                          * ahead of us in the queue, so we need to re-check
10976                          * for any states that can be caused by SCSI
10977                          * commands.
10978                          */
10979                         if (ctl_scsiio_lun_check(lun, entry,
10980                                                  &cur_blocked->scsiio) == 0) {
10981                                 cur_blocked->io_hdr.flags |=
10982                                                       CTL_FLAG_IS_WAS_ON_RTR;
10983                                 ctl_enqueue_rtr(cur_blocked);
10984                         } else
10985                                 ctl_done(cur_blocked);
10986                         break;
10987                 }
10988                 default:
10989                         /*
10990                          * This probably shouldn't happen -- we shouldn't
10991                          * get CTL_ACTION_ERROR, or anything else.
10992                          */
10993                         break;
10994                 }
10995         }
10996
10997         return (CTL_RETVAL_COMPLETE);
10998 }
10999
11000 /*
11001  * This routine (with one exception) checks LUN flags that can be set by
11002  * commands ahead of us in the OOA queue.  These flags have to be checked
11003  * when a command initially comes in, and when we pull a command off the
11004  * blocked queue and are preparing to execute it.  The reason we have to
11005  * check these flags for commands on the blocked queue is that the LUN
11006  * state may have been changed by a command ahead of us while we're on the
11007  * blocked queue.
11008  *
11009  * Ordering is somewhat important with these checks, so please pay
11010  * careful attention to the placement of any new checks.
11011  */
11012 static int
11013 ctl_scsiio_lun_check(struct ctl_lun *lun,
11014     const struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
11015 {
11016         struct ctl_softc *softc = lun->ctl_softc;
11017         int retval;
11018         uint32_t residx;
11019
11020         retval = 0;
11021
11022         mtx_assert(&lun->lun_lock, MA_OWNED);
11023
11024         /*
11025          * If this shelf is a secondary shelf controller, we may have to
11026          * reject some commands disallowed by HA mode and link state.
11027          */
11028         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
11029                 if (softc->ha_link == CTL_HA_LINK_OFFLINE &&
11030                     (entry->flags & CTL_CMD_FLAG_OK_ON_UNAVAIL) == 0) {
11031                         ctl_set_lun_unavail(ctsio);
11032                         retval = 1;
11033                         goto bailout;
11034                 }
11035                 if ((lun->flags & CTL_LUN_PEER_SC_PRIMARY) == 0 &&
11036                     (entry->flags & CTL_CMD_FLAG_OK_ON_UNAVAIL) == 0) {
11037                         ctl_set_lun_transit(ctsio);
11038                         retval = 1;
11039                         goto bailout;
11040                 }
11041                 if (softc->ha_mode == CTL_HA_MODE_ACT_STBY &&
11042                     (entry->flags & CTL_CMD_FLAG_OK_ON_STANDBY) == 0) {
11043                         ctl_set_lun_standby(ctsio);
11044                         retval = 1;
11045                         goto bailout;
11046                 }
11047
11048                 /* The rest of checks are only done on executing side */
11049                 if (softc->ha_mode == CTL_HA_MODE_XFER)
11050                         goto bailout;
11051         }
11052
11053         if (entry->pattern & CTL_LUN_PAT_WRITE) {
11054                 if (lun->be_lun &&
11055                     lun->be_lun->flags & CTL_LUN_FLAG_READONLY) {
11056                         ctl_set_hw_write_protected(ctsio);
11057                         retval = 1;
11058                         goto bailout;
11059                 }
11060                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT]
11061                     .eca_and_aen & SCP_SWP) != 0) {
11062                         ctl_set_sense(ctsio, /*current_error*/ 1,
11063                             /*sense_key*/ SSD_KEY_DATA_PROTECT,
11064                             /*asc*/ 0x27, /*ascq*/ 0x02, SSD_ELEM_NONE);
11065                         retval = 1;
11066                         goto bailout;
11067                 }
11068         }
11069
11070         /*
11071          * Check for a reservation conflict.  If this command isn't allowed
11072          * even on reserved LUNs, and if this initiator isn't the one who
11073          * reserved us, reject the command with a reservation conflict.
11074          */
11075         residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11076         if ((lun->flags & CTL_LUN_RESERVED)
11077          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
11078                 if (lun->res_idx != residx) {
11079                         ctl_set_reservation_conflict(ctsio);
11080                         retval = 1;
11081                         goto bailout;
11082                 }
11083         }
11084
11085         if ((lun->flags & CTL_LUN_PR_RESERVED) == 0 ||
11086             (entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV)) {
11087                 /* No reservation or command is allowed. */;
11088         } else if ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_WRESV) &&
11089             (lun->res_type == SPR_TYPE_WR_EX ||
11090              lun->res_type == SPR_TYPE_WR_EX_RO ||
11091              lun->res_type == SPR_TYPE_WR_EX_AR)) {
11092                 /* The command is allowed for Write Exclusive resv. */;
11093         } else {
11094                 /*
11095                  * if we aren't registered or it's a res holder type
11096                  * reservation and this isn't the res holder then set a
11097                  * conflict.
11098                  */
11099                 if (ctl_get_prkey(lun, residx) == 0
11100                  || (residx != lun->pr_res_idx && lun->res_type < 4)) {
11101                         ctl_set_reservation_conflict(ctsio);
11102                         retval = 1;
11103                         goto bailout;
11104                 }
11105         }
11106
11107         if ((lun->flags & CTL_LUN_OFFLINE)
11108          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STANDBY) == 0)) {
11109                 ctl_set_lun_not_ready(ctsio);
11110                 retval = 1;
11111                 goto bailout;
11112         }
11113
11114         if ((lun->flags & CTL_LUN_STOPPED)
11115          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
11116                 /* "Logical unit not ready, initializing cmd. required" */
11117                 ctl_set_lun_stopped(ctsio);
11118                 retval = 1;
11119                 goto bailout;
11120         }
11121
11122         if ((lun->flags & CTL_LUN_INOPERABLE)
11123          && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
11124                 /* "Medium format corrupted" */
11125                 ctl_set_medium_format_corrupted(ctsio);
11126                 retval = 1;
11127                 goto bailout;
11128         }
11129
11130 bailout:
11131         return (retval);
11132 }
11133
11134 static void
11135 ctl_failover_io(union ctl_io *io, int have_lock)
11136 {
11137         ctl_set_busy(&io->scsiio);
11138         ctl_done(io);
11139 }
11140
11141 static void
11142 ctl_failover_lun(struct ctl_lun *lun)
11143 {
11144         struct ctl_softc *softc = lun->ctl_softc;
11145         struct ctl_io_hdr *io, *next_io;
11146
11147         CTL_DEBUG_PRINT(("FAILOVER for lun %ju\n", lun->lun));
11148         if (softc->ha_mode == CTL_HA_MODE_XFER) {
11149                 TAILQ_FOREACH_SAFE(io, &lun->ooa_queue, ooa_links, next_io) {
11150                         /* We are master */
11151                         if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11152                                 if (io->flags & CTL_FLAG_IO_ACTIVE) {
11153                                         io->flags |= CTL_FLAG_ABORT;
11154                                         io->flags |= CTL_FLAG_FAILOVER;
11155                                 } else { /* This can be only due to DATAMOVE */
11156                                         io->msg_type = CTL_MSG_DATAMOVE_DONE;
11157                                         io->flags &= ~CTL_FLAG_DMA_INPROG;
11158                                         io->flags |= CTL_FLAG_IO_ACTIVE;
11159                                         io->port_status = 31340;
11160                                         ctl_enqueue_isc((union ctl_io *)io);
11161                                 }
11162                         }
11163                         /* We are slave */
11164                         if (io->flags & CTL_FLAG_SENT_2OTHER_SC) {
11165                                 io->flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11166                                 if (io->flags & CTL_FLAG_IO_ACTIVE) {
11167                                         io->flags |= CTL_FLAG_FAILOVER;
11168                                 } else {
11169                                         ctl_set_busy(&((union ctl_io *)io)->
11170                                             scsiio);
11171                                         ctl_done((union ctl_io *)io);
11172                                 }
11173                         }
11174                 }
11175         } else { /* SERIALIZE modes */
11176                 TAILQ_FOREACH_SAFE(io, &lun->blocked_queue, blocked_links,
11177                     next_io) {
11178                         /* We are master */
11179                         if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11180                                 TAILQ_REMOVE(&lun->blocked_queue, io,
11181                                     blocked_links);
11182                                 io->flags &= ~CTL_FLAG_BLOCKED;
11183                                 TAILQ_REMOVE(&lun->ooa_queue, io, ooa_links);
11184                                 ctl_free_io((union ctl_io *)io);
11185                         }
11186                 }
11187                 TAILQ_FOREACH_SAFE(io, &lun->ooa_queue, ooa_links, next_io) {
11188                         /* We are master */
11189                         if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11190                                 TAILQ_REMOVE(&lun->ooa_queue, io, ooa_links);
11191                                 ctl_free_io((union ctl_io *)io);
11192                         }
11193                         /* We are slave */
11194                         if (io->flags & CTL_FLAG_SENT_2OTHER_SC) {
11195                                 io->flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11196                                 if (!(io->flags & CTL_FLAG_IO_ACTIVE)) {
11197                                         ctl_set_busy(&((union ctl_io *)io)->
11198                                             scsiio);
11199                                         ctl_done((union ctl_io *)io);
11200                                 }
11201                         }
11202                 }
11203                 ctl_check_blocked(lun);
11204         }
11205 }
11206
11207 static int
11208 ctl_scsiio_precheck(struct ctl_softc *softc, struct ctl_scsiio *ctsio)
11209 {
11210         struct ctl_lun *lun;
11211         const struct ctl_cmd_entry *entry;
11212         uint32_t initidx, targ_lun;
11213         int retval;
11214
11215         retval = 0;
11216
11217         lun = NULL;
11218
11219         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
11220         if ((targ_lun < CTL_MAX_LUNS)
11221          && ((lun = softc->ctl_luns[targ_lun]) != NULL)) {
11222                 /*
11223                  * If the LUN is invalid, pretend that it doesn't exist.
11224                  * It will go away as soon as all pending I/O has been
11225                  * completed.
11226                  */
11227                 mtx_lock(&lun->lun_lock);
11228                 if (lun->flags & CTL_LUN_DISABLED) {
11229                         mtx_unlock(&lun->lun_lock);
11230                         lun = NULL;
11231                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11232                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11233                 } else {
11234                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
11235                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
11236                                 lun->be_lun;
11237
11238                         /*
11239                          * Every I/O goes into the OOA queue for a
11240                          * particular LUN, and stays there until completion.
11241                          */
11242 #ifdef CTL_TIME_IO
11243                         if (TAILQ_EMPTY(&lun->ooa_queue)) {
11244                                 lun->idle_time += getsbinuptime() -
11245                                     lun->last_busy;
11246                         }
11247 #endif
11248                         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr,
11249                             ooa_links);
11250                 }
11251         } else {
11252                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11253                 ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11254         }
11255
11256         /* Get command entry and return error if it is unsuppotyed. */
11257         entry = ctl_validate_command(ctsio);
11258         if (entry == NULL) {
11259                 if (lun)
11260                         mtx_unlock(&lun->lun_lock);
11261                 return (retval);
11262         }
11263
11264         ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
11265         ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
11266
11267         /*
11268          * Check to see whether we can send this command to LUNs that don't
11269          * exist.  This should pretty much only be the case for inquiry
11270          * and request sense.  Further checks, below, really require having
11271          * a LUN, so we can't really check the command anymore.  Just put
11272          * it on the rtr queue.
11273          */
11274         if (lun == NULL) {
11275                 if (entry->flags & CTL_CMD_FLAG_OK_ON_NO_LUN) {
11276                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11277                         ctl_enqueue_rtr((union ctl_io *)ctsio);
11278                         return (retval);
11279                 }
11280
11281                 ctl_set_unsupported_lun(ctsio);
11282                 ctl_done((union ctl_io *)ctsio);
11283                 CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
11284                 return (retval);
11285         } else {
11286                 /*
11287                  * Make sure we support this particular command on this LUN.
11288                  * e.g., we don't support writes to the control LUN.
11289                  */
11290                 if (!ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
11291                         mtx_unlock(&lun->lun_lock);
11292                         ctl_set_invalid_opcode(ctsio);
11293                         ctl_done((union ctl_io *)ctsio);
11294                         return (retval);
11295                 }
11296         }
11297
11298         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11299
11300 #ifdef CTL_WITH_CA
11301         /*
11302          * If we've got a request sense, it'll clear the contingent
11303          * allegiance condition.  Otherwise, if we have a CA condition for
11304          * this initiator, clear it, because it sent down a command other
11305          * than request sense.
11306          */
11307         if ((ctsio->cdb[0] != REQUEST_SENSE)
11308          && (ctl_is_set(lun->have_ca, initidx)))
11309                 ctl_clear_mask(lun->have_ca, initidx);
11310 #endif
11311
11312         /*
11313          * If the command has this flag set, it handles its own unit
11314          * attention reporting, we shouldn't do anything.  Otherwise we
11315          * check for any pending unit attentions, and send them back to the
11316          * initiator.  We only do this when a command initially comes in,
11317          * not when we pull it off the blocked queue.
11318          *
11319          * According to SAM-3, section 5.3.2, the order that things get
11320          * presented back to the host is basically unit attentions caused
11321          * by some sort of reset event, busy status, reservation conflicts
11322          * or task set full, and finally any other status.
11323          *
11324          * One issue here is that some of the unit attentions we report
11325          * don't fall into the "reset" category (e.g. "reported luns data
11326          * has changed").  So reporting it here, before the reservation
11327          * check, may be technically wrong.  I guess the only thing to do
11328          * would be to check for and report the reset events here, and then
11329          * check for the other unit attention types after we check for a
11330          * reservation conflict.
11331          *
11332          * XXX KDM need to fix this
11333          */
11334         if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
11335                 ctl_ua_type ua_type;
11336
11337                 ua_type = ctl_build_ua(lun, initidx, &ctsio->sense_data,
11338                     SSD_TYPE_NONE);
11339                 if (ua_type != CTL_UA_NONE) {
11340                         mtx_unlock(&lun->lun_lock);
11341                         ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
11342                         ctsio->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
11343                         ctsio->sense_len = SSD_FULL_SIZE;
11344                         ctl_done((union ctl_io *)ctsio);
11345                         return (retval);
11346                 }
11347         }
11348
11349
11350         if (ctl_scsiio_lun_check(lun, entry, ctsio) != 0) {
11351                 mtx_unlock(&lun->lun_lock);
11352                 ctl_done((union ctl_io *)ctsio);
11353                 return (retval);
11354         }
11355
11356         /*
11357          * XXX CHD this is where we want to send IO to other side if
11358          * this LUN is secondary on this SC. We will need to make a copy
11359          * of the IO and flag the IO on this side as SENT_2OTHER and the flag
11360          * the copy we send as FROM_OTHER.
11361          * We also need to stuff the address of the original IO so we can
11362          * find it easily. Something similar will need be done on the other
11363          * side so when we are done we can find the copy.
11364          */
11365         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
11366             (lun->flags & CTL_LUN_PEER_SC_PRIMARY) != 0 &&
11367             (entry->flags & CTL_CMD_FLAG_RUN_HERE) == 0) {
11368                 union ctl_ha_msg msg_info;
11369                 int isc_retval;
11370
11371                 ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11372                 ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11373                 mtx_unlock(&lun->lun_lock);
11374
11375                 msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
11376                 msg_info.hdr.original_sc = (union ctl_io *)ctsio;
11377                 msg_info.hdr.serializing_sc = NULL;
11378                 msg_info.hdr.nexus = ctsio->io_hdr.nexus;
11379                 msg_info.scsi.tag_num = ctsio->tag_num;
11380                 msg_info.scsi.tag_type = ctsio->tag_type;
11381                 msg_info.scsi.cdb_len = ctsio->cdb_len;
11382                 memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
11383
11384                 if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11385                     sizeof(msg_info.scsi) - sizeof(msg_info.scsi.sense_data),
11386                     M_WAITOK)) > CTL_HA_STATUS_SUCCESS) {
11387                         ctl_set_busy(ctsio);
11388                         ctl_done((union ctl_io *)ctsio);
11389                         return (retval);
11390                 }
11391                 return (retval);
11392         }
11393
11394         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
11395                               (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
11396                               ctl_ooaq, ooa_links))) {
11397         case CTL_ACTION_BLOCK:
11398                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
11399                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
11400                                   blocked_links);
11401                 mtx_unlock(&lun->lun_lock);
11402                 return (retval);
11403         case CTL_ACTION_PASS:
11404         case CTL_ACTION_SKIP:
11405                 ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11406                 mtx_unlock(&lun->lun_lock);
11407                 ctl_enqueue_rtr((union ctl_io *)ctsio);
11408                 break;
11409         case CTL_ACTION_OVERLAP:
11410                 mtx_unlock(&lun->lun_lock);
11411                 ctl_set_overlapped_cmd(ctsio);
11412                 ctl_done((union ctl_io *)ctsio);
11413                 break;
11414         case CTL_ACTION_OVERLAP_TAG:
11415                 mtx_unlock(&lun->lun_lock);
11416                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
11417                 ctl_done((union ctl_io *)ctsio);
11418                 break;
11419         case CTL_ACTION_ERROR:
11420         default:
11421                 mtx_unlock(&lun->lun_lock);
11422                 ctl_set_internal_failure(ctsio,
11423                                          /*sks_valid*/ 0,
11424                                          /*retry_count*/ 0);
11425                 ctl_done((union ctl_io *)ctsio);
11426                 break;
11427         }
11428         return (retval);
11429 }
11430
11431 const struct ctl_cmd_entry *
11432 ctl_get_cmd_entry(struct ctl_scsiio *ctsio, int *sa)
11433 {
11434         const struct ctl_cmd_entry *entry;
11435         int service_action;
11436
11437         entry = &ctl_cmd_table[ctsio->cdb[0]];
11438         if (sa)
11439                 *sa = ((entry->flags & CTL_CMD_FLAG_SA5) != 0);
11440         if (entry->flags & CTL_CMD_FLAG_SA5) {
11441                 service_action = ctsio->cdb[1] & SERVICE_ACTION_MASK;
11442                 entry = &((const struct ctl_cmd_entry *)
11443                     entry->execute)[service_action];
11444         }
11445         return (entry);
11446 }
11447
11448 const struct ctl_cmd_entry *
11449 ctl_validate_command(struct ctl_scsiio *ctsio)
11450 {
11451         const struct ctl_cmd_entry *entry;
11452         int i, sa;
11453         uint8_t diff;
11454
11455         entry = ctl_get_cmd_entry(ctsio, &sa);
11456         if (entry->execute == NULL) {
11457                 if (sa)
11458                         ctl_set_invalid_field(ctsio,
11459                                               /*sks_valid*/ 1,
11460                                               /*command*/ 1,
11461                                               /*field*/ 1,
11462                                               /*bit_valid*/ 1,
11463                                               /*bit*/ 4);
11464                 else
11465                         ctl_set_invalid_opcode(ctsio);
11466                 ctl_done((union ctl_io *)ctsio);
11467                 return (NULL);
11468         }
11469         KASSERT(entry->length > 0,
11470             ("Not defined length for command 0x%02x/0x%02x",
11471              ctsio->cdb[0], ctsio->cdb[1]));
11472         for (i = 1; i < entry->length; i++) {
11473                 diff = ctsio->cdb[i] & ~entry->usage[i - 1];
11474                 if (diff == 0)
11475                         continue;
11476                 ctl_set_invalid_field(ctsio,
11477                                       /*sks_valid*/ 1,
11478                                       /*command*/ 1,
11479                                       /*field*/ i,
11480                                       /*bit_valid*/ 1,
11481                                       /*bit*/ fls(diff) - 1);
11482                 ctl_done((union ctl_io *)ctsio);
11483                 return (NULL);
11484         }
11485         return (entry);
11486 }
11487
11488 static int
11489 ctl_cmd_applicable(uint8_t lun_type, const struct ctl_cmd_entry *entry)
11490 {
11491
11492         switch (lun_type) {
11493         case T_PROCESSOR:
11494                 if ((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0)
11495                         return (0);
11496                 break;
11497         case T_DIRECT:
11498                 if ((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0)
11499                         return (0);
11500                 break;
11501         default:
11502                 return (0);
11503         }
11504         return (1);
11505 }
11506
11507 static int
11508 ctl_scsiio(struct ctl_scsiio *ctsio)
11509 {
11510         int retval;
11511         const struct ctl_cmd_entry *entry;
11512
11513         retval = CTL_RETVAL_COMPLETE;
11514
11515         CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
11516
11517         entry = ctl_get_cmd_entry(ctsio, NULL);
11518
11519         /*
11520          * If this I/O has been aborted, just send it straight to
11521          * ctl_done() without executing it.
11522          */
11523         if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
11524                 ctl_done((union ctl_io *)ctsio);
11525                 goto bailout;
11526         }
11527
11528         /*
11529          * All the checks should have been handled by ctl_scsiio_precheck().
11530          * We should be clear now to just execute the I/O.
11531          */
11532         retval = entry->execute(ctsio);
11533
11534 bailout:
11535         return (retval);
11536 }
11537
11538 /*
11539  * Since we only implement one target right now, a bus reset simply resets
11540  * our single target.
11541  */
11542 static int
11543 ctl_bus_reset(struct ctl_softc *softc, union ctl_io *io)
11544 {
11545         return(ctl_target_reset(softc, io, CTL_UA_BUS_RESET));
11546 }
11547
11548 static int
11549 ctl_target_reset(struct ctl_softc *softc, union ctl_io *io,
11550                  ctl_ua_type ua_type)
11551 {
11552         struct ctl_port *port;
11553         struct ctl_lun *lun;
11554         int retval;
11555
11556         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11557                 union ctl_ha_msg msg_info;
11558
11559                 msg_info.hdr.nexus = io->io_hdr.nexus;
11560                 if (ua_type==CTL_UA_TARG_RESET)
11561                         msg_info.task.task_action = CTL_TASK_TARGET_RESET;
11562                 else
11563                         msg_info.task.task_action = CTL_TASK_BUS_RESET;
11564                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11565                 msg_info.hdr.original_sc = NULL;
11566                 msg_info.hdr.serializing_sc = NULL;
11567                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11568                     sizeof(msg_info.task), M_WAITOK);
11569         }
11570         retval = 0;
11571
11572         mtx_lock(&softc->ctl_lock);
11573         port = softc->ctl_ports[io->io_hdr.nexus.targ_port];
11574         STAILQ_FOREACH(lun, &softc->lun_list, links) {
11575                 if (port != NULL &&
11576                     ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
11577                         continue;
11578                 retval += ctl_do_lun_reset(lun, io, ua_type);
11579         }
11580         mtx_unlock(&softc->ctl_lock);
11581         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11582         return (retval);
11583 }
11584
11585 /*
11586  * The LUN should always be set.  The I/O is optional, and is used to
11587  * distinguish between I/Os sent by this initiator, and by other
11588  * initiators.  We set unit attention for initiators other than this one.
11589  * SAM-3 is vague on this point.  It does say that a unit attention should
11590  * be established for other initiators when a LUN is reset (see section
11591  * 5.7.3), but it doesn't specifically say that the unit attention should
11592  * be established for this particular initiator when a LUN is reset.  Here
11593  * is the relevant text, from SAM-3 rev 8:
11594  *
11595  * 5.7.2 When a SCSI initiator port aborts its own tasks
11596  *
11597  * When a SCSI initiator port causes its own task(s) to be aborted, no
11598  * notification that the task(s) have been aborted shall be returned to
11599  * the SCSI initiator port other than the completion response for the
11600  * command or task management function action that caused the task(s) to
11601  * be aborted and notification(s) associated with related effects of the
11602  * action (e.g., a reset unit attention condition).
11603  *
11604  * XXX KDM for now, we're setting unit attention for all initiators.
11605  */
11606 static int
11607 ctl_do_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
11608 {
11609         union ctl_io *xio;
11610 #if 0
11611         uint32_t initidx;
11612 #endif
11613 #ifdef CTL_WITH_CA
11614         int i;
11615 #endif
11616
11617         mtx_lock(&lun->lun_lock);
11618         /*
11619          * Run through the OOA queue and abort each I/O.
11620          */
11621         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11622              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11623                 xio->io_hdr.flags |= CTL_FLAG_ABORT | CTL_FLAG_ABORT_STATUS;
11624         }
11625
11626         /*
11627          * This version sets unit attention for every
11628          */
11629 #if 0
11630         initidx = ctl_get_initindex(&io->io_hdr.nexus);
11631         ctl_est_ua_all(lun, initidx, ua_type);
11632 #else
11633         ctl_est_ua_all(lun, -1, ua_type);
11634 #endif
11635
11636         /*
11637          * A reset (any kind, really) clears reservations established with
11638          * RESERVE/RELEASE.  It does not clear reservations established
11639          * with PERSISTENT RESERVE OUT, but we don't support that at the
11640          * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
11641          * reservations made with the RESERVE/RELEASE commands, because
11642          * those commands are obsolete in SPC-3.
11643          */
11644         lun->flags &= ~CTL_LUN_RESERVED;
11645
11646 #ifdef CTL_WITH_CA
11647         for (i = 0; i < CTL_MAX_INITIATORS; i++)
11648                 ctl_clear_mask(lun->have_ca, i);
11649 #endif
11650         mtx_unlock(&lun->lun_lock);
11651
11652         return (0);
11653 }
11654
11655 static int
11656 ctl_lun_reset(struct ctl_softc *softc, union ctl_io *io)
11657 {
11658         struct ctl_lun *lun;
11659         uint32_t targ_lun;
11660         int retval;
11661
11662         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11663         mtx_lock(&softc->ctl_lock);
11664         if ((targ_lun >= CTL_MAX_LUNS) ||
11665             (lun = softc->ctl_luns[targ_lun]) == NULL) {
11666                 mtx_unlock(&softc->ctl_lock);
11667                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11668                 return (1);
11669         }
11670         retval = ctl_do_lun_reset(lun, io, CTL_UA_LUN_RESET);
11671         mtx_unlock(&softc->ctl_lock);
11672         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11673
11674         if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0) {
11675                 union ctl_ha_msg msg_info;
11676
11677                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11678                 msg_info.hdr.nexus = io->io_hdr.nexus;
11679                 msg_info.task.task_action = CTL_TASK_LUN_RESET;
11680                 msg_info.hdr.original_sc = NULL;
11681                 msg_info.hdr.serializing_sc = NULL;
11682                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11683                     sizeof(msg_info.task), M_WAITOK);
11684         }
11685         return (retval);
11686 }
11687
11688 static void
11689 ctl_abort_tasks_lun(struct ctl_lun *lun, uint32_t targ_port, uint32_t init_id,
11690     int other_sc)
11691 {
11692         union ctl_io *xio;
11693
11694         mtx_assert(&lun->lun_lock, MA_OWNED);
11695
11696         /*
11697          * Run through the OOA queue and attempt to find the given I/O.
11698          * The target port, initiator ID, tag type and tag number have to
11699          * match the values that we got from the initiator.  If we have an
11700          * untagged command to abort, simply abort the first untagged command
11701          * we come to.  We only allow one untagged command at a time of course.
11702          */
11703         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11704              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11705
11706                 if ((targ_port == UINT32_MAX ||
11707                      targ_port == xio->io_hdr.nexus.targ_port) &&
11708                     (init_id == UINT32_MAX ||
11709                      init_id == xio->io_hdr.nexus.initid)) {
11710                         if (targ_port != xio->io_hdr.nexus.targ_port ||
11711                             init_id != xio->io_hdr.nexus.initid)
11712                                 xio->io_hdr.flags |= CTL_FLAG_ABORT_STATUS;
11713                         xio->io_hdr.flags |= CTL_FLAG_ABORT;
11714                         if (!other_sc && !(lun->flags & CTL_LUN_PRIMARY_SC)) {
11715                                 union ctl_ha_msg msg_info;
11716
11717                                 msg_info.hdr.nexus = xio->io_hdr.nexus;
11718                                 msg_info.task.task_action = CTL_TASK_ABORT_TASK;
11719                                 msg_info.task.tag_num = xio->scsiio.tag_num;
11720                                 msg_info.task.tag_type = xio->scsiio.tag_type;
11721                                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11722                                 msg_info.hdr.original_sc = NULL;
11723                                 msg_info.hdr.serializing_sc = NULL;
11724                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11725                                     sizeof(msg_info.task), M_NOWAIT);
11726                         }
11727                 }
11728         }
11729 }
11730
11731 static int
11732 ctl_abort_task_set(union ctl_io *io)
11733 {
11734         struct ctl_softc *softc = control_softc;
11735         struct ctl_lun *lun;
11736         uint32_t targ_lun;
11737
11738         /*
11739          * Look up the LUN.
11740          */
11741         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11742         mtx_lock(&softc->ctl_lock);
11743         if ((targ_lun >= CTL_MAX_LUNS) ||
11744             (lun = softc->ctl_luns[targ_lun]) == NULL) {
11745                 mtx_unlock(&softc->ctl_lock);
11746                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11747                 return (1);
11748         }
11749
11750         mtx_lock(&lun->lun_lock);
11751         mtx_unlock(&softc->ctl_lock);
11752         if (io->taskio.task_action == CTL_TASK_ABORT_TASK_SET) {
11753                 ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
11754                     io->io_hdr.nexus.initid,
11755                     (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
11756         } else { /* CTL_TASK_CLEAR_TASK_SET */
11757                 ctl_abort_tasks_lun(lun, UINT32_MAX, UINT32_MAX,
11758                     (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
11759         }
11760         mtx_unlock(&lun->lun_lock);
11761         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11762         return (0);
11763 }
11764
11765 static int
11766 ctl_i_t_nexus_reset(union ctl_io *io)
11767 {
11768         struct ctl_softc *softc = control_softc;
11769         struct ctl_lun *lun;
11770         uint32_t initidx;
11771
11772         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11773                 union ctl_ha_msg msg_info;
11774
11775                 msg_info.hdr.nexus = io->io_hdr.nexus;
11776                 msg_info.task.task_action = CTL_TASK_I_T_NEXUS_RESET;
11777                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11778                 msg_info.hdr.original_sc = NULL;
11779                 msg_info.hdr.serializing_sc = NULL;
11780                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11781                     sizeof(msg_info.task), M_WAITOK);
11782         }
11783
11784         initidx = ctl_get_initindex(&io->io_hdr.nexus);
11785         mtx_lock(&softc->ctl_lock);
11786         STAILQ_FOREACH(lun, &softc->lun_list, links) {
11787                 mtx_lock(&lun->lun_lock);
11788                 ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
11789                     io->io_hdr.nexus.initid, 1);
11790 #ifdef CTL_WITH_CA
11791                 ctl_clear_mask(lun->have_ca, initidx);
11792 #endif
11793                 if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == initidx))
11794                         lun->flags &= ~CTL_LUN_RESERVED;
11795                 ctl_est_ua(lun, initidx, CTL_UA_I_T_NEXUS_LOSS);
11796                 mtx_unlock(&lun->lun_lock);
11797         }
11798         mtx_unlock(&softc->ctl_lock);
11799         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11800         return (0);
11801 }
11802
11803 static int
11804 ctl_abort_task(union ctl_io *io)
11805 {
11806         union ctl_io *xio;
11807         struct ctl_lun *lun;
11808         struct ctl_softc *softc;
11809 #if 0
11810         struct sbuf sb;
11811         char printbuf[128];
11812 #endif
11813         int found;
11814         uint32_t targ_lun;
11815
11816         softc = control_softc;
11817         found = 0;
11818
11819         /*
11820          * Look up the LUN.
11821          */
11822         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11823         mtx_lock(&softc->ctl_lock);
11824         if ((targ_lun >= CTL_MAX_LUNS) ||
11825             (lun = softc->ctl_luns[targ_lun]) == NULL) {
11826                 mtx_unlock(&softc->ctl_lock);
11827                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11828                 return (1);
11829         }
11830
11831 #if 0
11832         printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
11833                lun->lun, io->taskio.tag_num, io->taskio.tag_type);
11834 #endif
11835
11836         mtx_lock(&lun->lun_lock);
11837         mtx_unlock(&softc->ctl_lock);
11838         /*
11839          * Run through the OOA queue and attempt to find the given I/O.
11840          * The target port, initiator ID, tag type and tag number have to
11841          * match the values that we got from the initiator.  If we have an
11842          * untagged command to abort, simply abort the first untagged command
11843          * we come to.  We only allow one untagged command at a time of course.
11844          */
11845         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11846              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11847 #if 0
11848                 sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
11849
11850                 sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
11851                             lun->lun, xio->scsiio.tag_num,
11852                             xio->scsiio.tag_type,
11853                             (xio->io_hdr.blocked_links.tqe_prev
11854                             == NULL) ? "" : " BLOCKED",
11855                             (xio->io_hdr.flags &
11856                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
11857                             (xio->io_hdr.flags &
11858                             CTL_FLAG_ABORT) ? " ABORT" : "",
11859                             (xio->io_hdr.flags &
11860                             CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
11861                 ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
11862                 sbuf_finish(&sb);
11863                 printf("%s\n", sbuf_data(&sb));
11864 #endif
11865
11866                 if ((xio->io_hdr.nexus.targ_port != io->io_hdr.nexus.targ_port)
11867                  || (xio->io_hdr.nexus.initid != io->io_hdr.nexus.initid)
11868                  || (xio->io_hdr.flags & CTL_FLAG_ABORT))
11869                         continue;
11870
11871                 /*
11872                  * If the abort says that the task is untagged, the
11873                  * task in the queue must be untagged.  Otherwise,
11874                  * we just check to see whether the tag numbers
11875                  * match.  This is because the QLogic firmware
11876                  * doesn't pass back the tag type in an abort
11877                  * request.
11878                  */
11879 #if 0
11880                 if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
11881                   && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
11882                  || (xio->scsiio.tag_num == io->taskio.tag_num))
11883 #endif
11884                 /*
11885                  * XXX KDM we've got problems with FC, because it
11886                  * doesn't send down a tag type with aborts.  So we
11887                  * can only really go by the tag number...
11888                  * This may cause problems with parallel SCSI.
11889                  * Need to figure that out!!
11890                  */
11891                 if (xio->scsiio.tag_num == io->taskio.tag_num) {
11892                         xio->io_hdr.flags |= CTL_FLAG_ABORT;
11893                         found = 1;
11894                         if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0 &&
11895                             !(lun->flags & CTL_LUN_PRIMARY_SC)) {
11896                                 union ctl_ha_msg msg_info;
11897
11898                                 msg_info.hdr.nexus = io->io_hdr.nexus;
11899                                 msg_info.task.task_action = CTL_TASK_ABORT_TASK;
11900                                 msg_info.task.tag_num = io->taskio.tag_num;
11901                                 msg_info.task.tag_type = io->taskio.tag_type;
11902                                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11903                                 msg_info.hdr.original_sc = NULL;
11904                                 msg_info.hdr.serializing_sc = NULL;
11905 #if 0
11906                                 printf("Sent Abort to other side\n");
11907 #endif
11908                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11909                                     sizeof(msg_info.task), M_NOWAIT);
11910                         }
11911 #if 0
11912                         printf("ctl_abort_task: found I/O to abort\n");
11913 #endif
11914                 }
11915         }
11916         mtx_unlock(&lun->lun_lock);
11917
11918         if (found == 0) {
11919                 /*
11920                  * This isn't really an error.  It's entirely possible for
11921                  * the abort and command completion to cross on the wire.
11922                  * This is more of an informative/diagnostic error.
11923                  */
11924 #if 0
11925                 printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
11926                        "%u:%u:%u tag %d type %d\n",
11927                        io->io_hdr.nexus.initid,
11928                        io->io_hdr.nexus.targ_port,
11929                        io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
11930                        io->taskio.tag_type);
11931 #endif
11932         }
11933         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11934         return (0);
11935 }
11936
11937 static int
11938 ctl_query_task(union ctl_io *io, int task_set)
11939 {
11940         union ctl_io *xio;
11941         struct ctl_lun *lun;
11942         struct ctl_softc *softc;
11943         int found = 0;
11944         uint32_t targ_lun;
11945
11946         softc = control_softc;
11947         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11948         mtx_lock(&softc->ctl_lock);
11949         if ((targ_lun >= CTL_MAX_LUNS) ||
11950             (lun = softc->ctl_luns[targ_lun]) == NULL) {
11951                 mtx_unlock(&softc->ctl_lock);
11952                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11953                 return (1);
11954         }
11955         mtx_lock(&lun->lun_lock);
11956         mtx_unlock(&softc->ctl_lock);
11957         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11958              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11959
11960                 if ((xio->io_hdr.nexus.targ_port != io->io_hdr.nexus.targ_port)
11961                  || (xio->io_hdr.nexus.initid != io->io_hdr.nexus.initid)
11962                  || (xio->io_hdr.flags & CTL_FLAG_ABORT))
11963                         continue;
11964
11965                 if (task_set || xio->scsiio.tag_num == io->taskio.tag_num) {
11966                         found = 1;
11967                         break;
11968                 }
11969         }
11970         mtx_unlock(&lun->lun_lock);
11971         if (found)
11972                 io->taskio.task_status = CTL_TASK_FUNCTION_SUCCEEDED;
11973         else
11974                 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11975         return (0);
11976 }
11977
11978 static int
11979 ctl_query_async_event(union ctl_io *io)
11980 {
11981         struct ctl_lun *lun;
11982         struct ctl_softc *softc;
11983         ctl_ua_type ua;
11984         uint32_t targ_lun, initidx;
11985
11986         softc = control_softc;
11987         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11988         mtx_lock(&softc->ctl_lock);
11989         if ((targ_lun >= CTL_MAX_LUNS) ||
11990             (lun = softc->ctl_luns[targ_lun]) == NULL) {
11991                 mtx_unlock(&softc->ctl_lock);
11992                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11993                 return (1);
11994         }
11995         mtx_lock(&lun->lun_lock);
11996         mtx_unlock(&softc->ctl_lock);
11997         initidx = ctl_get_initindex(&io->io_hdr.nexus);
11998         ua = ctl_build_qae(lun, initidx, io->taskio.task_resp);
11999         mtx_unlock(&lun->lun_lock);
12000         if (ua != CTL_UA_NONE)
12001                 io->taskio.task_status = CTL_TASK_FUNCTION_SUCCEEDED;
12002         else
12003                 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
12004         return (0);
12005 }
12006
12007 static void
12008 ctl_run_task(union ctl_io *io)
12009 {
12010         struct ctl_softc *softc = control_softc;
12011         int retval = 1;
12012
12013         CTL_DEBUG_PRINT(("ctl_run_task\n"));
12014         KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
12015             ("ctl_run_task: Unextected io_type %d\n", io->io_hdr.io_type));
12016         io->taskio.task_status = CTL_TASK_FUNCTION_NOT_SUPPORTED;
12017         bzero(io->taskio.task_resp, sizeof(io->taskio.task_resp));
12018         switch (io->taskio.task_action) {
12019         case CTL_TASK_ABORT_TASK:
12020                 retval = ctl_abort_task(io);
12021                 break;
12022         case CTL_TASK_ABORT_TASK_SET:
12023         case CTL_TASK_CLEAR_TASK_SET:
12024                 retval = ctl_abort_task_set(io);
12025                 break;
12026         case CTL_TASK_CLEAR_ACA:
12027                 break;
12028         case CTL_TASK_I_T_NEXUS_RESET:
12029                 retval = ctl_i_t_nexus_reset(io);
12030                 break;
12031         case CTL_TASK_LUN_RESET:
12032                 retval = ctl_lun_reset(softc, io);
12033                 break;
12034         case CTL_TASK_TARGET_RESET:
12035                 retval = ctl_target_reset(softc, io, CTL_UA_TARG_RESET);
12036                 break;
12037         case CTL_TASK_BUS_RESET:
12038                 retval = ctl_bus_reset(softc, io);
12039                 break;
12040         case CTL_TASK_PORT_LOGIN:
12041                 break;
12042         case CTL_TASK_PORT_LOGOUT:
12043                 break;
12044         case CTL_TASK_QUERY_TASK:
12045                 retval = ctl_query_task(io, 0);
12046                 break;
12047         case CTL_TASK_QUERY_TASK_SET:
12048                 retval = ctl_query_task(io, 1);
12049                 break;
12050         case CTL_TASK_QUERY_ASYNC_EVENT:
12051                 retval = ctl_query_async_event(io);
12052                 break;
12053         default:
12054                 printf("%s: got unknown task management event %d\n",
12055                        __func__, io->taskio.task_action);
12056                 break;
12057         }
12058         if (retval == 0)
12059                 io->io_hdr.status = CTL_SUCCESS;
12060         else
12061                 io->io_hdr.status = CTL_ERROR;
12062         ctl_done(io);
12063 }
12064
12065 /*
12066  * For HA operation.  Handle commands that come in from the other
12067  * controller.
12068  */
12069 static void
12070 ctl_handle_isc(union ctl_io *io)
12071 {
12072         int free_io;
12073         struct ctl_lun *lun;
12074         struct ctl_softc *softc;
12075         uint32_t targ_lun;
12076
12077         softc = control_softc;
12078
12079         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12080         lun = softc->ctl_luns[targ_lun];
12081
12082         switch (io->io_hdr.msg_type) {
12083         case CTL_MSG_SERIALIZE:
12084                 free_io = ctl_serialize_other_sc_cmd(&io->scsiio);
12085                 break;
12086         case CTL_MSG_R2R: {
12087                 const struct ctl_cmd_entry *entry;
12088
12089                 /*
12090                  * This is only used in SER_ONLY mode.
12091                  */
12092                 free_io = 0;
12093                 entry = ctl_get_cmd_entry(&io->scsiio, NULL);
12094                 mtx_lock(&lun->lun_lock);
12095                 if (ctl_scsiio_lun_check(lun,
12096                     entry, (struct ctl_scsiio *)io) != 0) {
12097                         mtx_unlock(&lun->lun_lock);
12098                         ctl_done(io);
12099                         break;
12100                 }
12101                 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
12102                 mtx_unlock(&lun->lun_lock);
12103                 ctl_enqueue_rtr(io);
12104                 break;
12105         }
12106         case CTL_MSG_FINISH_IO:
12107                 if (softc->ha_mode == CTL_HA_MODE_XFER) {
12108                         free_io = 0;
12109                         ctl_done(io);
12110                 } else {
12111                         free_io = 1;
12112                         mtx_lock(&lun->lun_lock);
12113                         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
12114                                      ooa_links);
12115                         ctl_check_blocked(lun);
12116                         mtx_unlock(&lun->lun_lock);
12117                 }
12118                 break;
12119         case CTL_MSG_PERS_ACTION:
12120                 ctl_hndl_per_res_out_on_other_sc(
12121                         (union ctl_ha_msg *)&io->presio.pr_msg);
12122                 free_io = 1;
12123                 break;
12124         case CTL_MSG_BAD_JUJU:
12125                 free_io = 0;
12126                 ctl_done(io);
12127                 break;
12128         case CTL_MSG_DATAMOVE:
12129                 /* Only used in XFER mode */
12130                 free_io = 0;
12131                 ctl_datamove_remote(io);
12132                 break;
12133         case CTL_MSG_DATAMOVE_DONE:
12134                 /* Only used in XFER mode */
12135                 free_io = 0;
12136                 io->scsiio.be_move_done(io);
12137                 break;
12138         case CTL_MSG_FAILOVER:
12139                 mtx_lock(&lun->lun_lock);
12140                 ctl_failover_lun(lun);
12141                 mtx_unlock(&lun->lun_lock);
12142                 free_io = 1;
12143                 break;
12144         default:
12145                 free_io = 1;
12146                 printf("%s: Invalid message type %d\n",
12147                        __func__, io->io_hdr.msg_type);
12148                 break;
12149         }
12150         if (free_io)
12151                 ctl_free_io(io);
12152
12153 }
12154
12155
12156 /*
12157  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
12158  * there is no match.
12159  */
12160 static ctl_lun_error_pattern
12161 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
12162 {
12163         const struct ctl_cmd_entry *entry;
12164         ctl_lun_error_pattern filtered_pattern, pattern;
12165
12166         pattern = desc->error_pattern;
12167
12168         /*
12169          * XXX KDM we need more data passed into this function to match a
12170          * custom pattern, and we actually need to implement custom pattern
12171          * matching.
12172          */
12173         if (pattern & CTL_LUN_PAT_CMD)
12174                 return (CTL_LUN_PAT_CMD);
12175
12176         if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
12177                 return (CTL_LUN_PAT_ANY);
12178
12179         entry = ctl_get_cmd_entry(ctsio, NULL);
12180
12181         filtered_pattern = entry->pattern & pattern;
12182
12183         /*
12184          * If the user requested specific flags in the pattern (e.g.
12185          * CTL_LUN_PAT_RANGE), make sure the command supports all of those
12186          * flags.
12187          *
12188          * If the user did not specify any flags, it doesn't matter whether
12189          * or not the command supports the flags.
12190          */
12191         if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
12192              (pattern & ~CTL_LUN_PAT_MASK))
12193                 return (CTL_LUN_PAT_NONE);
12194
12195         /*
12196          * If the user asked for a range check, see if the requested LBA
12197          * range overlaps with this command's LBA range.
12198          */
12199         if (filtered_pattern & CTL_LUN_PAT_RANGE) {
12200                 uint64_t lba1;
12201                 uint64_t len1;
12202                 ctl_action action;
12203                 int retval;
12204
12205                 retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
12206                 if (retval != 0)
12207                         return (CTL_LUN_PAT_NONE);
12208
12209                 action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
12210                                               desc->lba_range.len, FALSE);
12211                 /*
12212                  * A "pass" means that the LBA ranges don't overlap, so
12213                  * this doesn't match the user's range criteria.
12214                  */
12215                 if (action == CTL_ACTION_PASS)
12216                         return (CTL_LUN_PAT_NONE);
12217         }
12218
12219         return (filtered_pattern);
12220 }
12221
12222 static void
12223 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
12224 {
12225         struct ctl_error_desc *desc, *desc2;
12226
12227         mtx_assert(&lun->lun_lock, MA_OWNED);
12228
12229         STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
12230                 ctl_lun_error_pattern pattern;
12231                 /*
12232                  * Check to see whether this particular command matches
12233                  * the pattern in the descriptor.
12234                  */
12235                 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
12236                 if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
12237                         continue;
12238
12239                 switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
12240                 case CTL_LUN_INJ_ABORTED:
12241                         ctl_set_aborted(&io->scsiio);
12242                         break;
12243                 case CTL_LUN_INJ_MEDIUM_ERR:
12244                         ctl_set_medium_error(&io->scsiio,
12245                             (io->io_hdr.flags & CTL_FLAG_DATA_MASK) !=
12246                              CTL_FLAG_DATA_OUT);
12247                         break;
12248                 case CTL_LUN_INJ_UA:
12249                         /* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
12250                          * OCCURRED */
12251                         ctl_set_ua(&io->scsiio, 0x29, 0x00);
12252                         break;
12253                 case CTL_LUN_INJ_CUSTOM:
12254                         /*
12255                          * We're assuming the user knows what he is doing.
12256                          * Just copy the sense information without doing
12257                          * checks.
12258                          */
12259                         bcopy(&desc->custom_sense, &io->scsiio.sense_data,
12260                               MIN(sizeof(desc->custom_sense),
12261                                   sizeof(io->scsiio.sense_data)));
12262                         io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
12263                         io->scsiio.sense_len = SSD_FULL_SIZE;
12264                         io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
12265                         break;
12266                 case CTL_LUN_INJ_NONE:
12267                 default:
12268                         /*
12269                          * If this is an error injection type we don't know
12270                          * about, clear the continuous flag (if it is set)
12271                          * so it will get deleted below.
12272                          */
12273                         desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
12274                         break;
12275                 }
12276                 /*
12277                  * By default, each error injection action is a one-shot
12278                  */
12279                 if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
12280                         continue;
12281
12282                 STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
12283
12284                 free(desc, M_CTL);
12285         }
12286 }
12287
12288 #ifdef CTL_IO_DELAY
12289 static void
12290 ctl_datamove_timer_wakeup(void *arg)
12291 {
12292         union ctl_io *io;
12293
12294         io = (union ctl_io *)arg;
12295
12296         ctl_datamove(io);
12297 }
12298 #endif /* CTL_IO_DELAY */
12299
12300 void
12301 ctl_datamove(union ctl_io *io)
12302 {
12303         struct ctl_lun *lun;
12304         void (*fe_datamove)(union ctl_io *io);
12305
12306         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
12307
12308         CTL_DEBUG_PRINT(("ctl_datamove\n"));
12309
12310         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12311 #ifdef CTL_TIME_IO
12312         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12313                 char str[256];
12314                 char path_str[64];
12315                 struct sbuf sb;
12316
12317                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12318                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12319
12320                 sbuf_cat(&sb, path_str);
12321                 switch (io->io_hdr.io_type) {
12322                 case CTL_IO_SCSI:
12323                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12324                         sbuf_printf(&sb, "\n");
12325                         sbuf_cat(&sb, path_str);
12326                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12327                                     io->scsiio.tag_num, io->scsiio.tag_type);
12328                         break;
12329                 case CTL_IO_TASK:
12330                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12331                                     "Tag Type: %d\n", io->taskio.task_action,
12332                                     io->taskio.tag_num, io->taskio.tag_type);
12333                         break;
12334                 default:
12335                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12336                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12337                         break;
12338                 }
12339                 sbuf_cat(&sb, path_str);
12340                 sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
12341                             (intmax_t)time_uptime - io->io_hdr.start_time);
12342                 sbuf_finish(&sb);
12343                 printf("%s", sbuf_data(&sb));
12344         }
12345 #endif /* CTL_TIME_IO */
12346
12347 #ifdef CTL_IO_DELAY
12348         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12349                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12350         } else {
12351                 if ((lun != NULL)
12352                  && (lun->delay_info.datamove_delay > 0)) {
12353
12354                         callout_init(&io->io_hdr.delay_callout, /*mpsafe*/ 1);
12355                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12356                         callout_reset(&io->io_hdr.delay_callout,
12357                                       lun->delay_info.datamove_delay * hz,
12358                                       ctl_datamove_timer_wakeup, io);
12359                         if (lun->delay_info.datamove_type ==
12360                             CTL_DELAY_TYPE_ONESHOT)
12361                                 lun->delay_info.datamove_delay = 0;
12362                         return;
12363                 }
12364         }
12365 #endif
12366
12367         /*
12368          * This command has been aborted.  Set the port status, so we fail
12369          * the data move.
12370          */
12371         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12372                 printf("ctl_datamove: tag 0x%04x on (%u:%u:%u) aborted\n",
12373                        io->scsiio.tag_num, io->io_hdr.nexus.initid,
12374                        io->io_hdr.nexus.targ_port,
12375                        io->io_hdr.nexus.targ_lun);
12376                 io->io_hdr.port_status = 31337;
12377                 /*
12378                  * Note that the backend, in this case, will get the
12379                  * callback in its context.  In other cases it may get
12380                  * called in the frontend's interrupt thread context.
12381                  */
12382                 io->scsiio.be_move_done(io);
12383                 return;
12384         }
12385
12386         /* Don't confuse frontend with zero length data move. */
12387         if (io->scsiio.kern_data_len == 0) {
12388                 io->scsiio.be_move_done(io);
12389                 return;
12390         }
12391
12392         /*
12393          * If we're in XFER mode and this I/O is from the other shelf
12394          * controller, we need to send the DMA to the other side to
12395          * actually transfer the data to/from the host.  In serialize only
12396          * mode the transfer happens below CTL and ctl_datamove() is only
12397          * called on the machine that originally received the I/O.
12398          */
12399         if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
12400          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12401                 union ctl_ha_msg msg;
12402                 uint32_t sg_entries_sent;
12403                 int do_sg_copy;
12404                 int i;
12405
12406                 memset(&msg, 0, sizeof(msg));
12407                 msg.hdr.msg_type = CTL_MSG_DATAMOVE;
12408                 msg.hdr.original_sc = io->io_hdr.original_sc;
12409                 msg.hdr.serializing_sc = io;
12410                 msg.hdr.nexus = io->io_hdr.nexus;
12411                 msg.hdr.status = io->io_hdr.status;
12412                 msg.dt.flags = io->io_hdr.flags;
12413                 /*
12414                  * We convert everything into a S/G list here.  We can't
12415                  * pass by reference, only by value between controllers.
12416                  * So we can't pass a pointer to the S/G list, only as many
12417                  * S/G entries as we can fit in here.  If it's possible for
12418                  * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
12419                  * then we need to break this up into multiple transfers.
12420                  */
12421                 if (io->scsiio.kern_sg_entries == 0) {
12422                         msg.dt.kern_sg_entries = 1;
12423 #if 0
12424                         /*
12425                          * Convert to a physical address if this is a
12426                          * virtual address.
12427                          */
12428                         if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
12429                                 msg.dt.sg_list[0].addr =
12430                                         io->scsiio.kern_data_ptr;
12431                         } else {
12432                                 /*
12433                                  * XXX KDM use busdma here!
12434                                  */
12435                                 msg.dt.sg_list[0].addr = (void *)
12436                                         vtophys(io->scsiio.kern_data_ptr);
12437                         }
12438 #else
12439                         KASSERT((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0,
12440                             ("HA does not support BUS_ADDR"));
12441                         msg.dt.sg_list[0].addr = io->scsiio.kern_data_ptr;
12442 #endif
12443
12444                         msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
12445                         do_sg_copy = 0;
12446                 } else {
12447                         msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
12448                         do_sg_copy = 1;
12449                 }
12450
12451                 msg.dt.kern_data_len = io->scsiio.kern_data_len;
12452                 msg.dt.kern_total_len = io->scsiio.kern_total_len;
12453                 msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
12454                 msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
12455                 msg.dt.sg_sequence = 0;
12456
12457                 /*
12458                  * Loop until we've sent all of the S/G entries.  On the
12459                  * other end, we'll recompose these S/G entries into one
12460                  * contiguous list before passing it to the
12461                  */
12462                 for (sg_entries_sent = 0; sg_entries_sent <
12463                      msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
12464                         msg.dt.cur_sg_entries = MIN((sizeof(msg.dt.sg_list)/
12465                                 sizeof(msg.dt.sg_list[0])),
12466                                 msg.dt.kern_sg_entries - sg_entries_sent);
12467
12468                         if (do_sg_copy != 0) {
12469                                 struct ctl_sg_entry *sgl;
12470                                 int j;
12471
12472                                 sgl = (struct ctl_sg_entry *)
12473                                         io->scsiio.kern_data_ptr;
12474                                 /*
12475                                  * If this is in cached memory, flush the cache
12476                                  * before we send the DMA request to the other
12477                                  * controller.  We want to do this in either
12478                                  * the * read or the write case.  The read
12479                                  * case is straightforward.  In the write
12480                                  * case, we want to make sure nothing is
12481                                  * in the local cache that could overwrite
12482                                  * the DMAed data.
12483                                  */
12484
12485                                 for (i = sg_entries_sent, j = 0;
12486                                      i < msg.dt.cur_sg_entries; i++, j++) {
12487 #if 0
12488                                         if ((io->io_hdr.flags &
12489                                              CTL_FLAG_BUS_ADDR) == 0) {
12490                                                 /*
12491                                                  * XXX KDM use busdma.
12492                                                  */
12493                                                 msg.dt.sg_list[j].addr =(void *)
12494                                                        vtophys(sgl[i].addr);
12495                                         } else {
12496                                                 msg.dt.sg_list[j].addr =
12497                                                         sgl[i].addr;
12498                                         }
12499 #else
12500                                         KASSERT((io->io_hdr.flags &
12501                                             CTL_FLAG_BUS_ADDR) == 0,
12502                                             ("HA does not support BUS_ADDR"));
12503                                         msg.dt.sg_list[j].addr = sgl[i].addr;
12504 #endif
12505                                         msg.dt.sg_list[j].len = sgl[i].len;
12506                                 }
12507                         }
12508
12509                         sg_entries_sent += msg.dt.cur_sg_entries;
12510                         if (sg_entries_sent >= msg.dt.kern_sg_entries)
12511                                 msg.dt.sg_last = 1;
12512                         else
12513                                 msg.dt.sg_last = 0;
12514
12515                         if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12516                             sizeof(msg.dt) - sizeof(msg.dt.sg_list) +
12517                             sizeof(struct ctl_sg_entry)*msg.dt.cur_sg_entries,
12518                             M_WAITOK) > CTL_HA_STATUS_SUCCESS) {
12519                                 io->io_hdr.port_status = 31341;
12520                                 io->scsiio.be_move_done(io);
12521                                 return;
12522                         }
12523
12524                         msg.dt.sent_sg_entries = sg_entries_sent;
12525                 }
12526
12527                 /*
12528                  * Officially handover the request from us to peer.
12529                  * If failover has just happened, then we must return error.
12530                  * If failover happen just after, then it is not our problem.
12531                  */
12532                 if (lun)
12533                         mtx_lock(&lun->lun_lock);
12534                 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12535                         if (lun)
12536                                 mtx_unlock(&lun->lun_lock);
12537                         io->io_hdr.port_status = 31342;
12538                         io->scsiio.be_move_done(io);
12539                         return;
12540                 }
12541                 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12542                 io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
12543                 if (lun)
12544                         mtx_unlock(&lun->lun_lock);
12545         } else {
12546
12547                 /*
12548                  * Lookup the fe_datamove() function for this particular
12549                  * front end.
12550                  */
12551                 fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
12552
12553                 fe_datamove(io);
12554         }
12555 }
12556
12557 static void
12558 ctl_send_datamove_done(union ctl_io *io, int have_lock)
12559 {
12560         union ctl_ha_msg msg;
12561
12562         memset(&msg, 0, sizeof(msg));
12563
12564         msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
12565         msg.hdr.original_sc = io;
12566         msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
12567         msg.hdr.nexus = io->io_hdr.nexus;
12568         msg.hdr.status = io->io_hdr.status;
12569         msg.scsi.tag_num = io->scsiio.tag_num;
12570         msg.scsi.tag_type = io->scsiio.tag_type;
12571         msg.scsi.scsi_status = io->scsiio.scsi_status;
12572         memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12573                io->scsiio.sense_len);
12574         msg.scsi.sense_len = io->scsiio.sense_len;
12575         msg.scsi.sense_residual = io->scsiio.sense_residual;
12576         msg.scsi.fetd_status = io->io_hdr.port_status;
12577         msg.scsi.residual = io->scsiio.residual;
12578         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12579
12580         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12581                 ctl_failover_io(io, /*have_lock*/ have_lock);
12582                 return;
12583         }
12584
12585         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12586             sizeof(msg.scsi) - sizeof(msg.scsi.sense_data) +
12587             msg.scsi.sense_len, M_WAITOK);
12588 }
12589
12590 /*
12591  * The DMA to the remote side is done, now we need to tell the other side
12592  * we're done so it can continue with its data movement.
12593  */
12594 static void
12595 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
12596 {
12597         union ctl_io *io;
12598         int i;
12599
12600         io = rq->context;
12601
12602         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12603                 printf("%s: ISC DMA write failed with error %d", __func__,
12604                        rq->ret);
12605                 ctl_set_internal_failure(&io->scsiio,
12606                                          /*sks_valid*/ 1,
12607                                          /*retry_count*/ rq->ret);
12608         }
12609
12610         ctl_dt_req_free(rq);
12611
12612         for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12613                 free(io->io_hdr.local_sglist[i].addr, M_CTL);
12614         free(io->io_hdr.remote_sglist, M_CTL);
12615         io->io_hdr.remote_sglist = NULL;
12616         io->io_hdr.local_sglist = NULL;
12617
12618         /*
12619          * The data is in local and remote memory, so now we need to send
12620          * status (good or back) back to the other side.
12621          */
12622         ctl_send_datamove_done(io, /*have_lock*/ 0);
12623 }
12624
12625 /*
12626  * We've moved the data from the host/controller into local memory.  Now we
12627  * need to push it over to the remote controller's memory.
12628  */
12629 static int
12630 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
12631 {
12632         int retval;
12633
12634         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
12635                                           ctl_datamove_remote_write_cb);
12636         return (retval);
12637 }
12638
12639 static void
12640 ctl_datamove_remote_write(union ctl_io *io)
12641 {
12642         int retval;
12643         void (*fe_datamove)(union ctl_io *io);
12644
12645         /*
12646          * - Get the data from the host/HBA into local memory.
12647          * - DMA memory from the local controller to the remote controller.
12648          * - Send status back to the remote controller.
12649          */
12650
12651         retval = ctl_datamove_remote_sgl_setup(io);
12652         if (retval != 0)
12653                 return;
12654
12655         /* Switch the pointer over so the FETD knows what to do */
12656         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12657
12658         /*
12659          * Use a custom move done callback, since we need to send completion
12660          * back to the other controller, not to the backend on this side.
12661          */
12662         io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
12663
12664         fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
12665         fe_datamove(io);
12666 }
12667
12668 static int
12669 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
12670 {
12671 #if 0
12672         char str[256];
12673         char path_str[64];
12674         struct sbuf sb;
12675 #endif
12676         int i;
12677
12678         for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12679                 free(io->io_hdr.local_sglist[i].addr, M_CTL);
12680         free(io->io_hdr.remote_sglist, M_CTL);
12681         io->io_hdr.remote_sglist = NULL;
12682         io->io_hdr.local_sglist = NULL;
12683
12684 #if 0
12685         scsi_path_string(io, path_str, sizeof(path_str));
12686         sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12687         sbuf_cat(&sb, path_str);
12688         scsi_command_string(&io->scsiio, NULL, &sb);
12689         sbuf_printf(&sb, "\n");
12690         sbuf_cat(&sb, path_str);
12691         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12692                     io->scsiio.tag_num, io->scsiio.tag_type);
12693         sbuf_cat(&sb, path_str);
12694         sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
12695                     io->io_hdr.flags, io->io_hdr.status);
12696         sbuf_finish(&sb);
12697         printk("%s", sbuf_data(&sb));
12698 #endif
12699
12700
12701         /*
12702          * The read is done, now we need to send status (good or bad) back
12703          * to the other side.
12704          */
12705         ctl_send_datamove_done(io, /*have_lock*/ 0);
12706
12707         return (0);
12708 }
12709
12710 static void
12711 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
12712 {
12713         union ctl_io *io;
12714         void (*fe_datamove)(union ctl_io *io);
12715
12716         io = rq->context;
12717
12718         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12719                 printf("%s: ISC DMA read failed with error %d\n", __func__,
12720                        rq->ret);
12721                 ctl_set_internal_failure(&io->scsiio,
12722                                          /*sks_valid*/ 1,
12723                                          /*retry_count*/ rq->ret);
12724         }
12725
12726         ctl_dt_req_free(rq);
12727
12728         /* Switch the pointer over so the FETD knows what to do */
12729         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12730
12731         /*
12732          * Use a custom move done callback, since we need to send completion
12733          * back to the other controller, not to the backend on this side.
12734          */
12735         io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
12736
12737         /* XXX KDM add checks like the ones in ctl_datamove? */
12738
12739         fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
12740         fe_datamove(io);
12741 }
12742
12743 static int
12744 ctl_datamove_remote_sgl_setup(union ctl_io *io)
12745 {
12746         struct ctl_sg_entry *local_sglist;
12747         struct ctl_softc *softc;
12748         uint32_t len_to_go;
12749         int retval;
12750         int i;
12751
12752         retval = 0;
12753         softc = control_softc;
12754         local_sglist = io->io_hdr.local_sglist;
12755         len_to_go = io->scsiio.kern_data_len;
12756
12757         /*
12758          * The difficult thing here is that the size of the various
12759          * S/G segments may be different than the size from the
12760          * remote controller.  That'll make it harder when DMAing
12761          * the data back to the other side.
12762          */
12763         for (i = 0; len_to_go > 0; i++) {
12764                 local_sglist[i].len = MIN(len_to_go, CTL_HA_DATAMOVE_SEGMENT);
12765                 local_sglist[i].addr =
12766                     malloc(local_sglist[i].len, M_CTL, M_WAITOK);
12767
12768                 len_to_go -= local_sglist[i].len;
12769         }
12770         /*
12771          * Reset the number of S/G entries accordingly.  The original
12772          * number of S/G entries is available in rem_sg_entries.
12773          */
12774         io->scsiio.kern_sg_entries = i;
12775
12776 #if 0
12777         printf("%s: kern_sg_entries = %d\n", __func__,
12778                io->scsiio.kern_sg_entries);
12779         for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12780                 printf("%s: sg[%d] = %p, %lu\n", __func__, i,
12781                        local_sglist[i].addr, local_sglist[i].len);
12782 #endif
12783
12784         return (retval);
12785 }
12786
12787 static int
12788 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
12789                          ctl_ha_dt_cb callback)
12790 {
12791         struct ctl_ha_dt_req *rq;
12792         struct ctl_sg_entry *remote_sglist, *local_sglist;
12793         uint32_t local_used, remote_used, total_used;
12794         int i, j, isc_ret;
12795
12796         rq = ctl_dt_req_alloc();
12797
12798         /*
12799          * If we failed to allocate the request, and if the DMA didn't fail
12800          * anyway, set busy status.  This is just a resource allocation
12801          * failure.
12802          */
12803         if ((rq == NULL)
12804          && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
12805              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS))
12806                 ctl_set_busy(&io->scsiio);
12807
12808         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
12809             (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) {
12810
12811                 if (rq != NULL)
12812                         ctl_dt_req_free(rq);
12813
12814                 /*
12815                  * The data move failed.  We need to return status back
12816                  * to the other controller.  No point in trying to DMA
12817                  * data to the remote controller.
12818                  */
12819
12820                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12821
12822                 return (1);
12823         }
12824
12825         local_sglist = io->io_hdr.local_sglist;
12826         remote_sglist = io->io_hdr.remote_sglist;
12827         local_used = 0;
12828         remote_used = 0;
12829         total_used = 0;
12830
12831         /*
12832          * Pull/push the data over the wire from/to the other controller.
12833          * This takes into account the possibility that the local and
12834          * remote sglists may not be identical in terms of the size of
12835          * the elements and the number of elements.
12836          *
12837          * One fundamental assumption here is that the length allocated for
12838          * both the local and remote sglists is identical.  Otherwise, we've
12839          * essentially got a coding error of some sort.
12840          */
12841         isc_ret = CTL_HA_STATUS_SUCCESS;
12842         for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
12843                 uint32_t cur_len;
12844                 uint8_t *tmp_ptr;
12845
12846                 rq->command = command;
12847                 rq->context = io;
12848
12849                 /*
12850                  * Both pointers should be aligned.  But it is possible
12851                  * that the allocation length is not.  They should both
12852                  * also have enough slack left over at the end, though,
12853                  * to round up to the next 8 byte boundary.
12854                  */
12855                 cur_len = MIN(local_sglist[i].len - local_used,
12856                               remote_sglist[j].len - remote_used);
12857                 rq->size = cur_len;
12858
12859                 tmp_ptr = (uint8_t *)local_sglist[i].addr;
12860                 tmp_ptr += local_used;
12861
12862 #if 0
12863                 /* Use physical addresses when talking to ISC hardware */
12864                 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
12865                         /* XXX KDM use busdma */
12866                         rq->local = vtophys(tmp_ptr);
12867                 } else
12868                         rq->local = tmp_ptr;
12869 #else
12870                 KASSERT((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0,
12871                     ("HA does not support BUS_ADDR"));
12872                 rq->local = tmp_ptr;
12873 #endif
12874
12875                 tmp_ptr = (uint8_t *)remote_sglist[j].addr;
12876                 tmp_ptr += remote_used;
12877                 rq->remote = tmp_ptr;
12878
12879                 rq->callback = NULL;
12880
12881                 local_used += cur_len;
12882                 if (local_used >= local_sglist[i].len) {
12883                         i++;
12884                         local_used = 0;
12885                 }
12886
12887                 remote_used += cur_len;
12888                 if (remote_used >= remote_sglist[j].len) {
12889                         j++;
12890                         remote_used = 0;
12891                 }
12892                 total_used += cur_len;
12893
12894                 if (total_used >= io->scsiio.kern_data_len)
12895                         rq->callback = callback;
12896
12897 #if 0
12898                 printf("%s: %s: local %p remote %p size %d\n", __func__,
12899                        (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
12900                        rq->local, rq->remote, rq->size);
12901 #endif
12902
12903                 isc_ret = ctl_dt_single(rq);
12904                 if (isc_ret > CTL_HA_STATUS_SUCCESS)
12905                         break;
12906         }
12907         if (isc_ret != CTL_HA_STATUS_WAIT) {
12908                 rq->ret = isc_ret;
12909                 callback(rq);
12910         }
12911
12912         return (0);
12913 }
12914
12915 static void
12916 ctl_datamove_remote_read(union ctl_io *io)
12917 {
12918         int retval;
12919         int i;
12920
12921         /*
12922          * This will send an error to the other controller in the case of a
12923          * failure.
12924          */
12925         retval = ctl_datamove_remote_sgl_setup(io);
12926         if (retval != 0)
12927                 return;
12928
12929         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
12930                                           ctl_datamove_remote_read_cb);
12931         if (retval != 0) {
12932                 /*
12933                  * Make sure we free memory if there was an error..  The
12934                  * ctl_datamove_remote_xfer() function will send the
12935                  * datamove done message, or call the callback with an
12936                  * error if there is a problem.
12937                  */
12938                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12939                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12940                 free(io->io_hdr.remote_sglist, M_CTL);
12941                 io->io_hdr.remote_sglist = NULL;
12942                 io->io_hdr.local_sglist = NULL;
12943         }
12944 }
12945
12946 /*
12947  * Process a datamove request from the other controller.  This is used for
12948  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
12949  * first.  Once that is complete, the data gets DMAed into the remote
12950  * controller's memory.  For reads, we DMA from the remote controller's
12951  * memory into our memory first, and then move it out to the FETD.
12952  */
12953 static void
12954 ctl_datamove_remote(union ctl_io *io)
12955 {
12956
12957         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
12958
12959         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12960                 ctl_failover_io(io, /*have_lock*/ 0);
12961                 return;
12962         }
12963
12964         /*
12965          * Note that we look for an aborted I/O here, but don't do some of
12966          * the other checks that ctl_datamove() normally does.
12967          * We don't need to run the datamove delay code, since that should
12968          * have been done if need be on the other controller.
12969          */
12970         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12971                 printf("%s: tag 0x%04x on (%u:%u:%u) aborted\n", __func__,
12972                        io->scsiio.tag_num, io->io_hdr.nexus.initid,
12973                        io->io_hdr.nexus.targ_port,
12974                        io->io_hdr.nexus.targ_lun);
12975                 io->io_hdr.port_status = 31338;
12976                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12977                 return;
12978         }
12979
12980         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
12981                 ctl_datamove_remote_write(io);
12982         else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
12983                 ctl_datamove_remote_read(io);
12984         else {
12985                 io->io_hdr.port_status = 31339;
12986                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12987         }
12988 }
12989
12990 static int
12991 ctl_process_done(union ctl_io *io)
12992 {
12993         struct ctl_lun *lun;
12994         struct ctl_softc *softc = control_softc;
12995         void (*fe_done)(union ctl_io *io);
12996         union ctl_ha_msg msg;
12997         uint32_t targ_port = io->io_hdr.nexus.targ_port;
12998
12999         CTL_DEBUG_PRINT(("ctl_process_done\n"));
13000
13001         if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0)
13002                 fe_done = softc->ctl_ports[targ_port]->fe_done;
13003         else
13004                 fe_done = NULL;
13005
13006 #ifdef CTL_TIME_IO
13007         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
13008                 char str[256];
13009                 char path_str[64];
13010                 struct sbuf sb;
13011
13012                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
13013                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
13014
13015                 sbuf_cat(&sb, path_str);
13016                 switch (io->io_hdr.io_type) {
13017                 case CTL_IO_SCSI:
13018                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
13019                         sbuf_printf(&sb, "\n");
13020                         sbuf_cat(&sb, path_str);
13021                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
13022                                     io->scsiio.tag_num, io->scsiio.tag_type);
13023                         break;
13024                 case CTL_IO_TASK:
13025                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
13026                                     "Tag Type: %d\n", io->taskio.task_action,
13027                                     io->taskio.tag_num, io->taskio.tag_type);
13028                         break;
13029                 default:
13030                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
13031                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
13032                         break;
13033                 }
13034                 sbuf_cat(&sb, path_str);
13035                 sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
13036                             (intmax_t)time_uptime - io->io_hdr.start_time);
13037                 sbuf_finish(&sb);
13038                 printf("%s", sbuf_data(&sb));
13039         }
13040 #endif /* CTL_TIME_IO */
13041
13042         switch (io->io_hdr.io_type) {
13043         case CTL_IO_SCSI:
13044                 break;
13045         case CTL_IO_TASK:
13046                 if (ctl_debug & CTL_DEBUG_INFO)
13047                         ctl_io_error_print(io, NULL);
13048                 if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
13049                         ctl_free_io(io);
13050                 else
13051                         fe_done(io);
13052                 return (CTL_RETVAL_COMPLETE);
13053         default:
13054                 panic("ctl_process_done: invalid io type %d\n",
13055                       io->io_hdr.io_type);
13056                 break; /* NOTREACHED */
13057         }
13058
13059         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13060         if (lun == NULL) {
13061                 CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
13062                                  io->io_hdr.nexus.targ_mapped_lun));
13063                 goto bailout;
13064         }
13065
13066         mtx_lock(&lun->lun_lock);
13067
13068         /*
13069          * Check to see if we have any errors to inject here.  We only
13070          * inject errors for commands that don't already have errors set.
13071          */
13072         if ((STAILQ_FIRST(&lun->error_list) != NULL) &&
13073             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) &&
13074             ((io->io_hdr.flags & CTL_FLAG_STATUS_SENT) == 0))
13075                 ctl_inject_error(lun, io);
13076
13077         /*
13078          * XXX KDM how do we treat commands that aren't completed
13079          * successfully?
13080          *
13081          * XXX KDM should we also track I/O latency?
13082          */
13083         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
13084             io->io_hdr.io_type == CTL_IO_SCSI) {
13085 #ifdef CTL_TIME_IO
13086                 struct bintime cur_bt;
13087 #endif
13088                 int type;
13089
13090                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13091                     CTL_FLAG_DATA_IN)
13092                         type = CTL_STATS_READ;
13093                 else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13094                     CTL_FLAG_DATA_OUT)
13095                         type = CTL_STATS_WRITE;
13096                 else
13097                         type = CTL_STATS_NO_IO;
13098
13099                 lun->stats.ports[targ_port].bytes[type] +=
13100                     io->scsiio.kern_total_len;
13101                 lun->stats.ports[targ_port].operations[type]++;
13102 #ifdef CTL_TIME_IO
13103                 bintime_add(&lun->stats.ports[targ_port].dma_time[type],
13104                    &io->io_hdr.dma_bt);
13105                 lun->stats.ports[targ_port].num_dmas[type] +=
13106                     io->io_hdr.num_dmas;
13107                 getbintime(&cur_bt);
13108                 bintime_sub(&cur_bt, &io->io_hdr.start_bt);
13109                 bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
13110 #endif
13111         }
13112
13113         /*
13114          * Remove this from the OOA queue.
13115          */
13116         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
13117 #ifdef CTL_TIME_IO
13118         if (TAILQ_EMPTY(&lun->ooa_queue))
13119                 lun->last_busy = getsbinuptime();
13120 #endif
13121
13122         /*
13123          * Run through the blocked queue on this LUN and see if anything
13124          * has become unblocked, now that this transaction is done.
13125          */
13126         ctl_check_blocked(lun);
13127
13128         /*
13129          * If the LUN has been invalidated, free it if there is nothing
13130          * left on its OOA queue.
13131          */
13132         if ((lun->flags & CTL_LUN_INVALID)
13133          && TAILQ_EMPTY(&lun->ooa_queue)) {
13134                 mtx_unlock(&lun->lun_lock);
13135                 mtx_lock(&softc->ctl_lock);
13136                 ctl_free_lun(lun);
13137                 mtx_unlock(&softc->ctl_lock);
13138         } else
13139                 mtx_unlock(&lun->lun_lock);
13140
13141 bailout:
13142
13143         /*
13144          * If this command has been aborted, make sure we set the status
13145          * properly.  The FETD is responsible for freeing the I/O and doing
13146          * whatever it needs to do to clean up its state.
13147          */
13148         if (io->io_hdr.flags & CTL_FLAG_ABORT)
13149                 ctl_set_task_aborted(&io->scsiio);
13150
13151         /*
13152          * If enabled, print command error status.
13153          */
13154         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS &&
13155             (ctl_debug & CTL_DEBUG_INFO) != 0)
13156                 ctl_io_error_print(io, NULL);
13157
13158         /*
13159          * Tell the FETD or the other shelf controller we're done with this
13160          * command.  Note that only SCSI commands get to this point.  Task
13161          * management commands are completed above.
13162          */
13163         if ((softc->ha_mode != CTL_HA_MODE_XFER) &&
13164             (io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)) {
13165                 memset(&msg, 0, sizeof(msg));
13166                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
13167                 msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
13168                 msg.hdr.nexus = io->io_hdr.nexus;
13169                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13170                     sizeof(msg.scsi) - sizeof(msg.scsi.sense_data),
13171                     M_WAITOK);
13172         }
13173         if ((softc->ha_mode == CTL_HA_MODE_XFER)
13174          && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
13175                 memset(&msg, 0, sizeof(msg));
13176                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
13177                 msg.hdr.original_sc = io->io_hdr.original_sc;
13178                 msg.hdr.nexus = io->io_hdr.nexus;
13179                 msg.hdr.status = io->io_hdr.status;
13180                 msg.scsi.scsi_status = io->scsiio.scsi_status;
13181                 msg.scsi.tag_num = io->scsiio.tag_num;
13182                 msg.scsi.tag_type = io->scsiio.tag_type;
13183                 msg.scsi.sense_len = io->scsiio.sense_len;
13184                 msg.scsi.sense_residual = io->scsiio.sense_residual;
13185                 msg.scsi.residual = io->scsiio.residual;
13186                 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
13187                        io->scsiio.sense_len);
13188
13189                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13190                     sizeof(msg.scsi) - sizeof(msg.scsi.sense_data) +
13191                     msg.scsi.sense_len, M_WAITOK);
13192                 ctl_free_io(io);
13193         } else
13194                 fe_done(io);
13195
13196         return (CTL_RETVAL_COMPLETE);
13197 }
13198
13199 #ifdef CTL_WITH_CA
13200 /*
13201  * Front end should call this if it doesn't do autosense.  When the request
13202  * sense comes back in from the initiator, we'll dequeue this and send it.
13203  */
13204 int
13205 ctl_queue_sense(union ctl_io *io)
13206 {
13207         struct ctl_lun *lun;
13208         struct ctl_port *port;
13209         struct ctl_softc *softc;
13210         uint32_t initidx, targ_lun;
13211
13212         softc = control_softc;
13213
13214         CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
13215
13216         /*
13217          * LUN lookup will likely move to the ctl_work_thread() once we
13218          * have our new queueing infrastructure (that doesn't put things on
13219          * a per-LUN queue initially).  That is so that we can handle
13220          * things like an INQUIRY to a LUN that we don't have enabled.  We
13221          * can't deal with that right now.
13222          */
13223         mtx_lock(&softc->ctl_lock);
13224
13225         /*
13226          * If we don't have a LUN for this, just toss the sense
13227          * information.
13228          */
13229         port = ctl_io_port(&ctsio->io_hdr);
13230         targ_lun = ctl_lun_map_from_port(port, io->io_hdr.nexus.targ_lun);
13231         if ((targ_lun < CTL_MAX_LUNS)
13232          && (softc->ctl_luns[targ_lun] != NULL))
13233                 lun = softc->ctl_luns[targ_lun];
13234         else
13235                 goto bailout;
13236
13237         initidx = ctl_get_initindex(&io->io_hdr.nexus);
13238
13239         mtx_lock(&lun->lun_lock);
13240         /*
13241          * Already have CA set for this LUN...toss the sense information.
13242          */
13243         if (ctl_is_set(lun->have_ca, initidx)) {
13244                 mtx_unlock(&lun->lun_lock);
13245                 goto bailout;
13246         }
13247
13248         memcpy(&lun->pending_sense[initidx], &io->scsiio.sense_data,
13249                MIN(sizeof(lun->pending_sense[initidx]),
13250                sizeof(io->scsiio.sense_data)));
13251         ctl_set_mask(lun->have_ca, initidx);
13252         mtx_unlock(&lun->lun_lock);
13253
13254 bailout:
13255         mtx_unlock(&softc->ctl_lock);
13256
13257         ctl_free_io(io);
13258
13259         return (CTL_RETVAL_COMPLETE);
13260 }
13261 #endif
13262
13263 /*
13264  * Primary command inlet from frontend ports.  All SCSI and task I/O
13265  * requests must go through this function.
13266  */
13267 int
13268 ctl_queue(union ctl_io *io)
13269 {
13270         struct ctl_port *port;
13271
13272         CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
13273
13274 #ifdef CTL_TIME_IO
13275         io->io_hdr.start_time = time_uptime;
13276         getbintime(&io->io_hdr.start_bt);
13277 #endif /* CTL_TIME_IO */
13278
13279         /* Map FE-specific LUN ID into global one. */
13280         port = ctl_io_port(&io->io_hdr);
13281         io->io_hdr.nexus.targ_mapped_lun =
13282             ctl_lun_map_from_port(port, io->io_hdr.nexus.targ_lun);
13283
13284         switch (io->io_hdr.io_type) {
13285         case CTL_IO_SCSI:
13286         case CTL_IO_TASK:
13287                 if (ctl_debug & CTL_DEBUG_CDB)
13288                         ctl_io_print(io);
13289                 ctl_enqueue_incoming(io);
13290                 break;
13291         default:
13292                 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
13293                 return (EINVAL);
13294         }
13295
13296         return (CTL_RETVAL_COMPLETE);
13297 }
13298
13299 #ifdef CTL_IO_DELAY
13300 static void
13301 ctl_done_timer_wakeup(void *arg)
13302 {
13303         union ctl_io *io;
13304
13305         io = (union ctl_io *)arg;
13306         ctl_done(io);
13307 }
13308 #endif /* CTL_IO_DELAY */
13309
13310 void
13311 ctl_serseq_done(union ctl_io *io)
13312 {
13313         struct ctl_lun *lun;
13314
13315         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13316         if (lun->be_lun == NULL ||
13317             lun->be_lun->serseq == CTL_LUN_SERSEQ_OFF)
13318                 return;
13319         mtx_lock(&lun->lun_lock);
13320         io->io_hdr.flags |= CTL_FLAG_SERSEQ_DONE;
13321         ctl_check_blocked(lun);
13322         mtx_unlock(&lun->lun_lock);
13323 }
13324
13325 void
13326 ctl_done(union ctl_io *io)
13327 {
13328
13329         /*
13330          * Enable this to catch duplicate completion issues.
13331          */
13332 #if 0
13333         if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
13334                 printf("%s: type %d msg %d cdb %x iptl: "
13335                        "%u:%u:%u tag 0x%04x "
13336                        "flag %#x status %x\n",
13337                         __func__,
13338                         io->io_hdr.io_type,
13339                         io->io_hdr.msg_type,
13340                         io->scsiio.cdb[0],
13341                         io->io_hdr.nexus.initid,
13342                         io->io_hdr.nexus.targ_port,
13343                         io->io_hdr.nexus.targ_lun,
13344                         (io->io_hdr.io_type ==
13345                         CTL_IO_TASK) ?
13346                         io->taskio.tag_num :
13347                         io->scsiio.tag_num,
13348                         io->io_hdr.flags,
13349                         io->io_hdr.status);
13350         } else
13351                 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
13352 #endif
13353
13354         /*
13355          * This is an internal copy of an I/O, and should not go through
13356          * the normal done processing logic.
13357          */
13358         if (io->io_hdr.flags & CTL_FLAG_INT_COPY)
13359                 return;
13360
13361 #ifdef CTL_IO_DELAY
13362         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
13363                 struct ctl_lun *lun;
13364
13365                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13366
13367                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
13368         } else {
13369                 struct ctl_lun *lun;
13370
13371                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13372
13373                 if ((lun != NULL)
13374                  && (lun->delay_info.done_delay > 0)) {
13375
13376                         callout_init(&io->io_hdr.delay_callout, /*mpsafe*/ 1);
13377                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
13378                         callout_reset(&io->io_hdr.delay_callout,
13379                                       lun->delay_info.done_delay * hz,
13380                                       ctl_done_timer_wakeup, io);
13381                         if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
13382                                 lun->delay_info.done_delay = 0;
13383                         return;
13384                 }
13385         }
13386 #endif /* CTL_IO_DELAY */
13387
13388         ctl_enqueue_done(io);
13389 }
13390
13391 static void
13392 ctl_work_thread(void *arg)
13393 {
13394         struct ctl_thread *thr = (struct ctl_thread *)arg;
13395         struct ctl_softc *softc = thr->ctl_softc;
13396         union ctl_io *io;
13397         int retval;
13398
13399         CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
13400
13401         for (;;) {
13402                 retval = 0;
13403
13404                 /*
13405                  * We handle the queues in this order:
13406                  * - ISC
13407                  * - done queue (to free up resources, unblock other commands)
13408                  * - RtR queue
13409                  * - incoming queue
13410                  *
13411                  * If those queues are empty, we break out of the loop and
13412                  * go to sleep.
13413                  */
13414                 mtx_lock(&thr->queue_lock);
13415                 io = (union ctl_io *)STAILQ_FIRST(&thr->isc_queue);
13416                 if (io != NULL) {
13417                         STAILQ_REMOVE_HEAD(&thr->isc_queue, links);
13418                         mtx_unlock(&thr->queue_lock);
13419                         ctl_handle_isc(io);
13420                         continue;
13421                 }
13422                 io = (union ctl_io *)STAILQ_FIRST(&thr->done_queue);
13423                 if (io != NULL) {
13424                         STAILQ_REMOVE_HEAD(&thr->done_queue, links);
13425                         /* clear any blocked commands, call fe_done */
13426                         mtx_unlock(&thr->queue_lock);
13427                         retval = ctl_process_done(io);
13428                         continue;
13429                 }
13430                 io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
13431                 if (io != NULL) {
13432                         STAILQ_REMOVE_HEAD(&thr->incoming_queue, links);
13433                         mtx_unlock(&thr->queue_lock);
13434                         if (io->io_hdr.io_type == CTL_IO_TASK)
13435                                 ctl_run_task(io);
13436                         else
13437                                 ctl_scsiio_precheck(softc, &io->scsiio);
13438                         continue;
13439                 }
13440                 io = (union ctl_io *)STAILQ_FIRST(&thr->rtr_queue);
13441                 if (io != NULL) {
13442                         STAILQ_REMOVE_HEAD(&thr->rtr_queue, links);
13443                         mtx_unlock(&thr->queue_lock);
13444                         retval = ctl_scsiio(&io->scsiio);
13445                         if (retval != CTL_RETVAL_COMPLETE)
13446                                 CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
13447                         continue;
13448                 }
13449
13450                 /* Sleep until we have something to do. */
13451                 mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0);
13452         }
13453 }
13454
13455 static void
13456 ctl_lun_thread(void *arg)
13457 {
13458         struct ctl_softc *softc = (struct ctl_softc *)arg;
13459         struct ctl_be_lun *be_lun;
13460         int retval;
13461
13462         CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
13463
13464         for (;;) {
13465                 retval = 0;
13466                 mtx_lock(&softc->ctl_lock);
13467                 be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
13468                 if (be_lun != NULL) {
13469                         STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
13470                         mtx_unlock(&softc->ctl_lock);
13471                         ctl_create_lun(be_lun);
13472                         continue;
13473                 }
13474
13475                 /* Sleep until we have something to do. */
13476                 mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock,
13477                     PDROP | PRIBIO, "-", 0);
13478         }
13479 }
13480
13481 static void
13482 ctl_thresh_thread(void *arg)
13483 {
13484         struct ctl_softc *softc = (struct ctl_softc *)arg;
13485         struct ctl_lun *lun;
13486         struct ctl_be_lun *be_lun;
13487         struct scsi_da_rw_recovery_page *rwpage;
13488         struct ctl_logical_block_provisioning_page *page;
13489         const char *attr;
13490         union ctl_ha_msg msg;
13491         uint64_t thres, val;
13492         int i, e, set;
13493
13494         CTL_DEBUG_PRINT(("ctl_thresh_thread starting\n"));
13495
13496         for (;;) {
13497                 mtx_lock(&softc->ctl_lock);
13498                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
13499                         be_lun = lun->be_lun;
13500                         if ((lun->flags & CTL_LUN_DISABLED) ||
13501                             (lun->flags & CTL_LUN_OFFLINE) ||
13502                             lun->backend->lun_attr == NULL)
13503                                 continue;
13504                         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
13505                             softc->ha_mode == CTL_HA_MODE_XFER)
13506                                 continue;
13507                         rwpage = &lun->mode_pages.rw_er_page[CTL_PAGE_CURRENT];
13508                         if ((rwpage->byte8 & SMS_RWER_LBPERE) == 0)
13509                                 continue;
13510                         e = 0;
13511                         page = &lun->mode_pages.lbp_page[CTL_PAGE_CURRENT];
13512                         for (i = 0; i < CTL_NUM_LBP_THRESH; i++) {
13513                                 if ((page->descr[i].flags & SLBPPD_ENABLED) == 0)
13514                                         continue;
13515                                 thres = scsi_4btoul(page->descr[i].count);
13516                                 thres <<= CTL_LBP_EXPONENT;
13517                                 switch (page->descr[i].resource) {
13518                                 case 0x01:
13519                                         attr = "blocksavail";
13520                                         break;
13521                                 case 0x02:
13522                                         attr = "blocksused";
13523                                         break;
13524                                 case 0xf1:
13525                                         attr = "poolblocksavail";
13526                                         break;
13527                                 case 0xf2:
13528                                         attr = "poolblocksused";
13529                                         break;
13530                                 default:
13531                                         continue;
13532                                 }
13533                                 mtx_unlock(&softc->ctl_lock); // XXX
13534                                 val = lun->backend->lun_attr(
13535                                     lun->be_lun->be_lun, attr);
13536                                 mtx_lock(&softc->ctl_lock);
13537                                 if (val == UINT64_MAX)
13538                                         continue;
13539                                 if ((page->descr[i].flags & SLBPPD_ARMING_MASK)
13540                                     == SLBPPD_ARMING_INC)
13541                                         e = (val >= thres);
13542                                 else
13543                                         e = (val <= thres);
13544                                 if (e)
13545                                         break;
13546                         }
13547                         mtx_lock(&lun->lun_lock);
13548                         if (e) {
13549                                 scsi_u64to8b((uint8_t *)&page->descr[i] -
13550                                     (uint8_t *)page, lun->ua_tpt_info);
13551                                 if (lun->lasttpt == 0 ||
13552                                     time_uptime - lun->lasttpt >= CTL_LBP_UA_PERIOD) {
13553                                         lun->lasttpt = time_uptime;
13554                                         ctl_est_ua_all(lun, -1, CTL_UA_THIN_PROV_THRES);
13555                                         set = 1;
13556                                 } else
13557                                         set = 0;
13558                         } else {
13559                                 lun->lasttpt = 0;
13560                                 ctl_clr_ua_all(lun, -1, CTL_UA_THIN_PROV_THRES);
13561                                 set = -1;
13562                         }
13563                         mtx_unlock(&lun->lun_lock);
13564                         if (set != 0 &&
13565                             lun->ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
13566                                 /* Send msg to other side. */
13567                                 bzero(&msg.ua, sizeof(msg.ua));
13568                                 msg.hdr.msg_type = CTL_MSG_UA;
13569                                 msg.hdr.nexus.initid = -1;
13570                                 msg.hdr.nexus.targ_port = -1;
13571                                 msg.hdr.nexus.targ_lun = lun->lun;
13572                                 msg.hdr.nexus.targ_mapped_lun = lun->lun;
13573                                 msg.ua.ua_all = 1;
13574                                 msg.ua.ua_set = (set > 0);
13575                                 msg.ua.ua_type = CTL_UA_THIN_PROV_THRES;
13576                                 memcpy(msg.ua.ua_info, lun->ua_tpt_info, 8);
13577                                 mtx_unlock(&softc->ctl_lock); // XXX
13578                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13579                                     sizeof(msg.ua), M_WAITOK);
13580                                 mtx_lock(&softc->ctl_lock);
13581                         }
13582                 }
13583                 mtx_unlock(&softc->ctl_lock);
13584                 pause("-", CTL_LBP_PERIOD * hz);
13585         }
13586 }
13587
13588 static void
13589 ctl_enqueue_incoming(union ctl_io *io)
13590 {
13591         struct ctl_softc *softc = control_softc;
13592         struct ctl_thread *thr;
13593         u_int idx;
13594
13595         idx = (io->io_hdr.nexus.targ_port * 127 +
13596                io->io_hdr.nexus.initid) % worker_threads;
13597         thr = &softc->threads[idx];
13598         mtx_lock(&thr->queue_lock);
13599         STAILQ_INSERT_TAIL(&thr->incoming_queue, &io->io_hdr, links);
13600         mtx_unlock(&thr->queue_lock);
13601         wakeup(thr);
13602 }
13603
13604 static void
13605 ctl_enqueue_rtr(union ctl_io *io)
13606 {
13607         struct ctl_softc *softc = control_softc;
13608         struct ctl_thread *thr;
13609
13610         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13611         mtx_lock(&thr->queue_lock);
13612         STAILQ_INSERT_TAIL(&thr->rtr_queue, &io->io_hdr, links);
13613         mtx_unlock(&thr->queue_lock);
13614         wakeup(thr);
13615 }
13616
13617 static void
13618 ctl_enqueue_done(union ctl_io *io)
13619 {
13620         struct ctl_softc *softc = control_softc;
13621         struct ctl_thread *thr;
13622
13623         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13624         mtx_lock(&thr->queue_lock);
13625         STAILQ_INSERT_TAIL(&thr->done_queue, &io->io_hdr, links);
13626         mtx_unlock(&thr->queue_lock);
13627         wakeup(thr);
13628 }
13629
13630 static void
13631 ctl_enqueue_isc(union ctl_io *io)
13632 {
13633         struct ctl_softc *softc = control_softc;
13634         struct ctl_thread *thr;
13635
13636         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13637         mtx_lock(&thr->queue_lock);
13638         STAILQ_INSERT_TAIL(&thr->isc_queue, &io->io_hdr, links);
13639         mtx_unlock(&thr->queue_lock);
13640         wakeup(thr);
13641 }
13642
13643 /*
13644  *  vim: ts=8
13645  */