sys/cam/ctl/ctl.c

   1 /*-
   2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
   3  * Copyright (c) 2012 The FreeBSD Foundation
   4  * Copyright (c) 2015 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Portions of this software were developed by Edward Tomasz Napierala
   8  * under sponsorship from the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions, and the following disclaimer,
  15  *    without modification.
  16  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  17  *    substantially similar to the "NO WARRANTY" disclaimer below
  18  *    ("Disclaimer") and any redistribution must be conditioned upon
  19  *    including a substantially similar Disclaimer requirement for further
  20  *    binary redistribution.
  21  *
  22  * NO WARRANTY
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  32  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  33  * POSSIBILITY OF SUCH DAMAGES.
  34  *
  35  * $Id$
  36  */
  37 /*
  38  * CAM Target Layer, a SCSI device emulation subsystem.
  39  *
  40  * Author: Ken Merry <ken@FreeBSD.org>
  41  */
  42
  43 #define _CTL_C
  44
  45 #include <sys/cdefs.h>
  46 __FBSDID("$FreeBSD$");
  47
  48 #include <sys/param.h>
  49 #include <sys/systm.h>
  50 #include <sys/ctype.h>
  51 #include <sys/kernel.h>
  52 #include <sys/types.h>
  53 #include <sys/kthread.h>
  54 #include <sys/bio.h>
  55 #include <sys/fcntl.h>
  56 #include <sys/lock.h>
  57 #include <sys/module.h>
  58 #include <sys/mutex.h>
  59 #include <sys/condvar.h>
  60 #include <sys/malloc.h>
  61 #include <sys/conf.h>
  62 #include <sys/ioccom.h>
  63 #include <sys/queue.h>
  64 #include <sys/sbuf.h>
  65 #include <sys/smp.h>
  66 #include <sys/endian.h>
  67 #include <sys/sysctl.h>
  68 #include <vm/uma.h>
  69
  70 #include <cam/cam.h>
  71 #include <cam/scsi/scsi_all.h>
  72 #include <cam/scsi/scsi_da.h>
  73 #include <cam/ctl/ctl_io.h>
  74 #include <cam/ctl/ctl.h>
  75 #include <cam/ctl/ctl_frontend.h>
  76 #include <cam/ctl/ctl_util.h>
  77 #include <cam/ctl/ctl_backend.h>
  78 #include <cam/ctl/ctl_ioctl.h>
  79 #include <cam/ctl/ctl_ha.h>
  80 #include <cam/ctl/ctl_private.h>
  81 #include <cam/ctl/ctl_debug.h>
  82 #include <cam/ctl/ctl_scsi_all.h>
  83 #include <cam/ctl/ctl_error.h>
  84
  85 struct ctl_softc *control_softc = NULL;
  86
  87 /*
  88  * Template mode pages.
  89  */
  90
  91 /*
  92  * Note that these are default values only.  The actual values will be
  93  * filled in when the user does a mode sense.
  94  */
  95 const static struct copan_debugconf_subpage debugconf_page_default = {
  96         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
  97         DBGCNF_SUBPAGE_CODE,            /* subpage */
  98         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
  99          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 100         DBGCNF_VERSION,                 /* page_version */
 101         {CTL_TIME_IO_DEFAULT_SECS>>8,
 102          CTL_TIME_IO_DEFAULT_SECS>>0},  /* ctl_time_io_secs */
 103 };
 104
 105 const static struct copan_debugconf_subpage debugconf_page_changeable = {
 106         DBGCNF_PAGE_CODE | SMPH_SPF,    /* page_code */
 107         DBGCNF_SUBPAGE_CODE,            /* subpage */
 108         {(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
 109          (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
 110         0,                              /* page_version */
 111         {0xff,0xff},                    /* ctl_time_io_secs */
 112 };
 113
 114 const static struct scsi_da_rw_recovery_page rw_er_page_default = {
 115         /*page_code*/SMS_RW_ERROR_RECOVERY_PAGE,
 116         /*page_length*/sizeof(struct scsi_da_rw_recovery_page) - 2,
 117         /*byte3*/SMS_RWER_AWRE|SMS_RWER_ARRE,
 118         /*read_retry_count*/0,
 119         /*correction_span*/0,
 120         /*head_offset_count*/0,
 121         /*data_strobe_offset_cnt*/0,
 122         /*byte8*/SMS_RWER_LBPERE,
 123         /*write_retry_count*/0,
 124         /*reserved2*/0,
 125         /*recovery_time_limit*/{0, 0},
 126 };
 127
 128 const static struct scsi_da_rw_recovery_page rw_er_page_changeable = {
 129         /*page_code*/SMS_RW_ERROR_RECOVERY_PAGE,
 130         /*page_length*/sizeof(struct scsi_da_rw_recovery_page) - 2,
 131         /*byte3*/0,
 132         /*read_retry_count*/0,
 133         /*correction_span*/0,
 134         /*head_offset_count*/0,
 135         /*data_strobe_offset_cnt*/0,
 136         /*byte8*/0,
 137         /*write_retry_count*/0,
 138         /*reserved2*/0,
 139         /*recovery_time_limit*/{0, 0},
 140 };
 141
 142 const static struct scsi_format_page format_page_default = {
 143         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 144         /*page_length*/sizeof(struct scsi_format_page) - 2,
 145         /*tracks_per_zone*/ {0, 0},
 146         /*alt_sectors_per_zone*/ {0, 0},
 147         /*alt_tracks_per_zone*/ {0, 0},
 148         /*alt_tracks_per_lun*/ {0, 0},
 149         /*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
 150                                 CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
 151         /*bytes_per_sector*/ {0, 0},
 152         /*interleave*/ {0, 0},
 153         /*track_skew*/ {0, 0},
 154         /*cylinder_skew*/ {0, 0},
 155         /*flags*/ SFP_HSEC,
 156         /*reserved*/ {0, 0, 0}
 157 };
 158
 159 const static struct scsi_format_page format_page_changeable = {
 160         /*page_code*/SMS_FORMAT_DEVICE_PAGE,
 161         /*page_length*/sizeof(struct scsi_format_page) - 2,
 162         /*tracks_per_zone*/ {0, 0},
 163         /*alt_sectors_per_zone*/ {0, 0},
 164         /*alt_tracks_per_zone*/ {0, 0},
 165         /*alt_tracks_per_lun*/ {0, 0},
 166         /*sectors_per_track*/ {0, 0},
 167         /*bytes_per_sector*/ {0, 0},
 168         /*interleave*/ {0, 0},
 169         /*track_skew*/ {0, 0},
 170         /*cylinder_skew*/ {0, 0},
 171         /*flags*/ 0,
 172         /*reserved*/ {0, 0, 0}
 173 };
 174
 175 const static struct scsi_rigid_disk_page rigid_disk_page_default = {
 176         /*page_code*/SMS_RIGID_DISK_PAGE,
 177         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 178         /*cylinders*/ {0, 0, 0},
 179         /*heads*/ CTL_DEFAULT_HEADS,
 180         /*start_write_precomp*/ {0, 0, 0},
 181         /*start_reduced_current*/ {0, 0, 0},
 182         /*step_rate*/ {0, 0},
 183         /*landing_zone_cylinder*/ {0, 0, 0},
 184         /*rpl*/ SRDP_RPL_DISABLED,
 185         /*rotational_offset*/ 0,
 186         /*reserved1*/ 0,
 187         /*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
 188                            CTL_DEFAULT_ROTATION_RATE & 0xff},
 189         /*reserved2*/ {0, 0}
 190 };
 191
 192 const static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
 193         /*page_code*/SMS_RIGID_DISK_PAGE,
 194         /*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
 195         /*cylinders*/ {0, 0, 0},
 196         /*heads*/ 0,
 197         /*start_write_precomp*/ {0, 0, 0},
 198         /*start_reduced_current*/ {0, 0, 0},
 199         /*step_rate*/ {0, 0},
 200         /*landing_zone_cylinder*/ {0, 0, 0},
 201         /*rpl*/ 0,
 202         /*rotational_offset*/ 0,
 203         /*reserved1*/ 0,
 204         /*rotation_rate*/ {0, 0},
 205         /*reserved2*/ {0, 0}
 206 };
 207
 208 const static struct scsi_caching_page caching_page_default = {
 209         /*page_code*/SMS_CACHING_PAGE,
 210         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 211         /*flags1*/ SCP_DISC | SCP_WCE,
 212         /*ret_priority*/ 0,
 213         /*disable_pf_transfer_len*/ {0xff, 0xff},
 214         /*min_prefetch*/ {0, 0},
 215         /*max_prefetch*/ {0xff, 0xff},
 216         /*max_pf_ceiling*/ {0xff, 0xff},
 217         /*flags2*/ 0,
 218         /*cache_segments*/ 0,
 219         /*cache_seg_size*/ {0, 0},
 220         /*reserved*/ 0,
 221         /*non_cache_seg_size*/ {0, 0, 0}
 222 };
 223
 224 const static struct scsi_caching_page caching_page_changeable = {
 225         /*page_code*/SMS_CACHING_PAGE,
 226         /*page_length*/sizeof(struct scsi_caching_page) - 2,
 227         /*flags1*/ SCP_WCE | SCP_RCD,
 228         /*ret_priority*/ 0,
 229         /*disable_pf_transfer_len*/ {0, 0},
 230         /*min_prefetch*/ {0, 0},
 231         /*max_prefetch*/ {0, 0},
 232         /*max_pf_ceiling*/ {0, 0},
 233         /*flags2*/ 0,
 234         /*cache_segments*/ 0,
 235         /*cache_seg_size*/ {0, 0},
 236         /*reserved*/ 0,
 237         /*non_cache_seg_size*/ {0, 0, 0}
 238 };
 239
 240 const static struct scsi_control_page control_page_default = {
 241         /*page_code*/SMS_CONTROL_MODE_PAGE,
 242         /*page_length*/sizeof(struct scsi_control_page) - 2,
 243         /*rlec*/0,
 244         /*queue_flags*/SCP_QUEUE_ALG_RESTRICTED,
 245         /*eca_and_aen*/0,
 246         /*flags4*/SCP_TAS,
 247         /*aen_holdoff_period*/{0, 0},
 248         /*busy_timeout_period*/{0, 0},
 249         /*extended_selftest_completion_time*/{0, 0}
 250 };
 251
 252 const static struct scsi_control_page control_page_changeable = {
 253         /*page_code*/SMS_CONTROL_MODE_PAGE,
 254         /*page_length*/sizeof(struct scsi_control_page) - 2,
 255         /*rlec*/SCP_DSENSE,
 256         /*queue_flags*/SCP_QUEUE_ALG_MASK,
 257         /*eca_and_aen*/SCP_SWP,
 258         /*flags4*/0,
 259         /*aen_holdoff_period*/{0, 0},
 260         /*busy_timeout_period*/{0, 0},
 261         /*extended_selftest_completion_time*/{0, 0}
 262 };
 263
 264 #define CTL_CEM_LEN     (sizeof(struct scsi_control_ext_page) - 4)
 265
 266 const static struct scsi_control_ext_page control_ext_page_default = {
 267         /*page_code*/SMS_CONTROL_MODE_PAGE | SMPH_SPF,
 268         /*subpage_code*/0x01,
 269         /*page_length*/{CTL_CEM_LEN >> 8, CTL_CEM_LEN},
 270         /*flags*/0,
 271         /*prio*/0,
 272         /*max_sense*/0
 273 };
 274
 275 const static struct scsi_control_ext_page control_ext_page_changeable = {
 276         /*page_code*/SMS_CONTROL_MODE_PAGE | SMPH_SPF,
 277         /*subpage_code*/0x01,
 278         /*page_length*/{CTL_CEM_LEN >> 8, CTL_CEM_LEN},
 279         /*flags*/0,
 280         /*prio*/0,
 281         /*max_sense*/0
 282 };
 283
 284 const static struct scsi_info_exceptions_page ie_page_default = {
 285         /*page_code*/SMS_INFO_EXCEPTIONS_PAGE,
 286         /*page_length*/sizeof(struct scsi_info_exceptions_page) - 2,
 287         /*info_flags*/SIEP_FLAGS_DEXCPT,
 288         /*mrie*/0,
 289         /*interval_timer*/{0, 0, 0, 0},
 290         /*report_count*/{0, 0, 0, 0}
 291 };
 292
 293 const static struct scsi_info_exceptions_page ie_page_changeable = {
 294         /*page_code*/SMS_INFO_EXCEPTIONS_PAGE,
 295         /*page_length*/sizeof(struct scsi_info_exceptions_page) - 2,
 296         /*info_flags*/0,
 297         /*mrie*/0,
 298         /*interval_timer*/{0, 0, 0, 0},
 299         /*report_count*/{0, 0, 0, 0}
 300 };
 301
 302 #define CTL_LBPM_LEN    (sizeof(struct ctl_logical_block_provisioning_page) - 4)
 303
 304 const static struct ctl_logical_block_provisioning_page lbp_page_default = {{
 305         /*page_code*/SMS_INFO_EXCEPTIONS_PAGE | SMPH_SPF,
 306         /*subpage_code*/0x02,
 307         /*page_length*/{CTL_LBPM_LEN >> 8, CTL_LBPM_LEN},
 308         /*flags*/0,
 309         /*reserved*/{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 310         /*descr*/{}},
 311         {{/*flags*/0,
 312           /*resource*/0x01,
 313           /*reserved*/{0, 0},
 314           /*count*/{0, 0, 0, 0}},
 315          {/*flags*/0,
 316           /*resource*/0x02,
 317           /*reserved*/{0, 0},
 318           /*count*/{0, 0, 0, 0}},
 319          {/*flags*/0,
 320           /*resource*/0xf1,
 321           /*reserved*/{0, 0},
 322           /*count*/{0, 0, 0, 0}},
 323          {/*flags*/0,
 324           /*resource*/0xf2,
 325           /*reserved*/{0, 0},
 326           /*count*/{0, 0, 0, 0}}
 327         }
 328 };
 329
 330 const static struct ctl_logical_block_provisioning_page lbp_page_changeable = {{
 331         /*page_code*/SMS_INFO_EXCEPTIONS_PAGE | SMPH_SPF,
 332         /*subpage_code*/0x02,
 333         /*page_length*/{CTL_LBPM_LEN >> 8, CTL_LBPM_LEN},
 334         /*flags*/0,
 335         /*reserved*/{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 336         /*descr*/{}},
 337         {{/*flags*/0,
 338           /*resource*/0,
 339           /*reserved*/{0, 0},
 340           /*count*/{0, 0, 0, 0}},
 341          {/*flags*/0,
 342           /*resource*/0,
 343           /*reserved*/{0, 0},
 344           /*count*/{0, 0, 0, 0}},
 345          {/*flags*/0,
 346           /*resource*/0,
 347           /*reserved*/{0, 0},
 348           /*count*/{0, 0, 0, 0}},
 349          {/*flags*/0,
 350           /*resource*/0,
 351           /*reserved*/{0, 0},
 352           /*count*/{0, 0, 0, 0}}
 353         }
 354 };
 355
 356 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 357 static int worker_threads = -1;
 358 TUNABLE_INT("kern.cam.ctl.worker_threads", &worker_threads);
 359 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
 360     &worker_threads, 1, "Number of worker threads");
 361 static int ctl_debug = CTL_DEBUG_NONE;
 362 TUNABLE_INT("kern.cam.ctl.debug", &ctl_debug);
 363 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, debug, CTLFLAG_RWTUN,
 364     &ctl_debug, 0, "Enabled debug flags");
 365
 366 /*
 367  * Supported pages (0x00), Serial number (0x80), Device ID (0x83),
 368  * Extended INQUIRY Data (0x86), Mode Page Policy (0x87),
 369  * SCSI Ports (0x88), Third-party Copy (0x8F), Block limits (0xB0),
 370  * Block Device Characteristics (0xB1) and Logical Block Provisioning (0xB2)
 371  */
 372 #define SCSI_EVPD_NUM_SUPPORTED_PAGES   10
 373
 374 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 375                                   int param);
 376 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 377 static void ctl_copy_sense_data_back(union ctl_io *src, union ctl_ha_msg *dest);
 378 static int ctl_init(void);
 379 void ctl_shutdown(void);
 380 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 381 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 382 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
 383 static void ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 384                               struct ctl_ooa *ooa_hdr,
 385                               struct ctl_ooa_entry *kern_entries);
 386 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 387                      struct thread *td);
 388 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
 389                          struct ctl_be_lun *be_lun);
 390 static int ctl_free_lun(struct ctl_lun *lun);
 391 static void ctl_create_lun(struct ctl_be_lun *be_lun);
 392 static struct ctl_port * ctl_io_port(struct ctl_io_hdr *io_hdr);
 393
 394 static int ctl_do_mode_select(union ctl_io *io);
 395 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
 396                            uint64_t res_key, uint64_t sa_res_key,
 397                            uint8_t type, uint32_t residx,
 398                            struct ctl_scsiio *ctsio,
 399                            struct scsi_per_res_out *cdb,
 400                            struct scsi_per_res_out_parms* param);
 401 static void ctl_pro_preempt_other(struct ctl_lun *lun,
 402                                   union ctl_ha_msg *msg);
 403 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
 404 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
 405 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
 406 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
 407 static int ctl_inquiry_evpd_eid(struct ctl_scsiio *ctsio, int alloc_len);
 408 static int ctl_inquiry_evpd_mpp(struct ctl_scsiio *ctsio, int alloc_len);
 409 static int ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio,
 410                                          int alloc_len);
 411 static int ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio,
 412                                          int alloc_len);
 413 static int ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len);
 414 static int ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len);
 415 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
 416 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
 417 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len);
 418 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2,
 419     bool seq);
 420 static ctl_action ctl_extent_check_seq(union ctl_io *io1, union ctl_io *io2);
 421 static ctl_action ctl_check_for_blockage(struct ctl_lun *lun,
 422     union ctl_io *pending_io, union ctl_io *ooa_io);
 423 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
 424                                 union ctl_io *starting_io);
 425 static int ctl_check_blocked(struct ctl_lun *lun);
 426 static int ctl_scsiio_lun_check(struct ctl_lun *lun,
 427                                 const struct ctl_cmd_entry *entry,
 428                                 struct ctl_scsiio *ctsio);
 429 static void ctl_failover_lun(union ctl_io *io);
 430 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 431                                struct ctl_scsiio *ctsio);
 432 static int ctl_scsiio(struct ctl_scsiio *ctsio);
 433
 434 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 435 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 436                             ctl_ua_type ua_type);
 437 static int ctl_do_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 438                          ctl_ua_type ua_type);
 439 static int ctl_lun_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 440 static int ctl_abort_task(union ctl_io *io);
 441 static int ctl_abort_task_set(union ctl_io *io);
 442 static int ctl_query_task(union ctl_io *io, int task_set);
 443 static int ctl_i_t_nexus_reset(union ctl_io *io);
 444 static int ctl_query_async_event(union ctl_io *io);
 445 static void ctl_run_task(union ctl_io *io);
 446 #ifdef CTL_IO_DELAY
 447 static void ctl_datamove_timer_wakeup(void *arg);
 448 static void ctl_done_timer_wakeup(void *arg);
 449 #endif /* CTL_IO_DELAY */
 450
 451 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 452 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
 453 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
 454 static void ctl_datamove_remote_write(union ctl_io *io);
 455 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
 456 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 457 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 458 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 459                                     ctl_ha_dt_cb callback);
 460 static void ctl_datamove_remote_read(union ctl_io *io);
 461 static void ctl_datamove_remote(union ctl_io *io);
 462 static void ctl_process_done(union ctl_io *io);
 463 static void ctl_lun_thread(void *arg);
 464 static void ctl_thresh_thread(void *arg);
 465 static void ctl_work_thread(void *arg);
 466 static void ctl_enqueue_incoming(union ctl_io *io);
 467 static void ctl_enqueue_rtr(union ctl_io *io);
 468 static void ctl_enqueue_done(union ctl_io *io);
 469 static void ctl_enqueue_isc(union ctl_io *io);
 470 static const struct ctl_cmd_entry *
 471     ctl_get_cmd_entry(struct ctl_scsiio *ctsio, int *sa);
 472 static const struct ctl_cmd_entry *
 473     ctl_validate_command(struct ctl_scsiio *ctsio);
 474 static int ctl_cmd_applicable(uint8_t lun_type,
 475     const struct ctl_cmd_entry *entry);
 476
 477 static uint64_t ctl_get_prkey(struct ctl_lun *lun, uint32_t residx);
 478 static void ctl_clr_prkey(struct ctl_lun *lun, uint32_t residx);
 479 static void ctl_alloc_prkey(struct ctl_lun *lun, uint32_t residx);
 480 static void ctl_set_prkey(struct ctl_lun *lun, uint32_t residx, uint64_t key);
 481
 482 /*
 483  * Load the serialization table.  This isn't very pretty, but is probably
 484  * the easiest way to do it.
 485  */
 486 #include "ctl_ser_table.c"
 487
 488 /*
 489  * We only need to define open, close and ioctl routines for this driver.
 490  */
 491 static struct cdevsw ctl_cdevsw = {
 492         .d_version =    D_VERSION,
 493         .d_flags =      0,
 494         .d_open =       ctl_open,
 495         .d_close =      ctl_close,
 496         .d_ioctl =      ctl_ioctl,
 497         .d_name =       "ctl",
 498 };
 499
 500
 501 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 502
 503 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 504
 505 static moduledata_t ctl_moduledata = {
 506         "ctl",
 507         ctl_module_event_handler,
 508         NULL
 509 };
 510
 511 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
 512 MODULE_VERSION(ctl, 1);
 513
 514 static struct ctl_frontend ha_frontend =
 515 {
 516         .name = "ha",
 517 };
 518
 519 static void
 520 ctl_ha_datamove(union ctl_io *io)
 521 {
 522         struct ctl_lun *lun;
 523         struct ctl_sg_entry *sgl;
 524         union ctl_ha_msg msg;
 525         uint32_t sg_entries_sent;
 526         int do_sg_copy, i, j;
 527
 528         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
 529         memset(&msg.dt, 0, sizeof(msg.dt));
 530         msg.hdr.msg_type = CTL_MSG_DATAMOVE;
 531         msg.hdr.original_sc = io->io_hdr.original_sc;
 532         msg.hdr.serializing_sc = io;
 533         msg.hdr.nexus = io->io_hdr.nexus;
 534         msg.hdr.status = io->io_hdr.status;
 535         msg.dt.flags = io->io_hdr.flags;
 536
 537         /*
 538          * We convert everything into a S/G list here.  We can't
 539          * pass by reference, only by value between controllers.
 540          * So we can't pass a pointer to the S/G list, only as many
 541          * S/G entries as we can fit in here.  If it's possible for
 542          * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
 543          * then we need to break this up into multiple transfers.
 544          */
 545         if (io->scsiio.kern_sg_entries == 0) {
 546                 msg.dt.kern_sg_entries = 1;
 547 #if 0
 548                 if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
 549                         msg.dt.sg_list[0].addr = io->scsiio.kern_data_ptr;
 550                 } else {
 551                         /* XXX KDM use busdma here! */
 552                         msg.dt.sg_list[0].addr =
 553                             (void *)vtophys(io->scsiio.kern_data_ptr);
 554                 }
 555 #else
 556                 KASSERT((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0,
 557                     ("HA does not support BUS_ADDR"));
 558                 msg.dt.sg_list[0].addr = io->scsiio.kern_data_ptr;
 559 #endif
 560                 msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
 561                 do_sg_copy = 0;
 562         } else {
 563                 msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
 564                 do_sg_copy = 1;
 565         }
 566
 567         msg.dt.kern_data_len = io->scsiio.kern_data_len;
 568         msg.dt.kern_total_len = io->scsiio.kern_total_len;
 569         msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
 570         msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
 571         msg.dt.sg_sequence = 0;
 572
 573         /*
 574          * Loop until we've sent all of the S/G entries.  On the
 575          * other end, we'll recompose these S/G entries into one
 576          * contiguous list before processing.
 577          */
 578         for (sg_entries_sent = 0; sg_entries_sent < msg.dt.kern_sg_entries;
 579             msg.dt.sg_sequence++) {
 580                 msg.dt.cur_sg_entries = MIN((sizeof(msg.dt.sg_list) /
 581                     sizeof(msg.dt.sg_list[0])),
 582                     msg.dt.kern_sg_entries - sg_entries_sent);
 583                 if (do_sg_copy != 0) {
 584                         sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
 585                         for (i = sg_entries_sent, j = 0;
 586                              i < msg.dt.cur_sg_entries; i++, j++) {
 587 #if 0
 588                                 if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
 589                                         msg.dt.sg_list[j].addr = sgl[i].addr;
 590                                 } else {
 591                                         /* XXX KDM use busdma here! */
 592                                         msg.dt.sg_list[j].addr =
 593                                             (void *)vtophys(sgl[i].addr);
 594                                 }
 595 #else
 596                                 KASSERT((io->io_hdr.flags &
 597                                     CTL_FLAG_BUS_ADDR) == 0,
 598                                     ("HA does not support BUS_ADDR"));
 599                                 msg.dt.sg_list[j].addr = sgl[i].addr;
 600 #endif
 601                                 msg.dt.sg_list[j].len = sgl[i].len;
 602                         }
 603                 }
 604
 605                 sg_entries_sent += msg.dt.cur_sg_entries;
 606                 msg.dt.sg_last = (sg_entries_sent >= msg.dt.kern_sg_entries);
 607                 if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
 608                     sizeof(msg.dt) - sizeof(msg.dt.sg_list) +
 609                     sizeof(struct ctl_sg_entry) * msg.dt.cur_sg_entries,
 610                     M_WAITOK) > CTL_HA_STATUS_SUCCESS) {
 611                         io->io_hdr.port_status = 31341;
 612                         io->scsiio.be_move_done(io);
 613                         return;
 614                 }
 615                 msg.dt.sent_sg_entries = sg_entries_sent;
 616         }
 617
 618         /*
 619          * Officially handover the request from us to peer.
 620          * If failover has just happened, then we must return error.
 621          * If failover happen just after, then it is not our problem.
 622          */
 623         if (lun)
 624                 mtx_lock(&lun->lun_lock);
 625         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
 626                 if (lun)
 627                         mtx_unlock(&lun->lun_lock);
 628                 io->io_hdr.port_status = 31342;
 629                 io->scsiio.be_move_done(io);
 630                 return;
 631         }
 632         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
 633         io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
 634         if (lun)
 635                 mtx_unlock(&lun->lun_lock);
 636 }
 637
 638 static void
 639 ctl_ha_done(union ctl_io *io)
 640 {
 641         union ctl_ha_msg msg;
 642
 643         if (io->io_hdr.io_type == CTL_IO_SCSI) {
 644                 memset(&msg, 0, sizeof(msg));
 645                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
 646                 msg.hdr.original_sc = io->io_hdr.original_sc;
 647                 msg.hdr.nexus = io->io_hdr.nexus;
 648                 msg.hdr.status = io->io_hdr.status;
 649                 msg.scsi.scsi_status = io->scsiio.scsi_status;
 650                 msg.scsi.tag_num = io->scsiio.tag_num;
 651                 msg.scsi.tag_type = io->scsiio.tag_type;
 652                 msg.scsi.sense_len = io->scsiio.sense_len;
 653                 msg.scsi.sense_residual = io->scsiio.sense_residual;
 654                 msg.scsi.residual = io->scsiio.residual;
 655                 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
 656                     io->scsiio.sense_len);
 657                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
 658                     sizeof(msg.scsi) - sizeof(msg.scsi.sense_data) +
 659                     msg.scsi.sense_len, M_WAITOK);
 660         }
 661         ctl_free_io(io);
 662 }
 663
 664 static void
 665 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 666                             union ctl_ha_msg *msg_info)
 667 {
 668         struct ctl_scsiio *ctsio;
 669
 670         if (msg_info->hdr.original_sc == NULL) {
 671                 printf("%s: original_sc == NULL!\n", __func__);
 672                 /* XXX KDM now what? */
 673                 return;
 674         }
 675
 676         ctsio = &msg_info->hdr.original_sc->scsiio;
 677         ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 678         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 679         ctsio->io_hdr.status = msg_info->hdr.status;
 680         ctsio->scsi_status = msg_info->scsi.scsi_status;
 681         ctsio->sense_len = msg_info->scsi.sense_len;
 682         ctsio->sense_residual = msg_info->scsi.sense_residual;
 683         ctsio->residual = msg_info->scsi.residual;
 684         memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 685                msg_info->scsi.sense_len);
 686         ctl_enqueue_isc((union ctl_io *)ctsio);
 687 }
 688
 689 static void
 690 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
 691                                 union ctl_ha_msg *msg_info)
 692 {
 693         struct ctl_scsiio *ctsio;
 694
 695         if (msg_info->hdr.serializing_sc == NULL) {
 696                 printf("%s: serializing_sc == NULL!\n", __func__);
 697                 /* XXX KDM now what? */
 698                 return;
 699         }
 700
 701         ctsio = &msg_info->hdr.serializing_sc->scsiio;
 702         ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
 703         ctl_enqueue_isc((union ctl_io *)ctsio);
 704 }
 705
 706 void
 707 ctl_isc_announce_lun(struct ctl_lun *lun)
 708 {
 709         struct ctl_softc *softc = lun->ctl_softc;
 710         union ctl_ha_msg *msg;
 711         struct ctl_ha_msg_lun_pr_key pr_key;
 712         int i, k;
 713
 714         if (softc->ha_link != CTL_HA_LINK_ONLINE)
 715                 return;
 716         mtx_lock(&lun->lun_lock);
 717         i = sizeof(msg->lun);
 718         if (lun->lun_devid)
 719                 i += lun->lun_devid->len;
 720         i += sizeof(pr_key) * lun->pr_key_count;
 721 alloc:
 722         mtx_unlock(&lun->lun_lock);
 723         msg = malloc(i, M_CTL, M_WAITOK);
 724         mtx_lock(&lun->lun_lock);
 725         k = sizeof(msg->lun);
 726         if (lun->lun_devid)
 727                 k += lun->lun_devid->len;
 728         k += sizeof(pr_key) * lun->pr_key_count;
 729         if (i < k) {
 730                 free(msg, M_CTL);
 731                 i = k;
 732                 goto alloc;
 733         }
 734         bzero(&msg->lun, sizeof(msg->lun));
 735         msg->hdr.msg_type = CTL_MSG_LUN_SYNC;
 736         msg->hdr.nexus.targ_lun = lun->lun;
 737         msg->hdr.nexus.targ_mapped_lun = lun->lun;
 738         msg->lun.flags = lun->flags;
 739         msg->lun.pr_generation = lun->PRGeneration;
 740         msg->lun.pr_res_idx = lun->pr_res_idx;
 741         msg->lun.pr_res_type = lun->res_type;
 742         msg->lun.pr_key_count = lun->pr_key_count;
 743         i = 0;
 744         if (lun->lun_devid) {
 745                 msg->lun.lun_devid_len = lun->lun_devid->len;
 746                 memcpy(&msg->lun.data[i], lun->lun_devid->data,
 747                     msg->lun.lun_devid_len);
 748                 i += msg->lun.lun_devid_len;
 749         }
 750         for (k = 0; k < CTL_MAX_INITIATORS; k++) {
 751                 if ((pr_key.pr_key = ctl_get_prkey(lun, k)) == 0)
 752                         continue;
 753                 pr_key.pr_iid = k;
 754                 memcpy(&msg->lun.data[i], &pr_key, sizeof(pr_key));
 755                 i += sizeof(pr_key);
 756         }
 757         mtx_unlock(&lun->lun_lock);
 758         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->port, sizeof(msg->port) + i,
 759             M_WAITOK);
 760         free(msg, M_CTL);
 761
 762         if (lun->flags & CTL_LUN_PRIMARY_SC) {
 763                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
 764                         ctl_isc_announce_mode(lun, -1,
 765                             lun->mode_pages.index[i].page_code & SMPH_PC_MASK,
 766                             lun->mode_pages.index[i].subpage);
 767                 }
 768         }
 769 }
 770
 771 void
 772 ctl_isc_announce_port(struct ctl_port *port)
 773 {
 774         struct ctl_softc *softc = port->ctl_softc;
 775         union ctl_ha_msg *msg;
 776         int i;
 777
 778         if (port->targ_port < softc->port_min ||
 779             port->targ_port >= softc->port_max ||
 780             softc->ha_link != CTL_HA_LINK_ONLINE)
 781                 return;
 782         i = sizeof(msg->port) + strlen(port->port_name) + 1;
 783         if (port->lun_map)
 784                 i += sizeof(uint32_t) * CTL_MAX_LUNS;
 785         if (port->port_devid)
 786                 i += port->port_devid->len;
 787         if (port->target_devid)
 788                 i += port->target_devid->len;
 789         if (port->init_devid)
 790                 i += port->init_devid->len;
 791         msg = malloc(i, M_CTL, M_WAITOK);
 792         bzero(&msg->port, sizeof(msg->port));
 793         msg->hdr.msg_type = CTL_MSG_PORT_SYNC;
 794         msg->hdr.nexus.targ_port = port->targ_port;
 795         msg->port.port_type = port->port_type;
 796         msg->port.physical_port = port->physical_port;
 797         msg->port.virtual_port = port->virtual_port;
 798         msg->port.status = port->status;
 799         i = 0;
 800         msg->port.name_len = sprintf(&msg->port.data[i],
 801             "%d:%s", softc->ha_id, port->port_name) + 1;
 802         i += msg->port.name_len;
 803         if (port->lun_map) {
 804                 msg->port.lun_map_len = sizeof(uint32_t) * CTL_MAX_LUNS;
 805                 memcpy(&msg->port.data[i], port->lun_map,
 806                     msg->port.lun_map_len);
 807                 i += msg->port.lun_map_len;
 808         }
 809         if (port->port_devid) {
 810                 msg->port.port_devid_len = port->port_devid->len;
 811                 memcpy(&msg->port.data[i], port->port_devid->data,
 812                     msg->port.port_devid_len);
 813                 i += msg->port.port_devid_len;
 814         }
 815         if (port->target_devid) {
 816                 msg->port.target_devid_len = port->target_devid->len;
 817                 memcpy(&msg->port.data[i], port->target_devid->data,
 818                     msg->port.target_devid_len);
 819                 i += msg->port.target_devid_len;
 820         }
 821         if (port->init_devid) {
 822                 msg->port.init_devid_len = port->init_devid->len;
 823                 memcpy(&msg->port.data[i], port->init_devid->data,
 824                     msg->port.init_devid_len);
 825                 i += msg->port.init_devid_len;
 826         }
 827         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->port, sizeof(msg->port) + i,
 828             M_WAITOK);
 829         free(msg, M_CTL);
 830 }
 831
 832 void
 833 ctl_isc_announce_iid(struct ctl_port *port, int iid)
 834 {
 835         struct ctl_softc *softc = port->ctl_softc;
 836         union ctl_ha_msg *msg;
 837         int i, l;
 838
 839         if (port->targ_port < softc->port_min ||
 840             port->targ_port >= softc->port_max ||
 841             softc->ha_link != CTL_HA_LINK_ONLINE)
 842                 return;
 843         mtx_lock(&softc->ctl_lock);
 844         i = sizeof(msg->iid);
 845         l = 0;
 846         if (port->wwpn_iid[iid].name)
 847                 l = strlen(port->wwpn_iid[iid].name) + 1;
 848         i += l;
 849         msg = malloc(i, M_CTL, M_NOWAIT);
 850         if (msg == NULL) {
 851                 mtx_unlock(&softc->ctl_lock);
 852                 return;
 853         }
 854         bzero(&msg->iid, sizeof(msg->iid));
 855         msg->hdr.msg_type = CTL_MSG_IID_SYNC;
 856         msg->hdr.nexus.targ_port = port->targ_port;
 857         msg->hdr.nexus.initid = iid;
 858         msg->iid.in_use = port->wwpn_iid[iid].in_use;
 859         msg->iid.name_len = l;
 860         msg->iid.wwpn = port->wwpn_iid[iid].wwpn;
 861         if (port->wwpn_iid[iid].name)
 862                 strlcpy(msg->iid.data, port->wwpn_iid[iid].name, l);
 863         mtx_unlock(&softc->ctl_lock);
 864         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->iid, i, M_NOWAIT);
 865         free(msg, M_CTL);
 866 }
 867
 868 void
 869 ctl_isc_announce_mode(struct ctl_lun *lun, uint32_t initidx,
 870     uint8_t page, uint8_t subpage)
 871 {
 872         struct ctl_softc *softc = lun->ctl_softc;
 873         union ctl_ha_msg msg;
 874         int i;
 875
 876         if (softc->ha_link != CTL_HA_LINK_ONLINE)
 877                 return;
 878         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
 879                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) ==
 880                     page && lun->mode_pages.index[i].subpage == subpage)
 881                         break;
 882         }
 883         if (i == CTL_NUM_MODE_PAGES)
 884                 return;
 885         bzero(&msg.mode, sizeof(msg.mode));
 886         msg.hdr.msg_type = CTL_MSG_MODE_SYNC;
 887         msg.hdr.nexus.targ_port = initidx / CTL_MAX_INIT_PER_PORT;
 888         msg.hdr.nexus.initid = initidx % CTL_MAX_INIT_PER_PORT;
 889         msg.hdr.nexus.targ_lun = lun->lun;
 890         msg.hdr.nexus.targ_mapped_lun = lun->lun;
 891         msg.mode.page_code = page;
 892         msg.mode.subpage = subpage;
 893         msg.mode.page_len = lun->mode_pages.index[i].page_len;
 894         memcpy(msg.mode.data, lun->mode_pages.index[i].page_data,
 895             msg.mode.page_len);
 896         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg.mode, sizeof(msg.mode),
 897             M_WAITOK);
 898 }
 899
 900 static void
 901 ctl_isc_ha_link_up(struct ctl_softc *softc)
 902 {
 903         struct ctl_port *port;
 904         struct ctl_lun *lun;
 905         union ctl_ha_msg msg;
 906         int i;
 907
 908         /* Announce this node parameters to peer for validation. */
 909         msg.login.msg_type = CTL_MSG_LOGIN;
 910         msg.login.version = CTL_HA_VERSION;
 911         msg.login.ha_mode = softc->ha_mode;
 912         msg.login.ha_id = softc->ha_id;
 913         msg.login.max_luns = CTL_MAX_LUNS;
 914         msg.login.max_ports = CTL_MAX_PORTS;
 915         msg.login.max_init_per_port = CTL_MAX_INIT_PER_PORT;
 916         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg.login, sizeof(msg.login),
 917             M_WAITOK);
 918
 919         STAILQ_FOREACH(port, &softc->port_list, links) {
 920                 ctl_isc_announce_port(port);
 921                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
 922                         if (port->wwpn_iid[i].in_use)
 923                                 ctl_isc_announce_iid(port, i);
 924                 }
 925         }
 926         STAILQ_FOREACH(lun, &softc->lun_list, links)
 927                 ctl_isc_announce_lun(lun);
 928 }
 929
 930 static void
 931 ctl_isc_ha_link_down(struct ctl_softc *softc)
 932 {
 933         struct ctl_port *port;
 934         struct ctl_lun *lun;
 935         union ctl_io *io;
 936         int i;
 937
 938         mtx_lock(&softc->ctl_lock);
 939         STAILQ_FOREACH(lun, &softc->lun_list, links) {
 940                 mtx_lock(&lun->lun_lock);
 941                 if (lun->flags & CTL_LUN_PEER_SC_PRIMARY) {
 942                         lun->flags &= ~CTL_LUN_PEER_SC_PRIMARY;
 943                         ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
 944                 }
 945                 mtx_unlock(&lun->lun_lock);
 946
 947                 mtx_unlock(&softc->ctl_lock);
 948                 io = ctl_alloc_io(softc->othersc_pool);
 949                 mtx_lock(&softc->ctl_lock);
 950                 ctl_zero_io(io);
 951                 io->io_hdr.msg_type = CTL_MSG_FAILOVER;
 952                 io->io_hdr.nexus.targ_mapped_lun = lun->lun;
 953                 ctl_enqueue_isc(io);
 954         }
 955
 956         STAILQ_FOREACH(port, &softc->port_list, links) {
 957                 if (port->targ_port >= softc->port_min &&
 958                     port->targ_port < softc->port_max)
 959                         continue;
 960                 port->status &= ~CTL_PORT_STATUS_ONLINE;
 961                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
 962                         port->wwpn_iid[i].in_use = 0;
 963                         free(port->wwpn_iid[i].name, M_CTL);
 964                         port->wwpn_iid[i].name = NULL;
 965                 }
 966         }
 967         mtx_unlock(&softc->ctl_lock);
 968 }
 969
 970 static void
 971 ctl_isc_ua(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 972 {
 973         struct ctl_lun *lun;
 974         uint32_t iid = ctl_get_initindex(&msg->hdr.nexus);
 975
 976         mtx_lock(&softc->ctl_lock);
 977         if (msg->hdr.nexus.targ_lun < CTL_MAX_LUNS &&
 978             (lun = softc->ctl_luns[msg->hdr.nexus.targ_mapped_lun]) != NULL) {
 979                 mtx_lock(&lun->lun_lock);
 980                 mtx_unlock(&softc->ctl_lock);
 981                 if (msg->ua.ua_type == CTL_UA_THIN_PROV_THRES &&
 982                     msg->ua.ua_set)
 983                         memcpy(lun->ua_tpt_info, msg->ua.ua_info, 8);
 984                 if (msg->ua.ua_all) {
 985                         if (msg->ua.ua_set)
 986                                 ctl_est_ua_all(lun, iid, msg->ua.ua_type);
 987                         else
 988                                 ctl_clr_ua_all(lun, iid, msg->ua.ua_type);
 989                 } else {
 990                         if (msg->ua.ua_set)
 991                                 ctl_est_ua(lun, iid, msg->ua.ua_type);
 992                         else
 993                                 ctl_clr_ua(lun, iid, msg->ua.ua_type);
 994                 }
 995                 mtx_unlock(&lun->lun_lock);
 996         } else
 997                 mtx_unlock(&softc->ctl_lock);
 998 }
 999
1000 static void
1001 ctl_isc_lun_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
1002 {
1003         struct ctl_lun *lun;
1004         struct ctl_ha_msg_lun_pr_key pr_key;
1005         int i, k;
1006         ctl_lun_flags oflags;
1007         uint32_t targ_lun;
1008
1009         targ_lun = msg->hdr.nexus.targ_mapped_lun;
1010         mtx_lock(&softc->ctl_lock);
1011         if ((targ_lun >= CTL_MAX_LUNS) ||
1012             ((lun = softc->ctl_luns[targ_lun]) == NULL)) {
1013                 mtx_unlock(&softc->ctl_lock);
1014                 return;
1015         }
1016         mtx_lock(&lun->lun_lock);
1017         mtx_unlock(&softc->ctl_lock);
1018         if (lun->flags & CTL_LUN_DISABLED) {
1019                 mtx_unlock(&lun->lun_lock);
1020                 return;
1021         }
1022         i = (lun->lun_devid != NULL) ? lun->lun_devid->len : 0;
1023         if (msg->lun.lun_devid_len != i || (i > 0 &&
1024             memcmp(&msg->lun.data[0], lun->lun_devid->data, i) != 0)) {
1025                 mtx_unlock(&lun->lun_lock);
1026                 printf("%s: Received conflicting HA LUN %d\n",
1027                     __func__, msg->hdr.nexus.targ_lun);
1028                 return;
1029         } else {
1030                 /* Record whether peer is primary. */
1031                 oflags = lun->flags;
1032                 if ((msg->lun.flags & CTL_LUN_PRIMARY_SC) &&
1033                     (msg->lun.flags & CTL_LUN_DISABLED) == 0)
1034                         lun->flags |= CTL_LUN_PEER_SC_PRIMARY;
1035                 else
1036                         lun->flags &= ~CTL_LUN_PEER_SC_PRIMARY;
1037                 if (oflags != lun->flags)
1038                         ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
1039
1040                 /* If peer is primary and we are not -- use data */
1041                 if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
1042                     (lun->flags & CTL_LUN_PEER_SC_PRIMARY)) {
1043                         lun->PRGeneration = msg->lun.pr_generation;
1044                         lun->pr_res_idx = msg->lun.pr_res_idx;
1045                         lun->res_type = msg->lun.pr_res_type;
1046                         lun->pr_key_count = msg->lun.pr_key_count;
1047                         for (k = 0; k < CTL_MAX_INITIATORS; k++)
1048                                 ctl_clr_prkey(lun, k);
1049                         for (k = 0; k < msg->lun.pr_key_count; k++) {
1050                                 memcpy(&pr_key, &msg->lun.data[i],
1051                                     sizeof(pr_key));
1052                                 ctl_alloc_prkey(lun, pr_key.pr_iid);
1053                                 ctl_set_prkey(lun, pr_key.pr_iid,
1054                                     pr_key.pr_key);
1055                                 i += sizeof(pr_key);
1056                         }
1057                 }
1058
1059                 mtx_unlock(&lun->lun_lock);
1060                 CTL_DEBUG_PRINT(("%s: Known LUN %d, peer is %s\n",
1061                     __func__, msg->hdr.nexus.targ_lun,
1062                     (msg->lun.flags & CTL_LUN_PRIMARY_SC) ?
1063                     "primary" : "secondary"));
1064
1065                 /* If we are primary but peer doesn't know -- notify */
1066                 if ((lun->flags & CTL_LUN_PRIMARY_SC) &&
1067                     (msg->lun.flags & CTL_LUN_PEER_SC_PRIMARY) == 0)
1068                         ctl_isc_announce_lun(lun);
1069         }
1070 }
1071
1072 static void
1073 ctl_isc_port_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
1074 {
1075         struct ctl_port *port;
1076         struct ctl_lun *lun;
1077         int i, new;
1078
1079         port = softc->ctl_ports[msg->hdr.nexus.targ_port];
1080         if (port == NULL) {
1081                 CTL_DEBUG_PRINT(("%s: New port %d\n", __func__,
1082                     msg->hdr.nexus.targ_port));
1083                 new = 1;
1084                 port = malloc(sizeof(*port), M_CTL, M_WAITOK | M_ZERO);
1085                 port->frontend = &ha_frontend;
1086                 port->targ_port = msg->hdr.nexus.targ_port;
1087                 port->fe_datamove = ctl_ha_datamove;
1088                 port->fe_done = ctl_ha_done;
1089         } else if (port->frontend == &ha_frontend) {
1090                 CTL_DEBUG_PRINT(("%s: Updated port %d\n", __func__,
1091                     msg->hdr.nexus.targ_port));
1092                 new = 0;
1093         } else {
1094                 printf("%s: Received conflicting HA port %d\n",
1095                     __func__, msg->hdr.nexus.targ_port);
1096                 return;
1097         }
1098         port->port_type = msg->port.port_type;
1099         port->physical_port = msg->port.physical_port;
1100         port->virtual_port = msg->port.virtual_port;
1101         port->status = msg->port.status;
1102         i = 0;
1103         free(port->port_name, M_CTL);
1104         port->port_name = strndup(&msg->port.data[i], msg->port.name_len,
1105             M_CTL);
1106         i += msg->port.name_len;
1107         if (msg->port.lun_map_len != 0) {
1108                 if (port->lun_map == NULL)
1109                         port->lun_map = malloc(sizeof(uint32_t) * CTL_MAX_LUNS,
1110                             M_CTL, M_WAITOK);
1111                 memcpy(port->lun_map, &msg->port.data[i],
1112                     sizeof(uint32_t) * CTL_MAX_LUNS);
1113                 i += msg->port.lun_map_len;
1114         } else {
1115                 free(port->lun_map, M_CTL);
1116                 port->lun_map = NULL;
1117         }
1118         if (msg->port.port_devid_len != 0) {
1119                 if (port->port_devid == NULL ||
1120                     port->port_devid->len != msg->port.port_devid_len) {
1121                         free(port->port_devid, M_CTL);
1122                         port->port_devid = malloc(sizeof(struct ctl_devid) +
1123                             msg->port.port_devid_len, M_CTL, M_WAITOK);
1124                 }
1125                 memcpy(port->port_devid->data, &msg->port.data[i],
1126                     msg->port.port_devid_len);
1127                 port->port_devid->len = msg->port.port_devid_len;
1128                 i += msg->port.port_devid_len;
1129         } else {
1130                 free(port->port_devid, M_CTL);
1131                 port->port_devid = NULL;
1132         }
1133         if (msg->port.target_devid_len != 0) {
1134                 if (port->target_devid == NULL ||
1135                     port->target_devid->len != msg->port.target_devid_len) {
1136                         free(port->target_devid, M_CTL);
1137                         port->target_devid = malloc(sizeof(struct ctl_devid) +
1138                             msg->port.target_devid_len, M_CTL, M_WAITOK);
1139                 }
1140                 memcpy(port->target_devid->data, &msg->port.data[i],
1141                     msg->port.target_devid_len);
1142                 port->target_devid->len = msg->port.target_devid_len;
1143                 i += msg->port.target_devid_len;
1144         } else {
1145                 free(port->target_devid, M_CTL);
1146                 port->target_devid = NULL;
1147         }
1148         if (msg->port.init_devid_len != 0) {
1149                 if (port->init_devid == NULL ||
1150                     port->init_devid->len != msg->port.init_devid_len) {
1151                         free(port->init_devid, M_CTL);
1152                         port->init_devid = malloc(sizeof(struct ctl_devid) +
1153                             msg->port.init_devid_len, M_CTL, M_WAITOK);
1154                 }
1155                 memcpy(port->init_devid->data, &msg->port.data[i],
1156                     msg->port.init_devid_len);
1157                 port->init_devid->len = msg->port.init_devid_len;
1158                 i += msg->port.init_devid_len;
1159         } else {
1160                 free(port->init_devid, M_CTL);
1161                 port->init_devid = NULL;
1162         }
1163         if (new) {
1164                 if (ctl_port_register(port) != 0) {
1165                         printf("%s: ctl_port_register() failed with error\n",
1166                             __func__);
1167                 }
1168         }
1169         mtx_lock(&softc->ctl_lock);
1170         STAILQ_FOREACH(lun, &softc->lun_list, links) {
1171                 if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
1172                         continue;
1173                 mtx_lock(&lun->lun_lock);
1174                 ctl_est_ua_all(lun, -1, CTL_UA_INQ_CHANGE);
1175                 mtx_unlock(&lun->lun_lock);
1176         }
1177         mtx_unlock(&softc->ctl_lock);
1178 }
1179
1180 static void
1181 ctl_isc_iid_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
1182 {
1183         struct ctl_port *port;
1184         int iid;
1185
1186         port = softc->ctl_ports[msg->hdr.nexus.targ_port];
1187         if (port == NULL) {
1188                 printf("%s: Received IID for unknown port %d\n",
1189                     __func__, msg->hdr.nexus.targ_port);
1190                 return;
1191         }
1192         iid = msg->hdr.nexus.initid;
1193         port->wwpn_iid[iid].in_use = msg->iid.in_use;
1194         port->wwpn_iid[iid].wwpn = msg->iid.wwpn;
1195         free(port->wwpn_iid[iid].name, M_CTL);
1196         if (msg->iid.name_len) {
1197                 port->wwpn_iid[iid].name = strndup(&msg->iid.data[0],
1198                     msg->iid.name_len, M_CTL);
1199         } else
1200                 port->wwpn_iid[iid].name = NULL;
1201 }
1202
1203 static void
1204 ctl_isc_login(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
1205 {
1206
1207         if (msg->login.version != CTL_HA_VERSION) {
1208                 printf("CTL HA peers have different versions %d != %d\n",
1209                     msg->login.version, CTL_HA_VERSION);
1210                 ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1211                 return;
1212         }
1213         if (msg->login.ha_mode != softc->ha_mode) {
1214                 printf("CTL HA peers have different ha_mode %d != %d\n",
1215                     msg->login.ha_mode, softc->ha_mode);
1216                 ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1217                 return;
1218         }
1219         if (msg->login.ha_id == softc->ha_id) {
1220                 printf("CTL HA peers have same ha_id %d\n", msg->login.ha_id);
1221                 ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1222                 return;
1223         }
1224         if (msg->login.max_luns != CTL_MAX_LUNS ||
1225             msg->login.max_ports != CTL_MAX_PORTS ||
1226             msg->login.max_init_per_port != CTL_MAX_INIT_PER_PORT) {
1227                 printf("CTL HA peers have different limits\n");
1228                 ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1229                 return;
1230         }
1231 }
1232
1233 static void
1234 ctl_isc_mode_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
1235 {
1236         struct ctl_lun *lun;
1237         int i;
1238         uint32_t initidx, targ_lun;
1239
1240         targ_lun = msg->hdr.nexus.targ_mapped_lun;
1241         mtx_lock(&softc->ctl_lock);
1242         if ((targ_lun >= CTL_MAX_LUNS) ||
1243             ((lun = softc->ctl_luns[targ_lun]) == NULL)) {
1244                 mtx_unlock(&softc->ctl_lock);
1245                 return;
1246         }
1247         mtx_lock(&lun->lun_lock);
1248         mtx_unlock(&softc->ctl_lock);
1249         if (lun->flags & CTL_LUN_DISABLED) {
1250                 mtx_unlock(&lun->lun_lock);
1251                 return;
1252         }
1253         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
1254                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) ==
1255                     msg->mode.page_code &&
1256                     lun->mode_pages.index[i].subpage == msg->mode.subpage)
1257                         break;
1258         }
1259         if (i == CTL_NUM_MODE_PAGES) {
1260                 mtx_unlock(&lun->lun_lock);
1261                 return;
1262         }
1263         memcpy(lun->mode_pages.index[i].page_data, msg->mode.data,
1264             lun->mode_pages.index[i].page_len);
1265         initidx = ctl_get_initindex(&msg->hdr.nexus);
1266         if (initidx != -1)
1267                 ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
1268         mtx_unlock(&lun->lun_lock);
1269 }
1270
1271 /*
1272  * ISC (Inter Shelf Communication) event handler.  Events from the HA
1273  * subsystem come in here.
1274  */
1275 static void
1276 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
1277 {
1278         struct ctl_softc *softc = control_softc;
1279         union ctl_io *io;
1280         struct ctl_prio *presio;
1281         ctl_ha_status isc_status;
1282
1283         CTL_DEBUG_PRINT(("CTL: Isc Msg event %d\n", event));
1284         if (event == CTL_HA_EVT_MSG_RECV) {
1285                 union ctl_ha_msg *msg, msgbuf;
1286
1287                 if (param > sizeof(msgbuf))
1288                         msg = malloc(param, M_CTL, M_WAITOK);
1289                 else
1290                         msg = &msgbuf;
1291                 isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, msg, param,
1292                     M_WAITOK);
1293                 if (isc_status != CTL_HA_STATUS_SUCCESS) {
1294                         printf("%s: Error receiving message: %d\n",
1295                             __func__, isc_status);
1296                         if (msg != &msgbuf)
1297                                 free(msg, M_CTL);
1298                         return;
1299                 }
1300
1301                 CTL_DEBUG_PRINT(("CTL: msg_type %d\n", msg->msg_type));
1302                 switch (msg->hdr.msg_type) {
1303                 case CTL_MSG_SERIALIZE:
1304                         io = ctl_alloc_io(softc->othersc_pool);
1305                         ctl_zero_io(io);
1306                         // populate ctsio from msg
1307                         io->io_hdr.io_type = CTL_IO_SCSI;
1308                         io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
1309                         io->io_hdr.original_sc = msg->hdr.original_sc;
1310                         io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
1311                                             CTL_FLAG_IO_ACTIVE;
1312                         /*
1313                          * If we're in serialization-only mode, we don't
1314                          * want to go through full done processing.  Thus
1315                          * the COPY flag.
1316                          *
1317                          * XXX KDM add another flag that is more specific.
1318                          */
1319                         if (softc->ha_mode != CTL_HA_MODE_XFER)
1320                                 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
1321                         io->io_hdr.nexus = msg->hdr.nexus;
1322 #if 0
1323                         printf("port %u, iid %u, lun %u\n",
1324                                io->io_hdr.nexus.targ_port,
1325                                io->io_hdr.nexus.initid,
1326                                io->io_hdr.nexus.targ_lun);
1327 #endif
1328                         io->scsiio.tag_num = msg->scsi.tag_num;
1329                         io->scsiio.tag_type = msg->scsi.tag_type;
1330 #ifdef CTL_TIME_IO
1331                         io->io_hdr.start_time = time_uptime;
1332                         getbinuptime(&io->io_hdr.start_bt);
1333 #endif /* CTL_TIME_IO */
1334                         io->scsiio.cdb_len = msg->scsi.cdb_len;
1335                         memcpy(io->scsiio.cdb, msg->scsi.cdb,
1336                                CTL_MAX_CDBLEN);
1337                         if (softc->ha_mode == CTL_HA_MODE_XFER) {
1338                                 const struct ctl_cmd_entry *entry;
1339
1340                                 entry = ctl_get_cmd_entry(&io->scsiio, NULL);
1341                                 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
1342                                 io->io_hdr.flags |=
1343                                         entry->flags & CTL_FLAG_DATA_MASK;
1344                         }
1345                         ctl_enqueue_isc(io);
1346                         break;
1347
1348                 /* Performed on the Originating SC, XFER mode only */
1349                 case CTL_MSG_DATAMOVE: {
1350                         struct ctl_sg_entry *sgl;
1351                         int i, j;
1352
1353                         io = msg->hdr.original_sc;
1354                         if (io == NULL) {
1355                                 printf("%s: original_sc == NULL!\n", __func__);
1356                                 /* XXX KDM do something here */
1357                                 break;
1358                         }
1359                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
1360                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1361                         /*
1362                          * Keep track of this, we need to send it back over
1363                          * when the datamove is complete.
1364                          */
1365                         io->io_hdr.serializing_sc = msg->hdr.serializing_sc;
1366                         if (msg->hdr.status == CTL_SUCCESS)
1367                                 io->io_hdr.status = msg->hdr.status;
1368
1369                         if (msg->dt.sg_sequence == 0) {
1370 #ifdef CTL_TIME_IO
1371                                 getbinuptime(&io->io_hdr.dma_start_bt);
1372 #endif
1373                                 i = msg->dt.kern_sg_entries +
1374                                     msg->dt.kern_data_len /
1375                                     CTL_HA_DATAMOVE_SEGMENT + 1;
1376                                 sgl = malloc(sizeof(*sgl) * i, M_CTL,
1377                                     M_WAITOK | M_ZERO);
1378                                 io->io_hdr.remote_sglist = sgl;
1379                                 io->io_hdr.local_sglist =
1380                                     &sgl[msg->dt.kern_sg_entries];
1381
1382                                 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
1383
1384                                 io->scsiio.kern_sg_entries =
1385                                         msg->dt.kern_sg_entries;
1386                                 io->scsiio.rem_sg_entries =
1387                                         msg->dt.kern_sg_entries;
1388                                 io->scsiio.kern_data_len =
1389                                         msg->dt.kern_data_len;
1390                                 io->scsiio.kern_total_len =
1391                                         msg->dt.kern_total_len;
1392                                 io->scsiio.kern_data_resid =
1393                                         msg->dt.kern_data_resid;
1394                                 io->scsiio.kern_rel_offset =
1395                                         msg->dt.kern_rel_offset;
1396                                 io->io_hdr.flags &= ~CTL_FLAG_BUS_ADDR;
1397                                 io->io_hdr.flags |= msg->dt.flags &
1398                                     CTL_FLAG_BUS_ADDR;
1399                         } else
1400                                 sgl = (struct ctl_sg_entry *)
1401                                         io->scsiio.kern_data_ptr;
1402
1403                         for (i = msg->dt.sent_sg_entries, j = 0;
1404                              i < (msg->dt.sent_sg_entries +
1405                              msg->dt.cur_sg_entries); i++, j++) {
1406                                 sgl[i].addr = msg->dt.sg_list[j].addr;
1407                                 sgl[i].len = msg->dt.sg_list[j].len;
1408
1409 #if 0
1410                                 printf("%s: DATAMOVE: %p,%lu j=%d, i=%d\n",
1411                                     __func__, sgl[i].addr, sgl[i].len, j, i);
1412 #endif
1413                         }
1414
1415                         /*
1416                          * If this is the last piece of the I/O, we've got
1417                          * the full S/G list.  Queue processing in the thread.
1418                          * Otherwise wait for the next piece.
1419                          */
1420                         if (msg->dt.sg_last != 0)
1421                                 ctl_enqueue_isc(io);
1422                         break;
1423                 }
1424                 /* Performed on the Serializing (primary) SC, XFER mode only */
1425                 case CTL_MSG_DATAMOVE_DONE: {
1426                         if (msg->hdr.serializing_sc == NULL) {
1427                                 printf("%s: serializing_sc == NULL!\n",
1428                                        __func__);
1429                                 /* XXX KDM now what? */
1430                                 break;
1431                         }
1432                         /*
1433                          * We grab the sense information here in case
1434                          * there was a failure, so we can return status
1435                          * back to the initiator.
1436                          */
1437                         io = msg->hdr.serializing_sc;
1438                         io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
1439                         io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1440                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1441                         io->io_hdr.port_status = msg->scsi.fetd_status;
1442                         io->scsiio.residual = msg->scsi.residual;
1443                         if (msg->hdr.status != CTL_STATUS_NONE) {
1444                                 io->io_hdr.status = msg->hdr.status;
1445                                 io->scsiio.scsi_status = msg->scsi.scsi_status;
1446                                 io->scsiio.sense_len = msg->scsi.sense_len;
1447                                 io->scsiio.sense_residual =msg->scsi.sense_residual;
1448                                 memcpy(&io->scsiio.sense_data,
1449                                     &msg->scsi.sense_data,
1450                                     msg->scsi.sense_len);
1451                                 if (msg->hdr.status == CTL_SUCCESS)
1452                                         io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
1453                         }
1454                         ctl_enqueue_isc(io);
1455                         break;
1456                 }
1457
1458                 /* Preformed on Originating SC, SER_ONLY mode */
1459                 case CTL_MSG_R2R:
1460                         io = msg->hdr.original_sc;
1461                         if (io == NULL) {
1462                                 printf("%s: original_sc == NULL!\n",
1463                                     __func__);
1464                                 break;
1465                         }
1466                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1467                         io->io_hdr.msg_type = CTL_MSG_R2R;
1468                         io->io_hdr.serializing_sc = msg->hdr.serializing_sc;
1469                         ctl_enqueue_isc(io);
1470                         break;
1471
1472                 /*
1473                  * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
1474                  * mode.
1475                  * Performed on the Originating (i.e. secondary) SC in XFER
1476                  * mode
1477                  */
1478                 case CTL_MSG_FINISH_IO:
1479                         if (softc->ha_mode == CTL_HA_MODE_XFER)
1480                                 ctl_isc_handler_finish_xfer(softc, msg);
1481                         else
1482                                 ctl_isc_handler_finish_ser_only(softc, msg);
1483                         break;
1484
1485                 /* Preformed on Originating SC */
1486                 case CTL_MSG_BAD_JUJU:
1487                         io = msg->hdr.original_sc;
1488                         if (io == NULL) {
1489                                 printf("%s: Bad JUJU!, original_sc is NULL!\n",
1490                                        __func__);
1491                                 break;
1492                         }
1493                         ctl_copy_sense_data(msg, io);
1494                         /*
1495                          * IO should have already been cleaned up on other
1496                          * SC so clear this flag so we won't send a message
1497                          * back to finish the IO there.
1498                          */
1499                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
1500                         io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1501
1502                         /* io = msg->hdr.serializing_sc; */
1503                         io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
1504                         ctl_enqueue_isc(io);
1505                         break;
1506
1507                 /* Handle resets sent from the other side */
1508                 case CTL_MSG_MANAGE_TASKS: {
1509                         struct ctl_taskio *taskio;
1510                         taskio = (struct ctl_taskio *)ctl_alloc_io(
1511                             softc->othersc_pool);
1512                         ctl_zero_io((union ctl_io *)taskio);
1513                         taskio->io_hdr.io_type = CTL_IO_TASK;
1514                         taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
1515                         taskio->io_hdr.nexus = msg->hdr.nexus;
1516                         taskio->task_action = msg->task.task_action;
1517                         taskio->tag_num = msg->task.tag_num;
1518                         taskio->tag_type = msg->task.tag_type;
1519 #ifdef CTL_TIME_IO
1520                         taskio->io_hdr.start_time = time_uptime;
1521                         getbinuptime(&taskio->io_hdr.start_bt);
1522 #endif /* CTL_TIME_IO */
1523                         ctl_run_task((union ctl_io *)taskio);
1524                         break;
1525                 }
1526                 /* Persistent Reserve action which needs attention */
1527                 case CTL_MSG_PERS_ACTION:
1528                         presio = (struct ctl_prio *)ctl_alloc_io(
1529                             softc->othersc_pool);
1530                         ctl_zero_io((union ctl_io *)presio);
1531                         presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
1532                         presio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
1533                         presio->io_hdr.nexus = msg->hdr.nexus;
1534                         presio->pr_msg = msg->pr;
1535                         ctl_enqueue_isc((union ctl_io *)presio);
1536                         break;
1537                 case CTL_MSG_UA:
1538                         ctl_isc_ua(softc, msg, param);
1539                         break;
1540                 case CTL_MSG_PORT_SYNC:
1541                         ctl_isc_port_sync(softc, msg, param);
1542                         break;
1543                 case CTL_MSG_LUN_SYNC:
1544                         ctl_isc_lun_sync(softc, msg, param);
1545                         break;
1546                 case CTL_MSG_IID_SYNC:
1547                         ctl_isc_iid_sync(softc, msg, param);
1548                         break;
1549                 case CTL_MSG_LOGIN:
1550                         ctl_isc_login(softc, msg, param);
1551                         break;
1552                 case CTL_MSG_MODE_SYNC:
1553                         ctl_isc_mode_sync(softc, msg, param);
1554                         break;
1555                 default:
1556                         printf("Received HA message of unknown type %d\n",
1557                             msg->hdr.msg_type);
1558                         ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
1559                         break;
1560                 }
1561                 if (msg != &msgbuf)
1562                         free(msg, M_CTL);
1563         } else if (event == CTL_HA_EVT_LINK_CHANGE) {
1564                 printf("CTL: HA link status changed from %d to %d\n",
1565                     softc->ha_link, param);
1566                 if (param == softc->ha_link)
1567                         return;
1568                 if (softc->ha_link == CTL_HA_LINK_ONLINE) {
1569                         softc->ha_link = param;
1570                         ctl_isc_ha_link_down(softc);
1571                 } else {
1572                         softc->ha_link = param;
1573                         if (softc->ha_link == CTL_HA_LINK_ONLINE)
1574                                 ctl_isc_ha_link_up(softc);
1575                 }
1576                 return;
1577         } else {
1578                 printf("ctl_isc_event_handler: Unknown event %d\n", event);
1579                 return;
1580         }
1581 }
1582
1583 static void
1584 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
1585 {
1586
1587         memcpy(&dest->scsiio.sense_data, &src->scsi.sense_data,
1588             src->scsi.sense_len);
1589         dest->scsiio.scsi_status = src->scsi.scsi_status;
1590         dest->scsiio.sense_len = src->scsi.sense_len;
1591         dest->io_hdr.status = src->hdr.status;
1592 }
1593
1594 static void
1595 ctl_copy_sense_data_back(union ctl_io *src, union ctl_ha_msg *dest)
1596 {
1597
1598         memcpy(&dest->scsi.sense_data, &src->scsiio.sense_data,
1599             src->scsiio.sense_len);
1600         dest->scsi.scsi_status = src->scsiio.scsi_status;
1601         dest->scsi.sense_len = src->scsiio.sense_len;
1602         dest->hdr.status = src->io_hdr.status;
1603 }
1604
1605 void
1606 ctl_est_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
1607 {
1608         struct ctl_softc *softc = lun->ctl_softc;
1609         ctl_ua_type *pu;
1610
1611         if (initidx < softc->init_min || initidx >= softc->init_max)
1612                 return;
1613         mtx_assert(&lun->lun_lock, MA_OWNED);
1614         pu = lun->pending_ua[initidx / CTL_MAX_INIT_PER_PORT];
1615         if (pu == NULL)
1616                 return;
1617         pu[initidx % CTL_MAX_INIT_PER_PORT] |= ua;
1618 }
1619
1620 void
1621 ctl_est_ua_port(struct ctl_lun *lun, int port, uint32_t except, ctl_ua_type ua)
1622 {
1623         int i;
1624
1625         mtx_assert(&lun->lun_lock, MA_OWNED);
1626         if (lun->pending_ua[port] == NULL)
1627                 return;
1628         for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1629                 if (port * CTL_MAX_INIT_PER_PORT + i == except)
1630                         continue;
1631                 lun->pending_ua[port][i] |= ua;
1632         }
1633 }
1634
1635 void
1636 ctl_est_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
1637 {
1638         struct ctl_softc *softc = lun->ctl_softc;
1639         int i;
1640
1641         mtx_assert(&lun->lun_lock, MA_OWNED);
1642         for (i = softc->port_min; i < softc->port_max; i++)
1643                 ctl_est_ua_port(lun, i, except, ua);
1644 }
1645
1646 void
1647 ctl_clr_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
1648 {
1649         struct ctl_softc *softc = lun->ctl_softc;
1650         ctl_ua_type *pu;
1651
1652         if (initidx < softc->init_min || initidx >= softc->init_max)
1653                 return;
1654         mtx_assert(&lun->lun_lock, MA_OWNED);
1655         pu = lun->pending_ua[initidx / CTL_MAX_INIT_PER_PORT];
1656         if (pu == NULL)
1657                 return;
1658         pu[initidx % CTL_MAX_INIT_PER_PORT] &= ~ua;
1659 }
1660
1661 void
1662 ctl_clr_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
1663 {
1664         struct ctl_softc *softc = lun->ctl_softc;
1665         int i, j;
1666
1667         mtx_assert(&lun->lun_lock, MA_OWNED);
1668         for (i = softc->port_min; i < softc->port_max; i++) {
1669                 if (lun->pending_ua[i] == NULL)
1670                         continue;
1671                 for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
1672                         if (i * CTL_MAX_INIT_PER_PORT + j == except)
1673                                 continue;
1674                         lun->pending_ua[i][j] &= ~ua;
1675                 }
1676         }
1677 }
1678
1679 void
1680 ctl_clr_ua_allluns(struct ctl_softc *ctl_softc, uint32_t initidx,
1681     ctl_ua_type ua_type)
1682 {
1683         struct ctl_lun *lun;
1684
1685         mtx_assert(&ctl_softc->ctl_lock, MA_OWNED);
1686         STAILQ_FOREACH(lun, &ctl_softc->lun_list, links) {
1687                 mtx_lock(&lun->lun_lock);
1688                 ctl_clr_ua(lun, initidx, ua_type);
1689                 mtx_unlock(&lun->lun_lock);
1690         }
1691 }
1692
1693 static int
1694 ctl_ha_role_sysctl(SYSCTL_HANDLER_ARGS)
1695 {
1696         struct ctl_softc *softc = (struct ctl_softc *)arg1;
1697         struct ctl_lun *lun;
1698         struct ctl_lun_req ireq;
1699         int error, value;
1700
1701         value = (softc->flags & CTL_FLAG_ACTIVE_SHELF) ? 0 : 1;
1702         error = sysctl_handle_int(oidp, &value, 0, req);
1703         if ((error != 0) || (req->newptr == NULL))
1704                 return (error);
1705
1706         mtx_lock(&softc->ctl_lock);
1707         if (value == 0)
1708                 softc->flags |= CTL_FLAG_ACTIVE_SHELF;
1709         else
1710                 softc->flags &= ~CTL_FLAG_ACTIVE_SHELF;
1711         STAILQ_FOREACH(lun, &softc->lun_list, links) {
1712                 mtx_unlock(&softc->ctl_lock);
1713                 bzero(&ireq, sizeof(ireq));
1714                 ireq.reqtype = CTL_LUNREQ_MODIFY;
1715                 ireq.reqdata.modify.lun_id = lun->lun;
1716                 lun->backend->ioctl(NULL, CTL_LUN_REQ, (caddr_t)&ireq, 0,
1717                     curthread);
1718                 if (ireq.status != CTL_LUN_OK) {
1719                         printf("%s: CTL_LUNREQ_MODIFY returned %d '%s'\n",
1720                             __func__, ireq.status, ireq.error_str);
1721                 }
1722                 mtx_lock(&softc->ctl_lock);
1723         }
1724         mtx_unlock(&softc->ctl_lock);
1725         return (0);
1726 }
1727
1728 static int
1729 ctl_init(void)
1730 {
1731         struct ctl_softc *softc;
1732         void *other_pool;
1733         int i, error;
1734
1735         softc = control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
1736                                M_WAITOK | M_ZERO);
1737
1738         softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
1739                               "cam/ctl");
1740         softc->dev->si_drv1 = softc;
1741
1742         sysctl_ctx_init(&softc->sysctl_ctx);
1743         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
1744                 SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
1745                 CTLFLAG_RD, 0, "CAM Target Layer");
1746
1747         if (softc->sysctl_tree == NULL) {
1748                 printf("%s: unable to allocate sysctl tree\n", __func__);
1749                 destroy_dev(softc->dev);
1750                 free(control_softc, M_DEVBUF);
1751                 control_softc = NULL;
1752                 return (ENOMEM);
1753         }
1754
1755         mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
1756         softc->io_zone = uma_zcreate("CTL IO", sizeof(union ctl_io),
1757             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1758         softc->open_count = 0;
1759
1760         /*
1761          * Default to actually sending a SYNCHRONIZE CACHE command down to
1762          * the drive.
1763          */
1764         softc->flags = CTL_FLAG_REAL_SYNC;
1765
1766         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1767             OID_AUTO, "ha_mode", CTLFLAG_RDTUN, (int *)&softc->ha_mode, 0,
1768             "HA mode (0 - act/stby, 1 - serialize only, 2 - xfer)");
1769
1770         /*
1771          * In Copan's HA scheme, the "master" and "slave" roles are
1772          * figured out through the slot the controller is in.  Although it
1773          * is an active/active system, someone has to be in charge.
1774          */
1775         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1776             OID_AUTO, "ha_id", CTLFLAG_RDTUN, &softc->ha_id, 0,
1777             "HA head ID (0 - no HA)");
1778         if (softc->ha_id == 0 || softc->ha_id > NUM_TARGET_PORT_GROUPS) {
1779                 softc->flags |= CTL_FLAG_ACTIVE_SHELF;
1780                 softc->is_single = 1;
1781                 softc->port_cnt = CTL_MAX_PORTS;
1782                 softc->port_min = 0;
1783         } else {
1784                 softc->port_cnt = CTL_MAX_PORTS / NUM_TARGET_PORT_GROUPS;
1785                 softc->port_min = (softc->ha_id - 1) * softc->port_cnt;
1786         }
1787         softc->port_max = softc->port_min + softc->port_cnt;
1788         softc->init_min = softc->port_min * CTL_MAX_INIT_PER_PORT;
1789         softc->init_max = softc->port_max * CTL_MAX_INIT_PER_PORT;
1790
1791         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1792             OID_AUTO, "ha_link", CTLFLAG_RD, (int *)&softc->ha_link, 0,
1793             "HA link state (0 - offline, 1 - unknown, 2 - online)");
1794
1795         STAILQ_INIT(&softc->lun_list);
1796         STAILQ_INIT(&softc->pending_lun_queue);
1797         STAILQ_INIT(&softc->fe_list);
1798         STAILQ_INIT(&softc->port_list);
1799         STAILQ_INIT(&softc->be_list);
1800         ctl_tpc_init(softc);
1801
1802         if (ctl_pool_create(softc, "othersc", CTL_POOL_ENTRIES_OTHER_SC,
1803                             &other_pool) != 0)
1804         {
1805                 printf("ctl: can't allocate %d entry other SC pool, "
1806                        "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1807                 return (ENOMEM);
1808         }
1809         softc->othersc_pool = other_pool;
1810
1811         if (worker_threads <= 0)
1812                 worker_threads = max(1, mp_ncpus / 4);
1813         if (worker_threads > CTL_MAX_THREADS)
1814                 worker_threads = CTL_MAX_THREADS;
1815
1816         for (i = 0; i < worker_threads; i++) {
1817                 struct ctl_thread *thr = &softc->threads[i];
1818
1819                 mtx_init(&thr->queue_lock, "CTL queue mutex", NULL, MTX_DEF);
1820                 thr->ctl_softc = softc;
1821                 STAILQ_INIT(&thr->incoming_queue);
1822                 STAILQ_INIT(&thr->rtr_queue);
1823                 STAILQ_INIT(&thr->done_queue);
1824                 STAILQ_INIT(&thr->isc_queue);
1825
1826                 error = kproc_kthread_add(ctl_work_thread, thr,
1827                     &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i);
1828                 if (error != 0) {
1829                         printf("error creating CTL work thread!\n");
1830                         ctl_pool_free(other_pool);
1831                         return (error);
1832                 }
1833         }
1834         error = kproc_kthread_add(ctl_lun_thread, softc,
1835             &softc->ctl_proc, NULL, 0, 0, "ctl", "lun");
1836         if (error != 0) {
1837                 printf("error creating CTL lun thread!\n");
1838                 ctl_pool_free(other_pool);
1839                 return (error);
1840         }
1841         error = kproc_kthread_add(ctl_thresh_thread, softc,
1842             &softc->ctl_proc, NULL, 0, 0, "ctl", "thresh");
1843         if (error != 0) {
1844                 printf("error creating CTL threshold thread!\n");
1845                 ctl_pool_free(other_pool);
1846                 return (error);
1847         }
1848
1849         SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
1850             OID_AUTO, "ha_role", CTLTYPE_INT | CTLFLAG_RWTUN,
1851             softc, 0, ctl_ha_role_sysctl, "I", "HA role for this head");
1852
1853         if (softc->is_single == 0) {
1854                 ctl_frontend_register(&ha_frontend);
1855                 if (ctl_ha_msg_init(softc) != CTL_HA_STATUS_SUCCESS) {
1856                         printf("ctl_init: ctl_ha_msg_init failed.\n");
1857                         softc->is_single = 1;
1858                 } else
1859                 if (ctl_ha_msg_register(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
1860                     != CTL_HA_STATUS_SUCCESS) {
1861                         printf("ctl_init: ctl_ha_msg_register failed.\n");
1862                         softc->is_single = 1;
1863                 }
1864         }
1865         return (0);
1866 }
1867
1868 void
1869 ctl_shutdown(void)
1870 {
1871         struct ctl_softc *softc = control_softc;
1872         struct ctl_lun *lun, *next_lun;
1873
1874         if (softc->is_single == 0) {
1875                 ctl_ha_msg_shutdown(softc);
1876                 if (ctl_ha_msg_deregister(CTL_HA_CHAN_CTL)
1877                     != CTL_HA_STATUS_SUCCESS)
1878                         printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
1879                 if (ctl_ha_msg_destroy(softc) != CTL_HA_STATUS_SUCCESS)
1880                         printf("%s: ctl_ha_msg_destroy failed.\n", __func__);
1881                 ctl_frontend_deregister(&ha_frontend);
1882         }
1883
1884         mtx_lock(&softc->ctl_lock);
1885
1886         /*
1887          * Free up each LUN.
1888          */
1889         for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1890                 next_lun = STAILQ_NEXT(lun, links);
1891                 ctl_free_lun(lun);
1892         }
1893
1894         mtx_unlock(&softc->ctl_lock);
1895
1896 #if 0
1897         ctl_shutdown_thread(softc->work_thread);
1898         mtx_destroy(&softc->queue_lock);
1899 #endif
1900
1901         ctl_tpc_shutdown(softc);
1902         uma_zdestroy(softc->io_zone);
1903         mtx_destroy(&softc->ctl_lock);
1904
1905         destroy_dev(softc->dev);
1906
1907         sysctl_ctx_free(&softc->sysctl_ctx);
1908
1909         free(control_softc, M_DEVBUF);
1910         control_softc = NULL;
1911 }
1912
1913 static int
1914 ctl_module_event_handler(module_t mod, int what, void *arg)
1915 {
1916
1917         switch (what) {
1918         case MOD_LOAD:
1919                 return (ctl_init());
1920         case MOD_UNLOAD:
1921                 return (EBUSY);
1922         default:
1923                 return (EOPNOTSUPP);
1924         }
1925 }
1926
1927 /*
1928  * XXX KDM should we do some access checks here?  Bump a reference count to
1929  * prevent a CTL module from being unloaded while someone has it open?
1930  */
1931 static int
1932 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1933 {
1934         return (0);
1935 }
1936
1937 static int
1938 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1939 {
1940         return (0);
1941 }
1942
1943 /*
1944  * Remove an initiator by port number and initiator ID.
1945  * Returns 0 for success, -1 for failure.
1946  */
1947 int
1948 ctl_remove_initiator(struct ctl_port *port, int iid)
1949 {
1950         struct ctl_softc *softc = port->ctl_softc;
1951
1952         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1953
1954         if (iid > CTL_MAX_INIT_PER_PORT) {
1955                 printf("%s: initiator ID %u > maximun %u!\n",
1956                        __func__, iid, CTL_MAX_INIT_PER_PORT);
1957                 return (-1);
1958         }
1959
1960         mtx_lock(&softc->ctl_lock);
1961         port->wwpn_iid[iid].in_use--;
1962         port->wwpn_iid[iid].last_use = time_uptime;
1963         mtx_unlock(&softc->ctl_lock);
1964         ctl_isc_announce_iid(port, iid);
1965
1966         return (0);
1967 }
1968
1969 /*
1970  * Add an initiator to the initiator map.
1971  * Returns iid for success, < 0 for failure.
1972  */
1973 int
1974 ctl_add_initiator(struct ctl_port *port, int iid, uint64_t wwpn, char *name)
1975 {
1976         struct ctl_softc *softc = port->ctl_softc;
1977         time_t best_time;
1978         int i, best;
1979
1980         mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1981
1982         if (iid >= CTL_MAX_INIT_PER_PORT) {
1983                 printf("%s: WWPN %#jx initiator ID %u > maximum %u!\n",
1984                        __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1985                 free(name, M_CTL);
1986                 return (-1);
1987         }
1988
1989         mtx_lock(&softc->ctl_lock);
1990
1991         if (iid < 0 && (wwpn != 0 || name != NULL)) {
1992                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1993                         if (wwpn != 0 && wwpn == port->wwpn_iid[i].wwpn) {
1994                                 iid = i;
1995                                 break;
1996                         }
1997                         if (name != NULL && port->wwpn_iid[i].name != NULL &&
1998                             strcmp(name, port->wwpn_iid[i].name) == 0) {
1999                                 iid = i;
2000                                 break;
2001                         }
2002                 }
2003         }
2004
2005         if (iid < 0) {
2006                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
2007                         if (port->wwpn_iid[i].in_use == 0 &&
2008                             port->wwpn_iid[i].wwpn == 0 &&
2009                             port->wwpn_iid[i].name == NULL) {
2010                                 iid = i;
2011                                 break;
2012                         }
2013                 }
2014         }
2015
2016         if (iid < 0) {
2017                 best = -1;
2018                 best_time = INT32_MAX;
2019                 for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
2020                         if (port->wwpn_iid[i].in_use == 0) {
2021                                 if (port->wwpn_iid[i].last_use < best_time) {
2022                                         best = i;
2023                                         best_time = port->wwpn_iid[i].last_use;
2024                                 }
2025                         }
2026                 }
2027                 iid = best;
2028         }
2029
2030         if (iid < 0) {
2031                 mtx_unlock(&softc->ctl_lock);
2032                 free(name, M_CTL);
2033                 return (-2);
2034         }
2035
2036         if (port->wwpn_iid[iid].in_use > 0 && (wwpn != 0 || name != NULL)) {
2037                 /*
2038                  * This is not an error yet.
2039                  */
2040                 if (wwpn != 0 && wwpn == port->wwpn_iid[iid].wwpn) {
2041 #if 0
2042                         printf("%s: port %d iid %u WWPN %#jx arrived"
2043                             " again\n", __func__, port->targ_port,
2044                             iid, (uintmax_t)wwpn);
2045 #endif
2046                         goto take;
2047                 }
2048                 if (name != NULL && port->wwpn_iid[iid].name != NULL &&
2049                     strcmp(name, port->wwpn_iid[iid].name) == 0) {
2050 #if 0
2051                         printf("%s: port %d iid %u name '%s' arrived"
2052                             " again\n", __func__, port->targ_port,
2053                             iid, name);
2054 #endif
2055                         goto take;
2056                 }
2057
2058                 /*
2059                  * This is an error, but what do we do about it?  The
2060                  * driver is telling us we have a new WWPN for this
2061                  * initiator ID, so we pretty much need to use it.
2062                  */
2063                 printf("%s: port %d iid %u WWPN %#jx '%s' arrived,"
2064                     " but WWPN %#jx '%s' is still at that address\n",
2065                     __func__, port->targ_port, iid, wwpn, name,
2066                     (uintmax_t)port->wwpn_iid[iid].wwpn,
2067                     port->wwpn_iid[iid].name);
2068
2069                 /*
2070                  * XXX KDM clear have_ca and ua_pending on each LUN for
2071                  * this initiator.
2072                  */
2073         }
2074 take:
2075         free(port->wwpn_iid[iid].name, M_CTL);
2076         port->wwpn_iid[iid].name = name;
2077         port->wwpn_iid[iid].wwpn = wwpn;
2078         port->wwpn_iid[iid].in_use++;
2079         mtx_unlock(&softc->ctl_lock);
2080         ctl_isc_announce_iid(port, iid);
2081
2082         return (iid);
2083 }
2084
2085 static int
2086 ctl_create_iid(struct ctl_port *port, int iid, uint8_t *buf)
2087 {
2088         int len;
2089
2090         switch (port->port_type) {
2091         case CTL_PORT_FC:
2092         {
2093                 struct scsi_transportid_fcp *id =
2094                     (struct scsi_transportid_fcp *)buf;
2095                 if (port->wwpn_iid[iid].wwpn == 0)
2096                         return (0);
2097                 memset(id, 0, sizeof(*id));
2098                 id->format_protocol = SCSI_PROTO_FC;
2099                 scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->n_port_name);
2100                 return (sizeof(*id));
2101         }
2102         case CTL_PORT_ISCSI:
2103         {
2104                 struct scsi_transportid_iscsi_port *id =
2105                     (struct scsi_transportid_iscsi_port *)buf;
2106                 if (port->wwpn_iid[iid].name == NULL)
2107                         return (0);
2108                 memset(id, 0, 256);
2109                 id->format_protocol = SCSI_TRN_ISCSI_FORMAT_PORT |
2110                     SCSI_PROTO_ISCSI;
2111                 len = strlcpy(id->iscsi_name, port->wwpn_iid[iid].name, 252) + 1;
2112                 len = roundup2(min(len, 252), 4);
2113                 scsi_ulto2b(len, id->additional_length);
2114                 return (sizeof(*id) + len);
2115         }
2116         case CTL_PORT_SAS:
2117         {
2118                 struct scsi_transportid_sas *id =
2119                     (struct scsi_transportid_sas *)buf;
2120                 if (port->wwpn_iid[iid].wwpn == 0)
2121                         return (0);
2122                 memset(id, 0, sizeof(*id));
2123                 id->format_protocol = SCSI_PROTO_SAS;
2124                 scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->sas_address);
2125                 return (sizeof(*id));
2126         }
2127         default:
2128         {
2129                 struct scsi_transportid_spi *id =
2130                     (struct scsi_transportid_spi *)buf;
2131                 memset(id, 0, sizeof(*id));
2132                 id->format_protocol = SCSI_PROTO_SPI;
2133                 scsi_ulto2b(iid, id->scsi_addr);
2134                 scsi_ulto2b(port->targ_port, id->rel_trgt_port_id);
2135                 return (sizeof(*id));
2136         }
2137         }
2138 }
2139
2140 /*
2141  * Serialize a command that went down the "wrong" side, and so was sent to
2142  * this controller for execution.  The logic is a little different than the
2143  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
2144  * sent back to the other side, but in the success case, we execute the
2145  * command on this side (XFER mode) or tell the other side to execute it
2146  * (SER_ONLY mode).
2147  */
2148 static int
2149 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
2150 {
2151         struct ctl_softc *softc = control_softc;
2152         union ctl_ha_msg msg_info;
2153         struct ctl_port *port;
2154         struct ctl_lun *lun;
2155         const struct ctl_cmd_entry *entry;
2156         int retval = 0;
2157         uint32_t targ_lun;
2158
2159         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
2160         mtx_lock(&softc->ctl_lock);
2161
2162         /* Make sure that we know about this port. */
2163         port = ctl_io_port(&ctsio->io_hdr);
2164         if (port == NULL || (port->status & CTL_PORT_STATUS_ONLINE) == 0) {
2165                 ctl_set_internal_failure(ctsio, /*sks_valid*/ 0,
2166                                          /*retry_count*/ 1);
2167                 goto badjuju;
2168         }
2169
2170         /* Make sure that we know about this LUN. */
2171         if ((targ_lun < CTL_MAX_LUNS) &&
2172             ((lun = softc->ctl_luns[targ_lun]) != NULL)) {
2173                 mtx_lock(&lun->lun_lock);
2174                 mtx_unlock(&softc->ctl_lock);
2175                 /*
2176                  * If the LUN is invalid, pretend that it doesn't exist.
2177                  * It will go away as soon as all pending I/O has been
2178                  * completed.
2179                  */
2180                 if (lun->flags & CTL_LUN_DISABLED) {
2181                         mtx_unlock(&lun->lun_lock);
2182                         lun = NULL;
2183                 }
2184         } else {
2185                 mtx_unlock(&softc->ctl_lock);
2186                 lun = NULL;
2187         }
2188         if (lun == NULL) {
2189                 /*
2190                  * The other node would not send this request to us unless
2191                  * received announce that we are primary node for this LUN.
2192                  * If this LUN does not exist now, it is probably result of
2193                  * a race, so respond to initiator in the most opaque way.
2194                  */
2195                 ctl_set_busy(ctsio);
2196                 goto badjuju;
2197         }
2198
2199         entry = ctl_get_cmd_entry(ctsio, NULL);
2200         if (ctl_scsiio_lun_check(lun, entry, ctsio) != 0) {
2201                 mtx_unlock(&lun->lun_lock);
2202                 goto badjuju;
2203         }
2204
2205         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
2206         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = lun->be_lun;
2207
2208         /*
2209          * Every I/O goes into the OOA queue for a
2210          * particular LUN, and stays there until completion.
2211          */
2212 #ifdef CTL_TIME_IO
2213         if (TAILQ_EMPTY(&lun->ooa_queue))
2214                 lun->idle_time += getsbinuptime() - lun->last_busy;
2215 #endif
2216         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
2217
2218         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
2219                 (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
2220                  ooa_links))) {
2221         case CTL_ACTION_BLOCK:
2222                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
2223                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
2224                                   blocked_links);
2225                 mtx_unlock(&lun->lun_lock);
2226                 break;
2227         case CTL_ACTION_PASS:
2228         case CTL_ACTION_SKIP:
2229                 if (softc->ha_mode == CTL_HA_MODE_XFER) {
2230                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
2231                         ctl_enqueue_rtr((union ctl_io *)ctsio);
2232                         mtx_unlock(&lun->lun_lock);
2233                 } else {
2234                         ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
2235                         mtx_unlock(&lun->lun_lock);
2236
2237                         /* send msg back to other side */
2238                         msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
2239                         msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
2240                         msg_info.hdr.msg_type = CTL_MSG_R2R;
2241                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
2242                             sizeof(msg_info.hdr), M_WAITOK);
2243                 }
2244                 break;
2245         case CTL_ACTION_OVERLAP:
2246                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
2247                 mtx_unlock(&lun->lun_lock);
2248                 ctl_set_overlapped_cmd(ctsio);
2249                 goto badjuju;
2250         case CTL_ACTION_OVERLAP_TAG:
2251                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
2252                 mtx_unlock(&lun->lun_lock);
2253                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num);
2254                 goto badjuju;
2255         case CTL_ACTION_ERROR:
2256         default:
2257                 TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
2258                 mtx_unlock(&lun->lun_lock);
2259
2260                 ctl_set_internal_failure(ctsio, /*sks_valid*/ 0,
2261                                          /*retry_count*/ 0);
2262 badjuju:
2263                 ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
2264                 msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
2265                 msg_info.hdr.serializing_sc = NULL;
2266                 msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
2267                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
2268                     sizeof(msg_info.scsi), M_WAITOK);
2269                 retval = 1;
2270                 break;
2271         }
2272         return (retval);
2273 }
2274
2275 /*
2276  * Returns 0 for success, errno for failure.
2277  */
2278 static void
2279 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
2280                    struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
2281 {
2282         union ctl_io *io;
2283
2284         mtx_lock(&lun->lun_lock);
2285         for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
2286              (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2287              ooa_links)) {
2288                 struct ctl_ooa_entry *entry;
2289
2290                 /*
2291                  * If we've got more than we can fit, just count the
2292                  * remaining entries.
2293                  */
2294                 if (*cur_fill_num >= ooa_hdr->alloc_num)
2295                         continue;
2296
2297                 entry = &kern_entries[*cur_fill_num];
2298
2299                 entry->tag_num = io->scsiio.tag_num;
2300                 entry->lun_num = lun->lun;
2301 #ifdef CTL_TIME_IO
2302                 entry->start_bt = io->io_hdr.start_bt;
2303 #endif
2304                 bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
2305                 entry->cdb_len = io->scsiio.cdb_len;
2306                 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
2307                         entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
2308
2309                 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
2310                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
2311
2312                 if (io->io_hdr.flags & CTL_FLAG_ABORT)
2313                         entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
2314
2315                 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
2316                         entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
2317
2318                 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
2319                         entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
2320         }
2321         mtx_unlock(&lun->lun_lock);
2322 }
2323
2324 static void *
2325 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2326                  size_t error_str_len)
2327 {
2328         void *kptr;
2329
2330         kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2331
2332         if (copyin(user_addr, kptr, len) != 0) {
2333                 snprintf(error_str, error_str_len, "Error copying %d bytes "
2334                          "from user address %p to kernel address %p", len,
2335                          user_addr, kptr);
2336                 free(kptr, M_CTL);
2337                 return (NULL);
2338         }
2339
2340         return (kptr);
2341 }
2342
2343 static void
2344 ctl_free_args(int num_args, struct ctl_be_arg *args)
2345 {
2346         int i;
2347
2348         if (args == NULL)
2349                 return;
2350
2351         for (i = 0; i < num_args; i++) {
2352                 free(args[i].kname, M_CTL);
2353                 free(args[i].kvalue, M_CTL);
2354         }
2355
2356         free(args, M_CTL);
2357 }
2358
2359 static struct ctl_be_arg *
2360 ctl_copyin_args(int num_args, struct ctl_be_arg *uargs,
2361                 char *error_str, size_t error_str_len)
2362 {
2363         struct ctl_be_arg *args;
2364         int i;
2365
2366         args = ctl_copyin_alloc(uargs, num_args * sizeof(*args),
2367                                 error_str, error_str_len);
2368
2369         if (args == NULL)
2370                 goto bailout;
2371
2372         for (i = 0; i < num_args; i++) {
2373                 args[i].kname = NULL;
2374                 args[i].kvalue = NULL;
2375         }
2376
2377         for (i = 0; i < num_args; i++) {
2378                 uint8_t *tmpptr;
2379
2380                 args[i].kname = ctl_copyin_alloc(args[i].name,
2381                         args[i].namelen, error_str, error_str_len);
2382                 if (args[i].kname == NULL)
2383                         goto bailout;
2384
2385                 if (args[i].kname[args[i].namelen - 1] != '\0') {
2386                         snprintf(error_str, error_str_len, "Argument %d "
2387                                  "name is not NUL-terminated", i);
2388                         goto bailout;
2389                 }
2390
2391                 if (args[i].flags & CTL_BEARG_RD) {
2392                         tmpptr = ctl_copyin_alloc(args[i].value,
2393                                 args[i].vallen, error_str, error_str_len);
2394                         if (tmpptr == NULL)
2395                                 goto bailout;
2396                         if ((args[i].flags & CTL_BEARG_ASCII)
2397                          && (tmpptr[args[i].vallen - 1] != '\0')) {
2398                                 snprintf(error_str, error_str_len, "Argument "
2399                                     "%d value is not NUL-terminated", i);
2400                                 goto bailout;
2401                         }
2402                         args[i].kvalue = tmpptr;
2403                 } else {
2404                         args[i].kvalue = malloc(args[i].vallen,
2405                             M_CTL, M_WAITOK | M_ZERO);
2406                 }
2407         }
2408
2409         return (args);
2410 bailout:
2411
2412         ctl_free_args(num_args, args);
2413
2414         return (NULL);
2415 }
2416
2417 static void
2418 ctl_copyout_args(int num_args, struct ctl_be_arg *args)
2419 {
2420         int i;
2421
2422         for (i = 0; i < num_args; i++) {
2423                 if (args[i].flags & CTL_BEARG_WR)
2424                         copyout(args[i].kvalue, args[i].value, args[i].vallen);
2425         }
2426 }
2427
2428 /*
2429  * Escape characters that are illegal or not recommended in XML.
2430  */
2431 int
2432 ctl_sbuf_printf_esc(struct sbuf *sb, char *str, int size)
2433 {
2434         char *end = str + size;
2435         int retval;
2436
2437         retval = 0;
2438
2439         for (; *str && str < end; str++) {
2440                 switch (*str) {
2441                 case '&':
2442                         retval = sbuf_printf(sb, "&amp;");
2443                         break;
2444                 case '>':
2445                         retval = sbuf_printf(sb, "&gt;");
2446                         break;
2447                 case '<':
2448                         retval = sbuf_printf(sb, "&lt;");
2449                         break;
2450                 default:
2451                         retval = sbuf_putc(sb, *str);
2452                         break;
2453                 }
2454
2455                 if (retval != 0)
2456                         break;
2457
2458         }
2459
2460         return (retval);
2461 }
2462
2463 static void
2464 ctl_id_sbuf(struct ctl_devid *id, struct sbuf *sb)
2465 {
2466         struct scsi_vpd_id_descriptor *desc;
2467         int i;
2468
2469         if (id == NULL || id->len < 4)
2470                 return;
2471         desc = (struct scsi_vpd_id_descriptor *)id->data;
2472         switch (desc->id_type & SVPD_ID_TYPE_MASK) {
2473         case SVPD_ID_TYPE_T10:
2474                 sbuf_printf(sb, "t10.");
2475                 break;
2476         case SVPD_ID_TYPE_EUI64:
2477                 sbuf_printf(sb, "eui.");
2478                 break;
2479         case SVPD_ID_TYPE_NAA:
2480                 sbuf_printf(sb, "naa.");
2481                 break;
2482         case SVPD_ID_TYPE_SCSI_NAME:
2483                 break;
2484         }
2485         switch (desc->proto_codeset & SVPD_ID_CODESET_MASK) {
2486         case SVPD_ID_CODESET_BINARY:
2487                 for (i = 0; i < desc->length; i++)
2488                         sbuf_printf(sb, "%02x", desc->identifier[i]);
2489                 break;
2490         case SVPD_ID_CODESET_ASCII:
2491                 sbuf_printf(sb, "%.*s", (int)desc->length,
2492                     (char *)desc->identifier);
2493                 break;
2494         case SVPD_ID_CODESET_UTF8:
2495                 sbuf_printf(sb, "%s", (char *)desc->identifier);
2496                 break;
2497         }
2498 }
2499
2500 static int
2501 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2502           struct thread *td)
2503 {
2504         struct ctl_softc *softc = dev->si_drv1;
2505         struct ctl_lun *lun;
2506         int retval;
2507
2508         retval = 0;
2509
2510         switch (cmd) {
2511         case CTL_IO:
2512                 retval = ctl_ioctl_io(dev, cmd, addr, flag, td);
2513                 break;
2514         case CTL_ENABLE_PORT:
2515         case CTL_DISABLE_PORT:
2516         case CTL_SET_PORT_WWNS: {
2517                 struct ctl_port *port;
2518                 struct ctl_port_entry *entry;
2519
2520                 entry = (struct ctl_port_entry *)addr;
2521
2522                 mtx_lock(&softc->ctl_lock);
2523                 STAILQ_FOREACH(port, &softc->port_list, links) {
2524                         int action, done;
2525
2526                         if (port->targ_port < softc->port_min ||
2527                             port->targ_port >= softc->port_max)
2528                                 continue;
2529
2530                         action = 0;
2531                         done = 0;
2532                         if ((entry->port_type == CTL_PORT_NONE)
2533                          && (entry->targ_port == port->targ_port)) {
2534                                 /*
2535                                  * If the user only wants to enable or
2536                                  * disable or set WWNs on a specific port,
2537                                  * do the operation and we're done.
2538                                  */
2539                                 action = 1;
2540                                 done = 1;
2541                         } else if (entry->port_type & port->port_type) {
2542                                 /*
2543                                  * Compare the user's type mask with the
2544                                  * particular frontend type to see if we
2545                                  * have a match.
2546                                  */
2547                                 action = 1;
2548                                 done = 0;
2549
2550                                 /*
2551                                  * Make sure the user isn't trying to set
2552                                  * WWNs on multiple ports at the same time.
2553                                  */
2554                                 if (cmd == CTL_SET_PORT_WWNS) {
2555                                         printf("%s: Can't set WWNs on "
2556                                                "multiple ports\n", __func__);
2557                                         retval = EINVAL;
2558                                         break;
2559                                 }
2560                         }
2561                         if (action == 0)
2562                                 continue;
2563
2564                         /*
2565                          * XXX KDM we have to drop the lock here, because
2566                          * the online/offline operations can potentially
2567                          * block.  We need to reference count the frontends
2568                          * so they can't go away,
2569                          */
2570                         if (cmd == CTL_ENABLE_PORT) {
2571                                 mtx_unlock(&softc->ctl_lock);
2572                                 ctl_port_online(port);
2573                                 mtx_lock(&softc->ctl_lock);
2574                         } else if (cmd == CTL_DISABLE_PORT) {
2575                                 mtx_unlock(&softc->ctl_lock);
2576                                 ctl_port_offline(port);
2577                                 mtx_lock(&softc->ctl_lock);
2578                         } else if (cmd == CTL_SET_PORT_WWNS) {
2579                                 ctl_port_set_wwns(port,
2580                                     (entry->flags & CTL_PORT_WWNN_VALID) ?
2581                                     1 : 0, entry->wwnn,
2582                                     (entry->flags & CTL_PORT_WWPN_VALID) ?
2583                                     1 : 0, entry->wwpn);
2584                         }
2585                         if (done != 0)
2586                                 break;
2587                 }
2588                 mtx_unlock(&softc->ctl_lock);
2589                 break;
2590         }
2591         case CTL_GET_PORT_LIST: {
2592                 struct ctl_port *port;
2593                 struct ctl_port_list *list;
2594                 int i;
2595
2596                 list = (struct ctl_port_list *)addr;
2597
2598                 if (list->alloc_len != (list->alloc_num *
2599                     sizeof(struct ctl_port_entry))) {
2600                         printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2601                                "alloc_num %u * sizeof(struct ctl_port_entry) "
2602                                "%zu\n", __func__, list->alloc_len,
2603                                list->alloc_num, sizeof(struct ctl_port_entry));
2604                         retval = EINVAL;
2605                         break;
2606                 }
2607                 list->fill_len = 0;
2608                 list->fill_num = 0;
2609                 list->dropped_num = 0;
2610                 i = 0;
2611                 mtx_lock(&softc->ctl_lock);
2612                 STAILQ_FOREACH(port, &softc->port_list, links) {
2613                         struct ctl_port_entry entry, *list_entry;
2614
2615                         if (list->fill_num >= list->alloc_num) {
2616                                 list->dropped_num++;
2617                                 continue;
2618                         }
2619
2620                         entry.port_type = port->port_type;
2621                         strlcpy(entry.port_name, port->port_name,
2622                                 sizeof(entry.port_name));
2623                         entry.targ_port = port->targ_port;
2624                         entry.physical_port = port->physical_port;
2625                         entry.virtual_port = port->virtual_port;
2626                         entry.wwnn = port->wwnn;
2627                         entry.wwpn = port->wwpn;
2628                         if (port->status & CTL_PORT_STATUS_ONLINE)
2629                                 entry.online = 1;
2630                         else
2631                                 entry.online = 0;
2632
2633                         list_entry = &list->entries[i];
2634
2635                         retval = copyout(&entry, list_entry, sizeof(entry));
2636                         if (retval != 0) {
2637                                 printf("%s: CTL_GET_PORT_LIST: copyout "
2638                                        "returned %d\n", __func__, retval);
2639                                 break;
2640                         }
2641                         i++;
2642                         list->fill_num++;
2643                         list->fill_len += sizeof(entry);
2644                 }
2645                 mtx_unlock(&softc->ctl_lock);
2646
2647                 /*
2648                  * If this is non-zero, we had a copyout fault, so there's
2649                  * probably no point in attempting to set the status inside
2650                  * the structure.
2651                  */
2652                 if (retval != 0)
2653                         break;
2654
2655                 if (list->dropped_num > 0)
2656                         list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2657                 else
2658                         list->status = CTL_PORT_LIST_OK;
2659                 break;
2660         }
2661         case CTL_DUMP_OOA: {
2662                 union ctl_io *io;
2663                 char printbuf[128];
2664                 struct sbuf sb;
2665
2666                 mtx_lock(&softc->ctl_lock);
2667                 printf("Dumping OOA queues:\n");
2668                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
2669                         mtx_lock(&lun->lun_lock);
2670                         for (io = (union ctl_io *)TAILQ_FIRST(
2671                              &lun->ooa_queue); io != NULL;
2672                              io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2673                              ooa_links)) {
2674                                 sbuf_new(&sb, printbuf, sizeof(printbuf),
2675                                          SBUF_FIXEDLEN);
2676                                 sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2677                                             (intmax_t)lun->lun,
2678                                             io->scsiio.tag_num,
2679                                             (io->io_hdr.flags &
2680                                             CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2681                                             (io->io_hdr.flags &
2682                                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2683                                             (io->io_hdr.flags &
2684                                             CTL_FLAG_ABORT) ? " ABORT" : "",
2685                                             (io->io_hdr.flags &
2686                                         CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2687                                 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2688                                 sbuf_finish(&sb);
2689                                 printf("%s\n", sbuf_data(&sb));
2690                         }
2691                         mtx_unlock(&lun->lun_lock);
2692                 }
2693                 printf("OOA queues dump done\n");
2694                 mtx_unlock(&softc->ctl_lock);
2695                 break;
2696         }
2697         case CTL_GET_OOA: {
2698                 struct ctl_ooa *ooa_hdr;
2699                 struct ctl_ooa_entry *entries;
2700                 uint32_t cur_fill_num;
2701
2702                 ooa_hdr = (struct ctl_ooa *)addr;
2703
2704                 if ((ooa_hdr->alloc_len == 0)
2705                  || (ooa_hdr->alloc_num == 0)) {
2706                         printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2707                                "must be non-zero\n", __func__,
2708                                ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2709                         retval = EINVAL;
2710                         break;
2711                 }
2712
2713                 if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2714                     sizeof(struct ctl_ooa_entry))) {
2715                         printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2716                                "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2717                                __func__, ooa_hdr->alloc_len,
2718                                ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2719                         retval = EINVAL;
2720                         break;
2721                 }
2722
2723                 entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2724                 if (entries == NULL) {
2725                         printf("%s: could not allocate %d bytes for OOA "
2726                                "dump\n", __func__, ooa_hdr->alloc_len);
2727                         retval = ENOMEM;
2728                         break;
2729                 }
2730
2731                 mtx_lock(&softc->ctl_lock);
2732                 if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2733                  && ((ooa_hdr->lun_num >= CTL_MAX_LUNS)
2734                   || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2735                         mtx_unlock(&softc->ctl_lock);
2736                         free(entries, M_CTL);
2737                         printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2738                                __func__, (uintmax_t)ooa_hdr->lun_num);
2739                         retval = EINVAL;
2740                         break;
2741                 }
2742
2743                 cur_fill_num = 0;
2744
2745                 if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2746                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
2747                                 ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2748                                     ooa_hdr, entries);
2749                         }
2750                 } else {
2751                         lun = softc->ctl_luns[ooa_hdr->lun_num];
2752                         ctl_ioctl_fill_ooa(lun, &cur_fill_num, ooa_hdr,
2753                             entries);
2754                 }
2755                 mtx_unlock(&softc->ctl_lock);
2756
2757                 ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2758                 ooa_hdr->fill_len = ooa_hdr->fill_num *
2759                         sizeof(struct ctl_ooa_entry);
2760                 retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2761                 if (retval != 0) {
2762                         printf("%s: error copying out %d bytes for OOA dump\n",
2763                                __func__, ooa_hdr->fill_len);
2764                 }
2765
2766                 getbinuptime(&ooa_hdr->cur_bt);
2767
2768                 if (cur_fill_num > ooa_hdr->alloc_num) {
2769                         ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2770                         ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2771                 } else {
2772                         ooa_hdr->dropped_num = 0;
2773                         ooa_hdr->status = CTL_OOA_OK;
2774                 }
2775
2776                 free(entries, M_CTL);
2777                 break;
2778         }
2779         case CTL_CHECK_OOA: {
2780                 union ctl_io *io;
2781                 struct ctl_ooa_info *ooa_info;
2782
2783
2784                 ooa_info = (struct ctl_ooa_info *)addr;
2785
2786                 if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2787                         ooa_info->status = CTL_OOA_INVALID_LUN;
2788                         break;
2789                 }
2790                 mtx_lock(&softc->ctl_lock);
2791                 lun = softc->ctl_luns[ooa_info->lun_id];
2792                 if (lun == NULL) {
2793                         mtx_unlock(&softc->ctl_lock);
2794                         ooa_info->status = CTL_OOA_INVALID_LUN;
2795                         break;
2796                 }
2797                 mtx_lock(&lun->lun_lock);
2798                 mtx_unlock(&softc->ctl_lock);
2799                 ooa_info->num_entries = 0;
2800                 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2801                      io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2802                      &io->io_hdr, ooa_links)) {
2803                         ooa_info->num_entries++;
2804                 }
2805                 mtx_unlock(&lun->lun_lock);
2806
2807                 ooa_info->status = CTL_OOA_SUCCESS;
2808
2809                 break;
2810         }
2811         case CTL_DELAY_IO: {
2812                 struct ctl_io_delay_info *delay_info;
2813
2814                 delay_info = (struct ctl_io_delay_info *)addr;
2815
2816 #ifdef CTL_IO_DELAY
2817                 mtx_lock(&softc->ctl_lock);
2818
2819                 if ((delay_info->lun_id >= CTL_MAX_LUNS)
2820                  || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2821                         delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2822                 } else {
2823                         lun = softc->ctl_luns[delay_info->lun_id];
2824                         mtx_lock(&lun->lun_lock);
2825
2826                         delay_info->status = CTL_DELAY_STATUS_OK;
2827
2828                         switch (delay_info->delay_type) {
2829                         case CTL_DELAY_TYPE_CONT:
2830                                 break;
2831                         case CTL_DELAY_TYPE_ONESHOT:
2832                                 break;
2833                         default:
2834                                 delay_info->status =
2835                                         CTL_DELAY_STATUS_INVALID_TYPE;
2836                                 break;
2837                         }
2838
2839                         switch (delay_info->delay_loc) {
2840                         case CTL_DELAY_LOC_DATAMOVE:
2841                                 lun->delay_info.datamove_type =
2842                                         delay_info->delay_type;
2843                                 lun->delay_info.datamove_delay =
2844                                         delay_info->delay_secs;
2845                                 break;
2846                         case CTL_DELAY_LOC_DONE:
2847                                 lun->delay_info.done_type =
2848                                         delay_info->delay_type;
2849                                 lun->delay_info.done_delay =
2850                                         delay_info->delay_secs;
2851                                 break;
2852                         default:
2853                                 delay_info->status =
2854                                         CTL_DELAY_STATUS_INVALID_LOC;
2855                                 break;
2856                         }
2857                         mtx_unlock(&lun->lun_lock);
2858                 }
2859
2860                 mtx_unlock(&softc->ctl_lock);
2861 #else
2862                 delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2863 #endif /* CTL_IO_DELAY */
2864                 break;
2865         }
2866         case CTL_REALSYNC_SET: {
2867                 int *syncstate;
2868
2869                 syncstate = (int *)addr;
2870
2871                 mtx_lock(&softc->ctl_lock);
2872                 switch (*syncstate) {
2873                 case 0:
2874                         softc->flags &= ~CTL_FLAG_REAL_SYNC;
2875                         break;
2876                 case 1:
2877                         softc->flags |= CTL_FLAG_REAL_SYNC;
2878                         break;
2879                 default:
2880                         retval = EINVAL;
2881                         break;
2882                 }
2883                 mtx_unlock(&softc->ctl_lock);
2884                 break;
2885         }
2886         case CTL_REALSYNC_GET: {
2887                 int *syncstate;
2888
2889                 syncstate = (int*)addr;
2890
2891                 mtx_lock(&softc->ctl_lock);
2892                 if (softc->flags & CTL_FLAG_REAL_SYNC)
2893                         *syncstate = 1;
2894                 else
2895                         *syncstate = 0;
2896                 mtx_unlock(&softc->ctl_lock);
2897
2898                 break;
2899         }
2900         case CTL_SETSYNC:
2901         case CTL_GETSYNC: {
2902                 struct ctl_sync_info *sync_info;
2903
2904                 sync_info = (struct ctl_sync_info *)addr;
2905
2906                 mtx_lock(&softc->ctl_lock);
2907                 lun = softc->ctl_luns[sync_info->lun_id];
2908                 if (lun == NULL) {
2909                         mtx_unlock(&softc->ctl_lock);
2910                         sync_info->status = CTL_GS_SYNC_NO_LUN;
2911                         break;
2912                 }
2913                 /*
2914                  * Get or set the sync interval.  We're not bounds checking
2915                  * in the set case, hopefully the user won't do something
2916                  * silly.
2917                  */
2918                 mtx_lock(&lun->lun_lock);
2919                 mtx_unlock(&softc->ctl_lock);
2920                 if (cmd == CTL_GETSYNC)
2921                         sync_info->sync_interval = lun->sync_interval;
2922                 else
2923                         lun->sync_interval = sync_info->sync_interval;
2924                 mtx_unlock(&lun->lun_lock);
2925
2926                 sync_info->status = CTL_GS_SYNC_OK;
2927
2928                 break;
2929         }
2930         case CTL_GETSTATS: {
2931                 struct ctl_stats *stats;
2932                 int i;
2933
2934                 stats = (struct ctl_stats *)addr;
2935
2936                 if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2937                      stats->alloc_len) {
2938                         stats->status = CTL_SS_NEED_MORE_SPACE;
2939                         stats->num_luns = softc->num_luns;
2940                         break;
2941                 }
2942                 /*
2943                  * XXX KDM no locking here.  If the LUN list changes,
2944                  * things can blow up.
2945                  */
2946                 for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2947                      i++, lun = STAILQ_NEXT(lun, links)) {
2948                         retval = copyout(&lun->stats, &stats->lun_stats[i],
2949                                          sizeof(lun->stats));
2950                         if (retval != 0)
2951                                 break;
2952                 }
2953                 stats->num_luns = softc->num_luns;
2954                 stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2955                                  softc->num_luns;
2956                 stats->status = CTL_SS_OK;
2957 #ifdef CTL_TIME_IO
2958                 stats->flags = CTL_STATS_FLAG_TIME_VALID;
2959 #else
2960                 stats->flags = CTL_STATS_FLAG_NONE;
2961 #endif
2962                 getnanouptime(&stats->timestamp);
2963                 break;
2964         }
2965         case CTL_ERROR_INJECT: {
2966                 struct ctl_error_desc *err_desc, *new_err_desc;
2967
2968                 err_desc = (struct ctl_error_desc *)addr;
2969
2970                 new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2971                                       M_WAITOK | M_ZERO);
2972                 bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2973
2974                 mtx_lock(&softc->ctl_lock);
2975                 lun = softc->ctl_luns[err_desc->lun_id];
2976                 if (lun == NULL) {
2977                         mtx_unlock(&softc->ctl_lock);
2978                         free(new_err_desc, M_CTL);
2979                         printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2980                                __func__, (uintmax_t)err_desc->lun_id);
2981                         retval = EINVAL;
2982                         break;
2983                 }
2984                 mtx_lock(&lun->lun_lock);
2985                 mtx_unlock(&softc->ctl_lock);
2986
2987                 /*
2988                  * We could do some checking here to verify the validity
2989                  * of the request, but given the complexity of error
2990                  * injection requests, the checking logic would be fairly
2991                  * complex.
2992                  *
2993                  * For now, if the request is invalid, it just won't get
2994                  * executed and might get deleted.
2995                  */
2996                 STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2997
2998                 /*
2999                  * XXX KDM check to make sure the serial number is unique,
3000                  * in case we somehow manage to wrap.  That shouldn't
3001                  * happen for a very long time, but it's the right thing to
3002                  * do.
3003                  */
3004                 new_err_desc->serial = lun->error_serial;
3005                 err_desc->serial = lun->error_serial;
3006                 lun->error_serial++;
3007
3008                 mtx_unlock(&lun->lun_lock);
3009                 break;
3010         }
3011         case CTL_ERROR_INJECT_DELETE: {
3012                 struct ctl_error_desc *delete_desc, *desc, *desc2;
3013                 int delete_done;
3014
3015                 delete_desc = (struct ctl_error_desc *)addr;
3016                 delete_done = 0;
3017
3018                 mtx_lock(&softc->ctl_lock);
3019                 lun = softc->ctl_luns[delete_desc->lun_id];
3020                 if (lun == NULL) {
3021                         mtx_unlock(&softc->ctl_lock);
3022                         printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
3023                                __func__, (uintmax_t)delete_desc->lun_id);
3024                         retval = EINVAL;
3025                         break;
3026                 }
3027                 mtx_lock(&lun->lun_lock);
3028                 mtx_unlock(&softc->ctl_lock);
3029                 STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
3030                         if (desc->serial != delete_desc->serial)
3031                                 continue;
3032
3033                         STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
3034                                       links);
3035                         free(desc, M_CTL);
3036                         delete_done = 1;
3037                 }
3038                 mtx_unlock(&lun->lun_lock);
3039                 if (delete_done == 0) {
3040                         printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
3041                                "error serial %ju on LUN %u\n", __func__,
3042                                delete_desc->serial, delete_desc->lun_id);
3043                         retval = EINVAL;
3044                         break;
3045                 }
3046                 break;
3047         }
3048         case CTL_DUMP_STRUCTS: {
3049                 int i, j, k;
3050                 struct ctl_port *port;
3051                 struct ctl_frontend *fe;
3052
3053                 mtx_lock(&softc->ctl_lock);
3054                 printf("CTL Persistent Reservation information start:\n");
3055                 for (i = 0; i < CTL_MAX_LUNS; i++) {
3056                         lun = softc->ctl_luns[i];
3057
3058                         if ((lun == NULL)
3059                          || ((lun->flags & CTL_LUN_DISABLED) != 0))
3060                                 continue;
3061
3062                         for (j = 0; j < CTL_MAX_PORTS; j++) {
3063                                 if (lun->pr_keys[j] == NULL)
3064                                         continue;
3065                                 for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
3066                                         if (lun->pr_keys[j][k] == 0)
3067                                                 continue;
3068                                         printf("  LUN %d port %d iid %d key "
3069                                                "%#jx\n", i, j, k,
3070                                                (uintmax_t)lun->pr_keys[j][k]);
3071                                 }
3072                         }
3073                 }
3074                 printf("CTL Persistent Reservation information end\n");
3075                 printf("CTL Ports:\n");
3076                 STAILQ_FOREACH(port, &softc->port_list, links) {
3077                         printf("  Port %d '%s' Frontend '%s' Type %u pp %d vp %d WWNN "
3078                                "%#jx WWPN %#jx\n", port->targ_port, port->port_name,
3079                                port->frontend->name, port->port_type,
3080                                port->physical_port, port->virtual_port,
3081                                (uintmax_t)port->wwnn, (uintmax_t)port->wwpn);
3082                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
3083                                 if (port->wwpn_iid[j].in_use == 0 &&
3084                                     port->wwpn_iid[j].wwpn == 0 &&
3085                                     port->wwpn_iid[j].name == NULL)
3086                                         continue;
3087
3088                                 printf("    iid %u use %d WWPN %#jx '%s'\n",
3089                                     j, port->wwpn_iid[j].in_use,
3090                                     (uintmax_t)port->wwpn_iid[j].wwpn,
3091                                     port->wwpn_iid[j].name);
3092                         }
3093                 }
3094                 printf("CTL Port information end\n");
3095                 mtx_unlock(&softc->ctl_lock);
3096                 /*
3097                  * XXX KDM calling this without a lock.  We'd likely want
3098                  * to drop the lock before calling the frontend's dump
3099                  * routine anyway.
3100                  */
3101                 printf("CTL Frontends:\n");
3102                 STAILQ_FOREACH(fe, &softc->fe_list, links) {
3103                         printf("  Frontend '%s'\n", fe->name);
3104                         if (fe->fe_dump != NULL)
3105                                 fe->fe_dump();
3106                 }
3107                 printf("CTL Frontend information end\n");
3108                 break;
3109         }
3110         case CTL_LUN_REQ: {
3111                 struct ctl_lun_req *lun_req;
3112                 struct ctl_backend_driver *backend;
3113
3114                 lun_req = (struct ctl_lun_req *)addr;
3115
3116                 backend = ctl_backend_find(lun_req->backend);
3117                 if (backend == NULL) {
3118                         lun_req->status = CTL_LUN_ERROR;
3119                         snprintf(lun_req->error_str,
3120                                  sizeof(lun_req->error_str),
3121                                  "Backend \"%s\" not found.",
3122                                  lun_req->backend);
3123                         break;
3124                 }
3125                 if (lun_req->num_be_args > 0) {
3126                         lun_req->kern_be_args = ctl_copyin_args(
3127                                 lun_req->num_be_args,
3128                                 lun_req->be_args,
3129                                 lun_req->error_str,
3130                                 sizeof(lun_req->error_str));
3131                         if (lun_req->kern_be_args == NULL) {
3132                                 lun_req->status = CTL_LUN_ERROR;
3133                                 break;
3134                         }
3135                 }
3136
3137                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3138
3139                 if (lun_req->num_be_args > 0) {
3140                         ctl_copyout_args(lun_req->num_be_args,
3141                                       lun_req->kern_be_args);
3142                         ctl_free_args(lun_req->num_be_args,
3143                                       lun_req->kern_be_args);
3144                 }
3145                 break;
3146         }
3147         case CTL_LUN_LIST: {
3148                 struct sbuf *sb;
3149                 struct ctl_lun_list *list;
3150                 struct ctl_option *opt;
3151
3152                 list = (struct ctl_lun_list *)addr;
3153
3154                 /*
3155                  * Allocate a fixed length sbuf here, based on the length
3156                  * of the user's buffer.  We could allocate an auto-extending
3157                  * buffer, and then tell the user how much larger our
3158                  * amount of data is than his buffer, but that presents
3159                  * some problems:
3160                  *
3161                  * 1.  The sbuf(9) routines use a blocking malloc, and so
3162                  *     we can't hold a lock while calling them with an
3163                  *     auto-extending buffer.
3164                  *
3165                  * 2.  There is not currently a LUN reference counting
3166                  *     mechanism, outside of outstanding transactions on
3167                  *     the LUN's OOA queue.  So a LUN could go away on us
3168                  *     while we're getting the LUN number, backend-specific
3169                  *     information, etc.  Thus, given the way things
3170                  *     currently work, we need to hold the CTL lock while
3171                  *     grabbing LUN information.
3172                  *
3173                  * So, from the user's standpoint, the best thing to do is
3174                  * allocate what he thinks is a reasonable buffer length,
3175                  * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
3176                  * double the buffer length and try again.  (And repeat
3177                  * that until he succeeds.)
3178                  */
3179                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3180                 if (sb == NULL) {
3181                         list->status = CTL_LUN_LIST_ERROR;
3182                         snprintf(list->error_str, sizeof(list->error_str),
3183                                  "Unable to allocate %d bytes for LUN list",
3184                                  list->alloc_len);
3185                         break;
3186                 }
3187
3188                 sbuf_printf(sb, "<ctllunlist>\n");
3189
3190                 mtx_lock(&softc->ctl_lock);
3191                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
3192                         mtx_lock(&lun->lun_lock);
3193                         retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3194                                              (uintmax_t)lun->lun);
3195
3196                         /*
3197                          * Bail out as soon as we see that we've overfilled
3198                          * the buffer.
3199                          */
3200                         if (retval != 0)
3201                                 break;
3202
3203                         retval = sbuf_printf(sb, "\t<backend_type>%s"
3204                                              "</backend_type>\n",
3205                                              (lun->backend == NULL) ?  "none" :
3206                                              lun->backend->name);
3207
3208                         if (retval != 0)
3209                                 break;
3210
3211                         retval = sbuf_printf(sb, "\t<lun_type>%d</lun_type>\n",
3212                                              lun->be_lun->lun_type);
3213
3214                         if (retval != 0)
3215                                 break;
3216
3217                         if (lun->backend == NULL) {
3218                                 retval = sbuf_printf(sb, "</lun>\n");
3219                                 if (retval != 0)
3220                                         break;
3221                                 continue;
3222                         }
3223
3224                         retval = sbuf_printf(sb, "\t<size>%ju</size>\n",
3225                                              (lun->be_lun->maxlba > 0) ?
3226                                              lun->be_lun->maxlba + 1 : 0);
3227
3228                         if (retval != 0)
3229                                 break;
3230
3231                         retval = sbuf_printf(sb, "\t<blocksize>%u</blocksize>\n",
3232                                              lun->be_lun->blocksize);
3233
3234                         if (retval != 0)
3235                                 break;
3236
3237                         retval = sbuf_printf(sb, "\t<serial_number>");
3238
3239                         if (retval != 0)
3240                                 break;
3241
3242                         retval = ctl_sbuf_printf_esc(sb,
3243                             lun->be_lun->serial_num,
3244                             sizeof(lun->be_lun->serial_num));
3245
3246                         if (retval != 0)
3247                                 break;
3248
3249                         retval = sbuf_printf(sb, "</serial_number>\n");
3250
3251                         if (retval != 0)
3252                                 break;
3253
3254                         retval = sbuf_printf(sb, "\t<device_id>");
3255
3256                         if (retval != 0)
3257                                 break;
3258
3259                         retval = ctl_sbuf_printf_esc(sb,
3260                             lun->be_lun->device_id,
3261                             sizeof(lun->be_lun->device_id));
3262
3263                         if (retval != 0)
3264                                 break;
3265
3266                         retval = sbuf_printf(sb, "</device_id>\n");
3267
3268                         if (retval != 0)
3269                                 break;
3270
3271                         if (lun->backend->lun_info != NULL) {
3272                                 retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3273                                 if (retval != 0)
3274                                         break;
3275                         }
3276                         STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3277                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3278                                     opt->name, opt->value, opt->name);
3279                                 if (retval != 0)
3280                                         break;
3281                         }
3282
3283                         retval = sbuf_printf(sb, "</lun>\n");
3284
3285                         if (retval != 0)
3286                                 break;
3287                         mtx_unlock(&lun->lun_lock);
3288                 }
3289                 if (lun != NULL)
3290                         mtx_unlock(&lun->lun_lock);
3291                 mtx_unlock(&softc->ctl_lock);
3292
3293                 if ((retval != 0)
3294                  || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3295                         retval = 0;
3296                         sbuf_delete(sb);
3297                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3298                         snprintf(list->error_str, sizeof(list->error_str),
3299                                  "Out of space, %d bytes is too small",
3300                                  list->alloc_len);
3301                         break;
3302                 }
3303
3304                 sbuf_finish(sb);
3305
3306                 retval = copyout(sbuf_data(sb), list->lun_xml,
3307                                  sbuf_len(sb) + 1);
3308
3309                 list->fill_len = sbuf_len(sb) + 1;
3310                 list->status = CTL_LUN_LIST_OK;
3311                 sbuf_delete(sb);
3312                 break;
3313         }
3314         case CTL_ISCSI: {
3315                 struct ctl_iscsi *ci;
3316                 struct ctl_frontend *fe;
3317
3318                 ci = (struct ctl_iscsi *)addr;
3319
3320                 fe = ctl_frontend_find("iscsi");
3321                 if (fe == NULL) {
3322                         ci->status = CTL_ISCSI_ERROR;
3323                         snprintf(ci->error_str, sizeof(ci->error_str),
3324                             "Frontend \"iscsi\" not found.");
3325                         break;
3326                 }
3327
3328                 retval = fe->ioctl(dev, cmd, addr, flag, td);
3329                 break;
3330         }
3331         case CTL_PORT_REQ: {
3332                 struct ctl_req *req;
3333                 struct ctl_frontend *fe;
3334
3335                 req = (struct ctl_req *)addr;
3336
3337                 fe = ctl_frontend_find(req->driver);
3338                 if (fe == NULL) {
3339                         req->status = CTL_LUN_ERROR;
3340                         snprintf(req->error_str, sizeof(req->error_str),
3341                             "Frontend \"%s\" not found.", req->driver);
3342                         break;
3343                 }
3344                 if (req->num_args > 0) {
3345                         req->kern_args = ctl_copyin_args(req->num_args,
3346                             req->args, req->error_str, sizeof(req->error_str));
3347                         if (req->kern_args == NULL) {
3348                                 req->status = CTL_LUN_ERROR;
3349                                 break;
3350                         }
3351                 }
3352
3353                 if (fe->ioctl)
3354                         retval = fe->ioctl(dev, cmd, addr, flag, td);
3355                 else
3356                         retval = ENODEV;
3357
3358                 if (req->num_args > 0) {
3359                         ctl_copyout_args(req->num_args, req->kern_args);
3360                         ctl_free_args(req->num_args, req->kern_args);
3361                 }
3362                 break;
3363         }
3364         case CTL_PORT_LIST: {
3365                 struct sbuf *sb;
3366                 struct ctl_port *port;
3367                 struct ctl_lun_list *list;
3368                 struct ctl_option *opt;
3369                 int j;
3370                 uint32_t plun;
3371
3372                 list = (struct ctl_lun_list *)addr;
3373
3374                 sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3375                 if (sb == NULL) {
3376                         list->status = CTL_LUN_LIST_ERROR;
3377                         snprintf(list->error_str, sizeof(list->error_str),
3378                                  "Unable to allocate %d bytes for LUN list",
3379                                  list->alloc_len);
3380                         break;
3381                 }
3382
3383                 sbuf_printf(sb, "<ctlportlist>\n");
3384
3385                 mtx_lock(&softc->ctl_lock);
3386                 STAILQ_FOREACH(port, &softc->port_list, links) {
3387                         retval = sbuf_printf(sb, "<targ_port id=\"%ju\">\n",
3388                                              (uintmax_t)port->targ_port);
3389
3390                         /*
3391                          * Bail out as soon as we see that we've overfilled
3392                          * the buffer.
3393                          */
3394                         if (retval != 0)
3395                                 break;
3396
3397                         retval = sbuf_printf(sb, "\t<frontend_type>%s"
3398                             "</frontend_type>\n", port->frontend->name);
3399                         if (retval != 0)
3400                                 break;
3401
3402                         retval = sbuf_printf(sb, "\t<port_type>%d</port_type>\n",
3403                                              port->port_type);
3404                         if (retval != 0)
3405                                 break;
3406
3407                         retval = sbuf_printf(sb, "\t<online>%s</online>\n",
3408                             (port->status & CTL_PORT_STATUS_ONLINE) ? "YES" : "NO");
3409                         if (retval != 0)
3410                                 break;
3411
3412                         retval = sbuf_printf(sb, "\t<port_name>%s</port_name>\n",
3413                             port->port_name);
3414                         if (retval != 0)
3415                                 break;
3416
3417                         retval = sbuf_printf(sb, "\t<physical_port>%d</physical_port>\n",
3418                             port->physical_port);
3419                         if (retval != 0)
3420                                 break;
3421
3422                         retval = sbuf_printf(sb, "\t<virtual_port>%d</virtual_port>\n",
3423                             port->virtual_port);
3424                         if (retval != 0)
3425                                 break;
3426
3427                         if (port->target_devid != NULL) {
3428                                 sbuf_printf(sb, "\t<target>");
3429                                 ctl_id_sbuf(port->target_devid, sb);
3430                                 sbuf_printf(sb, "</target>\n");
3431                         }
3432
3433                         if (port->port_devid != NULL) {
3434                                 sbuf_printf(sb, "\t<port>");
3435                                 ctl_id_sbuf(port->port_devid, sb);
3436                                 sbuf_printf(sb, "</port>\n");
3437                         }
3438
3439                         if (port->port_info != NULL) {
3440                                 retval = port->port_info(port->onoff_arg, sb);
3441                                 if (retval != 0)
3442                                         break;
3443                         }
3444                         STAILQ_FOREACH(opt, &port->options, links) {
3445                                 retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3446                                     opt->name, opt->value, opt->name);
3447                                 if (retval != 0)
3448                                         break;
3449                         }
3450
3451                         if (port->lun_map != NULL) {
3452                                 sbuf_printf(sb, "\t<lun_map>on</lun_map>\n");
3453                                 for (j = 0; j < CTL_MAX_LUNS; j++) {
3454                                         plun = ctl_lun_map_from_port(port, j);
3455                                         if (plun >= CTL_MAX_LUNS)
3456                                                 continue;
3457                                         sbuf_printf(sb,
3458                                             "\t<lun id=\"%u\">%u</lun>\n",
3459                                             j, plun);
3460                                 }
3461                         }
3462
3463                         for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
3464                                 if (port->wwpn_iid[j].in_use == 0 ||
3465                                     (port->wwpn_iid[j].wwpn == 0 &&
3466                                      port->wwpn_iid[j].name == NULL))
3467                                         continue;
3468
3469                                 if (port->wwpn_iid[j].name != NULL)
3470                                         retval = sbuf_printf(sb,
3471                                             "\t<initiator id=\"%u\">%s</initiator>\n",
3472                                             j, port->wwpn_iid[j].name);
3473                                 else
3474                                         retval = sbuf_printf(sb,
3475                                             "\t<initiator id=\"%u\">naa.%08jx</initiator>\n",
3476                                             j, port->wwpn_iid[j].wwpn);
3477                                 if (retval != 0)
3478                                         break;
3479                         }
3480                         if (retval != 0)
3481                                 break;
3482
3483                         retval = sbuf_printf(sb, "</targ_port>\n");
3484                         if (retval != 0)
3485                                 break;
3486                 }
3487                 mtx_unlock(&softc->ctl_lock);
3488
3489                 if ((retval != 0)
3490                  || ((retval = sbuf_printf(sb, "</ctlportlist>\n")) != 0)) {
3491                         retval = 0;
3492                         sbuf_delete(sb);
3493                         list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3494                         snprintf(list->error_str, sizeof(list->error_str),
3495                                  "Out of space, %d bytes is too small",
3496                                  list->alloc_len);
3497                         break;
3498                 }
3499
3500                 sbuf_finish(sb);
3501
3502                 retval = copyout(sbuf_data(sb), list->lun_xml,
3503                                  sbuf_len(sb) + 1);
3504
3505                 list->fill_len = sbuf_len(sb) + 1;
3506                 list->status = CTL_LUN_LIST_OK;
3507                 sbuf_delete(sb);
3508                 break;
3509         }
3510         case CTL_LUN_MAP: {
3511                 struct ctl_lun_map *lm  = (struct ctl_lun_map *)addr;
3512                 struct ctl_port *port;
3513
3514                 mtx_lock(&softc->ctl_lock);
3515                 if (lm->port < softc->port_min ||
3516                     lm->port >= softc->port_max ||
3517                     (port = softc->ctl_ports[lm->port]) == NULL) {
3518                         mtx_unlock(&softc->ctl_lock);
3519                         return (ENXIO);
3520                 }
3521                 if (port->status & CTL_PORT_STATUS_ONLINE) {
3522                         STAILQ_FOREACH(lun, &softc->lun_list, links) {
3523                                 if (ctl_lun_map_to_port(port, lun->lun) >=
3524                                     CTL_MAX_LUNS)
3525                                         continue;
3526                                 mtx_lock(&lun->lun_lock);
3527                                 ctl_est_ua_port(lun, lm->port, -1,
3528                                     CTL_UA_LUN_CHANGE);
3529                                 mtx_unlock(&lun->lun_lock);
3530                         }
3531                 }
3532                 mtx_unlock(&softc->ctl_lock); // XXX: port_enable sleeps
3533                 if (lm->plun < CTL_MAX_LUNS) {
3534                         if (lm->lun == UINT32_MAX)
3535                                 retval = ctl_lun_map_unset(port, lm->plun);
3536                         else if (lm->lun < CTL_MAX_LUNS &&
3537                             softc->ctl_luns[lm->lun] != NULL)
3538                                 retval = ctl_lun_map_set(port, lm->plun, lm->lun);
3539                         else
3540                                 return (ENXIO);
3541                 } else if (lm->plun == UINT32_MAX) {
3542                         if (lm->lun == UINT32_MAX)
3543                                 retval = ctl_lun_map_deinit(port);
3544                         else
3545                                 retval = ctl_lun_map_init(port);
3546                 } else
3547                         return (ENXIO);
3548                 if (port->status & CTL_PORT_STATUS_ONLINE)
3549                         ctl_isc_announce_port(port);
3550                 break;
3551         }
3552         default: {
3553                 /* XXX KDM should we fix this? */
3554 #if 0
3555                 struct ctl_backend_driver *backend;
3556                 unsigned int type;
3557                 int found;
3558
3559                 found = 0;
3560
3561                 /*
3562                  * We encode the backend type as the ioctl type for backend
3563                  * ioctls.  So parse it out here, and then search for a
3564                  * backend of this type.
3565                  */
3566                 type = _IOC_TYPE(cmd);
3567
3568                 STAILQ_FOREACH(backend, &softc->be_list, links) {
3569                         if (backend->type == type) {
3570                                 found = 1;
3571                                 break;
3572                         }
3573                 }
3574                 if (found == 0) {
3575                         printf("ctl: unknown ioctl command %#lx or backend "
3576                                "%d\n", cmd, type);
3577                         retval = EINVAL;
3578                         break;
3579                 }
3580                 retval = backend->ioctl(dev, cmd, addr, flag, td);
3581 #endif
3582                 retval = ENOTTY;
3583                 break;
3584         }
3585         }
3586         return (retval);
3587 }
3588
3589 uint32_t
3590 ctl_get_initindex(struct ctl_nexus *nexus)
3591 {
3592         return (nexus->initid + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3593 }
3594
3595 int
3596 ctl_lun_map_init(struct ctl_port *port)
3597 {
3598         struct ctl_softc *softc = port->ctl_softc;
3599         struct ctl_lun *lun;
3600         uint32_t i;
3601
3602         if (port->lun_map == NULL)
3603                 port->lun_map = malloc(sizeof(uint32_t) * CTL_MAX_LUNS,
3604                     M_CTL, M_NOWAIT);
3605         if (port->lun_map == NULL)
3606                 return (ENOMEM);
3607         for (i = 0; i < CTL_MAX_LUNS; i++)
3608                 port->lun_map[i] = UINT32_MAX;
3609         if (port->status & CTL_PORT_STATUS_ONLINE) {
3610                 if (port->lun_disable != NULL) {
3611                         STAILQ_FOREACH(lun, &softc->lun_list, links)
3612                                 port->lun_disable(port->targ_lun_arg, lun->lun);
3613                 }
3614                 ctl_isc_announce_port(port);
3615         }
3616         return (0);
3617 }
3618
3619 int
3620 ctl_lun_map_deinit(struct ctl_port *port)
3621 {
3622         struct ctl_softc *softc = port->ctl_softc;
3623         struct ctl_lun *lun;
3624
3625         if (port->lun_map == NULL)
3626                 return (0);
3627         free(port->lun_map, M_CTL);
3628         port->lun_map = NULL;
3629         if (port->status & CTL_PORT_STATUS_ONLINE) {
3630                 if (port->lun_enable != NULL) {
3631                         STAILQ_FOREACH(lun, &softc->lun_list, links)
3632                                 port->lun_enable(port->targ_lun_arg, lun->lun);
3633                 }
3634                 ctl_isc_announce_port(port);
3635         }
3636         return (0);
3637 }
3638
3639 int
3640 ctl_lun_map_set(struct ctl_port *port, uint32_t plun, uint32_t glun)
3641 {
3642         int status;
3643         uint32_t old;
3644
3645         if (port->lun_map == NULL) {
3646                 status = ctl_lun_map_init(port);
3647                 if (status != 0)
3648                         return (status);
3649         }
3650         old = port->lun_map[plun];
3651         port->lun_map[plun] = glun;
3652         if ((port->status & CTL_PORT_STATUS_ONLINE) && old >= CTL_MAX_LUNS) {
3653                 if (port->lun_enable != NULL)
3654                         port->lun_enable(port->targ_lun_arg, plun);
3655                 ctl_isc_announce_port(port);
3656         }
3657         return (0);
3658 }
3659
3660 int
3661 ctl_lun_map_unset(struct ctl_port *port, uint32_t plun)
3662 {
3663         uint32_t old;
3664
3665         if (port->lun_map == NULL)
3666                 return (0);
3667         old = port->lun_map[plun];
3668         port->lun_map[plun] = UINT32_MAX;
3669         if ((port->status & CTL_PORT_STATUS_ONLINE) && old < CTL_MAX_LUNS) {
3670                 if (port->lun_disable != NULL)
3671                         port->lun_disable(port->targ_lun_arg, plun);
3672                 ctl_isc_announce_port(port);
3673         }
3674         return (0);
3675 }
3676
3677 uint32_t
3678 ctl_lun_map_from_port(struct ctl_port *port, uint32_t lun_id)
3679 {
3680
3681         if (port == NULL)
3682                 return (UINT32_MAX);
3683         if (port->lun_map == NULL || lun_id >= CTL_MAX_LUNS)
3684                 return (lun_id);
3685         return (port->lun_map[lun_id]);
3686 }
3687
3688 uint32_t
3689 ctl_lun_map_to_port(struct ctl_port *port, uint32_t lun_id)
3690 {
3691         uint32_t i;
3692
3693         if (port == NULL)
3694                 return (UINT32_MAX);
3695         if (port->lun_map == NULL)
3696                 return (lun_id);
3697         for (i = 0; i < CTL_MAX_LUNS; i++) {
3698                 if (port->lun_map[i] == lun_id)
3699                         return (i);
3700         }
3701         return (UINT32_MAX);
3702 }
3703
3704 static struct ctl_port *
3705 ctl_io_port(struct ctl_io_hdr *io_hdr)
3706 {
3707
3708         return (control_softc->ctl_ports[io_hdr->nexus.targ_port]);
3709 }
3710
3711 int
3712 ctl_ffz(uint32_t *mask, uint32_t first, uint32_t last)
3713 {
3714         int i;
3715
3716         for (i = first; i < last; i++) {
3717                 if ((mask[i / 32] & (1 << (i % 32))) == 0)
3718                         return (i);
3719         }
3720         return (-1);
3721 }
3722
3723 int
3724 ctl_set_mask(uint32_t *mask, uint32_t bit)
3725 {
3726         uint32_t chunk, piece;
3727
3728         chunk = bit >> 5;
3729         piece = bit % (sizeof(uint32_t) * 8);
3730
3731         if ((mask[chunk] & (1 << piece)) != 0)
3732                 return (-1);
3733         else
3734                 mask[chunk] |= (1 << piece);
3735
3736         return (0);
3737 }
3738
3739 int
3740 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3741 {
3742         uint32_t chunk, piece;
3743
3744         chunk = bit >> 5;
3745         piece = bit % (sizeof(uint32_t) * 8);
3746
3747         if ((mask[chunk] & (1 << piece)) == 0)
3748                 return (-1);
3749         else
3750                 mask[chunk] &= ~(1 << piece);
3751
3752         return (0);
3753 }
3754
3755 int
3756 ctl_is_set(uint32_t *mask, uint32_t bit)
3757 {
3758         uint32_t chunk, piece;
3759
3760         chunk = bit >> 5;
3761         piece = bit % (sizeof(uint32_t) * 8);
3762
3763         if ((mask[chunk] & (1 << piece)) == 0)
3764                 return (0);
3765         else
3766                 return (1);
3767 }
3768
3769 static uint64_t
3770 ctl_get_prkey(struct ctl_lun *lun, uint32_t residx)
3771 {
3772         uint64_t *t;
3773
3774         t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3775         if (t == NULL)
3776                 return (0);
3777         return (t[residx % CTL_MAX_INIT_PER_PORT]);
3778 }
3779
3780 static void
3781 ctl_clr_prkey(struct ctl_lun *lun, uint32_t residx)
3782 {
3783         uint64_t *t;
3784
3785         t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3786         if (t == NULL)
3787                 return;
3788         t[residx % CTL_MAX_INIT_PER_PORT] = 0;
3789 }
3790
3791 static void
3792 ctl_alloc_prkey(struct ctl_lun *lun, uint32_t residx)
3793 {
3794         uint64_t *p;
3795         u_int i;
3796
3797         i = residx/CTL_MAX_INIT_PER_PORT;
3798         if (lun->pr_keys[i] != NULL)
3799                 return;
3800         mtx_unlock(&lun->lun_lock);
3801         p = malloc(sizeof(uint64_t) * CTL_MAX_INIT_PER_PORT, M_CTL,
3802             M_WAITOK | M_ZERO);
3803         mtx_lock(&lun->lun_lock);
3804         if (lun->pr_keys[i] == NULL)
3805                 lun->pr_keys[i] = p;
3806         else
3807                 free(p, M_CTL);
3808 }
3809
3810 static void
3811 ctl_set_prkey(struct ctl_lun *lun, uint32_t residx, uint64_t key)
3812 {
3813         uint64_t *t;
3814
3815         t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3816         KASSERT(t != NULL, ("prkey %d is not allocated", residx));
3817         t[residx % CTL_MAX_INIT_PER_PORT] = key;
3818 }
3819
3820 /*
3821  * ctl_softc, pool_name, total_ctl_io are passed in.
3822  * npool is passed out.
3823  */
3824 int
3825 ctl_pool_create(struct ctl_softc *ctl_softc, const char *pool_name,
3826                 uint32_t total_ctl_io, void **npool)
3827 {
3828 #ifdef IO_POOLS
3829         struct ctl_io_pool *pool;
3830
3831         pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3832                                             M_NOWAIT | M_ZERO);
3833         if (pool == NULL)
3834                 return (ENOMEM);
3835
3836         snprintf(pool->name, sizeof(pool->name), "CTL IO %s", pool_name);
3837         pool->ctl_softc = ctl_softc;
3838         pool->zone = uma_zsecond_create(pool->name, NULL,
3839             NULL, NULL, NULL, ctl_softc->io_zone);
3840         /* uma_prealloc(pool->zone, total_ctl_io); */
3841
3842         *npool = pool;
3843 #else
3844         *npool = ctl_softc->io_zone;
3845 #endif
3846         return (0);
3847 }
3848
3849 void
3850 ctl_pool_free(struct ctl_io_pool *pool)
3851 {
3852
3853         if (pool == NULL)
3854                 return;
3855
3856 #ifdef IO_POOLS
3857         uma_zdestroy(pool->zone);
3858         free(pool, M_CTL);
3859 #endif
3860 }
3861
3862 union ctl_io *
3863 ctl_alloc_io(void *pool_ref)
3864 {
3865         union ctl_io *io;
3866 #ifdef IO_POOLS
3867         struct ctl_io_pool *pool = (struct ctl_io_pool *)pool_ref;
3868
3869         io = uma_zalloc(pool->zone, M_WAITOK);
3870 #else
3871         io = uma_zalloc((uma_zone_t)pool_ref, M_WAITOK);
3872 #endif
3873         if (io != NULL)
3874                 io->io_hdr.pool = pool_ref;
3875         return (io);
3876 }
3877
3878 union ctl_io *
3879 ctl_alloc_io_nowait(void *pool_ref)
3880 {
3881         union ctl_io *io;
3882 #ifdef IO_POOLS
3883         struct ctl_io_pool *pool = (struct ctl_io_pool *)pool_ref;
3884
3885         io = uma_zalloc(pool->zone, M_NOWAIT);
3886 #else
3887         io = uma_zalloc((uma_zone_t)pool_ref, M_NOWAIT);
3888 #endif
3889         if (io != NULL)
3890                 io->io_hdr.pool = pool_ref;
3891         return (io);
3892 }
3893
3894 void
3895 ctl_free_io(union ctl_io *io)
3896 {
3897 #ifdef IO_POOLS
3898         struct ctl_io_pool *pool;
3899 #endif
3900
3901         if (io == NULL)
3902                 return;
3903
3904 #ifdef IO_POOLS
3905         pool = (struct ctl_io_pool *)io->io_hdr.pool;
3906         uma_zfree(pool->zone, io);
3907 #else
3908         uma_zfree((uma_zone_t)io->io_hdr.pool, io);
3909 #endif
3910 }
3911
3912 void
3913 ctl_zero_io(union ctl_io *io)
3914 {
3915         void *pool_ref;
3916
3917         if (io == NULL)
3918                 return;
3919
3920         /*
3921          * May need to preserve linked list pointers at some point too.
3922          */
3923         pool_ref = io->io_hdr.pool;
3924         memset(io, 0, sizeof(*io));
3925         io->io_hdr.pool = pool_ref;
3926 }
3927
3928 /*
3929  * This routine is currently used for internal copies of ctl_ios that need
3930  * to persist for some reason after we've already returned status to the
3931  * FETD.  (Thus the flag set.)
3932  *
3933  * XXX XXX
3934  * Note that this makes a blind copy of all fields in the ctl_io, except
3935  * for the pool reference.  This includes any memory that has been
3936  * allocated!  That memory will no longer be valid after done has been
3937  * called, so this would be VERY DANGEROUS for command that actually does
3938  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3939  * start and stop commands, which don't transfer any data, so this is not a
3940  * problem.  If it is used for anything else, the caller would also need to
3941  * allocate data buffer space and this routine would need to be modified to
3942  * copy the data buffer(s) as well.
3943  */
3944 void
3945 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3946 {
3947         void *pool_ref;
3948
3949         if ((src == NULL)
3950          || (dest == NULL))
3951                 return;
3952
3953         /*
3954          * May need to preserve linked list pointers at some point too.
3955          */
3956         pool_ref = dest->io_hdr.pool;
3957
3958         memcpy(dest, src, MIN(sizeof(*src), sizeof(*dest)));
3959
3960         dest->io_hdr.pool = pool_ref;
3961         /*
3962          * We need to know that this is an internal copy, and doesn't need
3963          * to get passed back to the FETD that allocated it.
3964          */
3965         dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3966 }
3967
3968 int
3969 ctl_expand_number(const char *buf, uint64_t *num)
3970 {
3971         char *endptr;
3972         uint64_t number;
3973         unsigned shift;
3974
3975         number = strtoq(buf, &endptr, 0);
3976
3977         switch (tolower((unsigned char)*endptr)) {
3978         case 'e':
3979                 shift = 60;
3980                 break;
3981         case 'p':
3982                 shift = 50;
3983                 break;
3984         case 't':
3985                 shift = 40;
3986                 break;
3987         case 'g':
3988                 shift = 30;
3989                 break;
3990         case 'm':
3991                 shift = 20;
3992                 break;
3993         case 'k':
3994                 shift = 10;
3995                 break;
3996         case 'b':
3997         case '\0': /* No unit. */
3998                 *num = number;
3999                 return (0);
4000         default:
4001                 /* Unrecognized unit. */
4002                 return (-1);
4003         }
4004
4005         if ((number << shift) >> shift != number) {
4006                 /* Overflow */
4007                 return (-1);
4008         }
4009         *num = number << shift;
4010         return (0);
4011 }
4012
4013
4014 /*
4015  * This routine could be used in the future to load default and/or saved
4016  * mode page parameters for a particuar lun.
4017  */
4018 static int
4019 ctl_init_page_index(struct ctl_lun *lun)
4020 {
4021         int i;
4022         struct ctl_page_index *page_index;
4023         const char *value;
4024         uint64_t ival;
4025
4026         memcpy(&lun->mode_pages.index, page_index_template,
4027                sizeof(page_index_template));
4028
4029         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4030
4031                 page_index = &lun->mode_pages.index[i];
4032                 /*
4033                  * If this is a disk-only mode page, there's no point in
4034                  * setting it up.  For some pages, we have to have some
4035                  * basic information about the disk in order to calculate the
4036                  * mode page data.
4037                  */
4038                 if ((lun->be_lun->lun_type != T_DIRECT)
4039                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
4040                         continue;
4041
4042                 switch (page_index->page_code & SMPH_PC_MASK) {
4043                 case SMS_RW_ERROR_RECOVERY_PAGE: {
4044                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4045                                 panic("subpage is incorrect!");
4046                         memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_CURRENT],
4047                                &rw_er_page_default,
4048                                sizeof(rw_er_page_default));
4049                         memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_CHANGEABLE],
4050                                &rw_er_page_changeable,
4051                                sizeof(rw_er_page_changeable));
4052                         memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_DEFAULT],
4053                                &rw_er_page_default,
4054                                sizeof(rw_er_page_default));
4055                         memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_SAVED],
4056                                &rw_er_page_default,
4057                                sizeof(rw_er_page_default));
4058                         page_index->page_data =
4059                                 (uint8_t *)lun->mode_pages.rw_er_page;
4060                         break;
4061                 }
4062                 case SMS_FORMAT_DEVICE_PAGE: {
4063                         struct scsi_format_page *format_page;
4064
4065                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4066                                 panic("subpage is incorrect!");
4067
4068                         /*
4069                          * Sectors per track are set above.  Bytes per
4070                          * sector need to be set here on a per-LUN basis.
4071                          */
4072                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
4073                                &format_page_default,
4074                                sizeof(format_page_default));
4075                         memcpy(&lun->mode_pages.format_page[
4076                                CTL_PAGE_CHANGEABLE], &format_page_changeable,
4077                                sizeof(format_page_changeable));
4078                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
4079                                &format_page_default,
4080                                sizeof(format_page_default));
4081                         memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
4082                                &format_page_default,
4083                                sizeof(format_page_default));
4084
4085                         format_page = &lun->mode_pages.format_page[
4086                                 CTL_PAGE_CURRENT];
4087                         scsi_ulto2b(lun->be_lun->blocksize,
4088                                     format_page->bytes_per_sector);
4089
4090                         format_page = &lun->mode_pages.format_page[
4091                                 CTL_PAGE_DEFAULT];
4092                         scsi_ulto2b(lun->be_lun->blocksize,
4093                                     format_page->bytes_per_sector);
4094
4095                         format_page = &lun->mode_pages.format_page[
4096                                 CTL_PAGE_SAVED];
4097                         scsi_ulto2b(lun->be_lun->blocksize,
4098                                     format_page->bytes_per_sector);
4099
4100                         page_index->page_data =
4101                                 (uint8_t *)lun->mode_pages.format_page;
4102                         break;
4103                 }
4104                 case SMS_RIGID_DISK_PAGE: {
4105                         struct scsi_rigid_disk_page *rigid_disk_page;
4106                         uint32_t sectors_per_cylinder;
4107                         uint64_t cylinders;
4108 #ifndef __XSCALE__
4109                         int shift;
4110 #endif /* !__XSCALE__ */
4111
4112                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4113                                 panic("invalid subpage value %d",
4114                                       page_index->subpage);
4115
4116                         /*
4117                          * Rotation rate and sectors per track are set
4118                          * above.  We calculate the cylinders here based on
4119                          * capacity.  Due to the number of heads and
4120                          * sectors per track we're using, smaller arrays
4121                          * may turn out to have 0 cylinders.  Linux and
4122                          * FreeBSD don't pay attention to these mode pages
4123                          * to figure out capacity, but Solaris does.  It
4124                          * seems to deal with 0 cylinders just fine, and
4125                          * works out a fake geometry based on the capacity.
4126                          */
4127                         memcpy(&lun->mode_pages.rigid_disk_page[
4128                                CTL_PAGE_DEFAULT], &rigid_disk_page_default,
4129                                sizeof(rigid_disk_page_default));
4130                         memcpy(&lun->mode_pages.rigid_disk_page[
4131                                CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
4132                                sizeof(rigid_disk_page_changeable));
4133
4134                         sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
4135                                 CTL_DEFAULT_HEADS;
4136
4137                         /*
4138                          * The divide method here will be more accurate,
4139                          * probably, but results in floating point being
4140                          * used in the kernel on i386 (__udivdi3()).  On the
4141                          * XScale, though, __udivdi3() is implemented in
4142                          * software.
4143                          *
4144                          * The shift method for cylinder calculation is
4145                          * accurate if sectors_per_cylinder is a power of
4146                          * 2.  Otherwise it might be slightly off -- you
4147                          * might have a bit of a truncation problem.
4148                          */
4149 #ifdef  __XSCALE__
4150                         cylinders = (lun->be_lun->maxlba + 1) /
4151                                 sectors_per_cylinder;
4152 #else
4153                         for (shift = 31; shift > 0; shift--) {
4154                                 if (sectors_per_cylinder & (1 << shift))
4155                                         break;
4156                         }
4157                         cylinders = (lun->be_lun->maxlba + 1) >> shift;
4158 #endif
4159
4160                         /*
4161                          * We've basically got 3 bytes, or 24 bits for the
4162                          * cylinder size in the mode page.  If we're over,
4163                          * just round down to 2^24.
4164                          */
4165                         if (cylinders > 0xffffff)
4166                                 cylinders = 0xffffff;
4167
4168                         rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4169                                 CTL_PAGE_DEFAULT];
4170                         scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4171
4172                         if ((value = ctl_get_opt(&lun->be_lun->options,
4173                             "rpm")) != NULL) {
4174                                 scsi_ulto2b(strtol(value, NULL, 0),
4175                                      rigid_disk_page->rotation_rate);
4176                         }
4177
4178                         memcpy(&lun->mode_pages.rigid_disk_page[CTL_PAGE_CURRENT],
4179                                &lun->mode_pages.rigid_disk_page[CTL_PAGE_DEFAULT],
4180                                sizeof(rigid_disk_page_default));
4181                         memcpy(&lun->mode_pages.rigid_disk_page[CTL_PAGE_SAVED],
4182                                &lun->mode_pages.rigid_disk_page[CTL_PAGE_DEFAULT],
4183                                sizeof(rigid_disk_page_default));
4184
4185                         page_index->page_data =
4186                                 (uint8_t *)lun->mode_pages.rigid_disk_page;
4187                         break;
4188                 }
4189                 case SMS_CACHING_PAGE: {
4190                         struct scsi_caching_page *caching_page;
4191
4192                         if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4193                                 panic("invalid subpage value %d",
4194                                       page_index->subpage);
4195                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
4196                                &caching_page_default,
4197                                sizeof(caching_page_default));
4198                         memcpy(&lun->mode_pages.caching_page[
4199                                CTL_PAGE_CHANGEABLE], &caching_page_changeable,
4200                                sizeof(caching_page_changeable));
4201                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4202                                &caching_page_default,
4203                                sizeof(caching_page_default));
4204                         caching_page = &lun->mode_pages.caching_page[
4205                             CTL_PAGE_SAVED];
4206                         value = ctl_get_opt(&lun->be_lun->options, "writecache");
4207                         if (value != NULL && strcmp(value, "off") == 0)
4208                                 caching_page->flags1 &= ~SCP_WCE;
4209                         value = ctl_get_opt(&lun->be_lun->options, "readcache");
4210                         if (value != NULL && strcmp(value, "off") == 0)
4211                                 caching_page->flags1 |= SCP_RCD;
4212                         memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
4213                                &lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4214                                sizeof(caching_page_default));
4215                         page_index->page_data =
4216                                 (uint8_t *)lun->mode_pages.caching_page;
4217                         break;
4218                 }
4219                 case SMS_CONTROL_MODE_PAGE: {
4220                         switch (page_index->subpage) {
4221                         case SMS_SUBPAGE_PAGE_0: {
4222                                 struct scsi_control_page *control_page;
4223
4224                                 memcpy(&lun->mode_pages.control_page[
4225                                     CTL_PAGE_DEFAULT],
4226                                        &control_page_default,
4227                                        sizeof(control_page_default));
4228                                 memcpy(&lun->mode_pages.control_page[
4229                                     CTL_PAGE_CHANGEABLE],
4230                                        &control_page_changeable,
4231                                        sizeof(control_page_changeable));
4232                                 memcpy(&lun->mode_pages.control_page[
4233                                     CTL_PAGE_SAVED],
4234                                        &control_page_default,
4235                                        sizeof(control_page_default));
4236                                 control_page = &lun->mode_pages.control_page[
4237                                     CTL_PAGE_SAVED];
4238                                 value = ctl_get_opt(&lun->be_lun->options,
4239                                     "reordering");
4240                                 if (value != NULL &&
4241                                     strcmp(value, "unrestricted") == 0) {
4242                                         control_page->queue_flags &=
4243                                             ~SCP_QUEUE_ALG_MASK;
4244                                         control_page->queue_flags |=
4245                                             SCP_QUEUE_ALG_UNRESTRICTED;
4246                                 }
4247                                 memcpy(&lun->mode_pages.control_page[
4248                                     CTL_PAGE_CURRENT],
4249                                        &lun->mode_pages.control_page[
4250                                     CTL_PAGE_SAVED],
4251                                        sizeof(control_page_default));
4252                                 page_index->page_data =
4253                                     (uint8_t *)lun->mode_pages.control_page;
4254                                 break;
4255                         }
4256                         case 0x01:
4257                                 memcpy(&lun->mode_pages.control_ext_page[
4258                                     CTL_PAGE_DEFAULT],
4259                                        &control_ext_page_default,
4260                                        sizeof(control_ext_page_default));
4261                                 memcpy(&lun->mode_pages.control_ext_page[
4262                                     CTL_PAGE_CHANGEABLE],
4263                                        &control_ext_page_changeable,
4264                                        sizeof(control_ext_page_changeable));
4265                                 memcpy(&lun->mode_pages.control_ext_page[
4266                                     CTL_PAGE_SAVED],
4267                                        &control_ext_page_default,
4268                                        sizeof(control_ext_page_default));
4269                                 memcpy(&lun->mode_pages.control_ext_page[
4270                                     CTL_PAGE_CURRENT],
4271                                        &lun->mode_pages.control_ext_page[
4272                                     CTL_PAGE_SAVED],
4273                                        sizeof(control_ext_page_default));
4274                                 page_index->page_data =
4275                                     (uint8_t *)lun->mode_pages.control_ext_page;
4276                                 break;
4277                         }
4278                         break;
4279                 }
4280                 case SMS_INFO_EXCEPTIONS_PAGE: {
4281                         switch (page_index->subpage) {
4282                         case SMS_SUBPAGE_PAGE_0:
4283                                 memcpy(&lun->mode_pages.ie_page[CTL_PAGE_CURRENT],
4284                                        &ie_page_default,
4285                                        sizeof(ie_page_default));
4286                                 memcpy(&lun->mode_pages.ie_page[
4287                                        CTL_PAGE_CHANGEABLE], &ie_page_changeable,
4288                                        sizeof(ie_page_changeable));
4289                                 memcpy(&lun->mode_pages.ie_page[CTL_PAGE_DEFAULT],
4290                                        &ie_page_default,
4291                                        sizeof(ie_page_default));
4292                                 memcpy(&lun->mode_pages.ie_page[CTL_PAGE_SAVED],
4293                                        &ie_page_default,
4294                                        sizeof(ie_page_default));
4295                                 page_index->page_data =
4296                                         (uint8_t *)lun->mode_pages.ie_page;
4297                                 break;
4298                         case 0x02: {
4299                                 struct ctl_logical_block_provisioning_page *page;
4300
4301                                 memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_DEFAULT],
4302                                        &lbp_page_default,
4303                                        sizeof(lbp_page_default));
4304                                 memcpy(&lun->mode_pages.lbp_page[
4305                                        CTL_PAGE_CHANGEABLE], &lbp_page_changeable,
4306                                        sizeof(lbp_page_changeable));
4307                                 memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_SAVED],
4308                                        &lbp_page_default,
4309                                        sizeof(lbp_page_default));
4310                                 page = &lun->mode_pages.lbp_page[CTL_PAGE_SAVED];
4311                                 value = ctl_get_opt(&lun->be_lun->options,
4312                                     "avail-threshold");
4313                                 if (value != NULL &&
4314                                     ctl_expand_number(value, &ival) == 0) {
4315                                         page->descr[0].flags |= SLBPPD_ENABLED |
4316                                             SLBPPD_ARMING_DEC;
4317                                         if (lun->be_lun->blocksize)
4318                                                 ival /= lun->be_lun->blocksize;
4319                                         else
4320                                                 ival /= 512;
4321                                         scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4322                                             page->descr[0].count);
4323                                 }
4324                                 value = ctl_get_opt(&lun->be_lun->options,
4325                                     "used-threshold");
4326                                 if (value != NULL &&
4327                                     ctl_expand_number(value, &ival) == 0) {
4328                                         page->descr[1].flags |= SLBPPD_ENABLED |
4329                                             SLBPPD_ARMING_INC;
4330                                         if (lun->be_lun->blocksize)
4331                                                 ival /= lun->be_lun->blocksize;
4332                                         else
4333                                                 ival /= 512;
4334                                         scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4335                                             page->descr[1].count);
4336                                 }
4337                                 value = ctl_get_opt(&lun->be_lun->options,
4338                                     "pool-avail-threshold");
4339                                 if (value != NULL &&
4340                                     ctl_expand_number(value, &ival) == 0) {
4341                                         page->descr[2].flags |= SLBPPD_ENABLED |
4342                                             SLBPPD_ARMING_DEC;
4343                                         if (lun->be_lun->blocksize)
4344                                                 ival /= lun->be_lun->blocksize;
4345                                         else
4346                                                 ival /= 512;
4347                                         scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4348                                             page->descr[2].count);
4349                                 }
4350                                 value = ctl_get_opt(&lun->be_lun->options,
4351                                     "pool-used-threshold");
4352                                 if (value != NULL &&
4353                                     ctl_expand_number(value, &ival) == 0) {
4354                                         page->descr[3].flags |= SLBPPD_ENABLED |
4355                                             SLBPPD_ARMING_INC;
4356                                         if (lun->be_lun->blocksize)
4357                                                 ival /= lun->be_lun->blocksize;
4358                                         else
4359                                                 ival /= 512;
4360                                         scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4361                                             page->descr[3].count);
4362                                 }
4363                                 memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_CURRENT],
4364                                        &lun->mode_pages.lbp_page[CTL_PAGE_SAVED],
4365                                        sizeof(lbp_page_default));
4366                                 page_index->page_data =
4367                                         (uint8_t *)lun->mode_pages.lbp_page;
4368                         }}
4369                         break;
4370                 }
4371                 case SMS_VENDOR_SPECIFIC_PAGE:{
4372                         switch (page_index->subpage) {
4373                         case DBGCNF_SUBPAGE_CODE: {
4374                                 memcpy(&lun->mode_pages.debugconf_subpage[
4375                                        CTL_PAGE_CURRENT],
4376                                        &debugconf_page_default,
4377                                        sizeof(debugconf_page_default));
4378                                 memcpy(&lun->mode_pages.debugconf_subpage[
4379                                        CTL_PAGE_CHANGEABLE],
4380                                        &debugconf_page_changeable,
4381                                        sizeof(debugconf_page_changeable));
4382                                 memcpy(&lun->mode_pages.debugconf_subpage[
4383                                        CTL_PAGE_DEFAULT],
4384                                        &debugconf_page_default,
4385                                        sizeof(debugconf_page_default));
4386                                 memcpy(&lun->mode_pages.debugconf_subpage[
4387                                        CTL_PAGE_SAVED],
4388                                        &debugconf_page_default,
4389                                        sizeof(debugconf_page_default));
4390                                 page_index->page_data =
4391                                     (uint8_t *)lun->mode_pages.debugconf_subpage;
4392                                 break;
4393                         }
4394                         default:
4395                                 panic("invalid subpage value %d",
4396                                       page_index->subpage);
4397                                 break;
4398                         }
4399                         break;
4400                 }
4401                 default:
4402                         panic("invalid page value %d",
4403                               page_index->page_code & SMPH_PC_MASK);
4404                         break;
4405         }
4406         }
4407
4408         return (CTL_RETVAL_COMPLETE);
4409 }
4410
4411 static int
4412 ctl_init_log_page_index(struct ctl_lun *lun)
4413 {
4414         struct ctl_page_index *page_index;
4415         int i, j, k, prev;
4416
4417         memcpy(&lun->log_pages.index, log_page_index_template,
4418                sizeof(log_page_index_template));
4419
4420         prev = -1;
4421         for (i = 0, j = 0, k = 0; i < CTL_NUM_LOG_PAGES; i++) {
4422
4423                 page_index = &lun->log_pages.index[i];
4424                 /*
4425                  * If this is a disk-only mode page, there's no point in
4426                  * setting it up.  For some pages, we have to have some
4427                  * basic information about the disk in order to calculate the
4428                  * mode page data.
4429                  */
4430                 if ((lun->be_lun->lun_type != T_DIRECT)
4431                  && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
4432                         continue;
4433
4434                 if (page_index->page_code == SLS_LOGICAL_BLOCK_PROVISIONING &&
4435                      lun->backend->lun_attr == NULL)
4436                         continue;
4437
4438                 if (page_index->page_code != prev) {
4439                         lun->log_pages.pages_page[j] = page_index->page_code;
4440                         prev = page_index->page_code;
4441                         j++;
4442                 }
4443                 lun->log_pages.subpages_page[k*2] = page_index->page_code;
4444                 lun->log_pages.subpages_page[k*2+1] = page_index->subpage;
4445                 k++;
4446         }
4447         lun->log_pages.index[0].page_data = &lun->log_pages.pages_page[0];
4448         lun->log_pages.index[0].page_len = j;
4449         lun->log_pages.index[1].page_data = &lun->log_pages.subpages_page[0];
4450         lun->log_pages.index[1].page_len = k * 2;
4451         lun->log_pages.index[2].page_data = &lun->log_pages.lbp_page[0];
4452         lun->log_pages.index[2].page_len = 12*CTL_NUM_LBP_PARAMS;
4453         lun->log_pages.index[3].page_data = (uint8_t *)&lun->log_pages.stat_page;
4454         lun->log_pages.index[3].page_len = sizeof(lun->log_pages.stat_page);
4455
4456         return (CTL_RETVAL_COMPLETE);
4457 }
4458
4459 static int
4460 hex2bin(const char *str, uint8_t *buf, int buf_size)
4461 {
4462         int i;
4463         u_char c;
4464
4465         memset(buf, 0, buf_size);
4466         while (isspace(str[0]))
4467                 str++;
4468         if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X'))
4469                 str += 2;
4470         buf_size *= 2;
4471         for (i = 0; str[i] != 0 && i < buf_size; i++) {
4472                 c = str[i];
4473                 if (isdigit(c))
4474                         c -= '0';
4475                 else if (isalpha(c))
4476                         c -= isupper(c) ? 'A' - 10 : 'a' - 10;
4477                 else
4478                         break;
4479                 if (c >= 16)
4480                         break;
4481                 if ((i & 1) == 0)
4482                         buf[i / 2] |= (c << 4);
4483                 else
4484                         buf[i / 2] |= c;
4485         }
4486         return ((i + 1) / 2);
4487 }
4488
4489 /*
4490  * LUN allocation.
4491  *
4492  * Requirements:
4493  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4494  *   wants us to allocate the LUN and he can block.
4495  * - ctl_softc is always set
4496  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4497  *
4498  * Returns 0 for success, non-zero (errno) for failure.
4499  */
4500 static int
4501 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4502               struct ctl_be_lun *const be_lun)
4503 {
4504         struct ctl_lun *nlun, *lun;
4505         struct scsi_vpd_id_descriptor *desc;
4506         struct scsi_vpd_id_t10 *t10id;
4507         const char *eui, *naa, *scsiname, *vendor;
4508         int lun_number, i, lun_malloced;
4509         int devidlen, idlen1, idlen2 = 0, len;
4510
4511         if (be_lun == NULL)
4512                 return (EINVAL);
4513
4514         /*
4515          * We currently only support Direct Access or Processor LUN types.
4516          */
4517         switch (be_lun->lun_type) {
4518         case T_DIRECT:
4519                 break;
4520         case T_PROCESSOR:
4521                 break;
4522         case T_SEQUENTIAL:
4523         case T_CHANGER:
4524         default:
4525                 be_lun->lun_config_status(be_lun->be_lun,
4526                                           CTL_LUN_CONFIG_FAILURE);
4527                 break;
4528         }
4529         if (ctl_lun == NULL) {
4530                 lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4531                 lun_malloced = 1;
4532         } else {
4533                 lun_malloced = 0;
4534                 lun = ctl_lun;
4535         }
4536
4537         memset(lun, 0, sizeof(*lun));
4538         if (lun_malloced)
4539                 lun->flags = CTL_LUN_MALLOCED;
4540
4541         /* Generate LUN ID. */
4542         devidlen = max(CTL_DEVID_MIN_LEN,
4543             strnlen(be_lun->device_id, CTL_DEVID_LEN));
4544         idlen1 = sizeof(*t10id) + devidlen;
4545         len = sizeof(struct scsi_vpd_id_descriptor) + idlen1;
4546         scsiname = ctl_get_opt(&be_lun->options, "scsiname");
4547         if (scsiname != NULL) {
4548                 idlen2 = roundup2(strlen(scsiname) + 1, 4);
4549                 len += sizeof(struct scsi_vpd_id_descriptor) + idlen2;
4550         }
4551         eui = ctl_get_opt(&be_lun->options, "eui");
4552         if (eui != NULL) {
4553                 len += sizeof(struct scsi_vpd_id_descriptor) + 16;
4554         }
4555         naa = ctl_get_opt(&be_lun->options, "naa");
4556         if (naa != NULL) {
4557                 len += sizeof(struct scsi_vpd_id_descriptor) + 16;
4558         }
4559         lun->lun_devid = malloc(sizeof(struct ctl_devid) + len,
4560             M_CTL, M_WAITOK | M_ZERO);
4561         desc = (struct scsi_vpd_id_descriptor *)lun->lun_devid->data;
4562         desc->proto_codeset = SVPD_ID_CODESET_ASCII;
4563         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
4564         desc->length = idlen1;
4565         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
4566         memset(t10id->vendor, ' ', sizeof(t10id->vendor));
4567         if ((vendor = ctl_get_opt(&be_lun->options, "vendor")) == NULL) {
4568                 strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
4569         } else {
4570                 strncpy(t10id->vendor, vendor,
4571                     min(sizeof(t10id->vendor), strlen(vendor)));
4572         }
4573         strncpy((char *)t10id->vendor_spec_id,
4574             (char *)be_lun->device_id, devidlen);
4575         if (scsiname != NULL) {
4576                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4577                     desc->length);
4578                 desc->proto_codeset = SVPD_ID_CODESET_UTF8;
4579                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4580                     SVPD_ID_TYPE_SCSI_NAME;
4581                 desc->length = idlen2;
4582                 strlcpy(desc->identifier, scsiname, idlen2);
4583         }
4584         if (eui != NULL) {
4585                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4586                     desc->length);
4587                 desc->proto_codeset = SVPD_ID_CODESET_BINARY;
4588                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4589                     SVPD_ID_TYPE_EUI64;
4590                 desc->length = hex2bin(eui, desc->identifier, 16);
4591                 desc->length = desc->length > 12 ? 16 :
4592                     (desc->length > 8 ? 12 : 8);
4593                 len -= 16 - desc->length;
4594         }
4595         if (naa != NULL) {
4596                 desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4597                     desc->length);
4598                 desc->proto_codeset = SVPD_ID_CODESET_BINARY;
4599                 desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4600                     SVPD_ID_TYPE_NAA;
4601                 desc->length = hex2bin(naa, desc->identifier, 16);
4602                 desc->length = desc->length > 8 ? 16 : 8;
4603                 len -= 16 - desc->length;
4604         }
4605         lun->lun_devid->len = len;
4606
4607         mtx_lock(&ctl_softc->ctl_lock);
4608         /*
4609          * See if the caller requested a particular LUN number.  If so, see
4610          * if it is available.  Otherwise, allocate the first available LUN.
4611          */
4612         if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4613                 if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4614                  || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4615                         mtx_unlock(&ctl_softc->ctl_lock);
4616                         if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4617                                 printf("ctl: requested LUN ID %d is higher "
4618                                        "than CTL_MAX_LUNS - 1 (%d)\n",
4619                                        be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4620                         } else {
4621                                 /*
4622                                  * XXX KDM return an error, or just assign
4623                                  * another LUN ID in this case??
4624                                  */
4625                                 printf("ctl: requested LUN ID %d is already "
4626                                        "in use\n", be_lun->req_lun_id);
4627                         }
4628                         if (lun->flags & CTL_LUN_MALLOCED)
4629                                 free(lun, M_CTL);
4630                         be_lun->lun_config_status(be_lun->be_lun,
4631                                                   CTL_LUN_CONFIG_FAILURE);
4632                         return (ENOSPC);
4633                 }
4634                 lun_number = be_lun->req_lun_id;
4635         } else {
4636                 lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, 0, CTL_MAX_LUNS);
4637                 if (lun_number == -1) {
4638                         mtx_unlock(&ctl_softc->ctl_lock);
4639                         printf("ctl: can't allocate LUN, out of LUNs\n");
4640                         if (lun->flags & CTL_LUN_MALLOCED)
4641                                 free(lun, M_CTL);
4642                         be_lun->lun_config_status(be_lun->be_lun,
4643                                                   CTL_LUN_CONFIG_FAILURE);
4644                         return (ENOSPC);
4645                 }
4646         }
4647         ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4648
4649         mtx_init(&lun->lun_lock, "CTL LUN", NULL, MTX_DEF);
4650         lun->lun = lun_number;
4651         lun->be_lun = be_lun;
4652         /*
4653          * The processor LUN is always enabled.  Disk LUNs come on line
4654          * disabled, and must be enabled by the backend.
4655          */
4656         lun->flags |= CTL_LUN_DISABLED;
4657         lun->backend = be_lun->be;
4658         be_lun->ctl_lun = lun;
4659         be_lun->lun_id = lun_number;
4660         atomic_add_int(&be_lun->be->num_luns, 1);
4661         if (be_lun->flags & CTL_LUN_FLAG_OFFLINE)
4662                 lun->flags |= CTL_LUN_OFFLINE;
4663
4664         if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4665                 lun->flags |= CTL_LUN_STOPPED;
4666
4667         if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4668                 lun->flags |= CTL_LUN_INOPERABLE;
4669
4670         if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4671                 lun->flags |= CTL_LUN_PRIMARY_SC;
4672
4673         lun->ctl_softc = ctl_softc;
4674 #ifdef CTL_TIME_IO
4675         lun->last_busy = getsbinuptime();
4676 #endif
4677         TAILQ_INIT(&lun->ooa_queue);
4678         TAILQ_INIT(&lun->blocked_queue);
4679         STAILQ_INIT(&lun->error_list);
4680         ctl_tpc_lun_init(lun);
4681
4682         /*
4683          * Initialize the mode and log page index.
4684          */
4685         ctl_init_page_index(lun);
4686         ctl_init_log_page_index(lun);
4687
4688         /*
4689          * Now, before we insert this lun on the lun list, set the lun
4690          * inventory changed UA for all other luns.
4691          */
4692         STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4693                 mtx_lock(&nlun->lun_lock);
4694                 ctl_est_ua_all(nlun, -1, CTL_UA_LUN_CHANGE);
4695                 mtx_unlock(&nlun->lun_lock);
4696         }
4697
4698         STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4699
4700         ctl_softc->ctl_luns[lun_number] = lun;
4701
4702         ctl_softc->num_luns++;
4703
4704         /* Setup statistics gathering */
4705         lun->stats.device_type = be_lun->lun_type;
4706         lun->stats.lun_number = lun_number;
4707         if (lun->stats.device_type == T_DIRECT)
4708                 lun->stats.blocksize = be_lun->blocksize;
4709         else
4710                 lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4711         for (i = 0;i < CTL_MAX_PORTS;i++)
4712                 lun->stats.ports[i].targ_port = i;
4713
4714         mtx_unlock(&ctl_softc->ctl_lock);
4715
4716         lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4717         return (0);
4718 }
4719
4720 /*
4721  * Delete a LUN.
4722  * Assumptions:
4723  * - LUN has already been marked invalid and any pending I/O has been taken
4724  *   care of.
4725  */
4726 static int
4727 ctl_free_lun(struct ctl_lun *lun)
4728 {
4729         struct ctl_softc *softc;
4730         struct ctl_lun *nlun;
4731         int i;
4732
4733         softc = lun->ctl_softc;
4734
4735         mtx_assert(&softc->ctl_lock, MA_OWNED);
4736
4737         STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4738
4739         ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4740
4741         softc->ctl_luns[lun->lun] = NULL;
4742
4743         if (!TAILQ_EMPTY(&lun->ooa_queue))
4744                 panic("Freeing a LUN %p with outstanding I/O!!\n", lun);
4745
4746         softc->num_luns--;
4747
4748         /*
4749          * Tell the backend to free resources, if this LUN has a backend.
4750          */
4751         atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4752         lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4753
4754         ctl_tpc_lun_shutdown(lun);
4755         mtx_destroy(&lun->lun_lock);
4756         free(lun->lun_devid, M_CTL);
4757         for (i = 0; i < CTL_MAX_PORTS; i++)
4758                 free(lun->pending_ua[i], M_CTL);
4759         for (i = 0; i < CTL_MAX_PORTS; i++)
4760                 free(lun->pr_keys[i], M_CTL);
4761         free(lun->write_buffer, M_CTL);
4762         if (lun->flags & CTL_LUN_MALLOCED)
4763                 free(lun, M_CTL);
4764
4765         STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4766                 mtx_lock(&nlun->lun_lock);
4767                 ctl_est_ua_all(nlun, -1, CTL_UA_LUN_CHANGE);
4768                 mtx_unlock(&nlun->lun_lock);
4769         }
4770
4771         return (0);
4772 }
4773
4774 static void
4775 ctl_create_lun(struct ctl_be_lun *be_lun)
4776 {
4777
4778         /*
4779          * ctl_alloc_lun() should handle all potential failure cases.
4780          */
4781         ctl_alloc_lun(control_softc, NULL, be_lun);
4782 }
4783
4784 int
4785 ctl_add_lun(struct ctl_be_lun *be_lun)
4786 {
4787         struct ctl_softc *softc = control_softc;
4788
4789         mtx_lock(&softc->ctl_lock);
4790         STAILQ_INSERT_TAIL(&softc->pending_lun_queue, be_lun, links);
4791         mtx_unlock(&softc->ctl_lock);
4792         wakeup(&softc->pending_lun_queue);
4793
4794         return (0);
4795 }
4796
4797 int
4798 ctl_enable_lun(struct ctl_be_lun *be_lun)
4799 {
4800         struct ctl_softc *softc;
4801         struct ctl_port *port, *nport;
4802         struct ctl_lun *lun;
4803         int retval;
4804
4805         lun = (struct ctl_lun *)be_lun->ctl_lun;
4806         softc = lun->ctl_softc;
4807
4808         mtx_lock(&softc->ctl_lock);
4809         mtx_lock(&lun->lun_lock);
4810         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4811                 /*
4812                  * eh?  Why did we get called if the LUN is already
4813                  * enabled?
4814                  */
4815                 mtx_unlock(&lun->lun_lock);
4816                 mtx_unlock(&softc->ctl_lock);
4817                 return (0);
4818         }
4819         lun->flags &= ~CTL_LUN_DISABLED;
4820         mtx_unlock(&lun->lun_lock);
4821
4822         for (port = STAILQ_FIRST(&softc->port_list); port != NULL; port = nport) {
4823                 nport = STAILQ_NEXT(port, links);
4824                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0 ||
4825                     port->lun_map != NULL || port->lun_enable == NULL)
4826                         continue;
4827
4828                 /*
4829                  * Drop the lock while we call the FETD's enable routine.
4830                  * This can lead to a callback into CTL (at least in the
4831                  * case of the internal initiator frontend.
4832                  */
4833                 mtx_unlock(&softc->ctl_lock);
4834                 retval = port->lun_enable(port->targ_lun_arg, lun->lun);
4835                 mtx_lock(&softc->ctl_lock);
4836                 if (retval != 0) {
4837                         printf("%s: FETD %s port %d returned error "
4838                                "%d for lun_enable on lun %jd\n",
4839                                __func__, port->port_name, port->targ_port,
4840                                retval, (intmax_t)lun->lun);
4841                 }
4842         }
4843
4844         mtx_unlock(&softc->ctl_lock);
4845         ctl_isc_announce_lun(lun);
4846
4847         return (0);
4848 }
4849
4850 int
4851 ctl_disable_lun(struct ctl_be_lun *be_lun)
4852 {
4853         struct ctl_softc *softc;
4854         struct ctl_port *port;
4855         struct ctl_lun *lun;
4856         int retval;
4857
4858         lun = (struct ctl_lun *)be_lun->ctl_lun;
4859         softc = lun->ctl_softc;
4860
4861         mtx_lock(&softc->ctl_lock);
4862         mtx_lock(&lun->lun_lock);
4863         if (lun->flags & CTL_LUN_DISABLED) {
4864                 mtx_unlock(&lun->lun_lock);
4865                 mtx_unlock(&softc->ctl_lock);
4866                 return (0);
4867         }
4868         lun->flags |= CTL_LUN_DISABLED;
4869         mtx_unlock(&lun->lun_lock);
4870
4871         STAILQ_FOREACH(port, &softc->port_list, links) {
4872                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0 ||
4873                     port->lun_map != NULL || port->lun_disable == NULL)
4874                         continue;
4875
4876                 /*
4877                  * Drop the lock before we call the frontend's disable
4878                  * routine, to avoid lock order reversals.
4879                  *
4880                  * XXX KDM what happens if the frontend list changes while
4881                  * we're traversing it?  It's unlikely, but should be handled.
4882                  */
4883                 mtx_unlock(&softc->ctl_lock);
4884                 retval = port->lun_disable(port->targ_lun_arg, lun->lun);
4885                 mtx_lock(&softc->ctl_lock);
4886                 if (retval != 0) {
4887                         printf("%s: FETD %s port %d returned error "
4888                                "%d for lun_disable on lun %jd\n",
4889                                __func__, port->port_name, port->targ_port,
4890                                retval, (intmax_t)lun->lun);
4891                 }
4892         }
4893
4894         mtx_unlock(&softc->ctl_lock);
4895         ctl_isc_announce_lun(lun);
4896
4897         return (0);
4898 }
4899
4900 int
4901 ctl_start_lun(struct ctl_be_lun *be_lun)
4902 {
4903         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4904
4905         mtx_lock(&lun->lun_lock);
4906         lun->flags &= ~CTL_LUN_STOPPED;
4907         mtx_unlock(&lun->lun_lock);
4908         return (0);
4909 }
4910
4911 int
4912 ctl_stop_lun(struct ctl_be_lun *be_lun)
4913 {
4914         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4915
4916         mtx_lock(&lun->lun_lock);
4917         lun->flags |= CTL_LUN_STOPPED;
4918         mtx_unlock(&lun->lun_lock);
4919         return (0);
4920 }
4921
4922 int
4923 ctl_lun_offline(struct ctl_be_lun *be_lun)
4924 {
4925         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4926
4927         mtx_lock(&lun->lun_lock);
4928         lun->flags |= CTL_LUN_OFFLINE;
4929         mtx_unlock(&lun->lun_lock);
4930         return (0);
4931 }
4932
4933 int
4934 ctl_lun_online(struct ctl_be_lun *be_lun)
4935 {
4936         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4937
4938         mtx_lock(&lun->lun_lock);
4939         lun->flags &= ~CTL_LUN_OFFLINE;
4940         mtx_unlock(&lun->lun_lock);
4941         return (0);
4942 }
4943
4944 int
4945 ctl_lun_primary(struct ctl_be_lun *be_lun)
4946 {
4947         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4948
4949         mtx_lock(&lun->lun_lock);
4950         lun->flags |= CTL_LUN_PRIMARY_SC;
4951         ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
4952         mtx_unlock(&lun->lun_lock);
4953         ctl_isc_announce_lun(lun);
4954         return (0);
4955 }
4956
4957 int
4958 ctl_lun_secondary(struct ctl_be_lun *be_lun)
4959 {
4960         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4961
4962         mtx_lock(&lun->lun_lock);
4963         lun->flags &= ~CTL_LUN_PRIMARY_SC;
4964         ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
4965         mtx_unlock(&lun->lun_lock);
4966         ctl_isc_announce_lun(lun);
4967         return (0);
4968 }
4969
4970 int
4971 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4972 {
4973         struct ctl_softc *softc;
4974         struct ctl_lun *lun;
4975
4976         lun = (struct ctl_lun *)be_lun->ctl_lun;
4977         softc = lun->ctl_softc;
4978
4979         mtx_lock(&lun->lun_lock);
4980
4981         /*
4982          * The LUN needs to be disabled before it can be marked invalid.
4983          */
4984         if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4985                 mtx_unlock(&lun->lun_lock);
4986                 return (-1);
4987         }
4988         /*
4989          * Mark the LUN invalid.
4990          */
4991         lun->flags |= CTL_LUN_INVALID;
4992
4993         /*
4994          * If there is nothing in the OOA queue, go ahead and free the LUN.
4995          * If we have something in the OOA queue, we'll free it when the
4996          * last I/O completes.
4997          */
4998         if (TAILQ_EMPTY(&lun->ooa_queue)) {
4999                 mtx_unlock(&lun->lun_lock);
5000                 mtx_lock(&softc->ctl_lock);
5001                 ctl_free_lun(lun);
5002                 mtx_unlock(&softc->ctl_lock);
5003         } else
5004                 mtx_unlock(&lun->lun_lock);
5005
5006         return (0);
5007 }
5008
5009 int
5010 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
5011 {
5012         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
5013
5014         mtx_lock(&lun->lun_lock);
5015         lun->flags |= CTL_LUN_INOPERABLE;
5016         mtx_unlock(&lun->lun_lock);
5017         return (0);
5018 }
5019
5020 int
5021 ctl_lun_operable(struct ctl_be_lun *be_lun)
5022 {
5023         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
5024
5025         mtx_lock(&lun->lun_lock);
5026         lun->flags &= ~CTL_LUN_INOPERABLE;
5027         mtx_unlock(&lun->lun_lock);
5028         return (0);
5029 }
5030
5031 void
5032 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
5033 {
5034         struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
5035         union ctl_ha_msg msg;
5036
5037         mtx_lock(&lun->lun_lock);
5038         ctl_est_ua_all(lun, -1, CTL_UA_CAPACITY_CHANGED);
5039         mtx_unlock(&lun->lun_lock);
5040         if (lun->ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
5041                 /* Send msg to other side. */
5042                 bzero(&msg.ua, sizeof(msg.ua));
5043                 msg.hdr.msg_type = CTL_MSG_UA;
5044                 msg.hdr.nexus.initid = -1;
5045                 msg.hdr.nexus.targ_port = -1;
5046                 msg.hdr.nexus.targ_lun = lun->lun;
5047                 msg.hdr.nexus.targ_mapped_lun = lun->lun;
5048                 msg.ua.ua_all = 1;
5049                 msg.ua.ua_set = 1;
5050                 msg.ua.ua_type = CTL_UA_CAPACITY_CHANGED;
5051                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg.ua),
5052                     M_WAITOK);
5053         }
5054 }
5055
5056 /*
5057  * Backend "memory move is complete" callback for requests that never
5058  * make it down to say RAIDCore's configuration code.
5059  */
5060 int
5061 ctl_config_move_done(union ctl_io *io)
5062 {
5063         int retval;
5064
5065         CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
5066         KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
5067             ("Config I/O type isn't CTL_IO_SCSI (%d)!", io->io_hdr.io_type));
5068
5069         if ((io->io_hdr.port_status != 0) &&
5070             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5071              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5072                 /*
5073                  * For hardware error sense keys, the sense key
5074                  * specific value is defined to be a retry count,
5075                  * but we use it to pass back an internal FETD
5076                  * error code.  XXX KDM  Hopefully the FETD is only
5077                  * using 16 bits for an error code, since that's
5078                  * all the space we have in the sks field.
5079                  */
5080                 ctl_set_internal_failure(&io->scsiio,
5081                                          /*sks_valid*/ 1,
5082                                          /*retry_count*/
5083                                          io->io_hdr.port_status);
5084         }
5085
5086         if (ctl_debug & CTL_DEBUG_CDB_DATA)
5087                 ctl_data_print(io);
5088         if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) ||
5089             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
5090              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) ||
5091             ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
5092                 /*
5093                  * XXX KDM just assuming a single pointer here, and not a
5094                  * S/G list.  If we start using S/G lists for config data,
5095                  * we'll need to know how to clean them up here as well.
5096                  */
5097                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5098                         free(io->scsiio.kern_data_ptr, M_CTL);
5099                 ctl_done(io);
5100                 retval = CTL_RETVAL_COMPLETE;
5101         } else {
5102                 /*
5103                  * XXX KDM now we need to continue data movement.  Some
5104                  * options:
5105                  * - call ctl_scsiio() again?  We don't do this for data
5106                  *   writes, because for those at least we know ahead of
5107                  *   time where the write will go and how long it is.  For
5108                  *   config writes, though, that information is largely
5109                  *   contained within the write itself, thus we need to
5110                  *   parse out the data again.
5111                  *
5112                  * - Call some other function once the data is in?
5113                  */
5114
5115                 /*
5116                  * XXX KDM call ctl_scsiio() again for now, and check flag
5117                  * bits to see whether we're allocated or not.
5118                  */
5119                 retval = ctl_scsiio(&io->scsiio);
5120         }
5121         return (retval);
5122 }
5123
5124 /*
5125  * This gets called by a backend driver when it is done with a
5126  * data_submit method.
5127  */
5128 void
5129 ctl_data_submit_done(union ctl_io *io)
5130 {
5131         /*
5132          * If the IO_CONT flag is set, we need to call the supplied
5133          * function to continue processing the I/O, instead of completing
5134          * the I/O just yet.
5135          *
5136          * If there is an error, though, we don't want to keep processing.
5137          * Instead, just send status back to the initiator.
5138          */
5139         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5140             (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5141             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5142              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5143                 io->scsiio.io_cont(io);
5144                 return;
5145         }
5146         ctl_done(io);
5147 }
5148
5149 /*
5150  * This gets called by a backend driver when it is done with a
5151  * configuration write.
5152  */
5153 void
5154 ctl_config_write_done(union ctl_io *io)
5155 {
5156         uint8_t *buf;
5157
5158         /*
5159          * If the IO_CONT flag is set, we need to call the supplied
5160          * function to continue processing the I/O, instead of completing
5161          * the I/O just yet.
5162          *
5163          * If there is an error, though, we don't want to keep processing.
5164          * Instead, just send status back to the initiator.
5165          */
5166         if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5167             (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5168             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5169              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5170                 io->scsiio.io_cont(io);
5171                 return;
5172         }
5173         /*
5174          * Since a configuration write can be done for commands that actually
5175          * have data allocated, like write buffer, and commands that have
5176          * no data, like start/stop unit, we need to check here.
5177          */
5178         if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5179                 buf = io->scsiio.kern_data_ptr;
5180         else
5181                 buf = NULL;
5182         ctl_done(io);
5183         if (buf)
5184                 free(buf, M_CTL);
5185 }
5186
5187 void
5188 ctl_config_read_done(union ctl_io *io)
5189 {
5190         uint8_t *buf;
5191
5192         /*
5193          * If there is some error -- we are done, skip data transfer.
5194          */
5195         if ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0 ||
5196             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
5197              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
5198                 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5199                         buf = io->scsiio.kern_data_ptr;
5200                 else
5201                         buf = NULL;
5202                 ctl_done(io);
5203                 if (buf)
5204                         free(buf, M_CTL);
5205                 return;
5206         }
5207
5208         /*
5209          * If the IO_CONT flag is set, we need to call the supplied
5210          * function to continue processing the I/O, instead of completing
5211          * the I/O just yet.
5212          */
5213         if (io->io_hdr.flags & CTL_FLAG_IO_CONT) {
5214                 io->scsiio.io_cont(io);
5215                 return;
5216         }
5217
5218         ctl_datamove(io);
5219 }
5220
5221 /*
5222  * SCSI release command.
5223  */
5224 int
5225 ctl_scsi_release(struct ctl_scsiio *ctsio)
5226 {
5227         struct ctl_lun *lun;
5228         uint32_t residx;
5229
5230         CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
5231
5232         residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5233         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5234
5235         /*
5236          * XXX KDM right now, we only support LUN reservation.  We don't
5237          * support 3rd party reservations, or extent reservations, which
5238          * might actually need the parameter list.  If we've gotten this
5239          * far, we've got a LUN reservation.  Anything else got kicked out
5240          * above.  So, according to SPC, ignore the length.
5241          */
5242
5243         mtx_lock(&lun->lun_lock);
5244
5245         /*
5246          * According to SPC, it is not an error for an intiator to attempt
5247          * to release a reservation on a LUN that isn't reserved, or that
5248          * is reserved by another initiator.  The reservation can only be
5249          * released, though, by the initiator who made it or by one of
5250          * several reset type events.
5251          */
5252         if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == residx))
5253                         lun->flags &= ~CTL_LUN_RESERVED;
5254
5255         mtx_unlock(&lun->lun_lock);
5256
5257         ctl_set_success(ctsio);
5258         ctl_done((union ctl_io *)ctsio);
5259         return (CTL_RETVAL_COMPLETE);
5260 }
5261
5262 int
5263 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5264 {
5265         struct ctl_lun *lun;
5266         uint32_t residx;
5267
5268         CTL_DEBUG_PRINT(("ctl_reserve\n"));
5269
5270         residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5271         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5272
5273         /*
5274          * XXX KDM right now, we only support LUN reservation.  We don't
5275          * support 3rd party reservations, or extent reservations, which
5276          * might actually need the parameter list.  If we've gotten this
5277          * far, we've got a LUN reservation.  Anything else got kicked out
5278          * above.  So, according to SPC, ignore the length.
5279          */
5280
5281         mtx_lock(&lun->lun_lock);
5282         if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx != residx)) {
5283                 ctl_set_reservation_conflict(ctsio);
5284                 goto bailout;
5285         }
5286         lun->flags |= CTL_LUN_RESERVED;
5287         lun->res_idx = residx;
5288         ctl_set_success(ctsio);
5289
5290 bailout:
5291         mtx_unlock(&lun->lun_lock);
5292         ctl_done((union ctl_io *)ctsio);
5293         return (CTL_RETVAL_COMPLETE);
5294 }
5295
5296 int
5297 ctl_start_stop(struct ctl_scsiio *ctsio)
5298 {
5299         struct scsi_start_stop_unit *cdb;
5300         struct ctl_lun *lun;
5301         int retval;
5302
5303         CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5304
5305         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5306         retval = 0;
5307
5308         cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5309
5310         /*
5311          * XXX KDM
5312          * We don't support the immediate bit on a stop unit.  In order to
5313          * do that, we would need to code up a way to know that a stop is
5314          * pending, and hold off any new commands until it completes, one
5315          * way or another.  Then we could accept or reject those commands
5316          * depending on its status.  We would almost need to do the reverse
5317          * of what we do below for an immediate start -- return the copy of
5318          * the ctl_io to the FETD with status to send to the host (and to
5319          * free the copy!) and then free the original I/O once the stop
5320          * actually completes.  That way, the OOA queue mechanism can work
5321          * to block commands that shouldn't proceed.  Another alternative
5322          * would be to put the copy in the queue in place of the original,
5323          * and return the original back to the caller.  That could be
5324          * slightly safer..
5325          */
5326         if ((cdb->byte2 & SSS_IMMED)
5327          && ((cdb->how & SSS_START) == 0)) {
5328                 ctl_set_invalid_field(ctsio,
5329                                       /*sks_valid*/ 1,
5330                                       /*command*/ 1,
5331                                       /*field*/ 1,
5332                                       /*bit_valid*/ 1,
5333                                       /*bit*/ 0);
5334                 ctl_done((union ctl_io *)ctsio);
5335                 return (CTL_RETVAL_COMPLETE);
5336         }
5337
5338         if ((lun->flags & CTL_LUN_PR_RESERVED)
5339          && ((cdb->how & SSS_START)==0)) {
5340                 uint32_t residx;
5341
5342                 residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5343                 if (ctl_get_prkey(lun, residx) == 0
5344                  || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5345
5346                         ctl_set_reservation_conflict(ctsio);
5347                         ctl_done((union ctl_io *)ctsio);
5348                         return (CTL_RETVAL_COMPLETE);
5349                 }
5350         }
5351
5352         /*
5353          * If there is no backend on this device, we can't start or stop
5354          * it.  In theory we shouldn't get any start/stop commands in the
5355          * first place at this level if the LUN doesn't have a backend.
5356          * That should get stopped by the command decode code.
5357          */
5358         if (lun->backend == NULL) {
5359                 ctl_set_invalid_opcode(ctsio);
5360                 ctl_done((union ctl_io *)ctsio);
5361                 return (CTL_RETVAL_COMPLETE);
5362         }
5363
5364         /*
5365          * XXX KDM Copan-specific offline behavior.
5366          * Figure out a reasonable way to port this?
5367          */
5368 #ifdef NEEDTOPORT
5369         mtx_lock(&lun->lun_lock);
5370
5371         if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5372          && (lun->flags & CTL_LUN_OFFLINE)) {
5373                 /*
5374                  * If the LUN is offline, and the on/offline bit isn't set,
5375                  * reject the start or stop.  Otherwise, let it through.
5376                  */
5377                 mtx_unlock(&lun->lun_lock);
5378                 ctl_set_lun_not_ready(ctsio);
5379                 ctl_done((union ctl_io *)ctsio);
5380         } else {
5381                 mtx_unlock(&lun->lun_lock);
5382 #endif /* NEEDTOPORT */
5383                 /*
5384                  * This could be a start or a stop when we're online,
5385                  * or a stop/offline or start/online.  A start or stop when
5386                  * we're offline is covered in the case above.
5387                  */
5388                 /*
5389                  * In the non-immediate case, we send the request to
5390                  * the backend and return status to the user when
5391                  * it is done.
5392                  *
5393                  * In the immediate case, we allocate a new ctl_io
5394                  * to hold a copy of the request, and send that to
5395                  * the backend.  We then set good status on the
5396                  * user's request and return it immediately.
5397                  */
5398                 if (cdb->byte2 & SSS_IMMED) {
5399                         union ctl_io *new_io;
5400
5401                         new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5402                         ctl_copy_io((union ctl_io *)ctsio, new_io);
5403                         retval = lun->backend->config_write(new_io);
5404                         ctl_set_success(ctsio);
5405                         ctl_done((union ctl_io *)ctsio);
5406                 } else {
5407                         retval = lun->backend->config_write(
5408                                 (union ctl_io *)ctsio);
5409                 }
5410 #ifdef NEEDTOPORT
5411         }
5412 #endif
5413         return (retval);
5414 }
5415
5416 /*
5417  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5418  * we don't really do anything with the LBA and length fields if the user
5419  * passes them in.  Instead we'll just flush out the cache for the entire
5420  * LUN.
5421  */
5422 int
5423 ctl_sync_cache(struct ctl_scsiio *ctsio)
5424 {
5425         struct ctl_lun *lun;
5426         struct ctl_softc *softc;
5427         struct ctl_lba_len_flags *lbalen;
5428         uint64_t starting_lba;
5429         uint32_t block_count;
5430         int retval;
5431         uint8_t byte2;
5432
5433         CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5434
5435         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5436         softc = lun->ctl_softc;
5437         retval = 0;
5438
5439         switch (ctsio->cdb[0]) {
5440         case SYNCHRONIZE_CACHE: {
5441                 struct scsi_sync_cache *cdb;
5442                 cdb = (struct scsi_sync_cache *)ctsio->cdb;
5443
5444                 starting_lba = scsi_4btoul(cdb->begin_lba);
5445                 block_count = scsi_2btoul(cdb->lb_count);
5446                 byte2 = cdb->byte2;
5447                 break;
5448         }
5449         case SYNCHRONIZE_CACHE_16: {
5450                 struct scsi_sync_cache_16 *cdb;
5451                 cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5452
5453                 starting_lba = scsi_8btou64(cdb->begin_lba);
5454                 block_count = scsi_4btoul(cdb->lb_count);
5455                 byte2 = cdb->byte2;
5456                 break;
5457         }
5458         default:
5459                 ctl_set_invalid_opcode(ctsio);
5460                 ctl_done((union ctl_io *)ctsio);
5461                 goto bailout;
5462                 break; /* NOTREACHED */
5463         }
5464
5465         /*
5466          * We check the LBA and length, but don't do anything with them.
5467          * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5468          * get flushed.  This check will just help satisfy anyone who wants
5469          * to see an error for an out of range LBA.
5470          */
5471         if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5472                 ctl_set_lba_out_of_range(ctsio);
5473                 ctl_done((union ctl_io *)ctsio);
5474                 goto bailout;
5475         }
5476
5477         /*
5478          * If this LUN has no backend, we can't flush the cache anyway.
5479          */
5480         if (lun->backend == NULL) {
5481                 ctl_set_invalid_opcode(ctsio);
5482                 ctl_done((union ctl_io *)ctsio);
5483                 goto bailout;
5484         }
5485
5486         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5487         lbalen->lba = starting_lba;
5488         lbalen->len = block_count;
5489         lbalen->flags = byte2;
5490
5491         /*
5492          * Check to see whether we're configured to send the SYNCHRONIZE
5493          * CACHE command directly to the back end.
5494          */
5495         mtx_lock(&lun->lun_lock);
5496         if ((softc->flags & CTL_FLAG_REAL_SYNC)
5497          && (++(lun->sync_count) >= lun->sync_interval)) {
5498                 lun->sync_count = 0;
5499                 mtx_unlock(&lun->lun_lock);
5500                 retval = lun->backend->config_write((union ctl_io *)ctsio);
5501         } else {
5502                 mtx_unlock(&lun->lun_lock);
5503                 ctl_set_success(ctsio);
5504                 ctl_done((union ctl_io *)ctsio);
5505         }
5506
5507 bailout:
5508
5509         return (retval);
5510 }
5511
5512 int
5513 ctl_format(struct ctl_scsiio *ctsio)
5514 {
5515         struct scsi_format *cdb;
5516         struct ctl_lun *lun;
5517         int length, defect_list_len;
5518
5519         CTL_DEBUG_PRINT(("ctl_format\n"));
5520
5521         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5522
5523         cdb = (struct scsi_format *)ctsio->cdb;
5524
5525         length = 0;
5526         if (cdb->byte2 & SF_FMTDATA) {
5527                 if (cdb->byte2 & SF_LONGLIST)
5528                         length = sizeof(struct scsi_format_header_long);
5529                 else
5530                         length = sizeof(struct scsi_format_header_short);
5531         }
5532
5533         if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5534          && (length > 0)) {
5535                 ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5536                 ctsio->kern_data_len = length;
5537                 ctsio->kern_total_len = length;
5538                 ctsio->kern_data_resid = 0;
5539                 ctsio->kern_rel_offset = 0;
5540                 ctsio->kern_sg_entries = 0;
5541                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5542                 ctsio->be_move_done = ctl_config_move_done;
5543                 ctl_datamove((union ctl_io *)ctsio);
5544
5545                 return (CTL_RETVAL_COMPLETE);
5546         }
5547
5548         defect_list_len = 0;
5549
5550         if (cdb->byte2 & SF_FMTDATA) {
5551                 if (cdb->byte2 & SF_LONGLIST) {
5552                         struct scsi_format_header_long *header;
5553
5554                         header = (struct scsi_format_header_long *)
5555                                 ctsio->kern_data_ptr;
5556
5557                         defect_list_len = scsi_4btoul(header->defect_list_len);
5558                         if (defect_list_len != 0) {
5559                                 ctl_set_invalid_field(ctsio,
5560                                                       /*sks_valid*/ 1,
5561                                                       /*command*/ 0,
5562                                                       /*field*/ 2,
5563                                                       /*bit_valid*/ 0,
5564                                                       /*bit*/ 0);
5565                                 goto bailout;
5566                         }
5567                 } else {
5568                         struct scsi_format_header_short *header;
5569
5570                         header = (struct scsi_format_header_short *)
5571                                 ctsio->kern_data_ptr;
5572
5573                         defect_list_len = scsi_2btoul(header->defect_list_len);
5574                         if (defect_list_len != 0) {
5575                                 ctl_set_invalid_field(ctsio,
5576                                                       /*sks_valid*/ 1,
5577                                                       /*command*/ 0,
5578                                                       /*field*/ 2,
5579                                                       /*bit_valid*/ 0,
5580                                                       /*bit*/ 0);
5581                                 goto bailout;
5582                         }
5583                 }
5584         }
5585
5586         /*
5587          * The format command will clear out the "Medium format corrupted"
5588          * status if set by the configuration code.  That status is really
5589          * just a way to notify the host that we have lost the media, and
5590          * get them to issue a command that will basically make them think
5591          * they're blowing away the media.
5592          */
5593         mtx_lock(&lun->lun_lock);
5594         lun->flags &= ~CTL_LUN_INOPERABLE;
5595         mtx_unlock(&lun->lun_lock);
5596
5597         ctl_set_success(ctsio);
5598 bailout:
5599
5600         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5601                 free(ctsio->kern_data_ptr, M_CTL);
5602                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5603         }
5604
5605         ctl_done((union ctl_io *)ctsio);
5606         return (CTL_RETVAL_COMPLETE);
5607 }
5608
5609 int
5610 ctl_read_buffer(struct ctl_scsiio *ctsio)
5611 {
5612         struct ctl_lun *lun;
5613         uint64_t buffer_offset;
5614         uint32_t len;
5615         uint8_t byte2;
5616         static uint8_t descr[4];
5617         static uint8_t echo_descr[4] = { 0 };
5618
5619         CTL_DEBUG_PRINT(("ctl_read_buffer\n"));
5620         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5621         switch (ctsio->cdb[0]) {
5622         case READ_BUFFER: {
5623                 struct scsi_read_buffer *cdb;
5624
5625                 cdb = (struct scsi_read_buffer *)ctsio->cdb;
5626                 buffer_offset = scsi_3btoul(cdb->offset);
5627                 len = scsi_3btoul(cdb->length);
5628                 byte2 = cdb->byte2;
5629                 break;
5630         }
5631         case READ_BUFFER_16: {
5632                 struct scsi_read_buffer_16 *cdb;
5633
5634                 cdb = (struct scsi_read_buffer_16 *)ctsio->cdb;
5635                 buffer_offset = scsi_8btou64(cdb->offset);
5636                 len = scsi_4btoul(cdb->length);
5637                 byte2 = cdb->byte2;
5638                 break;
5639         }
5640         default: /* This shouldn't happen. */
5641                 ctl_set_invalid_opcode(ctsio);
5642                 ctl_done((union ctl_io *)ctsio);
5643                 return (CTL_RETVAL_COMPLETE);
5644         }
5645
5646         if ((byte2 & RWB_MODE) != RWB_MODE_DATA &&
5647             (byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
5648             (byte2 & RWB_MODE) != RWB_MODE_DESCR) {
5649                 ctl_set_invalid_field(ctsio,
5650                                       /*sks_valid*/ 1,
5651                                       /*command*/ 1,
5652                                       /*field*/ 1,
5653                                       /*bit_valid*/ 1,
5654                                       /*bit*/ 4);
5655                 ctl_done((union ctl_io *)ctsio);
5656                 return (CTL_RETVAL_COMPLETE);
5657         }
5658
5659         if (buffer_offset > CTL_WRITE_BUFFER_SIZE ||
5660             buffer_offset + len > CTL_WRITE_BUFFER_SIZE) {
5661                 ctl_set_invalid_field(ctsio,
5662                                       /*sks_valid*/ 1,
5663                                       /*command*/ 1,
5664                                       /*field*/ 6,
5665                                       /*bit_valid*/ 0,
5666                                       /*bit*/ 0);
5667                 ctl_done((union ctl_io *)ctsio);
5668                 return (CTL_RETVAL_COMPLETE);
5669         }
5670
5671         if ((byte2 & RWB_MODE) == RWB_MODE_DESCR) {
5672                 descr[0] = 0;
5673                 scsi_ulto3b(CTL_WRITE_BUFFER_SIZE, &descr[1]);
5674                 ctsio->kern_data_ptr = descr;
5675                 len = min(len, sizeof(descr));
5676         } else if ((byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
5677                 ctsio->kern_data_ptr = echo_descr;
5678                 len = min(len, sizeof(echo_descr));
5679         } else {
5680                 if (lun->write_buffer == NULL) {
5681                         lun->write_buffer = malloc(CTL_WRITE_BUFFER_SIZE,
5682                             M_CTL, M_WAITOK);
5683                 }
5684                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5685         }
5686         ctsio->kern_data_len = len;
5687         ctsio->kern_total_len = len;
5688         ctsio->kern_data_resid = 0;
5689         ctsio->kern_rel_offset = 0;
5690         ctsio->kern_sg_entries = 0;
5691         ctl_set_success(ctsio);
5692         ctsio->be_move_done = ctl_config_move_done;
5693         ctl_datamove((union ctl_io *)ctsio);
5694         return (CTL_RETVAL_COMPLETE);
5695 }
5696
5697 int
5698 ctl_write_buffer(struct ctl_scsiio *ctsio)
5699 {
5700         struct scsi_write_buffer *cdb;
5701         struct ctl_lun *lun;
5702         int buffer_offset, len;
5703
5704         CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5705
5706         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5707         cdb = (struct scsi_write_buffer *)ctsio->cdb;
5708
5709         if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5710                 ctl_set_invalid_field(ctsio,
5711                                       /*sks_valid*/ 1,
5712                                       /*command*/ 1,
5713                                       /*field*/ 1,
5714                                       /*bit_valid*/ 1,
5715                                       /*bit*/ 4);
5716                 ctl_done((union ctl_io *)ctsio);
5717                 return (CTL_RETVAL_COMPLETE);
5718         }
5719
5720         len = scsi_3btoul(cdb->length);
5721         buffer_offset = scsi_3btoul(cdb->offset);
5722
5723         if (buffer_offset + len > CTL_WRITE_BUFFER_SIZE) {
5724                 ctl_set_invalid_field(ctsio,
5725                                       /*sks_valid*/ 1,
5726                                       /*command*/ 1,
5727                                       /*field*/ 6,
5728                                       /*bit_valid*/ 0,
5729                                       /*bit*/ 0);
5730                 ctl_done((union ctl_io *)ctsio);
5731                 return (CTL_RETVAL_COMPLETE);
5732         }
5733
5734         /*
5735          * If we've got a kernel request that hasn't been malloced yet,
5736          * malloc it and tell the caller the data buffer is here.
5737          */
5738         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5739                 if (lun->write_buffer == NULL) {
5740                         lun->write_buffer = malloc(CTL_WRITE_BUFFER_SIZE,
5741                             M_CTL, M_WAITOK);
5742                 }
5743                 ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5744                 ctsio->kern_data_len = len;
5745                 ctsio->kern_total_len = len;
5746                 ctsio->kern_data_resid = 0;
5747                 ctsio->kern_rel_offset = 0;
5748                 ctsio->kern_sg_entries = 0;
5749                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5750                 ctsio->be_move_done = ctl_config_move_done;
5751                 ctl_datamove((union ctl_io *)ctsio);
5752
5753                 return (CTL_RETVAL_COMPLETE);
5754         }
5755
5756         ctl_set_success(ctsio);
5757         ctl_done((union ctl_io *)ctsio);
5758         return (CTL_RETVAL_COMPLETE);
5759 }
5760
5761 int
5762 ctl_write_same(struct ctl_scsiio *ctsio)
5763 {
5764         struct ctl_lun *lun;
5765         struct ctl_lba_len_flags *lbalen;
5766         uint64_t lba;
5767         uint32_t num_blocks;
5768         int len, retval;
5769         uint8_t byte2;
5770
5771         CTL_DEBUG_PRINT(("ctl_write_same\n"));
5772
5773         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5774
5775         switch (ctsio->cdb[0]) {
5776         case WRITE_SAME_10: {
5777                 struct scsi_write_same_10 *cdb;
5778
5779                 cdb = (struct scsi_write_same_10 *)ctsio->cdb;
5780
5781                 lba = scsi_4btoul(cdb->addr);
5782                 num_blocks = scsi_2btoul(cdb->length);
5783                 byte2 = cdb->byte2;
5784                 break;
5785         }
5786         case WRITE_SAME_16: {
5787                 struct scsi_write_same_16 *cdb;
5788
5789                 cdb = (struct scsi_write_same_16 *)ctsio->cdb;
5790
5791                 lba = scsi_8btou64(cdb->addr);
5792                 num_blocks = scsi_4btoul(cdb->length);
5793                 byte2 = cdb->byte2;
5794                 break;
5795         }
5796         default:
5797                 /*
5798                  * We got a command we don't support.  This shouldn't
5799                  * happen, commands should be filtered out above us.
5800                  */
5801                 ctl_set_invalid_opcode(ctsio);
5802                 ctl_done((union ctl_io *)ctsio);
5803
5804                 return (CTL_RETVAL_COMPLETE);
5805                 break; /* NOTREACHED */
5806         }
5807
5808         /* ANCHOR flag can be used only together with UNMAP */
5809         if ((byte2 & SWS_UNMAP) == 0 && (byte2 & SWS_ANCHOR) != 0) {
5810                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
5811                     /*command*/ 1, /*field*/ 1, /*bit_valid*/ 1, /*bit*/ 0);
5812                 ctl_done((union ctl_io *)ctsio);
5813                 return (CTL_RETVAL_COMPLETE);
5814         }
5815
5816         /*
5817          * The first check is to make sure we're in bounds, the second
5818          * check is to catch wrap-around problems.  If the lba + num blocks
5819          * is less than the lba, then we've wrapped around and the block
5820          * range is invalid anyway.
5821          */
5822         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5823          || ((lba + num_blocks) < lba)) {
5824                 ctl_set_lba_out_of_range(ctsio);
5825                 ctl_done((union ctl_io *)ctsio);
5826                 return (CTL_RETVAL_COMPLETE);
5827         }
5828
5829         /* Zero number of blocks means "to the last logical block" */
5830         if (num_blocks == 0) {
5831                 if ((lun->be_lun->maxlba + 1) - lba > UINT32_MAX) {
5832                         ctl_set_invalid_field(ctsio,
5833                                               /*sks_valid*/ 0,
5834                                               /*command*/ 1,
5835                                               /*field*/ 0,
5836                                               /*bit_valid*/ 0,
5837                                               /*bit*/ 0);
5838                         ctl_done((union ctl_io *)ctsio);
5839                         return (CTL_RETVAL_COMPLETE);
5840                 }
5841                 num_blocks = (lun->be_lun->maxlba + 1) - lba;
5842         }
5843
5844         len = lun->be_lun->blocksize;
5845
5846         /*
5847          * If we've got a kernel request that hasn't been malloced yet,
5848          * malloc it and tell the caller the data buffer is here.
5849          */
5850         if ((byte2 & SWS_NDOB) == 0 &&
5851             (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5852                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5853                 ctsio->kern_data_len = len;
5854                 ctsio->kern_total_len = len;
5855                 ctsio->kern_data_resid = 0;
5856                 ctsio->kern_rel_offset = 0;
5857                 ctsio->kern_sg_entries = 0;
5858                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5859                 ctsio->be_move_done = ctl_config_move_done;
5860                 ctl_datamove((union ctl_io *)ctsio);
5861
5862                 return (CTL_RETVAL_COMPLETE);
5863         }
5864
5865         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5866         lbalen->lba = lba;
5867         lbalen->len = num_blocks;
5868         lbalen->flags = byte2;
5869         retval = lun->backend->config_write((union ctl_io *)ctsio);
5870
5871         return (retval);
5872 }
5873
5874 int
5875 ctl_unmap(struct ctl_scsiio *ctsio)
5876 {
5877         struct ctl_lun *lun;
5878         struct scsi_unmap *cdb;
5879         struct ctl_ptr_len_flags *ptrlen;
5880         struct scsi_unmap_header *hdr;
5881         struct scsi_unmap_desc *buf, *end, *endnz, *range;
5882         uint64_t lba;
5883         uint32_t num_blocks;
5884         int len, retval;
5885         uint8_t byte2;
5886
5887         CTL_DEBUG_PRINT(("ctl_unmap\n"));
5888
5889         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5890         cdb = (struct scsi_unmap *)ctsio->cdb;
5891
5892         len = scsi_2btoul(cdb->length);
5893         byte2 = cdb->byte2;
5894
5895         /*
5896          * If we've got a kernel request that hasn't been malloced yet,
5897          * malloc it and tell the caller the data buffer is here.
5898          */
5899         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5900                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5901                 ctsio->kern_data_len = len;
5902                 ctsio->kern_total_len = len;
5903                 ctsio->kern_data_resid = 0;
5904                 ctsio->kern_rel_offset = 0;
5905                 ctsio->kern_sg_entries = 0;
5906                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5907                 ctsio->be_move_done = ctl_config_move_done;
5908                 ctl_datamove((union ctl_io *)ctsio);
5909
5910                 return (CTL_RETVAL_COMPLETE);
5911         }
5912
5913         len = ctsio->kern_total_len - ctsio->kern_data_resid;
5914         hdr = (struct scsi_unmap_header *)ctsio->kern_data_ptr;
5915         if (len < sizeof (*hdr) ||
5916             len < (scsi_2btoul(hdr->length) + sizeof(hdr->length)) ||
5917             len < (scsi_2btoul(hdr->desc_length) + sizeof (*hdr)) ||
5918             scsi_2btoul(hdr->desc_length) % sizeof(*buf) != 0) {
5919                 ctl_set_invalid_field(ctsio,
5920                                       /*sks_valid*/ 0,
5921                                       /*command*/ 0,
5922                                       /*field*/ 0,
5923                                       /*bit_valid*/ 0,
5924                                       /*bit*/ 0);
5925                 goto done;
5926         }
5927         len = scsi_2btoul(hdr->desc_length);
5928         buf = (struct scsi_unmap_desc *)(hdr + 1);
5929         end = buf + len / sizeof(*buf);
5930
5931         endnz = buf;
5932         for (range = buf; range < end; range++) {
5933                 lba = scsi_8btou64(range->lba);
5934                 num_blocks = scsi_4btoul(range->length);
5935                 if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5936                  || ((lba + num_blocks) < lba)) {
5937                         ctl_set_lba_out_of_range(ctsio);
5938                         ctl_done((union ctl_io *)ctsio);
5939                         return (CTL_RETVAL_COMPLETE);
5940                 }
5941                 if (num_blocks != 0)
5942                         endnz = range + 1;
5943         }
5944
5945         /*
5946          * Block backend can not handle zero last range.
5947          * Filter it out and return if there is nothing left.
5948          */
5949         len = (uint8_t *)endnz - (uint8_t *)buf;
5950         if (len == 0) {
5951                 ctl_set_success(ctsio);
5952                 goto done;
5953         }
5954
5955         mtx_lock(&lun->lun_lock);
5956         ptrlen = (struct ctl_ptr_len_flags *)
5957             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5958         ptrlen->ptr = (void *)buf;
5959         ptrlen->len = len;
5960         ptrlen->flags = byte2;
5961         ctl_check_blocked(lun);
5962         mtx_unlock(&lun->lun_lock);
5963
5964         retval = lun->backend->config_write((union ctl_io *)ctsio);
5965         return (retval);
5966
5967 done:
5968         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5969                 free(ctsio->kern_data_ptr, M_CTL);
5970                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5971         }
5972         ctl_done((union ctl_io *)ctsio);
5973         return (CTL_RETVAL_COMPLETE);
5974 }
5975
5976 /*
5977  * Note that this function currently doesn't actually do anything inside
5978  * CTL to enforce things if the DQue bit is turned on.
5979  *
5980  * Also note that this function can't be used in the default case, because
5981  * the DQue bit isn't set in the changeable mask for the control mode page
5982  * anyway.  This is just here as an example for how to implement a page
5983  * handler, and a placeholder in case we want to allow the user to turn
5984  * tagged queueing on and off.
5985  *
5986  * The D_SENSE bit handling is functional, however, and will turn
5987  * descriptor sense on and off for a given LUN.
5988  */
5989 int
5990 ctl_control_page_handler(struct ctl_scsiio *ctsio,
5991                          struct ctl_page_index *page_index, uint8_t *page_ptr)
5992 {
5993         struct scsi_control_page *current_cp, *saved_cp, *user_cp;
5994         struct ctl_lun *lun;
5995         int set_ua;
5996         uint32_t initidx;
5997
5998         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5999         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
6000         set_ua = 0;
6001
6002         user_cp = (struct scsi_control_page *)page_ptr;
6003         current_cp = (struct scsi_control_page *)
6004                 (page_index->page_data + (page_index->page_len *
6005                 CTL_PAGE_CURRENT));
6006         saved_cp = (struct scsi_control_page *)
6007                 (page_index->page_data + (page_index->page_len *
6008                 CTL_PAGE_SAVED));
6009
6010         mtx_lock(&lun->lun_lock);
6011         if (((current_cp->rlec & SCP_DSENSE) == 0)
6012          && ((user_cp->rlec & SCP_DSENSE) != 0)) {
6013                 /*
6014                  * Descriptor sense is currently turned off and the user
6015                  * wants to turn it on.
6016                  */
6017                 current_cp->rlec |= SCP_DSENSE;
6018                 saved_cp->rlec |= SCP_DSENSE;
6019                 lun->flags |= CTL_LUN_SENSE_DESC;
6020                 set_ua = 1;
6021         } else if (((current_cp->rlec & SCP_DSENSE) != 0)
6022                 && ((user_cp->rlec & SCP_DSENSE) == 0)) {
6023                 /*
6024                  * Descriptor sense is currently turned on, and the user
6025                  * wants to turn it off.
6026                  */
6027                 current_cp->rlec &= ~SCP_DSENSE;
6028                 saved_cp->rlec &= ~SCP_DSENSE;
6029                 lun->flags &= ~CTL_LUN_SENSE_DESC;
6030                 set_ua = 1;
6031         }
6032         if ((current_cp->queue_flags & SCP_QUEUE_ALG_MASK) !=
6033             (user_cp->queue_flags & SCP_QUEUE_ALG_MASK)) {
6034                 current_cp->queue_flags &= ~SCP_QUEUE_ALG_MASK;
6035                 current_cp->queue_flags |= user_cp->queue_flags & SCP_QUEUE_ALG_MASK;
6036                 saved_cp->queue_flags &= ~SCP_QUEUE_ALG_MASK;
6037                 saved_cp->queue_flags |= user_cp->queue_flags & SCP_QUEUE_ALG_MASK;
6038                 set_ua = 1;
6039         }
6040         if ((current_cp->eca_and_aen & SCP_SWP) !=
6041             (user_cp->eca_and_aen & SCP_SWP)) {
6042                 current_cp->eca_and_aen &= ~SCP_SWP;
6043                 current_cp->eca_and_aen |= user_cp->eca_and_aen & SCP_SWP;
6044                 saved_cp->eca_and_aen &= ~SCP_SWP;
6045                 saved_cp->eca_and_aen |= user_cp->eca_and_aen & SCP_SWP;
6046                 set_ua = 1;
6047         }
6048         if (set_ua != 0)
6049                 ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
6050         mtx_unlock(&lun->lun_lock);
6051         if (set_ua) {
6052                 ctl_isc_announce_mode(lun,
6053                     ctl_get_initindex(&ctsio->io_hdr.nexus),
6054                     page_index->page_code, page_index->subpage);
6055         }
6056         return (0);
6057 }
6058
6059 int
6060 ctl_caching_sp_handler(struct ctl_scsiio *ctsio,
6061                      struct ctl_page_index *page_index, uint8_t *page_ptr)
6062 {
6063         struct scsi_caching_page *current_cp, *saved_cp, *user_cp;
6064         struct ctl_lun *lun;
6065         int set_ua;
6066         uint32_t initidx;
6067
6068         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6069         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
6070         set_ua = 0;
6071
6072         user_cp = (struct scsi_caching_page *)page_ptr;
6073         current_cp = (struct scsi_caching_page *)
6074                 (page_index->page_data + (page_index->page_len *
6075                 CTL_PAGE_CURRENT));
6076         saved_cp = (struct scsi_caching_page *)
6077                 (page_index->page_data + (page_index->page_len *
6078                 CTL_PAGE_SAVED));
6079
6080         mtx_lock(&lun->lun_lock);
6081         if ((current_cp->flags1 & (SCP_WCE | SCP_RCD)) !=
6082             (user_cp->flags1 & (SCP_WCE | SCP_RCD))) {
6083                 current_cp->flags1 &= ~(SCP_WCE | SCP_RCD);
6084                 current_cp->flags1 |= user_cp->flags1 & (SCP_WCE | SCP_RCD);
6085                 saved_cp->flags1 &= ~(SCP_WCE | SCP_RCD);
6086                 saved_cp->flags1 |= user_cp->flags1 & (SCP_WCE | SCP_RCD);
6087                 set_ua = 1;
6088         }
6089         if (set_ua != 0)
6090                 ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
6091         mtx_unlock(&lun->lun_lock);
6092         if (set_ua) {
6093                 ctl_isc_announce_mode(lun,
6094                     ctl_get_initindex(&ctsio->io_hdr.nexus),
6095                     page_index->page_code, page_index->subpage);
6096         }
6097         return (0);
6098 }
6099
6100 int
6101 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6102                                 struct ctl_page_index *page_index,
6103                                 uint8_t *page_ptr)
6104 {
6105         uint8_t *c;
6106         int i;
6107
6108         c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6109         ctl_time_io_secs =
6110                 (c[0] << 8) |
6111                 (c[1] << 0) |
6112                 0;
6113         CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6114         printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6115         printf("page data:");
6116         for (i=0; i<8; i++)
6117                 printf(" %.2x",page_ptr[i]);
6118         printf("\n");
6119         return (0);
6120 }
6121
6122 int
6123 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6124                                struct ctl_page_index *page_index,
6125                                int pc)
6126 {
6127         struct copan_debugconf_subpage *page;
6128
6129         page = (struct copan_debugconf_subpage *)page_index->page_data +
6130                 (page_index->page_len * pc);
6131
6132         switch (pc) {
6133         case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6134         case SMS_PAGE_CTRL_DEFAULT >> 6:
6135         case SMS_PAGE_CTRL_SAVED >> 6:
6136                 /*
6137                  * We don't update the changable or default bits for this page.
6138                  */
6139                 break;
6140         case SMS_PAGE_CTRL_CURRENT >> 6:
6141                 page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6142                 page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6143                 break;
6144         default:
6145 #ifdef NEEDTOPORT
6146                 EPRINT(0, "Invalid PC %d!!", pc);
6147 #endif /* NEEDTOPORT */
6148                 break;
6149         }
6150         return (0);
6151 }
6152
6153
6154 static int
6155 ctl_do_mode_select(union ctl_io *io)
6156 {
6157         struct scsi_mode_page_header *page_header;
6158         struct ctl_page_index *page_index;
6159         struct ctl_scsiio *ctsio;
6160         int control_dev, page_len;
6161         int page_len_offset, page_len_size;
6162         union ctl_modepage_info *modepage_info;
6163         struct ctl_lun *lun;
6164         int *len_left, *len_used;
6165         int retval, i;
6166
6167         ctsio = &io->scsiio;
6168         page_index = NULL;
6169         page_len = 0;
6170         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6171
6172         if (lun->be_lun->lun_type != T_DIRECT)
6173                 control_dev = 1;
6174         else
6175                 control_dev = 0;
6176
6177         modepage_info = (union ctl_modepage_info *)
6178                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6179         len_left = &modepage_info->header.len_left;
6180         len_used = &modepage_info->header.len_used;
6181
6182 do_next_page:
6183
6184         page_header = (struct scsi_mode_page_header *)
6185                 (ctsio->kern_data_ptr + *len_used);
6186
6187         if (*len_left == 0) {
6188                 free(ctsio->kern_data_ptr, M_CTL);
6189                 ctl_set_success(ctsio);
6190                 ctl_done((union ctl_io *)ctsio);
6191                 return (CTL_RETVAL_COMPLETE);
6192         } else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6193
6194                 free(ctsio->kern_data_ptr, M_CTL);
6195                 ctl_set_param_len_error(ctsio);
6196                 ctl_done((union ctl_io *)ctsio);
6197                 return (CTL_RETVAL_COMPLETE);
6198
6199         } else if ((page_header->page_code & SMPH_SPF)
6200                 && (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6201
6202                 free(ctsio->kern_data_ptr, M_CTL);
6203                 ctl_set_param_len_error(ctsio);
6204                 ctl_done((union ctl_io *)ctsio);
6205                 return (CTL_RETVAL_COMPLETE);
6206         }
6207
6208
6209         /*
6210          * XXX KDM should we do something with the block descriptor?
6211          */
6212         for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6213
6214                 if ((control_dev != 0)
6215                  && (lun->mode_pages.index[i].page_flags &
6216                      CTL_PAGE_FLAG_DISK_ONLY))
6217                         continue;
6218
6219                 if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6220                     (page_header->page_code & SMPH_PC_MASK))
6221                         continue;
6222
6223                 /*
6224                  * If neither page has a subpage code, then we've got a
6225                  * match.
6226                  */
6227                 if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6228                  && ((page_header->page_code & SMPH_SPF) == 0)) {
6229                         page_index = &lun->mode_pages.index[i];
6230                         page_len = page_header->page_length;
6231                         break;
6232                 }
6233
6234                 /*
6235                  * If both pages have subpages, then the subpage numbers
6236                  * have to match.
6237                  */
6238                 if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6239                   && (page_header->page_code & SMPH_SPF)) {
6240                         struct scsi_mode_page_header_sp *sph;
6241
6242                         sph = (struct scsi_mode_page_header_sp *)page_header;
6243
6244                         if (lun->mode_pages.index[i].subpage ==
6245                             sph->subpage) {
6246                                 page_index = &lun->mode_pages.index[i];
6247                                 page_len = scsi_2btoul(sph->page_length);
6248                                 break;
6249                         }
6250                 }
6251         }
6252
6253         /*
6254          * If we couldn't find the page, or if we don't have a mode select
6255          * handler for it, send back an error to the user.
6256          */
6257         if ((page_index == NULL)
6258          || (page_index->select_handler == NULL)) {
6259                 ctl_set_invalid_field(ctsio,
6260                                       /*sks_valid*/ 1,
6261                                       /*command*/ 0,
6262                                       /*field*/ *len_used,
6263                                       /*bit_valid*/ 0,
6264                                       /*bit*/ 0);
6265                 free(ctsio->kern_data_ptr, M_CTL);
6266                 ctl_done((union ctl_io *)ctsio);
6267                 return (CTL_RETVAL_COMPLETE);
6268         }
6269
6270         if (page_index->page_code & SMPH_SPF) {
6271                 page_len_offset = 2;
6272                 page_len_size = 2;
6273         } else {
6274                 page_len_size = 1;
6275                 page_len_offset = 1;
6276         }
6277
6278         /*
6279          * If the length the initiator gives us isn't the one we specify in
6280          * the mode page header, or if they didn't specify enough data in
6281          * the CDB to avoid truncating this page, kick out the request.
6282          */
6283         if ((page_len != (page_index->page_len - page_len_offset -
6284                           page_len_size))
6285          || (*len_left < page_index->page_len)) {
6286
6287
6288                 ctl_set_invalid_field(ctsio,
6289                                       /*sks_valid*/ 1,
6290                                       /*command*/ 0,
6291                                       /*field*/ *len_used + page_len_offset,
6292                                       /*bit_valid*/ 0,
6293                                       /*bit*/ 0);
6294                 free(ctsio->kern_data_ptr, M_CTL);
6295                 ctl_done((union ctl_io *)ctsio);
6296                 return (CTL_RETVAL_COMPLETE);
6297         }
6298
6299         /*
6300          * Run through the mode page, checking to make sure that the bits
6301          * the user changed are actually legal for him to change.
6302          */
6303         for (i = 0; i < page_index->page_len; i++) {
6304                 uint8_t *user_byte, *change_mask, *current_byte;
6305                 int bad_bit;
6306                 int j;
6307
6308                 user_byte = (uint8_t *)page_header + i;
6309                 change_mask = page_index->page_data +
6310                               (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6311                 current_byte = page_index->page_data +
6312                                (page_index->page_len * CTL_PAGE_CURRENT) + i;
6313
6314                 /*
6315                  * Check to see whether the user set any bits in this byte
6316                  * that he is not allowed to set.
6317                  */
6318                 if ((*user_byte & ~(*change_mask)) ==
6319                     (*current_byte & ~(*change_mask)))
6320                         continue;
6321
6322                 /*
6323                  * Go through bit by bit to determine which one is illegal.
6324                  */
6325                 bad_bit = 0;
6326                 for (j = 7; j >= 0; j--) {
6327                         if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6328                             (((1 << i) & ~(*change_mask)) & *current_byte)) {
6329                                 bad_bit = i;
6330                                 break;
6331                         }
6332                 }
6333                 ctl_set_invalid_field(ctsio,
6334                                       /*sks_valid*/ 1,
6335                                       /*command*/ 0,
6336                                       /*field*/ *len_used + i,
6337                                       /*bit_valid*/ 1,
6338                                       /*bit*/ bad_bit);
6339                 free(ctsio->kern_data_ptr, M_CTL);
6340                 ctl_done((union ctl_io *)ctsio);
6341                 return (CTL_RETVAL_COMPLETE);
6342         }
6343
6344         /*
6345          * Decrement these before we call the page handler, since we may
6346          * end up getting called back one way or another before the handler
6347          * returns to this context.
6348          */
6349         *len_left -= page_index->page_len;
6350         *len_used += page_index->page_len;
6351
6352         retval = page_index->select_handler(ctsio, page_index,
6353                                             (uint8_t *)page_header);
6354
6355         /*
6356          * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6357          * wait until this queued command completes to finish processing
6358          * the mode page.  If it returns anything other than
6359          * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6360          * already set the sense information, freed the data pointer, and
6361          * completed the io for us.
6362          */
6363         if (retval != CTL_RETVAL_COMPLETE)
6364                 goto bailout_no_done;
6365
6366         /*
6367          * If the initiator sent us more than one page, parse the next one.
6368          */
6369         if (*len_left > 0)
6370                 goto do_next_page;
6371
6372         ctl_set_success(ctsio);
6373         free(ctsio->kern_data_ptr, M_CTL);
6374         ctl_done((union ctl_io *)ctsio);
6375
6376 bailout_no_done:
6377
6378         return (CTL_RETVAL_COMPLETE);
6379
6380 }
6381
6382 int
6383 ctl_mode_select(struct ctl_scsiio *ctsio)
6384 {
6385         int param_len, pf, sp;
6386         int header_size, bd_len;
6387         union ctl_modepage_info *modepage_info;
6388
6389         switch (ctsio->cdb[0]) {
6390         case MODE_SELECT_6: {
6391                 struct scsi_mode_select_6 *cdb;
6392
6393                 cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6394
6395                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6396                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6397                 param_len = cdb->length;
6398                 header_size = sizeof(struct scsi_mode_header_6);
6399                 break;
6400         }
6401         case MODE_SELECT_10: {
6402                 struct scsi_mode_select_10 *cdb;
6403
6404                 cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6405
6406                 pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6407                 sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6408                 param_len = scsi_2btoul(cdb->length);
6409                 header_size = sizeof(struct scsi_mode_header_10);
6410                 break;
6411         }
6412         default:
6413                 ctl_set_invalid_opcode(ctsio);
6414                 ctl_done((union ctl_io *)ctsio);
6415                 return (CTL_RETVAL_COMPLETE);
6416         }
6417
6418         /*
6419          * From SPC-3:
6420          * "A parameter list length of zero indicates that the Data-Out Buffer
6421          * shall be empty. This condition shall not be considered as an error."
6422          */
6423         if (param_len == 0) {
6424                 ctl_set_success(ctsio);
6425                 ctl_done((union ctl_io *)ctsio);
6426                 return (CTL_RETVAL_COMPLETE);
6427         }
6428
6429         /*
6430          * Since we'll hit this the first time through, prior to
6431          * allocation, we don't need to free a data buffer here.
6432          */
6433         if (param_len < header_size) {
6434                 ctl_set_param_len_error(ctsio);
6435                 ctl_done((union ctl_io *)ctsio);
6436                 return (CTL_RETVAL_COMPLETE);
6437         }
6438
6439         /*
6440          * Allocate the data buffer and grab the user's data.  In theory,
6441          * we shouldn't have to sanity check the parameter list length here
6442          * because the maximum size is 64K.  We should be able to malloc
6443          * that much without too many problems.
6444          */
6445         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6446                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6447                 ctsio->kern_data_len = param_len;
6448                 ctsio->kern_total_len = param_len;
6449                 ctsio->kern_data_resid = 0;
6450                 ctsio->kern_rel_offset = 0;
6451                 ctsio->kern_sg_entries = 0;
6452                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6453                 ctsio->be_move_done = ctl_config_move_done;
6454                 ctl_datamove((union ctl_io *)ctsio);
6455
6456                 return (CTL_RETVAL_COMPLETE);
6457         }
6458
6459         switch (ctsio->cdb[0]) {
6460         case MODE_SELECT_6: {
6461                 struct scsi_mode_header_6 *mh6;
6462
6463                 mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6464                 bd_len = mh6->blk_desc_len;
6465                 break;
6466         }
6467         case MODE_SELECT_10: {
6468                 struct scsi_mode_header_10 *mh10;
6469
6470                 mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6471                 bd_len = scsi_2btoul(mh10->blk_desc_len);
6472                 break;
6473         }
6474         default:
6475                 panic("Invalid CDB type %#x", ctsio->cdb[0]);
6476                 break;
6477         }
6478
6479         if (param_len < (header_size + bd_len)) {
6480                 free(ctsio->kern_data_ptr, M_CTL);
6481                 ctl_set_param_len_error(ctsio);
6482                 ctl_done((union ctl_io *)ctsio);
6483                 return (CTL_RETVAL_COMPLETE);
6484         }
6485
6486         /*
6487          * Set the IO_CONT flag, so that if this I/O gets passed to
6488          * ctl_config_write_done(), it'll get passed back to
6489          * ctl_do_mode_select() for further processing, or completion if
6490          * we're all done.
6491          */
6492         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6493         ctsio->io_cont = ctl_do_mode_select;
6494
6495         modepage_info = (union ctl_modepage_info *)
6496                 ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6497         memset(modepage_info, 0, sizeof(*modepage_info));
6498         modepage_info->header.len_left = param_len - header_size - bd_len;
6499         modepage_info->header.len_used = header_size + bd_len;
6500
6501         return (ctl_do_mode_select((union ctl_io *)ctsio));
6502 }
6503
6504 int
6505 ctl_mode_sense(struct ctl_scsiio *ctsio)
6506 {
6507         struct ctl_lun *lun;
6508         int pc, page_code, dbd, llba, subpage;
6509         int alloc_len, page_len, header_len, total_len;
6510         struct scsi_mode_block_descr *block_desc;
6511         struct ctl_page_index *page_index;
6512         int control_dev;
6513
6514         dbd = 0;
6515         llba = 0;
6516         block_desc = NULL;
6517         page_index = NULL;
6518
6519         CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6520
6521         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6522
6523         if (lun->be_lun->lun_type != T_DIRECT)
6524                 control_dev = 1;
6525         else
6526                 control_dev = 0;
6527
6528         switch (ctsio->cdb[0]) {
6529         case MODE_SENSE_6: {
6530                 struct scsi_mode_sense_6 *cdb;
6531
6532                 cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6533
6534                 header_len = sizeof(struct scsi_mode_hdr_6);
6535                 if (cdb->byte2 & SMS_DBD)
6536                         dbd = 1;
6537                 else
6538                         header_len += sizeof(struct scsi_mode_block_descr);
6539
6540                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6541                 page_code = cdb->page & SMS_PAGE_CODE;
6542                 subpage = cdb->subpage;
6543                 alloc_len = cdb->length;
6544                 break;
6545         }
6546         case MODE_SENSE_10: {
6547                 struct scsi_mode_sense_10 *cdb;
6548
6549                 cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6550
6551                 header_len = sizeof(struct scsi_mode_hdr_10);
6552
6553                 if (cdb->byte2 & SMS_DBD)
6554                         dbd = 1;
6555                 else
6556                         header_len += sizeof(struct scsi_mode_block_descr);
6557                 if (cdb->byte2 & SMS10_LLBAA)
6558                         llba = 1;
6559                 pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6560                 page_code = cdb->page & SMS_PAGE_CODE;
6561                 subpage = cdb->subpage;
6562                 alloc_len = scsi_2btoul(cdb->length);
6563                 break;
6564         }
6565         default:
6566                 ctl_set_invalid_opcode(ctsio);
6567                 ctl_done((union ctl_io *)ctsio);
6568                 return (CTL_RETVAL_COMPLETE);
6569                 break; /* NOTREACHED */
6570         }
6571
6572         /*
6573          * We have to make a first pass through to calculate the size of
6574          * the pages that match the user's query.  Then we allocate enough
6575          * memory to hold it, and actually copy the data into the buffer.
6576          */
6577         switch (page_code) {
6578         case SMS_ALL_PAGES_PAGE: {
6579                 int i;
6580
6581                 page_len = 0;
6582
6583                 /*
6584                  * At the moment, values other than 0 and 0xff here are
6585                  * reserved according to SPC-3.
6586                  */
6587                 if ((subpage != SMS_SUBPAGE_PAGE_0)
6588                  && (subpage != SMS_SUBPAGE_ALL)) {
6589                         ctl_set_invalid_field(ctsio,
6590                                               /*sks_valid*/ 1,
6591                                               /*command*/ 1,
6592                                               /*field*/ 3,
6593                                               /*bit_valid*/ 0,
6594                                               /*bit*/ 0);
6595                         ctl_done((union ctl_io *)ctsio);
6596                         return (CTL_RETVAL_COMPLETE);
6597                 }
6598
6599                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6600                         if ((control_dev != 0)
6601                          && (lun->mode_pages.index[i].page_flags &
6602                              CTL_PAGE_FLAG_DISK_ONLY))
6603                                 continue;
6604
6605                         /*
6606                          * We don't use this subpage if the user didn't
6607                          * request all subpages.
6608                          */
6609                         if ((lun->mode_pages.index[i].subpage != 0)
6610                          && (subpage == SMS_SUBPAGE_PAGE_0))
6611                                 continue;
6612
6613 #if 0
6614                         printf("found page %#x len %d\n",
6615                                lun->mode_pages.index[i].page_code &
6616                                SMPH_PC_MASK,
6617                                lun->mode_pages.index[i].page_len);
6618 #endif
6619                         page_len += lun->mode_pages.index[i].page_len;
6620                 }
6621                 break;
6622         }
6623         default: {
6624                 int i;
6625
6626                 page_len = 0;
6627
6628                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6629                         /* Look for the right page code */
6630                         if ((lun->mode_pages.index[i].page_code &
6631                              SMPH_PC_MASK) != page_code)
6632                                 continue;
6633
6634                         /* Look for the right subpage or the subpage wildcard*/
6635                         if ((lun->mode_pages.index[i].subpage != subpage)
6636                          && (subpage != SMS_SUBPAGE_ALL))
6637                                 continue;
6638
6639                         /* Make sure the page is supported for this dev type */
6640                         if ((control_dev != 0)
6641                          && (lun->mode_pages.index[i].page_flags &
6642                              CTL_PAGE_FLAG_DISK_ONLY))
6643                                 continue;
6644
6645 #if 0
6646                         printf("found page %#x len %d\n",
6647                                lun->mode_pages.index[i].page_code &
6648                                SMPH_PC_MASK,
6649                                lun->mode_pages.index[i].page_len);
6650 #endif
6651
6652                         page_len += lun->mode_pages.index[i].page_len;
6653                 }
6654
6655                 if (page_len == 0) {
6656                         ctl_set_invalid_field(ctsio,
6657                                               /*sks_valid*/ 1,
6658                                               /*command*/ 1,
6659                                               /*field*/ 2,
6660                                               /*bit_valid*/ 1,
6661                                               /*bit*/ 5);
6662                         ctl_done((union ctl_io *)ctsio);
6663                         return (CTL_RETVAL_COMPLETE);
6664                 }
6665                 break;
6666         }
6667         }
6668
6669         total_len = header_len + page_len;
6670 #if 0
6671         printf("header_len = %d, page_len = %d, total_len = %d\n",
6672                header_len, page_len, total_len);
6673 #endif
6674
6675         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6676         ctsio->kern_sg_entries = 0;
6677         ctsio->kern_data_resid = 0;
6678         ctsio->kern_rel_offset = 0;
6679         if (total_len < alloc_len) {
6680                 ctsio->residual = alloc_len - total_len;
6681                 ctsio->kern_data_len = total_len;
6682                 ctsio->kern_total_len = total_len;
6683         } else {
6684                 ctsio->residual = 0;
6685                 ctsio->kern_data_len = alloc_len;
6686                 ctsio->kern_total_len = alloc_len;
6687         }
6688
6689         switch (ctsio->cdb[0]) {
6690         case MODE_SENSE_6: {
6691                 struct scsi_mode_hdr_6 *header;
6692
6693                 header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6694
6695                 header->datalen = MIN(total_len - 1, 254);
6696                 if (control_dev == 0) {
6697                         header->dev_specific = 0x10; /* DPOFUA */
6698                         if ((lun->be_lun->flags & CTL_LUN_FLAG_READONLY) ||
6699                             (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
6700                             .eca_and_aen & SCP_SWP) != 0)
6701                                     header->dev_specific |= 0x80; /* WP */
6702                 }
6703                 if (dbd)
6704                         header->block_descr_len = 0;
6705                 else
6706                         header->block_descr_len =
6707                                 sizeof(struct scsi_mode_block_descr);
6708                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6709                 break;
6710         }
6711         case MODE_SENSE_10: {
6712                 struct scsi_mode_hdr_10 *header;
6713                 int datalen;
6714
6715                 header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6716
6717                 datalen = MIN(total_len - 2, 65533);
6718                 scsi_ulto2b(datalen, header->datalen);
6719                 if (control_dev == 0) {
6720                         header->dev_specific = 0x10; /* DPOFUA */
6721                         if ((lun->be_lun->flags & CTL_LUN_FLAG_READONLY) ||
6722                             (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
6723                             .eca_and_aen & SCP_SWP) != 0)
6724                                     header->dev_specific |= 0x80; /* WP */
6725                 }
6726                 if (dbd)
6727                         scsi_ulto2b(0, header->block_descr_len);
6728                 else
6729                         scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
6730                                     header->block_descr_len);
6731                 block_desc = (struct scsi_mode_block_descr *)&header[1];
6732                 break;
6733         }
6734         default:
6735                 panic("invalid CDB type %#x", ctsio->cdb[0]);
6736                 break; /* NOTREACHED */
6737         }
6738
6739         /*
6740          * If we've got a disk, use its blocksize in the block
6741          * descriptor.  Otherwise, just set it to 0.
6742          */
6743         if (dbd == 0) {
6744                 if (control_dev == 0)
6745                         scsi_ulto3b(lun->be_lun->blocksize,
6746                                     block_desc->block_len);
6747                 else
6748                         scsi_ulto3b(0, block_desc->block_len);
6749         }
6750
6751         switch (page_code) {
6752         case SMS_ALL_PAGES_PAGE: {
6753                 int i, data_used;
6754
6755                 data_used = header_len;
6756                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6757                         struct ctl_page_index *page_index;
6758
6759                         page_index = &lun->mode_pages.index[i];
6760
6761                         if ((control_dev != 0)
6762                          && (page_index->page_flags &
6763                             CTL_PAGE_FLAG_DISK_ONLY))
6764                                 continue;
6765
6766                         /*
6767                          * We don't use this subpage if the user didn't
6768                          * request all subpages.  We already checked (above)
6769                          * to make sure the user only specified a subpage
6770                          * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
6771                          */
6772                         if ((page_index->subpage != 0)
6773                          && (subpage == SMS_SUBPAGE_PAGE_0))
6774                                 continue;
6775
6776                         /*
6777                          * Call the handler, if it exists, to update the
6778                          * page to the latest values.
6779                          */
6780                         if (page_index->sense_handler != NULL)
6781                                 page_index->sense_handler(ctsio, page_index,pc);
6782
6783                         memcpy(ctsio->kern_data_ptr + data_used,
6784                                page_index->page_data +
6785                                (page_index->page_len * pc),
6786                                page_index->page_len);
6787                         data_used += page_index->page_len;
6788                 }
6789                 break;
6790         }
6791         default: {
6792                 int i, data_used;
6793
6794                 data_used = header_len;
6795
6796                 for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6797                         struct ctl_page_index *page_index;
6798
6799                         page_index = &lun->mode_pages.index[i];
6800
6801                         /* Look for the right page code */
6802                         if ((page_index->page_code & SMPH_PC_MASK) != page_code)
6803                                 continue;
6804
6805                         /* Look for the right subpage or the subpage wildcard*/
6806                         if ((page_index->subpage != subpage)
6807                          && (subpage != SMS_SUBPAGE_ALL))
6808                                 continue;
6809
6810                         /* Make sure the page is supported for this dev type */
6811                         if ((control_dev != 0)
6812                          && (page_index->page_flags &
6813                              CTL_PAGE_FLAG_DISK_ONLY))
6814                                 continue;
6815
6816                         /*
6817                          * Call the handler, if it exists, to update the
6818                          * page to the latest values.
6819                          */
6820                         if (page_index->sense_handler != NULL)
6821                                 page_index->sense_handler(ctsio, page_index,pc);
6822
6823                         memcpy(ctsio->kern_data_ptr + data_used,
6824                                page_index->page_data +
6825                                (page_index->page_len * pc),
6826                                page_index->page_len);
6827                         data_used += page_index->page_len;
6828                 }
6829                 break;
6830         }
6831         }
6832
6833         ctl_set_success(ctsio);
6834         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6835         ctsio->be_move_done = ctl_config_move_done;
6836         ctl_datamove((union ctl_io *)ctsio);
6837         return (CTL_RETVAL_COMPLETE);
6838 }
6839
6840 int
6841 ctl_lbp_log_sense_handler(struct ctl_scsiio *ctsio,
6842                                struct ctl_page_index *page_index,
6843                                int pc)
6844 {
6845         struct ctl_lun *lun;
6846         struct scsi_log_param_header *phdr;
6847         uint8_t *data;
6848         uint64_t val;
6849
6850         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6851         data = page_index->page_data;
6852
6853         if (lun->backend->lun_attr != NULL &&
6854             (val = lun->backend->lun_attr(lun->be_lun->be_lun, "blocksavail"))
6855              != UINT64_MAX) {
6856                 phdr = (struct scsi_log_param_header *)data;
6857                 scsi_ulto2b(0x0001, phdr->param_code);
6858                 phdr->param_control = SLP_LBIN | SLP_LP;
6859                 phdr->param_len = 8;
6860                 data = (uint8_t *)(phdr + 1);
6861                 scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6862                 data[4] = 0x02; /* per-pool */
6863                 data += phdr->param_len;
6864         }
6865
6866         if (lun->backend->lun_attr != NULL &&
6867             (val = lun->backend->lun_attr(lun->be_lun->be_lun, "blocksused"))
6868              != UINT64_MAX) {
6869                 phdr = (struct scsi_log_param_header *)data;
6870                 scsi_ulto2b(0x0002, phdr->param_code);
6871                 phdr->param_control = SLP_LBIN | SLP_LP;
6872                 phdr->param_len = 8;
6873                 data = (uint8_t *)(phdr + 1);
6874                 scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6875                 data[4] = 0x01; /* per-LUN */
6876                 data += phdr->param_len;
6877         }
6878
6879         if (lun->backend->lun_attr != NULL &&
6880             (val = lun->backend->lun_attr(lun->be_lun->be_lun, "poolblocksavail"))
6881              != UINT64_MAX) {
6882                 phdr = (struct scsi_log_param_header *)data;
6883                 scsi_ulto2b(0x00f1, phdr->param_code);
6884                 phdr->param_control = SLP_LBIN | SLP_LP;
6885                 phdr->param_len = 8;
6886                 data = (uint8_t *)(phdr + 1);
6887                 scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6888                 data[4] = 0x02; /* per-pool */
6889                 data += phdr->param_len;
6890         }
6891
6892         if (lun->backend->lun_attr != NULL &&
6893             (val = lun->backend->lun_attr(lun->be_lun->be_lun, "poolblocksused"))
6894              != UINT64_MAX) {
6895                 phdr = (struct scsi_log_param_header *)data;
6896                 scsi_ulto2b(0x00f2, phdr->param_code);
6897                 phdr->param_control = SLP_LBIN | SLP_LP;
6898                 phdr->param_len = 8;
6899                 data = (uint8_t *)(phdr + 1);
6900                 scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6901                 data[4] = 0x02; /* per-pool */
6902                 data += phdr->param_len;
6903         }
6904
6905         page_index->page_len = data - page_index->page_data;
6906         return (0);
6907 }
6908
6909 int
6910 ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
6911                                struct ctl_page_index *page_index,
6912                                int pc)
6913 {
6914         struct ctl_lun *lun;
6915         struct stat_page *data;
6916         uint64_t rn, wn, rb, wb;
6917         struct bintime rt, wt;
6918         int i;
6919
6920         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6921         data = (struct stat_page *)page_index->page_data;
6922
6923         scsi_ulto2b(SLP_SAP, data->sap.hdr.param_code);
6924         data->sap.hdr.param_control = SLP_LBIN;
6925         data->sap.hdr.param_len = sizeof(struct scsi_log_stat_and_perf) -
6926             sizeof(struct scsi_log_param_header);
6927         rn = wn = rb = wb = 0;
6928         bintime_clear(&rt);
6929         bintime_clear(&wt);
6930         for (i = 0; i < CTL_MAX_PORTS; i++) {
6931                 rn += lun->stats.ports[i].operations[CTL_STATS_READ];
6932                 wn += lun->stats.ports[i].operations[CTL_STATS_WRITE];
6933                 rb += lun->stats.ports[i].bytes[CTL_STATS_READ];
6934                 wb += lun->stats.ports[i].bytes[CTL_STATS_WRITE];
6935                 bintime_add(&rt, &lun->stats.ports[i].time[CTL_STATS_READ]);
6936                 bintime_add(&wt, &lun->stats.ports[i].time[CTL_STATS_WRITE]);
6937         }
6938         scsi_u64to8b(rn, data->sap.read_num);
6939         scsi_u64to8b(wn, data->sap.write_num);
6940         if (lun->stats.blocksize > 0) {
6941                 scsi_u64to8b(wb / lun->stats.blocksize,
6942                     data->sap.recvieved_lba);
6943                 scsi_u64to8b(rb / lun->stats.blocksize,
6944                     data->sap.transmitted_lba);
6945         }
6946         scsi_u64to8b((uint64_t)rt.sec * 1000 + rt.frac / (UINT64_MAX / 1000),
6947             data->sap.read_int);
6948         scsi_u64to8b((uint64_t)wt.sec * 1000 + wt.frac / (UINT64_MAX / 1000),
6949             data->sap.write_int);
6950         scsi_u64to8b(0, data->sap.weighted_num);
6951         scsi_u64to8b(0, data->sap.weighted_int);
6952         scsi_ulto2b(SLP_IT, data->it.hdr.param_code);
6953         data->it.hdr.param_control = SLP_LBIN;
6954         data->it.hdr.param_len = sizeof(struct scsi_log_idle_time) -
6955             sizeof(struct scsi_log_param_header);
6956 #ifdef CTL_TIME_IO
6957         scsi_u64to8b(lun->idle_time / SBT_1MS, data->it.idle_int);
6958 #endif
6959         scsi_ulto2b(SLP_TI, data->ti.hdr.param_code);
6960         data->it.hdr.param_control = SLP_LBIN;
6961         data->ti.hdr.param_len = sizeof(struct scsi_log_time_interval) -
6962             sizeof(struct scsi_log_param_header);
6963         scsi_ulto4b(3, data->ti.exponent);
6964         scsi_ulto4b(1, data->ti.integer);
6965
6966         page_index->page_len = sizeof(*data);
6967         return (0);
6968 }
6969
6970 int
6971 ctl_log_sense(struct ctl_scsiio *ctsio)
6972 {
6973         struct ctl_lun *lun;
6974         int i, pc, page_code, subpage;
6975         int alloc_len, total_len;
6976         struct ctl_page_index *page_index;
6977         struct scsi_log_sense *cdb;
6978         struct scsi_log_header *header;
6979
6980         CTL_DEBUG_PRINT(("ctl_log_sense\n"));
6981
6982         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6983         cdb = (struct scsi_log_sense *)ctsio->cdb;
6984         pc = (cdb->page & SLS_PAGE_CTRL_MASK) >> 6;
6985         page_code = cdb->page & SLS_PAGE_CODE;
6986         subpage = cdb->subpage;
6987         alloc_len = scsi_2btoul(cdb->length);
6988
6989         page_index = NULL;
6990         for (i = 0; i < CTL_NUM_LOG_PAGES; i++) {
6991                 page_index = &lun->log_pages.index[i];
6992
6993                 /* Look for the right page code */
6994                 if ((page_index->page_code & SL_PAGE_CODE) != page_code)
6995                         continue;
6996
6997                 /* Look for the right subpage or the subpage wildcard*/
6998                 if (page_index->subpage != subpage)
6999                         continue;
7000
7001                 break;
7002         }
7003         if (i >= CTL_NUM_LOG_PAGES) {
7004                 ctl_set_invalid_field(ctsio,
7005                                       /*sks_valid*/ 1,
7006                                       /*command*/ 1,
7007                                       /*field*/ 2,
7008                                       /*bit_valid*/ 0,
7009                                       /*bit*/ 0);
7010                 ctl_done((union ctl_io *)ctsio);
7011                 return (CTL_RETVAL_COMPLETE);
7012         }
7013
7014         total_len = sizeof(struct scsi_log_header) + page_index->page_len;
7015
7016         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7017         ctsio->kern_sg_entries = 0;
7018         ctsio->kern_data_resid = 0;
7019         ctsio->kern_rel_offset = 0;
7020         if (total_len < alloc_len) {
7021                 ctsio->residual = alloc_len - total_len;
7022                 ctsio->kern_data_len = total_len;
7023                 ctsio->kern_total_len = total_len;
7024         } else {
7025                 ctsio->residual = 0;
7026                 ctsio->kern_data_len = alloc_len;
7027                 ctsio->kern_total_len = alloc_len;
7028         }
7029
7030         header = (struct scsi_log_header *)ctsio->kern_data_ptr;
7031         header->page = page_index->page_code;
7032         if (page_index->subpage) {
7033                 header->page |= SL_SPF;
7034                 header->subpage = page_index->subpage;
7035         }
7036         scsi_ulto2b(page_index->page_len, header->datalen);
7037
7038         /*
7039          * Call the handler, if it exists, to update the
7040          * page to the latest values.
7041          */
7042         if (page_index->sense_handler != NULL)
7043                 page_index->sense_handler(ctsio, page_index, pc);
7044
7045         memcpy(header + 1, page_index->page_data, page_index->page_len);
7046
7047         ctl_set_success(ctsio);
7048         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7049         ctsio->be_move_done = ctl_config_move_done;
7050         ctl_datamove((union ctl_io *)ctsio);
7051         return (CTL_RETVAL_COMPLETE);
7052 }
7053
7054 int
7055 ctl_read_capacity(struct ctl_scsiio *ctsio)
7056 {
7057         struct scsi_read_capacity *cdb;
7058         struct scsi_read_capacity_data *data;
7059         struct ctl_lun *lun;
7060         uint32_t lba;
7061
7062         CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
7063
7064         cdb = (struct scsi_read_capacity *)ctsio->cdb;
7065
7066         lba = scsi_4btoul(cdb->addr);
7067         if (((cdb->pmi & SRC_PMI) == 0)
7068          && (lba != 0)) {
7069                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7070                                       /*sks_valid*/ 1,
7071                                       /*command*/ 1,
7072                                       /*field*/ 2,
7073                                       /*bit_valid*/ 0,
7074                                       /*bit*/ 0);
7075                 ctl_done((union ctl_io *)ctsio);
7076                 return (CTL_RETVAL_COMPLETE);
7077         }
7078
7079         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7080
7081         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7082         data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
7083         ctsio->residual = 0;
7084         ctsio->kern_data_len = sizeof(*data);
7085         ctsio->kern_total_len = sizeof(*data);
7086         ctsio->kern_data_resid = 0;
7087         ctsio->kern_rel_offset = 0;
7088         ctsio->kern_sg_entries = 0;
7089
7090         /*
7091          * If the maximum LBA is greater than 0xfffffffe, the user must
7092          * issue a SERVICE ACTION IN (16) command, with the read capacity
7093          * serivce action set.
7094          */
7095         if (lun->be_lun->maxlba > 0xfffffffe)
7096                 scsi_ulto4b(0xffffffff, data->addr);
7097         else
7098                 scsi_ulto4b(lun->be_lun->maxlba, data->addr);
7099
7100         /*
7101          * XXX KDM this may not be 512 bytes...
7102          */
7103         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7104
7105         ctl_set_success(ctsio);
7106         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7107         ctsio->be_move_done = ctl_config_move_done;
7108         ctl_datamove((union ctl_io *)ctsio);
7109         return (CTL_RETVAL_COMPLETE);
7110 }
7111
7112 int
7113 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
7114 {
7115         struct scsi_read_capacity_16 *cdb;
7116         struct scsi_read_capacity_data_long *data;
7117         struct ctl_lun *lun;
7118         uint64_t lba;
7119         uint32_t alloc_len;
7120
7121         CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
7122
7123         cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
7124
7125         alloc_len = scsi_4btoul(cdb->alloc_len);
7126         lba = scsi_8btou64(cdb->addr);
7127
7128         if ((cdb->reladr & SRC16_PMI)
7129          && (lba != 0)) {
7130                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7131                                       /*sks_valid*/ 1,
7132                                       /*command*/ 1,
7133                                       /*field*/ 2,
7134                                       /*bit_valid*/ 0,
7135                                       /*bit*/ 0);
7136                 ctl_done((union ctl_io *)ctsio);
7137                 return (CTL_RETVAL_COMPLETE);
7138         }
7139
7140         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7141
7142         ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7143         data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
7144
7145         if (sizeof(*data) < alloc_len) {
7146                 ctsio->residual = alloc_len - sizeof(*data);
7147                 ctsio->kern_data_len = sizeof(*data);
7148                 ctsio->kern_total_len = sizeof(*data);
7149         } else {
7150                 ctsio->residual = 0;
7151                 ctsio->kern_data_len = alloc_len;
7152                 ctsio->kern_total_len = alloc_len;
7153         }
7154         ctsio->kern_data_resid = 0;
7155         ctsio->kern_rel_offset = 0;
7156         ctsio->kern_sg_entries = 0;
7157
7158         scsi_u64to8b(lun->be_lun->maxlba, data->addr);
7159         /* XXX KDM this may not be 512 bytes... */
7160         scsi_ulto4b(lun->be_lun->blocksize, data->length);
7161         data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
7162         scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
7163         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
7164                 data->lalba_lbp[0] |= SRC16_LBPME | SRC16_LBPRZ;
7165
7166         ctl_set_success(ctsio);
7167         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7168         ctsio->be_move_done = ctl_config_move_done;
7169         ctl_datamove((union ctl_io *)ctsio);
7170         return (CTL_RETVAL_COMPLETE);
7171 }
7172
7173 int
7174 ctl_get_lba_status(struct ctl_scsiio *ctsio)
7175 {
7176         struct scsi_get_lba_status *cdb;
7177         struct scsi_get_lba_status_data *data;
7178         struct ctl_lun *lun;
7179         struct ctl_lba_len_flags *lbalen;
7180         uint64_t lba;
7181         uint32_t alloc_len, total_len;
7182         int retval;
7183
7184         CTL_DEBUG_PRINT(("ctl_get_lba_status\n"));
7185
7186         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7187         cdb = (struct scsi_get_lba_status *)ctsio->cdb;
7188         lba = scsi_8btou64(cdb->addr);
7189         alloc_len = scsi_4btoul(cdb->alloc_len);
7190
7191         if (lba > lun->be_lun->maxlba) {
7192                 ctl_set_lba_out_of_range(ctsio);
7193                 ctl_done((union ctl_io *)ctsio);
7194                 return (CTL_RETVAL_COMPLETE);
7195         }
7196
7197         total_len = sizeof(*data) + sizeof(data->descr[0]);
7198         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7199         data = (struct scsi_get_lba_status_data *)ctsio->kern_data_ptr;
7200
7201         if (total_len < alloc_len) {
7202                 ctsio->residual = alloc_len - total_len;
7203                 ctsio->kern_data_len = total_len;
7204                 ctsio->kern_total_len = total_len;
7205         } else {
7206                 ctsio->residual = 0;
7207                 ctsio->kern_data_len = alloc_len;
7208                 ctsio->kern_total_len = alloc_len;
7209         }
7210         ctsio->kern_data_resid = 0;
7211         ctsio->kern_rel_offset = 0;
7212         ctsio->kern_sg_entries = 0;
7213
7214         /* Fill dummy data in case backend can't tell anything. */
7215         scsi_ulto4b(4 + sizeof(data->descr[0]), data->length);
7216         scsi_u64to8b(lba, data->descr[0].addr);
7217         scsi_ulto4b(MIN(UINT32_MAX, lun->be_lun->maxlba + 1 - lba),
7218             data->descr[0].length);
7219         data->descr[0].status = 0; /* Mapped or unknown. */
7220
7221         ctl_set_success(ctsio);
7222         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7223         ctsio->be_move_done = ctl_config_move_done;
7224
7225         lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
7226         lbalen->lba = lba;
7227         lbalen->len = total_len;
7228         lbalen->flags = 0;
7229         retval = lun->backend->config_read((union ctl_io *)ctsio);
7230         return (CTL_RETVAL_COMPLETE);
7231 }
7232
7233 int
7234 ctl_read_defect(struct ctl_scsiio *ctsio)
7235 {
7236         struct scsi_read_defect_data_10 *ccb10;
7237         struct scsi_read_defect_data_12 *ccb12;
7238         struct scsi_read_defect_data_hdr_10 *data10;
7239         struct scsi_read_defect_data_hdr_12 *data12;
7240         uint32_t alloc_len, data_len;
7241         uint8_t format;
7242
7243         CTL_DEBUG_PRINT(("ctl_read_defect\n"));
7244
7245         if (ctsio->cdb[0] == READ_DEFECT_DATA_10) {
7246                 ccb10 = (struct scsi_read_defect_data_10 *)&ctsio->cdb;
7247                 format = ccb10->format;
7248                 alloc_len = scsi_2btoul(ccb10->alloc_length);
7249                 data_len = sizeof(*data10);
7250         } else {
7251                 ccb12 = (struct scsi_read_defect_data_12 *)&ctsio->cdb;
7252                 format = ccb12->format;
7253                 alloc_len = scsi_4btoul(ccb12->alloc_length);
7254                 data_len = sizeof(*data12);
7255         }
7256         if (alloc_len == 0) {
7257                 ctl_set_success(ctsio);
7258                 ctl_done((union ctl_io *)ctsio);
7259                 return (CTL_RETVAL_COMPLETE);
7260         }
7261
7262         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
7263         if (data_len < alloc_len) {
7264                 ctsio->residual = alloc_len - data_len;
7265                 ctsio->kern_data_len = data_len;
7266                 ctsio->kern_total_len = data_len;
7267         } else {
7268                 ctsio->residual = 0;
7269                 ctsio->kern_data_len = alloc_len;
7270                 ctsio->kern_total_len = alloc_len;
7271         }
7272         ctsio->kern_data_resid = 0;
7273         ctsio->kern_rel_offset = 0;
7274         ctsio->kern_sg_entries = 0;
7275
7276         if (ctsio->cdb[0] == READ_DEFECT_DATA_10) {
7277                 data10 = (struct scsi_read_defect_data_hdr_10 *)
7278                     ctsio->kern_data_ptr;
7279                 data10->format = format;
7280                 scsi_ulto2b(0, data10->length);
7281         } else {
7282                 data12 = (struct scsi_read_defect_data_hdr_12 *)
7283                     ctsio->kern_data_ptr;
7284                 data12->format = format;
7285                 scsi_ulto2b(0, data12->generation);
7286                 scsi_ulto4b(0, data12->length);
7287         }
7288
7289         ctl_set_success(ctsio);
7290         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7291         ctsio->be_move_done = ctl_config_move_done;
7292         ctl_datamove((union ctl_io *)ctsio);
7293         return (CTL_RETVAL_COMPLETE);
7294 }
7295
7296 int
7297 ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
7298 {
7299         struct scsi_maintenance_in *cdb;
7300         int retval;
7301         int alloc_len, ext, total_len = 0, g, pc, pg, gs, os;
7302         int num_target_port_groups, num_target_ports;
7303         struct ctl_lun *lun;
7304         struct ctl_softc *softc;
7305         struct ctl_port *port;
7306         struct scsi_target_group_data *rtg_ptr;
7307         struct scsi_target_group_data_extended *rtg_ext_ptr;
7308         struct scsi_target_port_group_descriptor *tpg_desc;
7309
7310         CTL_DEBUG_PRINT(("ctl_report_tagret_port_groups\n"));
7311
7312         cdb = (struct scsi_maintenance_in *)ctsio->cdb;
7313         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7314         softc = lun->ctl_softc;
7315
7316         retval = CTL_RETVAL_COMPLETE;
7317
7318         switch (cdb->byte2 & STG_PDF_MASK) {
7319         case STG_PDF_LENGTH:
7320                 ext = 0;
7321                 break;
7322         case STG_PDF_EXTENDED:
7323                 ext = 1;
7324                 break;
7325         default:
7326                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7327                                       /*sks_valid*/ 1,
7328                                       /*command*/ 1,
7329                                       /*field*/ 2,
7330                                       /*bit_valid*/ 1,
7331                                       /*bit*/ 5);
7332                 ctl_done((union ctl_io *)ctsio);
7333                 return(retval);
7334         }
7335
7336         if (softc->is_single)
7337                 num_target_port_groups = 1;
7338         else
7339                 num_target_port_groups = NUM_TARGET_PORT_GROUPS;
7340         num_target_ports = 0;
7341         mtx_lock(&softc->ctl_lock);
7342         STAILQ_FOREACH(port, &softc->port_list, links) {
7343                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7344                         continue;
7345                 if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
7346                         continue;
7347                 num_target_ports++;
7348         }
7349         mtx_unlock(&softc->ctl_lock);
7350
7351         if (ext)
7352                 total_len = sizeof(struct scsi_target_group_data_extended);
7353         else
7354                 total_len = sizeof(struct scsi_target_group_data);
7355         total_len += sizeof(struct scsi_target_port_group_descriptor) *
7356                 num_target_port_groups +
7357             sizeof(struct scsi_target_port_descriptor) * num_target_ports;
7358
7359         alloc_len = scsi_4btoul(cdb->length);
7360
7361         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7362
7363         ctsio->kern_sg_entries = 0;
7364
7365         if (total_len < alloc_len) {
7366                 ctsio->residual = alloc_len - total_len;
7367                 ctsio->kern_data_len = total_len;
7368                 ctsio->kern_total_len = total_len;
7369         } else {
7370                 ctsio->residual = 0;
7371                 ctsio->kern_data_len = alloc_len;
7372                 ctsio->kern_total_len = alloc_len;
7373         }
7374         ctsio->kern_data_resid = 0;
7375         ctsio->kern_rel_offset = 0;
7376
7377         if (ext) {
7378                 rtg_ext_ptr = (struct scsi_target_group_data_extended *)
7379                     ctsio->kern_data_ptr;
7380                 scsi_ulto4b(total_len - 4, rtg_ext_ptr->length);
7381                 rtg_ext_ptr->format_type = 0x10;
7382                 rtg_ext_ptr->implicit_transition_time = 0;
7383                 tpg_desc = &rtg_ext_ptr->groups[0];
7384         } else {
7385                 rtg_ptr = (struct scsi_target_group_data *)
7386                     ctsio->kern_data_ptr;
7387                 scsi_ulto4b(total_len - 4, rtg_ptr->length);
7388                 tpg_desc = &rtg_ptr->groups[0];
7389         }
7390
7391         mtx_lock(&softc->ctl_lock);
7392         pg = softc->port_min / softc->port_cnt;
7393         if (softc->ha_link == CTL_HA_LINK_OFFLINE)
7394                 gs = TPG_ASYMMETRIC_ACCESS_UNAVAILABLE;
7395         else if (softc->ha_link == CTL_HA_LINK_UNKNOWN)
7396                 gs = TPG_ASYMMETRIC_ACCESS_TRANSITIONING;
7397         else if (softc->ha_mode == CTL_HA_MODE_ACT_STBY)
7398                 gs = TPG_ASYMMETRIC_ACCESS_STANDBY;
7399         else
7400                 gs = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7401         if (lun->flags & CTL_LUN_PRIMARY_SC) {
7402                 os = gs;
7403                 gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7404         } else
7405                 os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7406         for (g = 0; g < num_target_port_groups; g++) {
7407                 tpg_desc->pref_state = (g == pg) ? gs : os;
7408                 tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP |
7409                     TPG_U_SUP | TPG_T_SUP;
7410                 scsi_ulto2b(g + 1, tpg_desc->target_port_group);
7411                 tpg_desc->status = TPG_IMPLICIT;
7412                 pc = 0;
7413                 STAILQ_FOREACH(port, &softc->port_list, links) {
7414                         if (port->targ_port < g * softc->port_cnt ||
7415                             port->targ_port >= (g + 1) * softc->port_cnt)
7416                                 continue;
7417                         if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7418                                 continue;
7419                         if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
7420                                 continue;
7421                         scsi_ulto2b(port->targ_port, tpg_desc->descriptors[pc].
7422                             relative_target_port_identifier);
7423                         pc++;
7424                 }
7425                 tpg_desc->target_port_count = pc;
7426                 tpg_desc = (struct scsi_target_port_group_descriptor *)
7427                     &tpg_desc->descriptors[pc];
7428         }
7429         mtx_unlock(&softc->ctl_lock);
7430
7431         ctl_set_success(ctsio);
7432         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7433         ctsio->be_move_done = ctl_config_move_done;
7434         ctl_datamove((union ctl_io *)ctsio);
7435         return(retval);
7436 }
7437
7438 int
7439 ctl_report_supported_opcodes(struct ctl_scsiio *ctsio)
7440 {
7441         struct ctl_lun *lun;
7442         struct scsi_report_supported_opcodes *cdb;
7443         const struct ctl_cmd_entry *entry, *sentry;
7444         struct scsi_report_supported_opcodes_all *all;
7445         struct scsi_report_supported_opcodes_descr *descr;
7446         struct scsi_report_supported_opcodes_one *one;
7447         int retval;
7448         int alloc_len, total_len;
7449         int opcode, service_action, i, j, num;
7450
7451         CTL_DEBUG_PRINT(("ctl_report_supported_opcodes\n"));
7452
7453         cdb = (struct scsi_report_supported_opcodes *)ctsio->cdb;
7454         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7455
7456         retval = CTL_RETVAL_COMPLETE;
7457
7458         opcode = cdb->requested_opcode;
7459         service_action = scsi_2btoul(cdb->requested_service_action);
7460         switch (cdb->options & RSO_OPTIONS_MASK) {
7461         case RSO_OPTIONS_ALL:
7462                 num = 0;
7463                 for (i = 0; i < 256; i++) {
7464                         entry = &ctl_cmd_table[i];
7465                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7466                                 for (j = 0; j < 32; j++) {
7467                                         sentry = &((const struct ctl_cmd_entry *)
7468                                             entry->execute)[j];
7469                                         if (ctl_cmd_applicable(
7470                                             lun->be_lun->lun_type, sentry))
7471                                                 num++;
7472                                 }
7473                         } else {
7474                                 if (ctl_cmd_applicable(lun->be_lun->lun_type,
7475                                     entry))
7476                                         num++;
7477                         }
7478                 }
7479                 total_len = sizeof(struct scsi_report_supported_opcodes_all) +
7480                     num * sizeof(struct scsi_report_supported_opcodes_descr);
7481                 break;
7482         case RSO_OPTIONS_OC:
7483                 if (ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) {
7484                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7485                                               /*sks_valid*/ 1,
7486                                               /*command*/ 1,
7487                                               /*field*/ 2,
7488                                               /*bit_valid*/ 1,
7489                                               /*bit*/ 2);
7490                         ctl_done((union ctl_io *)ctsio);
7491                         return (CTL_RETVAL_COMPLETE);
7492                 }
7493                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7494                 break;
7495         case RSO_OPTIONS_OC_SA:
7496                 if ((ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) == 0 ||
7497                     service_action >= 32) {
7498                         ctl_set_invalid_field(/*ctsio*/ ctsio,
7499                                               /*sks_valid*/ 1,
7500                                               /*command*/ 1,
7501                                               /*field*/ 2,
7502                                               /*bit_valid*/ 1,
7503                                               /*bit*/ 2);
7504                         ctl_done((union ctl_io *)ctsio);
7505                         return (CTL_RETVAL_COMPLETE);
7506                 }
7507                 total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7508                 break;
7509         default:
7510                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7511                                       /*sks_valid*/ 1,
7512                                       /*command*/ 1,
7513                                       /*field*/ 2,
7514                                       /*bit_valid*/ 1,
7515                                       /*bit*/ 2);
7516                 ctl_done((union ctl_io *)ctsio);
7517                 return (CTL_RETVAL_COMPLETE);
7518         }
7519
7520         alloc_len = scsi_4btoul(cdb->length);
7521
7522         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7523
7524         ctsio->kern_sg_entries = 0;
7525
7526         if (total_len < alloc_len) {
7527                 ctsio->residual = alloc_len - total_len;
7528                 ctsio->kern_data_len = total_len;
7529                 ctsio->kern_total_len = total_len;
7530         } else {
7531                 ctsio->residual = 0;
7532                 ctsio->kern_data_len = alloc_len;
7533                 ctsio->kern_total_len = alloc_len;
7534         }
7535         ctsio->kern_data_resid = 0;
7536         ctsio->kern_rel_offset = 0;
7537
7538         switch (cdb->options & RSO_OPTIONS_MASK) {
7539         case RSO_OPTIONS_ALL:
7540                 all = (struct scsi_report_supported_opcodes_all *)
7541                     ctsio->kern_data_ptr;
7542                 num = 0;
7543                 for (i = 0; i < 256; i++) {
7544                         entry = &ctl_cmd_table[i];
7545                         if (entry->flags & CTL_CMD_FLAG_SA5) {
7546                                 for (j = 0; j < 32; j++) {
7547                                         sentry = &((const struct ctl_cmd_entry *)
7548                                             entry->execute)[j];
7549                                         if (!ctl_cmd_applicable(
7550                                             lun->be_lun->lun_type, sentry))
7551                                                 continue;
7552                                         descr = &all->descr[num++];
7553                                         descr->opcode = i;
7554                                         scsi_ulto2b(j, descr->service_action);
7555                                         descr->flags = RSO_SERVACTV;
7556                                         scsi_ulto2b(sentry->length,
7557                                             descr->cdb_length);
7558                                 }
7559                         } else {
7560                                 if (!ctl_cmd_applicable(lun->be_lun->lun_type,
7561                                     entry))
7562                                         continue;
7563                                 descr = &all->descr[num++];
7564                                 descr->opcode = i;
7565                                 scsi_ulto2b(0, descr->service_action);
7566                                 descr->flags = 0;
7567                                 scsi_ulto2b(entry->length, descr->cdb_length);
7568                         }
7569                 }
7570                 scsi_ulto4b(
7571                     num * sizeof(struct scsi_report_supported_opcodes_descr),
7572                     all->length);
7573                 break;
7574         case RSO_OPTIONS_OC:
7575                 one = (struct scsi_report_supported_opcodes_one *)
7576                     ctsio->kern_data_ptr;
7577                 entry = &ctl_cmd_table[opcode];
7578                 goto fill_one;
7579         case RSO_OPTIONS_OC_SA:
7580                 one = (struct scsi_report_supported_opcodes_one *)
7581                     ctsio->kern_data_ptr;
7582                 entry = &ctl_cmd_table[opcode];
7583                 entry = &((const struct ctl_cmd_entry *)
7584                     entry->execute)[service_action];
7585 fill_one:
7586                 if (ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
7587                         one->support = 3;
7588                         scsi_ulto2b(entry->length, one->cdb_length);
7589                         one->cdb_usage[0] = opcode;
7590                         memcpy(&one->cdb_usage[1], entry->usage,
7591                             entry->length - 1);
7592                 } else
7593                         one->support = 1;
7594                 break;
7595         }
7596
7597         ctl_set_success(ctsio);
7598         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7599         ctsio->be_move_done = ctl_config_move_done;
7600         ctl_datamove((union ctl_io *)ctsio);
7601         return(retval);
7602 }
7603
7604 int
7605 ctl_report_supported_tmf(struct ctl_scsiio *ctsio)
7606 {
7607         struct scsi_report_supported_tmf *cdb;
7608         struct scsi_report_supported_tmf_data *data;
7609         int retval;
7610         int alloc_len, total_len;
7611
7612         CTL_DEBUG_PRINT(("ctl_report_supported_tmf\n"));
7613
7614         cdb = (struct scsi_report_supported_tmf *)ctsio->cdb;
7615
7616         retval = CTL_RETVAL_COMPLETE;
7617
7618         total_len = sizeof(struct scsi_report_supported_tmf_data);
7619         alloc_len = scsi_4btoul(cdb->length);
7620
7621         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7622
7623         ctsio->kern_sg_entries = 0;
7624
7625         if (total_len < alloc_len) {
7626                 ctsio->residual = alloc_len - total_len;
7627                 ctsio->kern_data_len = total_len;
7628                 ctsio->kern_total_len = total_len;
7629         } else {
7630                 ctsio->residual = 0;
7631                 ctsio->kern_data_len = alloc_len;
7632                 ctsio->kern_total_len = alloc_len;
7633         }
7634         ctsio->kern_data_resid = 0;
7635         ctsio->kern_rel_offset = 0;
7636
7637         data = (struct scsi_report_supported_tmf_data *)ctsio->kern_data_ptr;
7638         data->byte1 |= RST_ATS | RST_ATSS | RST_CTSS | RST_LURS | RST_QTS |
7639             RST_TRS;
7640         data->byte2 |= RST_QAES | RST_QTSS | RST_ITNRS;
7641
7642         ctl_set_success(ctsio);
7643         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7644         ctsio->be_move_done = ctl_config_move_done;
7645         ctl_datamove((union ctl_io *)ctsio);
7646         return (retval);
7647 }
7648
7649 int
7650 ctl_report_timestamp(struct ctl_scsiio *ctsio)
7651 {
7652         struct scsi_report_timestamp *cdb;
7653         struct scsi_report_timestamp_data *data;
7654         struct timeval tv;
7655         int64_t timestamp;
7656         int retval;
7657         int alloc_len, total_len;
7658
7659         CTL_DEBUG_PRINT(("ctl_report_timestamp\n"));
7660
7661         cdb = (struct scsi_report_timestamp *)ctsio->cdb;
7662
7663         retval = CTL_RETVAL_COMPLETE;
7664
7665         total_len = sizeof(struct scsi_report_timestamp_data);
7666         alloc_len = scsi_4btoul(cdb->length);
7667
7668         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7669
7670         ctsio->kern_sg_entries = 0;
7671
7672         if (total_len < alloc_len) {
7673                 ctsio->residual = alloc_len - total_len;
7674                 ctsio->kern_data_len = total_len;
7675                 ctsio->kern_total_len = total_len;
7676         } else {
7677                 ctsio->residual = 0;
7678                 ctsio->kern_data_len = alloc_len;
7679                 ctsio->kern_total_len = alloc_len;
7680         }
7681         ctsio->kern_data_resid = 0;
7682         ctsio->kern_rel_offset = 0;
7683
7684         data = (struct scsi_report_timestamp_data *)ctsio->kern_data_ptr;
7685         scsi_ulto2b(sizeof(*data) - 2, data->length);
7686         data->origin = RTS_ORIG_OUTSIDE;
7687         getmicrotime(&tv);
7688         timestamp = (int64_t)tv.tv_sec * 1000 + tv.tv_usec / 1000;
7689         scsi_ulto4b(timestamp >> 16, data->timestamp);
7690         scsi_ulto2b(timestamp & 0xffff, &data->timestamp[4]);
7691
7692         ctl_set_success(ctsio);
7693         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7694         ctsio->be_move_done = ctl_config_move_done;
7695         ctl_datamove((union ctl_io *)ctsio);
7696         return (retval);
7697 }
7698
7699 int
7700 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7701 {
7702         struct scsi_per_res_in *cdb;
7703         int alloc_len, total_len = 0;
7704         /* struct scsi_per_res_in_rsrv in_data; */
7705         struct ctl_lun *lun;
7706         struct ctl_softc *softc;
7707         uint64_t key;
7708
7709         CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7710
7711         cdb = (struct scsi_per_res_in *)ctsio->cdb;
7712
7713         alloc_len = scsi_2btoul(cdb->length);
7714
7715         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7716         softc = lun->ctl_softc;
7717
7718 retry:
7719         mtx_lock(&lun->lun_lock);
7720         switch (cdb->action) {
7721         case SPRI_RK: /* read keys */
7722                 total_len = sizeof(struct scsi_per_res_in_keys) +
7723                         lun->pr_key_count *
7724                         sizeof(struct scsi_per_res_key);
7725                 break;
7726         case SPRI_RR: /* read reservation */
7727                 if (lun->flags & CTL_LUN_PR_RESERVED)
7728                         total_len = sizeof(struct scsi_per_res_in_rsrv);
7729                 else
7730                         total_len = sizeof(struct scsi_per_res_in_header);
7731                 break;
7732         case SPRI_RC: /* report capabilities */
7733                 total_len = sizeof(struct scsi_per_res_cap);
7734                 break;
7735         case SPRI_RS: /* read full status */
7736                 total_len = sizeof(struct scsi_per_res_in_header) +
7737                     (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7738                     lun->pr_key_count;
7739                 break;
7740         default:
7741                 panic("Invalid PR type %x", cdb->action);
7742         }
7743         mtx_unlock(&lun->lun_lock);
7744
7745         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7746
7747         if (total_len < alloc_len) {
7748                 ctsio->residual = alloc_len - total_len;
7749                 ctsio->kern_data_len = total_len;
7750                 ctsio->kern_total_len = total_len;
7751         } else {
7752                 ctsio->residual = 0;
7753                 ctsio->kern_data_len = alloc_len;
7754                 ctsio->kern_total_len = alloc_len;
7755         }
7756
7757         ctsio->kern_data_resid = 0;
7758         ctsio->kern_rel_offset = 0;
7759         ctsio->kern_sg_entries = 0;
7760
7761         mtx_lock(&lun->lun_lock);
7762         switch (cdb->action) {
7763         case SPRI_RK: { // read keys
7764         struct scsi_per_res_in_keys *res_keys;
7765                 int i, key_count;
7766
7767                 res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7768
7769                 /*
7770                  * We had to drop the lock to allocate our buffer, which
7771                  * leaves time for someone to come in with another
7772                  * persistent reservation.  (That is unlikely, though,
7773                  * since this should be the only persistent reservation
7774                  * command active right now.)
7775                  */
7776                 if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7777                     (lun->pr_key_count *
7778                      sizeof(struct scsi_per_res_key)))){
7779                         mtx_unlock(&lun->lun_lock);
7780                         free(ctsio->kern_data_ptr, M_CTL);
7781                         printf("%s: reservation length changed, retrying\n",
7782                                __func__);
7783                         goto retry;
7784                 }
7785
7786                 scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7787
7788                 scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7789                              lun->pr_key_count, res_keys->header.length);
7790
7791                 for (i = 0, key_count = 0; i < CTL_MAX_INITIATORS; i++) {
7792                         if ((key = ctl_get_prkey(lun, i)) == 0)
7793                                 continue;
7794
7795                         /*
7796                          * We used lun->pr_key_count to calculate the
7797                          * size to allocate.  If it turns out the number of
7798                          * initiators with the registered flag set is
7799                          * larger than that (i.e. they haven't been kept in
7800                          * sync), we've got a problem.
7801                          */
7802                         if (key_count >= lun->pr_key_count) {
7803 #ifdef NEEDTOPORT
7804                                 csevent_log(CSC_CTL | CSC_SHELF_SW |
7805                                             CTL_PR_ERROR,
7806                                             csevent_LogType_Fault,
7807                                             csevent_AlertLevel_Yellow,
7808                                             csevent_FRU_ShelfController,
7809                                             csevent_FRU_Firmware,
7810                                         csevent_FRU_Unknown,
7811                                             "registered keys %d >= key "
7812                                             "count %d", key_count,
7813                                             lun->pr_key_count);
7814 #endif
7815                                 key_count++;
7816                                 continue;
7817                         }
7818                         scsi_u64to8b(key, res_keys->keys[key_count].key);
7819                         key_count++;
7820                 }
7821                 break;
7822         }
7823         case SPRI_RR: { // read reservation
7824                 struct scsi_per_res_in_rsrv *res;
7825                 int tmp_len, header_only;
7826
7827                 res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7828
7829                 scsi_ulto4b(lun->PRGeneration, res->header.generation);
7830
7831                 if (lun->flags & CTL_LUN_PR_RESERVED)
7832                 {
7833                         tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7834                         scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7835                                     res->header.length);
7836                         header_only = 0;
7837                 } else {
7838                         tmp_len = sizeof(struct scsi_per_res_in_header);
7839                         scsi_ulto4b(0, res->header.length);
7840                         header_only = 1;
7841                 }
7842
7843                 /*
7844                  * We had to drop the lock to allocate our buffer, which
7845                  * leaves time for someone to come in with another
7846                  * persistent reservation.  (That is unlikely, though,
7847                  * since this should be the only persistent reservation
7848                  * command active right now.)
7849                  */
7850                 if (tmp_len != total_len) {
7851                         mtx_unlock(&lun->lun_lock);
7852                         free(ctsio->kern_data_ptr, M_CTL);
7853                         printf("%s: reservation status changed, retrying\n",
7854                                __func__);
7855                         goto retry;
7856                 }
7857
7858                 /*
7859                  * No reservation held, so we're done.
7860                  */
7861                 if (header_only != 0)
7862                         break;
7863
7864                 /*
7865                  * If the registration is an All Registrants type, the key
7866                  * is 0, since it doesn't really matter.
7867                  */
7868                 if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7869                         scsi_u64to8b(ctl_get_prkey(lun, lun->pr_res_idx),
7870                             res->data.reservation);
7871                 }
7872                 res->data.scopetype = lun->res_type;
7873                 break;
7874         }
7875         case SPRI_RC:     //report capabilities
7876         {
7877                 struct scsi_per_res_cap *res_cap;
7878                 uint16_t type_mask;
7879
7880                 res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7881                 scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7882                 res_cap->flags2 |= SPRI_TMV | SPRI_ALLOW_5;
7883                 type_mask = SPRI_TM_WR_EX_AR |
7884                             SPRI_TM_EX_AC_RO |
7885                             SPRI_TM_WR_EX_RO |
7886                             SPRI_TM_EX_AC |
7887                             SPRI_TM_WR_EX |
7888                             SPRI_TM_EX_AC_AR;
7889                 scsi_ulto2b(type_mask, res_cap->type_mask);
7890                 break;
7891         }
7892         case SPRI_RS: { // read full status
7893                 struct scsi_per_res_in_full *res_status;
7894                 struct scsi_per_res_in_full_desc *res_desc;
7895                 struct ctl_port *port;
7896                 int i, len;
7897
7898                 res_status = (struct scsi_per_res_in_full*)ctsio->kern_data_ptr;
7899
7900                 /*
7901                  * We had to drop the lock to allocate our buffer, which
7902                  * leaves time for someone to come in with another
7903                  * persistent reservation.  (That is unlikely, though,
7904                  * since this should be the only persistent reservation
7905                  * command active right now.)
7906                  */
7907                 if (total_len < (sizeof(struct scsi_per_res_in_header) +
7908                     (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7909                      lun->pr_key_count)){
7910                         mtx_unlock(&lun->lun_lock);
7911                         free(ctsio->kern_data_ptr, M_CTL);
7912                         printf("%s: reservation length changed, retrying\n",
7913                                __func__);
7914                         goto retry;
7915                 }
7916
7917                 scsi_ulto4b(lun->PRGeneration, res_status->header.generation);
7918
7919                 res_desc = &res_status->desc[0];
7920                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
7921                         if ((key = ctl_get_prkey(lun, i)) == 0)
7922                                 continue;
7923
7924                         scsi_u64to8b(key, res_desc->res_key.key);
7925                         if ((lun->flags & CTL_LUN_PR_RESERVED) &&
7926                             (lun->pr_res_idx == i ||
7927                              lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS)) {
7928                                 res_desc->flags = SPRI_FULL_R_HOLDER;
7929                                 res_desc->scopetype = lun->res_type;
7930                         }
7931                         scsi_ulto2b(i / CTL_MAX_INIT_PER_PORT,
7932                             res_desc->rel_trgt_port_id);
7933                         len = 0;
7934                         port = softc->ctl_ports[i / CTL_MAX_INIT_PER_PORT];
7935                         if (port != NULL)
7936                                 len = ctl_create_iid(port,
7937                                     i % CTL_MAX_INIT_PER_PORT,
7938                                     res_desc->transport_id);
7939                         scsi_ulto4b(len, res_desc->additional_length);
7940                         res_desc = (struct scsi_per_res_in_full_desc *)
7941                             &res_desc->transport_id[len];
7942                 }
7943                 scsi_ulto4b((uint8_t *)res_desc - (uint8_t *)&res_status->desc[0],
7944                     res_status->header.length);
7945                 break;
7946         }
7947         default:
7948                 /*
7949                  * This is a bug, because we just checked for this above,
7950                  * and should have returned an error.
7951                  */
7952                 panic("Invalid PR type %x", cdb->action);
7953                 break; /* NOTREACHED */
7954         }
7955         mtx_unlock(&lun->lun_lock);
7956
7957         ctl_set_success(ctsio);
7958         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7959         ctsio->be_move_done = ctl_config_move_done;
7960         ctl_datamove((union ctl_io *)ctsio);
7961         return (CTL_RETVAL_COMPLETE);
7962 }
7963
7964 /*
7965  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
7966  * it should return.
7967  */
7968 static int
7969 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
7970                 uint64_t sa_res_key, uint8_t type, uint32_t residx,
7971                 struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
7972                 struct scsi_per_res_out_parms* param)
7973 {
7974         union ctl_ha_msg persis_io;
7975         int i;
7976
7977         mtx_lock(&lun->lun_lock);
7978         if (sa_res_key == 0) {
7979                 if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7980                         /* validate scope and type */
7981                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7982                              SPR_LU_SCOPE) {
7983                                 mtx_unlock(&lun->lun_lock);
7984                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7985                                                       /*sks_valid*/ 1,
7986                                                       /*command*/ 1,
7987                                                       /*field*/ 2,
7988                                                       /*bit_valid*/ 1,
7989                                                       /*bit*/ 4);
7990                                 ctl_done((union ctl_io *)ctsio);
7991                                 return (1);
7992                         }
7993
7994                         if (type>8 || type==2 || type==4 || type==0) {
7995                                 mtx_unlock(&lun->lun_lock);
7996                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
7997                                                       /*sks_valid*/ 1,
7998                                                       /*command*/ 1,
7999                                                       /*field*/ 2,
8000                                                       /*bit_valid*/ 1,
8001                                                       /*bit*/ 0);
8002                                 ctl_done((union ctl_io *)ctsio);
8003                                 return (1);
8004                         }
8005
8006                         /*
8007                          * Unregister everybody else and build UA for
8008                          * them
8009                          */
8010                         for(i = 0; i < CTL_MAX_INITIATORS; i++) {
8011                                 if (i == residx || ctl_get_prkey(lun, i) == 0)
8012                                         continue;
8013
8014                                 ctl_clr_prkey(lun, i);
8015                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8016                         }
8017                         lun->pr_key_count = 1;
8018                         lun->res_type = type;
8019                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8020                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8021                                 lun->pr_res_idx = residx;
8022                         lun->PRGeneration++;
8023                         mtx_unlock(&lun->lun_lock);
8024
8025                         /* send msg to other side */
8026                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8027                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8028                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8029                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8030                         persis_io.pr.pr_info.res_type = type;
8031                         memcpy(persis_io.pr.pr_info.sa_res_key,
8032                                param->serv_act_res_key,
8033                                sizeof(param->serv_act_res_key));
8034                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8035                             sizeof(persis_io.pr), M_WAITOK);
8036                 } else {
8037                         /* not all registrants */
8038                         mtx_unlock(&lun->lun_lock);
8039                         free(ctsio->kern_data_ptr, M_CTL);
8040                         ctl_set_invalid_field(ctsio,
8041                                               /*sks_valid*/ 1,
8042                                               /*command*/ 0,
8043                                               /*field*/ 8,
8044                                               /*bit_valid*/ 0,
8045                                               /*bit*/ 0);
8046                         ctl_done((union ctl_io *)ctsio);
8047                         return (1);
8048                 }
8049         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8050                 || !(lun->flags & CTL_LUN_PR_RESERVED)) {
8051                 int found = 0;
8052
8053                 if (res_key == sa_res_key) {
8054                         /* special case */
8055                         /*
8056                          * The spec implies this is not good but doesn't
8057                          * say what to do. There are two choices either
8058                          * generate a res conflict or check condition
8059                          * with illegal field in parameter data. Since
8060                          * that is what is done when the sa_res_key is
8061                          * zero I'll take that approach since this has
8062                          * to do with the sa_res_key.
8063                          */
8064                         mtx_unlock(&lun->lun_lock);
8065                         free(ctsio->kern_data_ptr, M_CTL);
8066                         ctl_set_invalid_field(ctsio,
8067                                               /*sks_valid*/ 1,
8068                                               /*command*/ 0,
8069                                               /*field*/ 8,
8070                                               /*bit_valid*/ 0,
8071                                               /*bit*/ 0);
8072                         ctl_done((union ctl_io *)ctsio);
8073                         return (1);
8074                 }
8075
8076                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8077                         if (ctl_get_prkey(lun, i) != sa_res_key)
8078                                 continue;
8079
8080                         found = 1;
8081                         ctl_clr_prkey(lun, i);
8082                         lun->pr_key_count--;
8083                         ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8084                 }
8085                 if (!found) {
8086                         mtx_unlock(&lun->lun_lock);
8087                         free(ctsio->kern_data_ptr, M_CTL);
8088                         ctl_set_reservation_conflict(ctsio);
8089                         ctl_done((union ctl_io *)ctsio);
8090                         return (CTL_RETVAL_COMPLETE);
8091                 }
8092                 lun->PRGeneration++;
8093                 mtx_unlock(&lun->lun_lock);
8094
8095                 /* send msg to other side */
8096                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8097                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8098                 persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8099                 persis_io.pr.pr_info.residx = lun->pr_res_idx;
8100                 persis_io.pr.pr_info.res_type = type;
8101                 memcpy(persis_io.pr.pr_info.sa_res_key,
8102                        param->serv_act_res_key,
8103                        sizeof(param->serv_act_res_key));
8104                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8105                     sizeof(persis_io.pr), M_WAITOK);
8106         } else {
8107                 /* Reserved but not all registrants */
8108                 /* sa_res_key is res holder */
8109                 if (sa_res_key == ctl_get_prkey(lun, lun->pr_res_idx)) {
8110                         /* validate scope and type */
8111                         if ((cdb->scope_type & SPR_SCOPE_MASK) !=
8112                              SPR_LU_SCOPE) {
8113                                 mtx_unlock(&lun->lun_lock);
8114                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8115                                                       /*sks_valid*/ 1,
8116                                                       /*command*/ 1,
8117                                                       /*field*/ 2,
8118                                                       /*bit_valid*/ 1,
8119                                                       /*bit*/ 4);
8120                                 ctl_done((union ctl_io *)ctsio);
8121                                 return (1);
8122                         }
8123
8124                         if (type>8 || type==2 || type==4 || type==0) {
8125                                 mtx_unlock(&lun->lun_lock);
8126                                 ctl_set_invalid_field(/*ctsio*/ ctsio,
8127                                                       /*sks_valid*/ 1,
8128                                                       /*command*/ 1,
8129                                                       /*field*/ 2,
8130                                                       /*bit_valid*/ 1,
8131                                                       /*bit*/ 0);
8132                                 ctl_done((union ctl_io *)ctsio);
8133                                 return (1);
8134                         }
8135
8136                         /*
8137                          * Do the following:
8138                          * if sa_res_key != res_key remove all
8139                          * registrants w/sa_res_key and generate UA
8140                          * for these registrants(Registrations
8141                          * Preempted) if it wasn't an exclusive
8142                          * reservation generate UA(Reservations
8143                          * Preempted) for all other registered nexuses
8144                          * if the type has changed. Establish the new
8145                          * reservation and holder. If res_key and
8146                          * sa_res_key are the same do the above
8147                          * except don't unregister the res holder.
8148                          */
8149
8150                         for(i = 0; i < CTL_MAX_INITIATORS; i++) {
8151                                 if (i == residx || ctl_get_prkey(lun, i) == 0)
8152                                         continue;
8153
8154                                 if (sa_res_key == ctl_get_prkey(lun, i)) {
8155                                         ctl_clr_prkey(lun, i);
8156                                         lun->pr_key_count--;
8157                                         ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8158                                 } else if (type != lun->res_type
8159                                         && (lun->res_type == SPR_TYPE_WR_EX_RO
8160                                          || lun->res_type ==SPR_TYPE_EX_AC_RO)){
8161                                         ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8162                                 }
8163                         }
8164                         lun->res_type = type;
8165                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8166                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8167                                 lun->pr_res_idx = residx;
8168                         else
8169                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8170                         lun->PRGeneration++;
8171                         mtx_unlock(&lun->lun_lock);
8172
8173                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8174                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8175                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8176                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8177                         persis_io.pr.pr_info.res_type = type;
8178                         memcpy(persis_io.pr.pr_info.sa_res_key,
8179                                param->serv_act_res_key,
8180                                sizeof(param->serv_act_res_key));
8181                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8182                             sizeof(persis_io.pr), M_WAITOK);
8183                 } else {
8184                         /*
8185                          * sa_res_key is not the res holder just
8186                          * remove registrants
8187                          */
8188                         int found=0;
8189
8190                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8191                                 if (sa_res_key != ctl_get_prkey(lun, i))
8192                                         continue;
8193
8194                                 found = 1;
8195                                 ctl_clr_prkey(lun, i);
8196                                 lun->pr_key_count--;
8197                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8198                         }
8199
8200                         if (!found) {
8201                                 mtx_unlock(&lun->lun_lock);
8202                                 free(ctsio->kern_data_ptr, M_CTL);
8203                                 ctl_set_reservation_conflict(ctsio);
8204                                 ctl_done((union ctl_io *)ctsio);
8205                                 return (1);
8206                         }
8207                         lun->PRGeneration++;
8208                         mtx_unlock(&lun->lun_lock);
8209
8210                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8211                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8212                         persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8213                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8214                         persis_io.pr.pr_info.res_type = type;
8215                         memcpy(persis_io.pr.pr_info.sa_res_key,
8216                                param->serv_act_res_key,
8217                                sizeof(param->serv_act_res_key));
8218                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8219                             sizeof(persis_io.pr), M_WAITOK);
8220                 }
8221         }
8222         return (0);
8223 }
8224
8225 static void
8226 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
8227 {
8228         uint64_t sa_res_key;
8229         int i;
8230
8231         sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
8232
8233         if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8234          || lun->pr_res_idx == CTL_PR_NO_RESERVATION
8235          || sa_res_key != ctl_get_prkey(lun, lun->pr_res_idx)) {
8236                 if (sa_res_key == 0) {
8237                         /*
8238                          * Unregister everybody else and build UA for
8239                          * them
8240                          */
8241                         for(i = 0; i < CTL_MAX_INITIATORS; i++) {
8242                                 if (i == msg->pr.pr_info.residx ||
8243                                     ctl_get_prkey(lun, i) == 0)
8244                                         continue;
8245
8246                                 ctl_clr_prkey(lun, i);
8247                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8248                         }
8249
8250                         lun->pr_key_count = 1;
8251                         lun->res_type = msg->pr.pr_info.res_type;
8252                         if (lun->res_type != SPR_TYPE_WR_EX_AR
8253                          && lun->res_type != SPR_TYPE_EX_AC_AR)
8254                                 lun->pr_res_idx = msg->pr.pr_info.residx;
8255                 } else {
8256                         for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8257                                 if (sa_res_key == ctl_get_prkey(lun, i))
8258                                         continue;
8259
8260                                 ctl_clr_prkey(lun, i);
8261                                 lun->pr_key_count--;
8262                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8263                         }
8264                 }
8265         } else {
8266                 for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8267                         if (i == msg->pr.pr_info.residx ||
8268                             ctl_get_prkey(lun, i) == 0)
8269                                 continue;
8270
8271                         if (sa_res_key == ctl_get_prkey(lun, i)) {
8272                                 ctl_clr_prkey(lun, i);
8273                                 lun->pr_key_count--;
8274                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8275                         } else if (msg->pr.pr_info.res_type != lun->res_type
8276                                 && (lun->res_type == SPR_TYPE_WR_EX_RO
8277                                  || lun->res_type == SPR_TYPE_EX_AC_RO)) {
8278                                 ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8279                         }
8280                 }
8281                 lun->res_type = msg->pr.pr_info.res_type;
8282                 if (lun->res_type != SPR_TYPE_WR_EX_AR
8283                  && lun->res_type != SPR_TYPE_EX_AC_AR)
8284                         lun->pr_res_idx = msg->pr.pr_info.residx;
8285                 else
8286                         lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8287         }
8288         lun->PRGeneration++;
8289
8290 }
8291
8292
8293 int
8294 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
8295 {
8296         int retval;
8297         u_int32_t param_len;
8298         struct scsi_per_res_out *cdb;
8299         struct ctl_lun *lun;
8300         struct scsi_per_res_out_parms* param;
8301         struct ctl_softc *softc;
8302         uint32_t residx;
8303         uint64_t res_key, sa_res_key, key;
8304         uint8_t type;
8305         union ctl_ha_msg persis_io;
8306         int    i;
8307
8308         CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
8309
8310         retval = CTL_RETVAL_COMPLETE;
8311
8312         cdb = (struct scsi_per_res_out *)ctsio->cdb;
8313         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8314         softc = lun->ctl_softc;
8315
8316         /*
8317          * We only support whole-LUN scope.  The scope & type are ignored for
8318          * register, register and ignore existing key and clear.
8319          * We sometimes ignore scope and type on preempts too!!
8320          * Verify reservation type here as well.
8321          */
8322         type = cdb->scope_type & SPR_TYPE_MASK;
8323         if ((cdb->action == SPRO_RESERVE)
8324          || (cdb->action == SPRO_RELEASE)) {
8325                 if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
8326                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8327                                               /*sks_valid*/ 1,
8328                                               /*command*/ 1,
8329                                               /*field*/ 2,
8330                                               /*bit_valid*/ 1,
8331                                               /*bit*/ 4);
8332                         ctl_done((union ctl_io *)ctsio);
8333                         return (CTL_RETVAL_COMPLETE);
8334                 }
8335
8336                 if (type>8 || type==2 || type==4 || type==0) {
8337                         ctl_set_invalid_field(/*ctsio*/ ctsio,
8338                                               /*sks_valid*/ 1,
8339                                               /*command*/ 1,
8340                                               /*field*/ 2,
8341                                               /*bit_valid*/ 1,
8342                                               /*bit*/ 0);
8343                         ctl_done((union ctl_io *)ctsio);
8344                         return (CTL_RETVAL_COMPLETE);
8345                 }
8346         }
8347
8348         param_len = scsi_4btoul(cdb->length);
8349
8350         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
8351                 ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
8352                 ctsio->kern_data_len = param_len;
8353                 ctsio->kern_total_len = param_len;
8354                 ctsio->kern_data_resid = 0;
8355                 ctsio->kern_rel_offset = 0;
8356                 ctsio->kern_sg_entries = 0;
8357                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8358                 ctsio->be_move_done = ctl_config_move_done;
8359                 ctl_datamove((union ctl_io *)ctsio);
8360
8361                 return (CTL_RETVAL_COMPLETE);
8362         }
8363
8364         param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
8365
8366         residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
8367         res_key = scsi_8btou64(param->res_key.key);
8368         sa_res_key = scsi_8btou64(param->serv_act_res_key);
8369
8370         /*
8371          * Validate the reservation key here except for SPRO_REG_IGNO
8372          * This must be done for all other service actions
8373          */
8374         if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
8375                 mtx_lock(&lun->lun_lock);
8376                 if ((key = ctl_get_prkey(lun, residx)) != 0) {
8377                         if (res_key != key) {
8378                                 /*
8379                                  * The current key passed in doesn't match
8380                                  * the one the initiator previously
8381                                  * registered.
8382                                  */
8383                                 mtx_unlock(&lun->lun_lock);
8384                                 free(ctsio->kern_data_ptr, M_CTL);
8385                                 ctl_set_reservation_conflict(ctsio);
8386                                 ctl_done((union ctl_io *)ctsio);
8387                                 return (CTL_RETVAL_COMPLETE);
8388                         }
8389                 } else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
8390                         /*
8391                          * We are not registered
8392                          */
8393                         mtx_unlock(&lun->lun_lock);
8394                         free(ctsio->kern_data_ptr, M_CTL);
8395                         ctl_set_reservation_conflict(ctsio);
8396                         ctl_done((union ctl_io *)ctsio);
8397                         return (CTL_RETVAL_COMPLETE);
8398                 } else if (res_key != 0) {
8399                         /*
8400                          * We are not registered and trying to register but
8401                          * the register key isn't zero.
8402                          */
8403                         mtx_unlock(&lun->lun_lock);
8404                         free(ctsio->kern_data_ptr, M_CTL);
8405                         ctl_set_reservation_conflict(ctsio);
8406                         ctl_done((union ctl_io *)ctsio);
8407                         return (CTL_RETVAL_COMPLETE);
8408                 }
8409                 mtx_unlock(&lun->lun_lock);
8410         }
8411
8412         switch (cdb->action & SPRO_ACTION_MASK) {
8413         case SPRO_REGISTER:
8414         case SPRO_REG_IGNO: {
8415
8416 #if 0
8417                 printf("Registration received\n");
8418 #endif
8419
8420                 /*
8421                  * We don't support any of these options, as we report in
8422                  * the read capabilities request (see
8423                  * ctl_persistent_reserve_in(), above).
8424                  */
8425                 if ((param->flags & SPR_SPEC_I_PT)
8426                  || (param->flags & SPR_ALL_TG_PT)
8427                  || (param->flags & SPR_APTPL)) {
8428                         int bit_ptr;
8429
8430                         if (param->flags & SPR_APTPL)
8431                                 bit_ptr = 0;
8432                         else if (param->flags & SPR_ALL_TG_PT)
8433                                 bit_ptr = 2;
8434                         else /* SPR_SPEC_I_PT */
8435                                 bit_ptr = 3;
8436
8437                         free(ctsio->kern_data_ptr, M_CTL);
8438                         ctl_set_invalid_field(ctsio,
8439                                               /*sks_valid*/ 1,
8440                                               /*command*/ 0,
8441                                               /*field*/ 20,
8442                                               /*bit_valid*/ 1,
8443                                               /*bit*/ bit_ptr);
8444                         ctl_done((union ctl_io *)ctsio);
8445                         return (CTL_RETVAL_COMPLETE);
8446                 }
8447
8448                 mtx_lock(&lun->lun_lock);
8449
8450                 /*
8451                  * The initiator wants to clear the
8452                  * key/unregister.
8453                  */
8454                 if (sa_res_key == 0) {
8455                         if ((res_key == 0
8456                           && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
8457                          || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
8458                           && ctl_get_prkey(lun, residx) == 0)) {
8459                                 mtx_unlock(&lun->lun_lock);
8460                                 goto done;
8461                         }
8462
8463                         ctl_clr_prkey(lun, residx);
8464                         lun->pr_key_count--;
8465
8466                         if (residx == lun->pr_res_idx) {
8467                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8468                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8469
8470                                 if ((lun->res_type == SPR_TYPE_WR_EX_RO
8471                                   || lun->res_type == SPR_TYPE_EX_AC_RO)
8472                                  && lun->pr_key_count) {
8473                                         /*
8474                                          * If the reservation is a registrants
8475                                          * only type we need to generate a UA
8476                                          * for other registered inits.  The
8477                                          * sense code should be RESERVATIONS
8478                                          * RELEASED
8479                                          */
8480
8481                                         for (i = softc->init_min; i < softc->init_max; i++){
8482                                                 if (ctl_get_prkey(lun, i) == 0)
8483                                                         continue;
8484                                                 ctl_est_ua(lun, i,
8485                                                     CTL_UA_RES_RELEASE);
8486                                         }
8487                                 }
8488                                 lun->res_type = 0;
8489                         } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8490                                 if (lun->pr_key_count==0) {
8491                                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8492                                         lun->res_type = 0;
8493                                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8494                                 }
8495                         }
8496                         lun->PRGeneration++;
8497                         mtx_unlock(&lun->lun_lock);
8498
8499                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8500                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8501                         persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8502                         persis_io.pr.pr_info.residx = residx;
8503                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8504                             sizeof(persis_io.pr), M_WAITOK);
8505                 } else /* sa_res_key != 0 */ {
8506
8507                         /*
8508                          * If we aren't registered currently then increment
8509                          * the key count and set the registered flag.
8510                          */
8511                         ctl_alloc_prkey(lun, residx);
8512                         if (ctl_get_prkey(lun, residx) == 0)
8513                                 lun->pr_key_count++;
8514                         ctl_set_prkey(lun, residx, sa_res_key);
8515                         lun->PRGeneration++;
8516                         mtx_unlock(&lun->lun_lock);
8517
8518                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8519                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8520                         persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8521                         persis_io.pr.pr_info.residx = residx;
8522                         memcpy(persis_io.pr.pr_info.sa_res_key,
8523                                param->serv_act_res_key,
8524                                sizeof(param->serv_act_res_key));
8525                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8526                             sizeof(persis_io.pr), M_WAITOK);
8527                 }
8528
8529                 break;
8530         }
8531         case SPRO_RESERVE:
8532 #if 0
8533                 printf("Reserve executed type %d\n", type);
8534 #endif
8535                 mtx_lock(&lun->lun_lock);
8536                 if (lun->flags & CTL_LUN_PR_RESERVED) {
8537                         /*
8538                          * if this isn't the reservation holder and it's
8539                          * not a "all registrants" type or if the type is
8540                          * different then we have a conflict
8541                          */
8542                         if ((lun->pr_res_idx != residx
8543                           && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8544                          || lun->res_type != type) {
8545                                 mtx_unlock(&lun->lun_lock);
8546                                 free(ctsio->kern_data_ptr, M_CTL);
8547                                 ctl_set_reservation_conflict(ctsio);
8548                                 ctl_done((union ctl_io *)ctsio);
8549                                 return (CTL_RETVAL_COMPLETE);
8550                         }
8551                         mtx_unlock(&lun->lun_lock);
8552                 } else /* create a reservation */ {
8553                         /*
8554                          * If it's not an "all registrants" type record
8555                          * reservation holder
8556                          */
8557                         if (type != SPR_TYPE_WR_EX_AR
8558                          && type != SPR_TYPE_EX_AC_AR)
8559                                 lun->pr_res_idx = residx; /* Res holder */
8560                         else
8561                                 lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8562
8563                         lun->flags |= CTL_LUN_PR_RESERVED;
8564                         lun->res_type = type;
8565
8566                         mtx_unlock(&lun->lun_lock);
8567
8568                         /* send msg to other side */
8569                         persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8570                         persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8571                         persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8572                         persis_io.pr.pr_info.residx = lun->pr_res_idx;
8573                         persis_io.pr.pr_info.res_type = type;
8574                         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8575                             sizeof(persis_io.pr), M_WAITOK);
8576                 }
8577                 break;
8578
8579         case SPRO_RELEASE:
8580                 mtx_lock(&lun->lun_lock);
8581                 if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8582                         /* No reservation exists return good status */
8583                         mtx_unlock(&lun->lun_lock);
8584                         goto done;
8585                 }
8586                 /*
8587                  * Is this nexus a reservation holder?
8588                  */
8589                 if (lun->pr_res_idx != residx
8590                  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8591                         /*
8592                          * not a res holder return good status but
8593                          * do nothing
8594                          */
8595                         mtx_unlock(&lun->lun_lock);
8596                         goto done;
8597                 }
8598
8599                 if (lun->res_type != type) {
8600                         mtx_unlock(&lun->lun_lock);
8601                         free(ctsio->kern_data_ptr, M_CTL);
8602                         ctl_set_illegal_pr_release(ctsio);
8603                         ctl_done((union ctl_io *)ctsio);
8604                         return (CTL_RETVAL_COMPLETE);
8605                 }
8606
8607                 /* okay to release */
8608                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8609                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8610                 lun->res_type = 0;
8611
8612                 /*
8613                  * if this isn't an exclusive access
8614                  * res generate UA for all other
8615                  * registrants.
8616                  */
8617                 if (type != SPR_TYPE_EX_AC
8618                  && type != SPR_TYPE_WR_EX) {
8619                         for (i = softc->init_min; i < softc->init_max; i++) {
8620                                 if (i == residx || ctl_get_prkey(lun, i) == 0)
8621                                         continue;
8622                                 ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8623                         }
8624                 }
8625                 mtx_unlock(&lun->lun_lock);
8626
8627                 /* Send msg to other side */
8628                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8629                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8630                 persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8631                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8632                      sizeof(persis_io.pr), M_WAITOK);
8633                 break;
8634
8635         case SPRO_CLEAR:
8636                 /* send msg to other side */
8637
8638                 mtx_lock(&lun->lun_lock);
8639                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8640                 lun->res_type = 0;
8641                 lun->pr_key_count = 0;
8642                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8643
8644                 ctl_clr_prkey(lun, residx);
8645                 for (i = 0; i < CTL_MAX_INITIATORS; i++)
8646                         if (ctl_get_prkey(lun, i) != 0) {
8647                                 ctl_clr_prkey(lun, i);
8648                                 ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8649                         }
8650                 lun->PRGeneration++;
8651                 mtx_unlock(&lun->lun_lock);
8652
8653                 persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8654                 persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8655                 persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8656                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8657                      sizeof(persis_io.pr), M_WAITOK);
8658                 break;
8659
8660         case SPRO_PREEMPT:
8661         case SPRO_PRE_ABO: {
8662                 int nretval;
8663
8664                 nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8665                                           residx, ctsio, cdb, param);
8666                 if (nretval != 0)
8667                         return (CTL_RETVAL_COMPLETE);
8668                 break;
8669         }
8670         default:
8671                 panic("Invalid PR type %x", cdb->action);
8672         }
8673
8674 done:
8675         free(ctsio->kern_data_ptr, M_CTL);
8676         ctl_set_success(ctsio);
8677         ctl_done((union ctl_io *)ctsio);
8678
8679         return (retval);
8680 }
8681
8682 /*
8683  * This routine is for handling a message from the other SC pertaining to
8684  * persistent reserve out. All the error checking will have been done
8685  * so only perorming the action need be done here to keep the two
8686  * in sync.
8687  */
8688 static void
8689 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8690 {
8691         struct ctl_softc *softc = control_softc;
8692         struct ctl_lun *lun;
8693         int i;
8694         uint32_t residx, targ_lun;
8695
8696         targ_lun = msg->hdr.nexus.targ_mapped_lun;
8697         mtx_lock(&softc->ctl_lock);
8698         if ((targ_lun >= CTL_MAX_LUNS) ||
8699             ((lun = softc->ctl_luns[targ_lun]) == NULL)) {
8700                 mtx_unlock(&softc->ctl_lock);
8701                 return;
8702         }
8703         mtx_lock(&lun->lun_lock);
8704         mtx_unlock(&softc->ctl_lock);
8705         if (lun->flags & CTL_LUN_DISABLED) {
8706                 mtx_unlock(&lun->lun_lock);
8707                 return;
8708         }
8709         residx = ctl_get_initindex(&msg->hdr.nexus);
8710         switch(msg->pr.pr_info.action) {
8711         case CTL_PR_REG_KEY:
8712                 ctl_alloc_prkey(lun, msg->pr.pr_info.residx);
8713                 if (ctl_get_prkey(lun, msg->pr.pr_info.residx) == 0)
8714                         lun->pr_key_count++;
8715                 ctl_set_prkey(lun, msg->pr.pr_info.residx,
8716                     scsi_8btou64(msg->pr.pr_info.sa_res_key));
8717                 lun->PRGeneration++;
8718                 break;
8719
8720         case CTL_PR_UNREG_KEY:
8721                 ctl_clr_prkey(lun, msg->pr.pr_info.residx);
8722                 lun->pr_key_count--;
8723
8724                 /* XXX Need to see if the reservation has been released */
8725                 /* if so do we need to generate UA? */
8726                 if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8727                         lun->flags &= ~CTL_LUN_PR_RESERVED;
8728                         lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8729
8730                         if ((lun->res_type == SPR_TYPE_WR_EX_RO
8731                           || lun->res_type == SPR_TYPE_EX_AC_RO)
8732                          && lun->pr_key_count) {
8733                                 /*
8734                                  * If the reservation is a registrants
8735                                  * only type we need to generate a UA
8736                                  * for other registered inits.  The
8737                                  * sense code should be RESERVATIONS
8738                                  * RELEASED
8739                                  */
8740
8741                                 for (i = softc->init_min; i < softc->init_max; i++) {
8742                                         if (ctl_get_prkey(lun, i) == 0)
8743                                                 continue;
8744
8745                                         ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8746                                 }
8747                         }
8748                         lun->res_type = 0;
8749                 } else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8750                         if (lun->pr_key_count==0) {
8751                                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8752                                 lun->res_type = 0;
8753                                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8754                         }
8755                 }
8756                 lun->PRGeneration++;
8757                 break;
8758
8759         case CTL_PR_RESERVE:
8760                 lun->flags |= CTL_LUN_PR_RESERVED;
8761                 lun->res_type = msg->pr.pr_info.res_type;
8762                 lun->pr_res_idx = msg->pr.pr_info.residx;
8763
8764                 break;
8765
8766         case CTL_PR_RELEASE:
8767                 /*
8768                  * if this isn't an exclusive access res generate UA for all
8769                  * other registrants.
8770                  */
8771                 if (lun->res_type != SPR_TYPE_EX_AC
8772                  && lun->res_type != SPR_TYPE_WR_EX) {
8773                         for (i = softc->init_min; i < softc->init_max; i++)
8774                                 if (i == residx || ctl_get_prkey(lun, i) == 0)
8775                                         continue;
8776                                 ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8777                 }
8778
8779                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8780                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8781                 lun->res_type = 0;
8782                 break;
8783
8784         case CTL_PR_PREEMPT:
8785                 ctl_pro_preempt_other(lun, msg);
8786                 break;
8787         case CTL_PR_CLEAR:
8788                 lun->flags &= ~CTL_LUN_PR_RESERVED;
8789                 lun->res_type = 0;
8790                 lun->pr_key_count = 0;
8791                 lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8792
8793                 for (i=0; i < CTL_MAX_INITIATORS; i++) {
8794                         if (ctl_get_prkey(lun, i) == 0)
8795                                 continue;
8796                         ctl_clr_prkey(lun, i);
8797                         ctl_est_ua(lun, i, CTL_UA_REG_PREEMPT);
8798                 }
8799                 lun->PRGeneration++;
8800                 break;
8801         }
8802
8803         mtx_unlock(&lun->lun_lock);
8804 }
8805
8806 int
8807 ctl_read_write(struct ctl_scsiio *ctsio)
8808 {
8809         struct ctl_lun *lun;
8810         struct ctl_lba_len_flags *lbalen;
8811         uint64_t lba;
8812         uint32_t num_blocks;
8813         int flags, retval;
8814         int isread;
8815
8816         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8817
8818         CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8819
8820         flags = 0;
8821         isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8822               || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8823         switch (ctsio->cdb[0]) {
8824         case READ_6:
8825         case WRITE_6: {
8826                 struct scsi_rw_6 *cdb;
8827
8828                 cdb = (struct scsi_rw_6 *)ctsio->cdb;
8829
8830                 lba = scsi_3btoul(cdb->addr);
8831                 /* only 5 bits are valid in the most significant address byte */
8832                 lba &= 0x1fffff;
8833                 num_blocks = cdb->length;
8834                 /*
8835                  * This is correct according to SBC-2.
8836                  */
8837                 if (num_blocks == 0)
8838                         num_blocks = 256;
8839                 break;
8840         }
8841         case READ_10:
8842         case WRITE_10: {
8843                 struct scsi_rw_10 *cdb;
8844
8845                 cdb = (struct scsi_rw_10 *)ctsio->cdb;
8846                 if (cdb->byte2 & SRW10_FUA)
8847                         flags |= CTL_LLF_FUA;
8848                 if (cdb->byte2 & SRW10_DPO)
8849                         flags |= CTL_LLF_DPO;
8850                 lba = scsi_4btoul(cdb->addr);
8851                 num_blocks = scsi_2btoul(cdb->length);
8852                 break;
8853         }
8854         case WRITE_VERIFY_10: {
8855                 struct scsi_write_verify_10 *cdb;
8856
8857                 cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
8858                 flags |= CTL_LLF_FUA;
8859                 if (cdb->byte2 & SWV_DPO)
8860                         flags |= CTL_LLF_DPO;
8861                 lba = scsi_4btoul(cdb->addr);
8862                 num_blocks = scsi_2btoul(cdb->length);
8863                 break;
8864         }
8865         case READ_12:
8866         case WRITE_12: {
8867                 struct scsi_rw_12 *cdb;
8868
8869                 cdb = (struct scsi_rw_12 *)ctsio->cdb;
8870                 if (cdb->byte2 & SRW12_FUA)
8871                         flags |= CTL_LLF_FUA;
8872                 if (cdb->byte2 & SRW12_DPO)
8873                         flags |= CTL_LLF_DPO;
8874                 lba = scsi_4btoul(cdb->addr);
8875                 num_blocks = scsi_4btoul(cdb->length);
8876                 break;
8877         }
8878         case WRITE_VERIFY_12: {
8879                 struct scsi_write_verify_12 *cdb;
8880
8881                 cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
8882                 flags |= CTL_LLF_FUA;
8883                 if (cdb->byte2 & SWV_DPO)
8884                         flags |= CTL_LLF_DPO;
8885                 lba = scsi_4btoul(cdb->addr);
8886                 num_blocks = scsi_4btoul(cdb->length);
8887                 break;
8888         }
8889         case READ_16:
8890         case WRITE_16: {
8891                 struct scsi_rw_16 *cdb;
8892
8893                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
8894                 if (cdb->byte2 & SRW12_FUA)
8895                         flags |= CTL_LLF_FUA;
8896                 if (cdb->byte2 & SRW12_DPO)
8897                         flags |= CTL_LLF_DPO;
8898                 lba = scsi_8btou64(cdb->addr);
8899                 num_blocks = scsi_4btoul(cdb->length);
8900                 break;
8901         }
8902         case WRITE_ATOMIC_16: {
8903                 struct scsi_write_atomic_16 *cdb;
8904
8905                 if (lun->be_lun->atomicblock == 0) {
8906                         ctl_set_invalid_opcode(ctsio);
8907                         ctl_done((union ctl_io *)ctsio);
8908                         return (CTL_RETVAL_COMPLETE);
8909                 }
8910
8911                 cdb = (struct scsi_write_atomic_16 *)ctsio->cdb;
8912                 if (cdb->byte2 & SRW12_FUA)
8913                         flags |= CTL_LLF_FUA;
8914                 if (cdb->byte2 & SRW12_DPO)
8915                         flags |= CTL_LLF_DPO;
8916                 lba = scsi_8btou64(cdb->addr);
8917                 num_blocks = scsi_2btoul(cdb->length);
8918                 if (num_blocks > lun->be_lun->atomicblock) {
8919                         ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
8920                             /*command*/ 1, /*field*/ 12, /*bit_valid*/ 0,
8921                             /*bit*/ 0);
8922                         ctl_done((union ctl_io *)ctsio);
8923                         return (CTL_RETVAL_COMPLETE);
8924                 }
8925                 break;
8926         }
8927         case WRITE_VERIFY_16: {
8928                 struct scsi_write_verify_16 *cdb;
8929
8930                 cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
8931                 flags |= CTL_LLF_FUA;
8932                 if (cdb->byte2 & SWV_DPO)
8933                         flags |= CTL_LLF_DPO;
8934                 lba = scsi_8btou64(cdb->addr);
8935                 num_blocks = scsi_4btoul(cdb->length);
8936                 break;
8937         }
8938         default:
8939                 /*
8940                  * We got a command we don't support.  This shouldn't
8941                  * happen, commands should be filtered out above us.
8942                  */
8943                 ctl_set_invalid_opcode(ctsio);
8944                 ctl_done((union ctl_io *)ctsio);
8945
8946                 return (CTL_RETVAL_COMPLETE);
8947                 break; /* NOTREACHED */
8948         }
8949
8950         /*
8951          * The first check is to make sure we're in bounds, the second
8952          * check is to catch wrap-around problems.  If the lba + num blocks
8953          * is less than the lba, then we've wrapped around and the block
8954          * range is invalid anyway.
8955          */
8956         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
8957          || ((lba + num_blocks) < lba)) {
8958                 ctl_set_lba_out_of_range(ctsio);
8959                 ctl_done((union ctl_io *)ctsio);
8960                 return (CTL_RETVAL_COMPLETE);
8961         }
8962
8963         /*
8964          * According to SBC-3, a transfer length of 0 is not an error.
8965          * Note that this cannot happen with WRITE(6) or READ(6), since 0
8966          * translates to 256 blocks for those commands.
8967          */
8968         if (num_blocks == 0) {
8969                 ctl_set_success(ctsio);
8970                 ctl_done((union ctl_io *)ctsio);
8971                 return (CTL_RETVAL_COMPLETE);
8972         }
8973
8974         /* Set FUA and/or DPO if caches are disabled. */
8975         if (isread) {
8976                 if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
8977                     SCP_RCD) != 0)
8978                         flags |= CTL_LLF_FUA | CTL_LLF_DPO;
8979         } else {
8980                 if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
8981                     SCP_WCE) == 0)
8982                         flags |= CTL_LLF_FUA;
8983         }
8984
8985         lbalen = (struct ctl_lba_len_flags *)
8986             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
8987         lbalen->lba = lba;
8988         lbalen->len = num_blocks;
8989         lbalen->flags = (isread ? CTL_LLF_READ : CTL_LLF_WRITE) | flags;
8990
8991         ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
8992         ctsio->kern_rel_offset = 0;
8993
8994         CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
8995
8996         retval = lun->backend->data_submit((union ctl_io *)ctsio);
8997         return (retval);
8998 }
8999
9000 static int
9001 ctl_cnw_cont(union ctl_io *io)
9002 {
9003         struct ctl_scsiio *ctsio;
9004         struct ctl_lun *lun;
9005         struct ctl_lba_len_flags *lbalen;
9006         int retval;
9007
9008         ctsio = &io->scsiio;
9009         ctsio->io_hdr.status = CTL_STATUS_NONE;
9010         ctsio->io_hdr.flags &= ~CTL_FLAG_IO_CONT;
9011         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9012         lbalen = (struct ctl_lba_len_flags *)
9013             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9014         lbalen->flags &= ~CTL_LLF_COMPARE;
9015         lbalen->flags |= CTL_LLF_WRITE;
9016
9017         CTL_DEBUG_PRINT(("ctl_cnw_cont: calling data_submit()\n"));
9018         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9019         return (retval);
9020 }
9021
9022 int
9023 ctl_cnw(struct ctl_scsiio *ctsio)
9024 {
9025         struct ctl_lun *lun;
9026         struct ctl_lba_len_flags *lbalen;
9027         uint64_t lba;
9028         uint32_t num_blocks;
9029         int flags, retval;
9030
9031         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9032
9033         CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
9034
9035         flags = 0;
9036         switch (ctsio->cdb[0]) {
9037         case COMPARE_AND_WRITE: {
9038                 struct scsi_compare_and_write *cdb;
9039
9040                 cdb = (struct scsi_compare_and_write *)ctsio->cdb;
9041                 if (cdb->byte2 & SRW10_FUA)
9042                         flags |= CTL_LLF_FUA;
9043                 if (cdb->byte2 & SRW10_DPO)
9044                         flags |= CTL_LLF_DPO;
9045                 lba = scsi_8btou64(cdb->addr);
9046                 num_blocks = cdb->length;
9047                 break;
9048         }
9049         default:
9050                 /*
9051                  * We got a command we don't support.  This shouldn't
9052                  * happen, commands should be filtered out above us.
9053                  */
9054                 ctl_set_invalid_opcode(ctsio);
9055                 ctl_done((union ctl_io *)ctsio);
9056
9057                 return (CTL_RETVAL_COMPLETE);
9058                 break; /* NOTREACHED */
9059         }
9060
9061         /*
9062          * The first check is to make sure we're in bounds, the second
9063          * check is to catch wrap-around problems.  If the lba + num blocks
9064          * is less than the lba, then we've wrapped around and the block
9065          * range is invalid anyway.
9066          */
9067         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9068          || ((lba + num_blocks) < lba)) {
9069                 ctl_set_lba_out_of_range(ctsio);
9070                 ctl_done((union ctl_io *)ctsio);
9071                 return (CTL_RETVAL_COMPLETE);
9072         }
9073
9074         /*
9075          * According to SBC-3, a transfer length of 0 is not an error.
9076          */
9077         if (num_blocks == 0) {
9078                 ctl_set_success(ctsio);
9079                 ctl_done((union ctl_io *)ctsio);
9080                 return (CTL_RETVAL_COMPLETE);
9081         }
9082
9083         /* Set FUA if write cache is disabled. */
9084         if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
9085             SCP_WCE) == 0)
9086                 flags |= CTL_LLF_FUA;
9087
9088         ctsio->kern_total_len = 2 * num_blocks * lun->be_lun->blocksize;
9089         ctsio->kern_rel_offset = 0;
9090
9091         /*
9092          * Set the IO_CONT flag, so that if this I/O gets passed to
9093          * ctl_data_submit_done(), it'll get passed back to
9094          * ctl_ctl_cnw_cont() for further processing.
9095          */
9096         ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
9097         ctsio->io_cont = ctl_cnw_cont;
9098
9099         lbalen = (struct ctl_lba_len_flags *)
9100             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9101         lbalen->lba = lba;
9102         lbalen->len = num_blocks;
9103         lbalen->flags = CTL_LLF_COMPARE | flags;
9104
9105         CTL_DEBUG_PRINT(("ctl_cnw: calling data_submit()\n"));
9106         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9107         return (retval);
9108 }
9109
9110 int
9111 ctl_verify(struct ctl_scsiio *ctsio)
9112 {
9113         struct ctl_lun *lun;
9114         struct ctl_lba_len_flags *lbalen;
9115         uint64_t lba;
9116         uint32_t num_blocks;
9117         int bytchk, flags;
9118         int retval;
9119
9120         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9121
9122         CTL_DEBUG_PRINT(("ctl_verify: command: %#x\n", ctsio->cdb[0]));
9123
9124         bytchk = 0;
9125         flags = CTL_LLF_FUA;
9126         switch (ctsio->cdb[0]) {
9127         case VERIFY_10: {
9128                 struct scsi_verify_10 *cdb;
9129
9130                 cdb = (struct scsi_verify_10 *)ctsio->cdb;
9131                 if (cdb->byte2 & SVFY_BYTCHK)
9132                         bytchk = 1;
9133                 if (cdb->byte2 & SVFY_DPO)
9134                         flags |= CTL_LLF_DPO;
9135                 lba = scsi_4btoul(cdb->addr);
9136                 num_blocks = scsi_2btoul(cdb->length);
9137                 break;
9138         }
9139         case VERIFY_12: {
9140                 struct scsi_verify_12 *cdb;
9141
9142                 cdb = (struct scsi_verify_12 *)ctsio->cdb;
9143                 if (cdb->byte2 & SVFY_BYTCHK)
9144                         bytchk = 1;
9145                 if (cdb->byte2 & SVFY_DPO)
9146                         flags |= CTL_LLF_DPO;
9147                 lba = scsi_4btoul(cdb->addr);
9148                 num_blocks = scsi_4btoul(cdb->length);
9149                 break;
9150         }
9151         case VERIFY_16: {
9152                 struct scsi_rw_16 *cdb;
9153
9154                 cdb = (struct scsi_rw_16 *)ctsio->cdb;
9155                 if (cdb->byte2 & SVFY_BYTCHK)
9156                         bytchk = 1;
9157                 if (cdb->byte2 & SVFY_DPO)
9158                         flags |= CTL_LLF_DPO;
9159                 lba = scsi_8btou64(cdb->addr);
9160                 num_blocks = scsi_4btoul(cdb->length);
9161                 break;
9162         }
9163         default:
9164                 /*
9165                  * We got a command we don't support.  This shouldn't
9166                  * happen, commands should be filtered out above us.
9167                  */
9168                 ctl_set_invalid_opcode(ctsio);
9169                 ctl_done((union ctl_io *)ctsio);
9170                 return (CTL_RETVAL_COMPLETE);
9171         }
9172
9173         /*
9174          * The first check is to make sure we're in bounds, the second
9175          * check is to catch wrap-around problems.  If the lba + num blocks
9176          * is less than the lba, then we've wrapped around and the block
9177          * range is invalid anyway.
9178          */
9179         if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9180          || ((lba + num_blocks) < lba)) {
9181                 ctl_set_lba_out_of_range(ctsio);
9182                 ctl_done((union ctl_io *)ctsio);
9183                 return (CTL_RETVAL_COMPLETE);
9184         }
9185
9186         /*
9187          * According to SBC-3, a transfer length of 0 is not an error.
9188          */
9189         if (num_blocks == 0) {
9190                 ctl_set_success(ctsio);
9191                 ctl_done((union ctl_io *)ctsio);
9192                 return (CTL_RETVAL_COMPLETE);
9193         }
9194
9195         lbalen = (struct ctl_lba_len_flags *)
9196             &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9197         lbalen->lba = lba;
9198         lbalen->len = num_blocks;
9199         if (bytchk) {
9200                 lbalen->flags = CTL_LLF_COMPARE | flags;
9201                 ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9202         } else {
9203                 lbalen->flags = CTL_LLF_VERIFY | flags;
9204                 ctsio->kern_total_len = 0;
9205         }
9206         ctsio->kern_rel_offset = 0;
9207
9208         CTL_DEBUG_PRINT(("ctl_verify: calling data_submit()\n"));
9209         retval = lun->backend->data_submit((union ctl_io *)ctsio);
9210         return (retval);
9211 }
9212
9213 int
9214 ctl_report_luns(struct ctl_scsiio *ctsio)
9215 {
9216         struct ctl_softc *softc;
9217         struct scsi_report_luns *cdb;
9218         struct scsi_report_luns_data *lun_data;
9219         struct ctl_lun *lun, *request_lun;
9220         struct ctl_port *port;
9221         int num_luns, retval;
9222         uint32_t alloc_len, lun_datalen;
9223         int num_filled;
9224         uint32_t initidx, targ_lun_id, lun_id;
9225
9226         retval = CTL_RETVAL_COMPLETE;
9227         cdb = (struct scsi_report_luns *)ctsio->cdb;
9228         port = ctl_io_port(&ctsio->io_hdr);
9229         softc = port->ctl_softc;
9230
9231         CTL_DEBUG_PRINT(("ctl_report_luns\n"));
9232
9233         mtx_lock(&softc->ctl_lock);
9234         num_luns = 0;
9235         for (targ_lun_id = 0; targ_lun_id < CTL_MAX_LUNS; targ_lun_id++) {
9236                 if (ctl_lun_map_from_port(port, targ_lun_id) < CTL_MAX_LUNS)
9237                         num_luns++;
9238         }
9239         mtx_unlock(&softc->ctl_lock);
9240
9241         switch (cdb->select_report) {
9242         case RPL_REPORT_DEFAULT:
9243         case RPL_REPORT_ALL:
9244         case RPL_REPORT_NONSUBSID:
9245                 break;
9246         case RPL_REPORT_WELLKNOWN:
9247         case RPL_REPORT_ADMIN:
9248         case RPL_REPORT_CONGLOM:
9249                 num_luns = 0;
9250                 break;
9251         default:
9252                 ctl_set_invalid_field(ctsio,
9253                                       /*sks_valid*/ 1,
9254                                       /*command*/ 1,
9255                                       /*field*/ 2,
9256                                       /*bit_valid*/ 0,
9257                                       /*bit*/ 0);
9258                 ctl_done((union ctl_io *)ctsio);
9259                 return (retval);
9260                 break; /* NOTREACHED */
9261         }
9262
9263         alloc_len = scsi_4btoul(cdb->length);
9264         /*
9265          * The initiator has to allocate at least 16 bytes for this request,
9266          * so he can at least get the header and the first LUN.  Otherwise
9267          * we reject the request (per SPC-3 rev 14, section 6.21).
9268          */
9269         if (alloc_len < (sizeof(struct scsi_report_luns_data) +
9270             sizeof(struct scsi_report_luns_lundata))) {
9271                 ctl_set_invalid_field(ctsio,
9272                                       /*sks_valid*/ 1,
9273                                       /*command*/ 1,
9274                                       /*field*/ 6,
9275                                       /*bit_valid*/ 0,
9276                                       /*bit*/ 0);
9277                 ctl_done((union ctl_io *)ctsio);
9278                 return (retval);
9279         }
9280
9281         request_lun = (struct ctl_lun *)
9282                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9283
9284         lun_datalen = sizeof(*lun_data) +
9285                 (num_luns * sizeof(struct scsi_report_luns_lundata));
9286
9287         ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
9288         lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
9289         ctsio->kern_sg_entries = 0;
9290
9291         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9292
9293         mtx_lock(&softc->ctl_lock);
9294         for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
9295                 lun_id = ctl_lun_map_from_port(port, targ_lun_id);
9296                 if (lun_id >= CTL_MAX_LUNS)
9297                         continue;
9298                 lun = softc->ctl_luns[lun_id];
9299                 if (lun == NULL)
9300                         continue;
9301
9302                 if (targ_lun_id <= 0xff) {
9303                         /*
9304                          * Peripheral addressing method, bus number 0.
9305                          */
9306                         lun_data->luns[num_filled].lundata[0] =
9307                                 RPL_LUNDATA_ATYP_PERIPH;
9308                         lun_data->luns[num_filled].lundata[1] = targ_lun_id;
9309                         num_filled++;
9310                 } else if (targ_lun_id <= 0x3fff) {
9311                         /*
9312                          * Flat addressing method.
9313                          */
9314                         lun_data->luns[num_filled].lundata[0] =
9315                                 RPL_LUNDATA_ATYP_FLAT | (targ_lun_id >> 8);
9316                         lun_data->luns[num_filled].lundata[1] =
9317                                 (targ_lun_id & 0xff);
9318                         num_filled++;
9319                 } else if (targ_lun_id <= 0xffffff) {
9320                         /*
9321                          * Extended flat addressing method.
9322                          */
9323                         lun_data->luns[num_filled].lundata[0] =
9324                             RPL_LUNDATA_ATYP_EXTLUN | 0x12;
9325                         scsi_ulto3b(targ_lun_id,
9326                             &lun_data->luns[num_filled].lundata[1]);
9327                         num_filled++;
9328                 } else {
9329                         printf("ctl_report_luns: bogus LUN number %jd, "
9330                                "skipping\n", (intmax_t)targ_lun_id);
9331                 }
9332                 /*
9333                  * According to SPC-3, rev 14 section 6.21:
9334                  *
9335                  * "The execution of a REPORT LUNS command to any valid and
9336                  * installed logical unit shall clear the REPORTED LUNS DATA
9337                  * HAS CHANGED unit attention condition for all logical
9338                  * units of that target with respect to the requesting
9339                  * initiator. A valid and installed logical unit is one
9340                  * having a PERIPHERAL QUALIFIER of 000b in the standard
9341                  * INQUIRY data (see 6.4.2)."
9342                  *
9343                  * If request_lun is NULL, the LUN this report luns command
9344                  * was issued to is either disabled or doesn't exist. In that
9345                  * case, we shouldn't clear any pending lun change unit
9346                  * attention.
9347                  */
9348                 if (request_lun != NULL) {
9349                         mtx_lock(&lun->lun_lock);
9350                         ctl_clr_ua(lun, initidx, CTL_UA_LUN_CHANGE);
9351                         mtx_unlock(&lun->lun_lock);
9352                 }
9353         }
9354         mtx_unlock(&softc->ctl_lock);
9355
9356         /*
9357          * It's quite possible that we've returned fewer LUNs than we allocated
9358          * space for.  Trim it.
9359          */
9360         lun_datalen = sizeof(*lun_data) +
9361                 (num_filled * sizeof(struct scsi_report_luns_lundata));
9362
9363         if (lun_datalen < alloc_len) {
9364                 ctsio->residual = alloc_len - lun_datalen;
9365                 ctsio->kern_data_len = lun_datalen;
9366                 ctsio->kern_total_len = lun_datalen;
9367         } else {
9368                 ctsio->residual = 0;
9369                 ctsio->kern_data_len = alloc_len;
9370                 ctsio->kern_total_len = alloc_len;
9371         }
9372         ctsio->kern_data_resid = 0;
9373         ctsio->kern_rel_offset = 0;
9374         ctsio->kern_sg_entries = 0;
9375
9376         /*
9377          * We set this to the actual data length, regardless of how much
9378          * space we actually have to return results.  If the user looks at
9379          * this value, he'll know whether or not he allocated enough space
9380          * and reissue the command if necessary.  We don't support well
9381          * known logical units, so if the user asks for that, return none.
9382          */
9383         scsi_ulto4b(lun_datalen - 8, lun_data->length);
9384
9385         /*
9386          * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
9387          * this request.
9388          */
9389         ctl_set_success(ctsio);
9390         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9391         ctsio->be_move_done = ctl_config_move_done;
9392         ctl_datamove((union ctl_io *)ctsio);
9393         return (retval);
9394 }
9395
9396 int
9397 ctl_request_sense(struct ctl_scsiio *ctsio)
9398 {
9399         struct scsi_request_sense *cdb;
9400         struct scsi_sense_data *sense_ptr;
9401         struct ctl_softc *ctl_softc;
9402         struct ctl_lun *lun;
9403         uint32_t initidx;
9404         int have_error;
9405         scsi_sense_data_type sense_format;
9406         ctl_ua_type ua_type;
9407
9408         cdb = (struct scsi_request_sense *)ctsio->cdb;
9409
9410         ctl_softc = control_softc;
9411         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9412
9413         CTL_DEBUG_PRINT(("ctl_request_sense\n"));
9414
9415         /*
9416          * Determine which sense format the user wants.
9417          */
9418         if (cdb->byte2 & SRS_DESC)
9419                 sense_format = SSD_TYPE_DESC;
9420         else
9421                 sense_format = SSD_TYPE_FIXED;
9422
9423         ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
9424         sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
9425         ctsio->kern_sg_entries = 0;
9426
9427         /*
9428          * struct scsi_sense_data, which is currently set to 256 bytes, is
9429          * larger than the largest allowed value for the length field in the
9430          * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
9431          */
9432         ctsio->residual = 0;
9433         ctsio->kern_data_len = cdb->length;
9434         ctsio->kern_total_len = cdb->length;
9435
9436         ctsio->kern_data_resid = 0;
9437         ctsio->kern_rel_offset = 0;
9438         ctsio->kern_sg_entries = 0;
9439
9440         /*
9441          * If we don't have a LUN, we don't have any pending sense.
9442          */
9443         if (lun == NULL)
9444                 goto no_sense;
9445
9446         have_error = 0;
9447         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9448         /*
9449          * Check for pending sense, and then for pending unit attentions.
9450          * Pending sense gets returned first, then pending unit attentions.
9451          */
9452         mtx_lock(&lun->lun_lock);
9453 #ifdef CTL_WITH_CA
9454         if (ctl_is_set(lun->have_ca, initidx)) {
9455                 scsi_sense_data_type stored_format;
9456
9457                 /*
9458                  * Check to see which sense format was used for the stored
9459                  * sense data.
9460                  */
9461                 stored_format = scsi_sense_type(&lun->pending_sense[initidx]);
9462
9463                 /*
9464                  * If the user requested a different sense format than the
9465                  * one we stored, then we need to convert it to the other
9466                  * format.  If we're going from descriptor to fixed format
9467                  * sense data, we may lose things in translation, depending
9468                  * on what options were used.
9469                  *
9470                  * If the stored format is SSD_TYPE_NONE (i.e. invalid),
9471                  * for some reason we'll just copy it out as-is.
9472                  */
9473                 if ((stored_format == SSD_TYPE_FIXED)
9474                  && (sense_format == SSD_TYPE_DESC))
9475                         ctl_sense_to_desc((struct scsi_sense_data_fixed *)
9476                             &lun->pending_sense[initidx],
9477                             (struct scsi_sense_data_desc *)sense_ptr);
9478                 else if ((stored_format == SSD_TYPE_DESC)
9479                       && (sense_format == SSD_TYPE_FIXED))
9480                         ctl_sense_to_fixed((struct scsi_sense_data_desc *)
9481                             &lun->pending_sense[initidx],
9482                             (struct scsi_sense_data_fixed *)sense_ptr);
9483                 else
9484                         memcpy(sense_ptr, &lun->pending_sense[initidx],
9485                                MIN(sizeof(*sense_ptr),
9486                                sizeof(lun->pending_sense[initidx])));
9487
9488                 ctl_clear_mask(lun->have_ca, initidx);
9489                 have_error = 1;
9490         } else
9491 #endif
9492         {
9493                 ua_type = ctl_build_ua(lun, initidx, sense_ptr, sense_format);
9494                 if (ua_type != CTL_UA_NONE)
9495                         have_error = 1;
9496                 if (ua_type == CTL_UA_LUN_CHANGE) {
9497                         mtx_unlock(&lun->lun_lock);
9498                         mtx_lock(&ctl_softc->ctl_lock);
9499                         ctl_clr_ua_allluns(ctl_softc, initidx, ua_type);
9500                         mtx_unlock(&ctl_softc->ctl_lock);
9501                         mtx_lock(&lun->lun_lock);
9502                 }
9503
9504         }
9505         mtx_unlock(&lun->lun_lock);
9506
9507         /*
9508          * We already have a pending error, return it.
9509          */
9510         if (have_error != 0) {
9511                 /*
9512                  * We report the SCSI status as OK, since the status of the
9513                  * request sense command itself is OK.
9514                  * We report 0 for the sense length, because we aren't doing
9515                  * autosense in this case.  We're reporting sense as
9516                  * parameter data.
9517                  */
9518                 ctl_set_success(ctsio);
9519                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9520                 ctsio->be_move_done = ctl_config_move_done;
9521                 ctl_datamove((union ctl_io *)ctsio);
9522                 return (CTL_RETVAL_COMPLETE);
9523         }
9524
9525 no_sense:
9526
9527         /*
9528          * No sense information to report, so we report that everything is
9529          * okay.
9530          */
9531         ctl_set_sense_data(sense_ptr,
9532                            lun,
9533                            sense_format,
9534                            /*current_error*/ 1,
9535                            /*sense_key*/ SSD_KEY_NO_SENSE,
9536                            /*asc*/ 0x00,
9537                            /*ascq*/ 0x00,
9538                            SSD_ELEM_NONE);
9539
9540         /*
9541          * We report 0 for the sense length, because we aren't doing
9542          * autosense in this case.  We're reporting sense as parameter data.
9543          */
9544         ctl_set_success(ctsio);
9545         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9546         ctsio->be_move_done = ctl_config_move_done;
9547         ctl_datamove((union ctl_io *)ctsio);
9548         return (CTL_RETVAL_COMPLETE);
9549 }
9550
9551 int
9552 ctl_tur(struct ctl_scsiio *ctsio)
9553 {
9554
9555         CTL_DEBUG_PRINT(("ctl_tur\n"));
9556
9557         ctl_set_success(ctsio);
9558         ctl_done((union ctl_io *)ctsio);
9559
9560         return (CTL_RETVAL_COMPLETE);
9561 }
9562
9563 /*
9564  * SCSI VPD page 0x00, the Supported VPD Pages page.
9565  */
9566 static int
9567 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
9568 {
9569         struct scsi_vpd_supported_pages *pages;
9570         int sup_page_size;
9571         struct ctl_lun *lun;
9572         int p;
9573
9574         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9575
9576         sup_page_size = sizeof(struct scsi_vpd_supported_pages) *
9577             SCSI_EVPD_NUM_SUPPORTED_PAGES;
9578         ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9579         pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9580         ctsio->kern_sg_entries = 0;
9581
9582         if (sup_page_size < alloc_len) {
9583                 ctsio->residual = alloc_len - sup_page_size;
9584                 ctsio->kern_data_len = sup_page_size;
9585                 ctsio->kern_total_len = sup_page_size;
9586         } else {
9587                 ctsio->residual = 0;
9588                 ctsio->kern_data_len = alloc_len;
9589                 ctsio->kern_total_len = alloc_len;
9590         }
9591         ctsio->kern_data_resid = 0;
9592         ctsio->kern_rel_offset = 0;
9593         ctsio->kern_sg_entries = 0;
9594
9595         /*
9596          * The control device is always connected.  The disk device, on the
9597          * other hand, may not be online all the time.  Need to change this
9598          * to figure out whether the disk device is actually online or not.
9599          */
9600         if (lun != NULL)
9601                 pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9602                                 lun->be_lun->lun_type;
9603         else
9604                 pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9605
9606         p = 0;
9607         /* Supported VPD pages */
9608         pages->page_list[p++] = SVPD_SUPPORTED_PAGES;
9609         /* Serial Number */
9610         pages->page_list[p++] = SVPD_UNIT_SERIAL_NUMBER;
9611         /* Device Identification */
9612         pages->page_list[p++] = SVPD_DEVICE_ID;
9613         /* Extended INQUIRY Data */
9614         pages->page_list[p++] = SVPD_EXTENDED_INQUIRY_DATA;
9615         /* Mode Page Policy */
9616         pages->page_list[p++] = SVPD_MODE_PAGE_POLICY;
9617         /* SCSI Ports */
9618         pages->page_list[p++] = SVPD_SCSI_PORTS;
9619         /* Third-party Copy */
9620         pages->page_list[p++] = SVPD_SCSI_TPC;
9621         if (lun != NULL && lun->be_lun->lun_type == T_DIRECT) {
9622                 /* Block limits */
9623                 pages->page_list[p++] = SVPD_BLOCK_LIMITS;
9624                 /* Block Device Characteristics */
9625                 pages->page_list[p++] = SVPD_BDC;
9626                 /* Logical Block Provisioning */
9627                 pages->page_list[p++] = SVPD_LBP;
9628         }
9629         pages->length = p;
9630
9631         ctl_set_success(ctsio);
9632         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9633         ctsio->be_move_done = ctl_config_move_done;
9634         ctl_datamove((union ctl_io *)ctsio);
9635         return (CTL_RETVAL_COMPLETE);
9636 }
9637
9638 /*
9639  * SCSI VPD page 0x80, the Unit Serial Number page.
9640  */
9641 static int
9642 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9643 {
9644         struct scsi_vpd_unit_serial_number *sn_ptr;
9645         struct ctl_lun *lun;
9646         int data_len;
9647
9648         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9649
9650         data_len = 4 + CTL_SN_LEN;
9651         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9652         sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9653         if (data_len < alloc_len) {
9654                 ctsio->residual = alloc_len - data_len;
9655                 ctsio->kern_data_len = data_len;
9656                 ctsio->kern_total_len = data_len;
9657         } else {
9658                 ctsio->residual = 0;
9659                 ctsio->kern_data_len = alloc_len;
9660                 ctsio->kern_total_len = alloc_len;
9661         }
9662         ctsio->kern_data_resid = 0;
9663         ctsio->kern_rel_offset = 0;
9664         ctsio->kern_sg_entries = 0;
9665
9666         /*
9667          * The control device is always connected.  The disk device, on the
9668          * other hand, may not be online all the time.  Need to change this
9669          * to figure out whether the disk device is actually online or not.
9670          */
9671         if (lun != NULL)
9672                 sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9673                                   lun->be_lun->lun_type;
9674         else
9675                 sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9676
9677         sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9678         sn_ptr->length = CTL_SN_LEN;
9679         /*
9680          * If we don't have a LUN, we just leave the serial number as
9681          * all spaces.
9682          */
9683         if (lun != NULL) {
9684                 strncpy((char *)sn_ptr->serial_num,
9685                         (char *)lun->be_lun->serial_num, CTL_SN_LEN);
9686         } else
9687                 memset(sn_ptr->serial_num, 0x20, CTL_SN_LEN);
9688
9689         ctl_set_success(ctsio);
9690         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9691         ctsio->be_move_done = ctl_config_move_done;
9692         ctl_datamove((union ctl_io *)ctsio);
9693         return (CTL_RETVAL_COMPLETE);
9694 }
9695
9696
9697 /*
9698  * SCSI VPD page 0x86, the Extended INQUIRY Data page.
9699  */
9700 static int
9701 ctl_inquiry_evpd_eid(struct ctl_scsiio *ctsio, int alloc_len)
9702 {
9703         struct scsi_vpd_extended_inquiry_data *eid_ptr;
9704         struct ctl_lun *lun;
9705         int data_len;
9706
9707         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9708
9709         data_len = sizeof(struct scsi_vpd_extended_inquiry_data);
9710         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9711         eid_ptr = (struct scsi_vpd_extended_inquiry_data *)ctsio->kern_data_ptr;
9712         ctsio->kern_sg_entries = 0;
9713
9714         if (data_len < alloc_len) {
9715                 ctsio->residual = alloc_len - data_len;
9716                 ctsio->kern_data_len = data_len;
9717                 ctsio->kern_total_len = data_len;
9718         } else {
9719                 ctsio->residual = 0;
9720                 ctsio->kern_data_len = alloc_len;
9721                 ctsio->kern_total_len = alloc_len;
9722         }
9723         ctsio->kern_data_resid = 0;
9724         ctsio->kern_rel_offset = 0;
9725         ctsio->kern_sg_entries = 0;
9726
9727         /*
9728          * The control device is always connected.  The disk device, on the
9729          * other hand, may not be online all the time.
9730          */
9731         if (lun != NULL)
9732                 eid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9733                                      lun->be_lun->lun_type;
9734         else
9735                 eid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9736         eid_ptr->page_code = SVPD_EXTENDED_INQUIRY_DATA;
9737         scsi_ulto2b(data_len - 4, eid_ptr->page_length);
9738         /*
9739          * We support head of queue, ordered and simple tags.
9740          */
9741         eid_ptr->flags2 = SVPD_EID_HEADSUP | SVPD_EID_ORDSUP | SVPD_EID_SIMPSUP;
9742         /*
9743          * Volatile cache supported.
9744          */
9745         eid_ptr->flags3 = SVPD_EID_V_SUP;
9746
9747         /*
9748          * This means that we clear the REPORTED LUNS DATA HAS CHANGED unit
9749          * attention for a particular IT nexus on all LUNs once we report
9750          * it to that nexus once.  This bit is required as of SPC-4.
9751          */
9752         eid_ptr->flags4 = SVPD_EID_LUICLT;
9753
9754         /*
9755          * XXX KDM in order to correctly answer this, we would need
9756          * information from the SIM to determine how much sense data it
9757          * can send.  So this would really be a path inquiry field, most
9758          * likely.  This can be set to a maximum of 252 according to SPC-4,
9759          * but the hardware may or may not be able to support that much.
9760          * 0 just means that the maximum sense data length is not reported.
9761          */
9762         eid_ptr->max_sense_length = 0;
9763
9764         ctl_set_success(ctsio);
9765         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9766         ctsio->be_move_done = ctl_config_move_done;
9767         ctl_datamove((union ctl_io *)ctsio);
9768         return (CTL_RETVAL_COMPLETE);
9769 }
9770
9771 static int
9772 ctl_inquiry_evpd_mpp(struct ctl_scsiio *ctsio, int alloc_len)
9773 {
9774         struct scsi_vpd_mode_page_policy *mpp_ptr;
9775         struct ctl_lun *lun;
9776         int data_len;
9777
9778         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9779
9780         data_len = sizeof(struct scsi_vpd_mode_page_policy) +
9781             sizeof(struct scsi_vpd_mode_page_policy_descr);
9782
9783         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9784         mpp_ptr = (struct scsi_vpd_mode_page_policy *)ctsio->kern_data_ptr;
9785         ctsio->kern_sg_entries = 0;
9786
9787         if (data_len < alloc_len) {
9788                 ctsio->residual = alloc_len - data_len;
9789                 ctsio->kern_data_len = data_len;
9790                 ctsio->kern_total_len = data_len;
9791         } else {
9792                 ctsio->residual = 0;
9793                 ctsio->kern_data_len = alloc_len;
9794                 ctsio->kern_total_len = alloc_len;
9795         }
9796         ctsio->kern_data_resid = 0;
9797         ctsio->kern_rel_offset = 0;
9798         ctsio->kern_sg_entries = 0;
9799
9800         /*
9801          * The control device is always connected.  The disk device, on the
9802          * other hand, may not be online all the time.
9803          */
9804         if (lun != NULL)
9805                 mpp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9806                                      lun->be_lun->lun_type;
9807         else
9808                 mpp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9809         mpp_ptr->page_code = SVPD_MODE_PAGE_POLICY;
9810         scsi_ulto2b(data_len - 4, mpp_ptr->page_length);
9811         mpp_ptr->descr[0].page_code = 0x3f;
9812         mpp_ptr->descr[0].subpage_code = 0xff;
9813         mpp_ptr->descr[0].policy = SVPD_MPP_SHARED;
9814
9815         ctl_set_success(ctsio);
9816         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9817         ctsio->be_move_done = ctl_config_move_done;
9818         ctl_datamove((union ctl_io *)ctsio);
9819         return (CTL_RETVAL_COMPLETE);
9820 }
9821
9822 /*
9823  * SCSI VPD page 0x83, the Device Identification page.
9824  */
9825 static int
9826 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9827 {
9828         struct scsi_vpd_device_id *devid_ptr;
9829         struct scsi_vpd_id_descriptor *desc;
9830         struct ctl_softc *softc;
9831         struct ctl_lun *lun;
9832         struct ctl_port *port;
9833         int data_len;
9834         uint8_t proto;
9835
9836         softc = control_softc;
9837
9838         port = ctl_io_port(&ctsio->io_hdr);
9839         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9840
9841         data_len = sizeof(struct scsi_vpd_device_id) +
9842             sizeof(struct scsi_vpd_id_descriptor) +
9843                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9844             sizeof(struct scsi_vpd_id_descriptor) +
9845                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9846         if (lun && lun->lun_devid)
9847                 data_len += lun->lun_devid->len;
9848         if (port && port->port_devid)
9849                 data_len += port->port_devid->len;
9850         if (port && port->target_devid)
9851                 data_len += port->target_devid->len;
9852
9853         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9854         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9855         ctsio->kern_sg_entries = 0;
9856
9857         if (data_len < alloc_len) {
9858                 ctsio->residual = alloc_len - data_len;
9859                 ctsio->kern_data_len = data_len;
9860                 ctsio->kern_total_len = data_len;
9861         } else {
9862                 ctsio->residual = 0;
9863                 ctsio->kern_data_len = alloc_len;
9864                 ctsio->kern_total_len = alloc_len;
9865         }
9866         ctsio->kern_data_resid = 0;
9867         ctsio->kern_rel_offset = 0;
9868         ctsio->kern_sg_entries = 0;
9869
9870         /*
9871          * The control device is always connected.  The disk device, on the
9872          * other hand, may not be online all the time.
9873          */
9874         if (lun != NULL)
9875                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9876                                      lun->be_lun->lun_type;
9877         else
9878                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9879         devid_ptr->page_code = SVPD_DEVICE_ID;
9880         scsi_ulto2b(data_len - 4, devid_ptr->length);
9881
9882         if (port && port->port_type == CTL_PORT_FC)
9883                 proto = SCSI_PROTO_FC << 4;
9884         else if (port && port->port_type == CTL_PORT_ISCSI)
9885                 proto = SCSI_PROTO_ISCSI << 4;
9886         else
9887                 proto = SCSI_PROTO_SPI << 4;
9888         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
9889
9890         /*
9891          * We're using a LUN association here.  i.e., this device ID is a
9892          * per-LUN identifier.
9893          */
9894         if (lun && lun->lun_devid) {
9895                 memcpy(desc, lun->lun_devid->data, lun->lun_devid->len);
9896                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
9897                     lun->lun_devid->len);
9898         }
9899
9900         /*
9901          * This is for the WWPN which is a port association.
9902          */
9903         if (port && port->port_devid) {
9904                 memcpy(desc, port->port_devid->data, port->port_devid->len);
9905                 desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
9906                     port->port_devid->len);
9907         }
9908
9909         /*
9910          * This is for the Relative Target Port(type 4h) identifier
9911          */
9912         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
9913         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
9914             SVPD_ID_TYPE_RELTARG;
9915         desc->length = 4;
9916         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port, &desc->identifier[2]);
9917         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9918             sizeof(struct scsi_vpd_id_rel_trgt_port_id));
9919
9920         /*
9921          * This is for the Target Port Group(type 5h) identifier
9922          */
9923         desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
9924         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
9925             SVPD_ID_TYPE_TPORTGRP;
9926         desc->length = 4;
9927         scsi_ulto2b(ctsio->io_hdr.nexus.targ_port / softc->port_cnt + 1,
9928             &desc->identifier[2]);
9929         desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9930             sizeof(struct scsi_vpd_id_trgt_port_grp_id));
9931
9932         /*
9933          * This is for the Target identifier
9934          */
9935         if (port && port->target_devid) {
9936                 memcpy(desc, port->target_devid->data, port->target_devid->len);
9937         }
9938
9939         ctl_set_success(ctsio);
9940         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9941         ctsio->be_move_done = ctl_config_move_done;
9942         ctl_datamove((union ctl_io *)ctsio);
9943         return (CTL_RETVAL_COMPLETE);
9944 }
9945
9946 static int
9947 ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len)
9948 {
9949         struct ctl_softc *softc = control_softc;
9950         struct scsi_vpd_scsi_ports *sp;
9951         struct scsi_vpd_port_designation *pd;
9952         struct scsi_vpd_port_designation_cont *pdc;
9953         struct ctl_lun *lun;
9954         struct ctl_port *port;
9955         int data_len, num_target_ports, iid_len, id_len;
9956
9957         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9958
9959         num_target_ports = 0;
9960         iid_len = 0;
9961         id_len = 0;
9962         mtx_lock(&softc->ctl_lock);
9963         STAILQ_FOREACH(port, &softc->port_list, links) {
9964                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
9965                         continue;
9966                 if (lun != NULL &&
9967                     ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
9968                         continue;
9969                 num_target_ports++;
9970                 if (port->init_devid)
9971                         iid_len += port->init_devid->len;
9972                 if (port->port_devid)
9973                         id_len += port->port_devid->len;
9974         }
9975         mtx_unlock(&softc->ctl_lock);
9976
9977         data_len = sizeof(struct scsi_vpd_scsi_ports) +
9978             num_target_ports * (sizeof(struct scsi_vpd_port_designation) +
9979              sizeof(struct scsi_vpd_port_designation_cont)) + iid_len + id_len;
9980         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9981         sp = (struct scsi_vpd_scsi_ports *)ctsio->kern_data_ptr;
9982         ctsio->kern_sg_entries = 0;
9983
9984         if (data_len < alloc_len) {
9985                 ctsio->residual = alloc_len - data_len;
9986                 ctsio->kern_data_len = data_len;
9987                 ctsio->kern_total_len = data_len;
9988         } else {
9989                 ctsio->residual = 0;
9990                 ctsio->kern_data_len = alloc_len;
9991                 ctsio->kern_total_len = alloc_len;
9992         }
9993         ctsio->kern_data_resid = 0;
9994         ctsio->kern_rel_offset = 0;
9995         ctsio->kern_sg_entries = 0;
9996
9997         /*
9998          * The control device is always connected.  The disk device, on the
9999          * other hand, may not be online all the time.  Need to change this
10000          * to figure out whether the disk device is actually online or not.
10001          */
10002         if (lun != NULL)
10003                 sp->device = (SID_QUAL_LU_CONNECTED << 5) |
10004                                   lun->be_lun->lun_type;
10005         else
10006                 sp->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10007
10008         sp->page_code = SVPD_SCSI_PORTS;
10009         scsi_ulto2b(data_len - sizeof(struct scsi_vpd_scsi_ports),
10010             sp->page_length);
10011         pd = &sp->design[0];
10012
10013         mtx_lock(&softc->ctl_lock);
10014         STAILQ_FOREACH(port, &softc->port_list, links) {
10015                 if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
10016                         continue;
10017                 if (lun != NULL &&
10018                     ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
10019                         continue;
10020                 scsi_ulto2b(port->targ_port, pd->relative_port_id);
10021                 if (port->init_devid) {
10022                         iid_len = port->init_devid->len;
10023                         memcpy(pd->initiator_transportid,
10024                             port->init_devid->data, port->init_devid->len);
10025                 } else
10026                         iid_len = 0;
10027                 scsi_ulto2b(iid_len, pd->initiator_transportid_length);
10028                 pdc = (struct scsi_vpd_port_designation_cont *)
10029                     (&pd->initiator_transportid[iid_len]);
10030                 if (port->port_devid) {
10031                         id_len = port->port_devid->len;
10032                         memcpy(pdc->target_port_descriptors,
10033                             port->port_devid->data, port->port_devid->len);
10034                 } else
10035                         id_len = 0;
10036                 scsi_ulto2b(id_len, pdc->target_port_descriptors_length);
10037                 pd = (struct scsi_vpd_port_designation *)
10038                     ((uint8_t *)pdc->target_port_descriptors + id_len);
10039         }
10040         mtx_unlock(&softc->ctl_lock);
10041
10042         ctl_set_success(ctsio);
10043         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10044         ctsio->be_move_done = ctl_config_move_done;
10045         ctl_datamove((union ctl_io *)ctsio);
10046         return (CTL_RETVAL_COMPLETE);
10047 }
10048
10049 static int
10050 ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
10051 {
10052         struct scsi_vpd_block_limits *bl_ptr;
10053         struct ctl_lun *lun;
10054
10055         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10056
10057         ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
10058         bl_ptr = (struct scsi_vpd_block_limits *)ctsio->kern_data_ptr;
10059         ctsio->kern_sg_entries = 0;
10060
10061         if (sizeof(*bl_ptr) < alloc_len) {
10062                 ctsio->residual = alloc_len - sizeof(*bl_ptr);
10063                 ctsio->kern_data_len = sizeof(*bl_ptr);
10064                 ctsio->kern_total_len = sizeof(*bl_ptr);
10065         } else {
10066                 ctsio->residual = 0;
10067                 ctsio->kern_data_len = alloc_len;
10068                 ctsio->kern_total_len = alloc_len;
10069         }
10070         ctsio->kern_data_resid = 0;
10071         ctsio->kern_rel_offset = 0;
10072         ctsio->kern_sg_entries = 0;
10073
10074         /*
10075          * The control device is always connected.  The disk device, on the
10076          * other hand, may not be online all the time.  Need to change this
10077          * to figure out whether the disk device is actually online or not.
10078          */
10079         if (lun != NULL)
10080                 bl_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10081                                   lun->be_lun->lun_type;
10082         else
10083                 bl_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10084
10085         bl_ptr->page_code = SVPD_BLOCK_LIMITS;
10086         scsi_ulto2b(sizeof(*bl_ptr) - 4, bl_ptr->page_length);
10087         bl_ptr->max_cmp_write_len = 0xff;
10088         scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
10089         if (lun != NULL) {
10090                 scsi_ulto4b(lun->be_lun->opttxferlen, bl_ptr->opt_txfer_len);
10091                 if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10092                         scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
10093                         scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_blk_cnt);
10094                         if (lun->be_lun->ublockexp != 0) {
10095                                 scsi_ulto4b((1 << lun->be_lun->ublockexp),
10096                                     bl_ptr->opt_unmap_grain);
10097                                 scsi_ulto4b(0x80000000 | lun->be_lun->ublockoff,
10098                                     bl_ptr->unmap_grain_align);
10099                         }
10100                 }
10101                 scsi_ulto4b(lun->be_lun->atomicblock,
10102                     bl_ptr->max_atomic_transfer_length);
10103                 scsi_ulto4b(0, bl_ptr->atomic_alignment);
10104                 scsi_ulto4b(0, bl_ptr->atomic_transfer_length_granularity);
10105                 scsi_ulto4b(0, bl_ptr->max_atomic_transfer_length_with_atomic_boundary);
10106                 scsi_ulto4b(0, bl_ptr->max_atomic_boundary_size);
10107         }
10108         scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
10109
10110         ctl_set_success(ctsio);
10111         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10112         ctsio->be_move_done = ctl_config_move_done;
10113         ctl_datamove((union ctl_io *)ctsio);
10114         return (CTL_RETVAL_COMPLETE);
10115 }
10116
10117 static int
10118 ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len)
10119 {
10120         struct scsi_vpd_block_device_characteristics *bdc_ptr;
10121         struct ctl_lun *lun;
10122         const char *value;
10123         u_int i;
10124
10125         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10126
10127         ctsio->kern_data_ptr = malloc(sizeof(*bdc_ptr), M_CTL, M_WAITOK | M_ZERO);
10128         bdc_ptr = (struct scsi_vpd_block_device_characteristics *)ctsio->kern_data_ptr;
10129         ctsio->kern_sg_entries = 0;
10130
10131         if (sizeof(*bdc_ptr) < alloc_len) {
10132                 ctsio->residual = alloc_len - sizeof(*bdc_ptr);
10133                 ctsio->kern_data_len = sizeof(*bdc_ptr);
10134                 ctsio->kern_total_len = sizeof(*bdc_ptr);
10135         } else {
10136                 ctsio->residual = 0;
10137                 ctsio->kern_data_len = alloc_len;
10138                 ctsio->kern_total_len = alloc_len;
10139         }
10140         ctsio->kern_data_resid = 0;
10141         ctsio->kern_rel_offset = 0;
10142         ctsio->kern_sg_entries = 0;
10143
10144         /*
10145          * The control device is always connected.  The disk device, on the
10146          * other hand, may not be online all the time.  Need to change this
10147          * to figure out whether the disk device is actually online or not.
10148          */
10149         if (lun != NULL)
10150                 bdc_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10151                                   lun->be_lun->lun_type;
10152         else
10153                 bdc_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10154         bdc_ptr->page_code = SVPD_BDC;
10155         scsi_ulto2b(sizeof(*bdc_ptr) - 4, bdc_ptr->page_length);
10156         if (lun != NULL &&
10157             (value = ctl_get_opt(&lun->be_lun->options, "rpm")) != NULL)
10158                 i = strtol(value, NULL, 0);
10159         else
10160                 i = CTL_DEFAULT_ROTATION_RATE;
10161         scsi_ulto2b(i, bdc_ptr->medium_rotation_rate);
10162         if (lun != NULL &&
10163             (value = ctl_get_opt(&lun->be_lun->options, "formfactor")) != NULL)
10164                 i = strtol(value, NULL, 0);
10165         else
10166                 i = 0;
10167         bdc_ptr->wab_wac_ff = (i & 0x0f);
10168         bdc_ptr->flags = SVPD_FUAB | SVPD_VBULS;
10169
10170         ctl_set_success(ctsio);
10171         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10172         ctsio->be_move_done = ctl_config_move_done;
10173         ctl_datamove((union ctl_io *)ctsio);
10174         return (CTL_RETVAL_COMPLETE);
10175 }
10176
10177 static int
10178 ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
10179 {
10180         struct scsi_vpd_logical_block_prov *lbp_ptr;
10181         struct ctl_lun *lun;
10182
10183         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10184
10185         ctsio->kern_data_ptr = malloc(sizeof(*lbp_ptr), M_CTL, M_WAITOK | M_ZERO);
10186         lbp_ptr = (struct scsi_vpd_logical_block_prov *)ctsio->kern_data_ptr;
10187         ctsio->kern_sg_entries = 0;
10188
10189         if (sizeof(*lbp_ptr) < alloc_len) {
10190                 ctsio->residual = alloc_len - sizeof(*lbp_ptr);
10191                 ctsio->kern_data_len = sizeof(*lbp_ptr);
10192                 ctsio->kern_total_len = sizeof(*lbp_ptr);
10193         } else {
10194                 ctsio->residual = 0;
10195                 ctsio->kern_data_len = alloc_len;
10196                 ctsio->kern_total_len = alloc_len;
10197         }
10198         ctsio->kern_data_resid = 0;
10199         ctsio->kern_rel_offset = 0;
10200         ctsio->kern_sg_entries = 0;
10201
10202         /*
10203          * The control device is always connected.  The disk device, on the
10204          * other hand, may not be online all the time.  Need to change this
10205          * to figure out whether the disk device is actually online or not.
10206          */
10207         if (lun != NULL)
10208                 lbp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10209                                   lun->be_lun->lun_type;
10210         else
10211                 lbp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10212
10213         lbp_ptr->page_code = SVPD_LBP;
10214         scsi_ulto2b(sizeof(*lbp_ptr) - 4, lbp_ptr->page_length);
10215         lbp_ptr->threshold_exponent = CTL_LBP_EXPONENT;
10216         if (lun != NULL && lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10217                 lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 |
10218                     SVPD_LBP_WS10 | SVPD_LBP_RZ | SVPD_LBP_ANC_SUP;
10219                 lbp_ptr->prov_type = SVPD_LBP_THIN;
10220         }
10221
10222         ctl_set_success(ctsio);
10223         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10224         ctsio->be_move_done = ctl_config_move_done;
10225         ctl_datamove((union ctl_io *)ctsio);
10226         return (CTL_RETVAL_COMPLETE);
10227 }
10228
10229 /*
10230  * INQUIRY with the EVPD bit set.
10231  */
10232 static int
10233 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
10234 {
10235         struct ctl_lun *lun;
10236         struct scsi_inquiry *cdb;
10237         int alloc_len, retval;
10238
10239         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10240         cdb = (struct scsi_inquiry *)ctsio->cdb;
10241         alloc_len = scsi_2btoul(cdb->length);
10242
10243         switch (cdb->page_code) {
10244         case SVPD_SUPPORTED_PAGES:
10245                 retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
10246                 break;
10247         case SVPD_UNIT_SERIAL_NUMBER:
10248                 retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
10249                 break;
10250         case SVPD_DEVICE_ID:
10251                 retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
10252                 break;
10253         case SVPD_EXTENDED_INQUIRY_DATA:
10254                 retval = ctl_inquiry_evpd_eid(ctsio, alloc_len);
10255                 break;
10256         case SVPD_MODE_PAGE_POLICY:
10257                 retval = ctl_inquiry_evpd_mpp(ctsio, alloc_len);
10258                 break;
10259         case SVPD_SCSI_PORTS:
10260                 retval = ctl_inquiry_evpd_scsi_ports(ctsio, alloc_len);
10261                 break;
10262         case SVPD_SCSI_TPC:
10263                 retval = ctl_inquiry_evpd_tpc(ctsio, alloc_len);
10264                 break;
10265         case SVPD_BLOCK_LIMITS:
10266                 if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10267                         goto err;
10268                 retval = ctl_inquiry_evpd_block_limits(ctsio, alloc_len);
10269                 break;
10270         case SVPD_BDC:
10271                 if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10272                         goto err;
10273                 retval = ctl_inquiry_evpd_bdc(ctsio, alloc_len);
10274                 break;
10275         case SVPD_LBP:
10276                 if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10277                         goto err;
10278                 retval = ctl_inquiry_evpd_lbp(ctsio, alloc_len);
10279                 break;
10280         default:
10281 err:
10282                 ctl_set_invalid_field(ctsio,
10283                                       /*sks_valid*/ 1,
10284                                       /*command*/ 1,
10285                                       /*field*/ 2,
10286                                       /*bit_valid*/ 0,
10287                                       /*bit*/ 0);
10288                 ctl_done((union ctl_io *)ctsio);
10289                 retval = CTL_RETVAL_COMPLETE;
10290                 break;
10291         }
10292
10293         return (retval);
10294 }
10295
10296 /*
10297  * Standard INQUIRY data.
10298  */
10299 static int
10300 ctl_inquiry_std(struct ctl_scsiio *ctsio)
10301 {
10302         struct scsi_inquiry_data *inq_ptr;
10303         struct scsi_inquiry *cdb;
10304         struct ctl_softc *softc = control_softc;
10305         struct ctl_port *port;
10306         struct ctl_lun *lun;
10307         char *val;
10308         uint32_t alloc_len, data_len;
10309         ctl_port_type port_type;
10310
10311         port = ctl_io_port(&ctsio->io_hdr);
10312         port_type = port->port_type;
10313         if (port_type == CTL_PORT_IOCTL || port_type == CTL_PORT_INTERNAL)
10314                 port_type = CTL_PORT_SCSI;
10315
10316         lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10317         cdb = (struct scsi_inquiry *)ctsio->cdb;
10318         alloc_len = scsi_2btoul(cdb->length);
10319
10320         /*
10321          * We malloc the full inquiry data size here and fill it
10322          * in.  If the user only asks for less, we'll give him
10323          * that much.
10324          */
10325         data_len = offsetof(struct scsi_inquiry_data, vendor_specific1);
10326         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10327         inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
10328         ctsio->kern_sg_entries = 0;
10329         ctsio->kern_data_resid = 0;
10330         ctsio->kern_rel_offset = 0;
10331
10332         if (data_len < alloc_len) {
10333                 ctsio->residual = alloc_len - data_len;
10334                 ctsio->kern_data_len = data_len;
10335                 ctsio->kern_total_len = data_len;
10336         } else {
10337                 ctsio->residual = 0;
10338                 ctsio->kern_data_len = alloc_len;
10339                 ctsio->kern_total_len = alloc_len;
10340         }
10341
10342         if (lun != NULL) {
10343                 if ((lun->flags & CTL_LUN_PRIMARY_SC) ||
10344                     softc->ha_link >= CTL_HA_LINK_UNKNOWN) {
10345                         inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10346                             lun->be_lun->lun_type;
10347                 } else {
10348                         inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) |
10349                             lun->be_lun->lun_type;
10350                 }
10351         } else
10352                 inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
10353
10354         /* RMB in byte 2 is 0 */
10355         inq_ptr->version = SCSI_REV_SPC4;
10356
10357         /*
10358          * According to SAM-3, even if a device only supports a single
10359          * level of LUN addressing, it should still set the HISUP bit:
10360          *
10361          * 4.9.1 Logical unit numbers overview
10362          *
10363          * All logical unit number formats described in this standard are
10364          * hierarchical in structure even when only a single level in that
10365          * hierarchy is used. The HISUP bit shall be set to one in the
10366          * standard INQUIRY data (see SPC-2) when any logical unit number
10367          * format described in this standard is used.  Non-hierarchical
10368          * formats are outside the scope of this standard.
10369          *
10370          * Therefore we set the HiSup bit here.
10371          *
10372          * The reponse format is 2, per SPC-3.
10373          */
10374         inq_ptr->response_format = SID_HiSup | 2;
10375
10376         inq_ptr->additional_length = data_len -
10377             (offsetof(struct scsi_inquiry_data, additional_length) + 1);
10378         CTL_DEBUG_PRINT(("additional_length = %d\n",
10379                          inq_ptr->additional_length));
10380
10381         inq_ptr->spc3_flags = SPC3_SID_3PC | SPC3_SID_TPGS_IMPLICIT;
10382         /* 16 bit addressing */
10383         if (port_type == CTL_PORT_SCSI)
10384                 inq_ptr->spc2_flags = SPC2_SID_ADDR16;
10385         /* XXX set the SID_MultiP bit here if we're actually going to
10386            respond on multiple ports */
10387         inq_ptr->spc2_flags |= SPC2_SID_MultiP;
10388
10389         /* 16 bit data bus, synchronous transfers */
10390         if (port_type == CTL_PORT_SCSI)
10391                 inq_ptr->flags = SID_WBus16 | SID_Sync;
10392         /*
10393          * XXX KDM do we want to support tagged queueing on the control
10394          * device at all?
10395          */
10396         if ((lun == NULL)
10397          || (lun->be_lun->lun_type != T_PROCESSOR))
10398                 inq_ptr->flags |= SID_CmdQue;
10399         /*
10400          * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
10401          * We have 8 bytes for the vendor name, and 16 bytes for the device
10402          * name and 4 bytes for the revision.
10403          */
10404         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10405             "vendor")) == NULL) {
10406                 strncpy(inq_ptr->vendor, CTL_VENDOR, sizeof(inq_ptr->vendor));
10407         } else {
10408                 memset(inq_ptr->vendor, ' ', sizeof(inq_ptr->vendor));
10409                 strncpy(inq_ptr->vendor, val,
10410                     min(sizeof(inq_ptr->vendor), strlen(val)));
10411         }
10412         if (lun == NULL) {
10413                 strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
10414                     sizeof(inq_ptr->product));
10415         } else if ((val = ctl_get_opt(&lun->be_lun->options, "product")) == NULL) {
10416                 switch (lun->be_lun->lun_type) {
10417                 case T_DIRECT:
10418                         strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
10419                             sizeof(inq_ptr->product));
10420                         break;
10421                 case T_PROCESSOR:
10422                         strncpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT,
10423                             sizeof(inq_ptr->product));
10424                         break;
10425                 default:
10426                         strncpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT,
10427                             sizeof(inq_ptr->product));
10428                         break;
10429                 }
10430         } else {
10431                 memset(inq_ptr->product, ' ', sizeof(inq_ptr->product));
10432                 strncpy(inq_ptr->product, val,
10433                     min(sizeof(inq_ptr->product), strlen(val)));
10434         }
10435
10436         /*
10437          * XXX make this a macro somewhere so it automatically gets
10438          * incremented when we make changes.
10439          */
10440         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10441             "revision")) == NULL) {
10442                 strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
10443         } else {
10444                 memset(inq_ptr->revision, ' ', sizeof(inq_ptr->revision));
10445                 strncpy(inq_ptr->revision, val,
10446                     min(sizeof(inq_ptr->revision), strlen(val)));
10447         }
10448
10449         /*
10450          * For parallel SCSI, we support double transition and single
10451          * transition clocking.  We also support QAS (Quick Arbitration
10452          * and Selection) and Information Unit transfers on both the
10453          * control and array devices.
10454          */
10455         if (port_type == CTL_PORT_SCSI)
10456                 inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
10457                                     SID_SPI_IUS;
10458
10459         /* SAM-5 (no version claimed) */
10460         scsi_ulto2b(0x00A0, inq_ptr->version1);
10461         /* SPC-4 (no version claimed) */
10462         scsi_ulto2b(0x0460, inq_ptr->version2);
10463         if (port_type == CTL_PORT_FC) {
10464                 /* FCP-2 ANSI INCITS.350:2003 */
10465                 scsi_ulto2b(0x0917, inq_ptr->version3);
10466         } else if (port_type == CTL_PORT_SCSI) {
10467                 /* SPI-4 ANSI INCITS.362:200x */
10468                 scsi_ulto2b(0x0B56, inq_ptr->version3);
10469         } else if (port_type == CTL_PORT_ISCSI) {
10470                 /* iSCSI (no version claimed) */
10471                 scsi_ulto2b(0x0960, inq_ptr->version3);
10472         } else if (port_type == CTL_PORT_SAS) {
10473                 /* SAS (no version claimed) */
10474                 scsi_ulto2b(0x0BE0, inq_ptr->version3);
10475         }
10476
10477         if (lun == NULL) {
10478                 /* SBC-4 (no version claimed) */
10479                 scsi_ulto2b(0x0600, inq_ptr->version4);
10480         } else {
10481                 switch (lun->be_lun->lun_type) {
10482                 case T_DIRECT:
10483                         /* SBC-4 (no version claimed) */
10484                         scsi_ulto2b(0x0600, inq_ptr->version4);
10485                         break;
10486                 case T_PROCESSOR:
10487                 default:
10488                         break;
10489                 }
10490         }
10491
10492         ctl_set_success(ctsio);
10493         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10494         ctsio->be_move_done = ctl_config_move_done;
10495         ctl_datamove((union ctl_io *)ctsio);
10496         return (CTL_RETVAL_COMPLETE);
10497 }
10498
10499 int
10500 ctl_inquiry(struct ctl_scsiio *ctsio)
10501 {
10502         struct scsi_inquiry *cdb;
10503         int retval;
10504
10505         CTL_DEBUG_PRINT(("ctl_inquiry\n"));
10506
10507         cdb = (struct scsi_inquiry *)ctsio->cdb;
10508         if (cdb->byte2 & SI_EVPD)
10509                 retval = ctl_inquiry_evpd(ctsio);
10510         else if (cdb->page_code == 0)
10511                 retval = ctl_inquiry_std(ctsio);
10512         else {
10513                 ctl_set_invalid_field(ctsio,
10514                                       /*sks_valid*/ 1,
10515                                       /*command*/ 1,
10516                                       /*field*/ 2,
10517                                       /*bit_valid*/ 0,
10518                                       /*bit*/ 0);
10519                 ctl_done((union ctl_io *)ctsio);
10520                 return (CTL_RETVAL_COMPLETE);
10521         }
10522
10523         return (retval);
10524 }
10525
10526 /*
10527  * For known CDB types, parse the LBA and length.
10528  */
10529 static int
10530 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len)
10531 {
10532         if (io->io_hdr.io_type != CTL_IO_SCSI)
10533                 return (1);
10534
10535         switch (io->scsiio.cdb[0]) {
10536         case COMPARE_AND_WRITE: {
10537                 struct scsi_compare_and_write *cdb;
10538
10539                 cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
10540
10541                 *lba = scsi_8btou64(cdb->addr);
10542                 *len = cdb->length;
10543                 break;
10544         }
10545         case READ_6:
10546         case WRITE_6: {
10547                 struct scsi_rw_6 *cdb;
10548
10549                 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
10550
10551                 *lba = scsi_3btoul(cdb->addr);
10552                 /* only 5 bits are valid in the most significant address byte */
10553                 *lba &= 0x1fffff;
10554                 *len = cdb->length;
10555                 break;
10556         }
10557         case READ_10:
10558         case WRITE_10: {
10559                 struct scsi_rw_10 *cdb;
10560
10561                 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
10562
10563                 *lba = scsi_4btoul(cdb->addr);
10564                 *len = scsi_2btoul(cdb->length);
10565                 break;
10566         }
10567         case WRITE_VERIFY_10: {
10568                 struct scsi_write_verify_10 *cdb;
10569
10570                 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
10571
10572                 *lba = scsi_4btoul(cdb->addr);
10573                 *len = scsi_2btoul(cdb->length);
10574                 break;
10575         }
10576         case READ_12:
10577         case WRITE_12: {
10578                 struct scsi_rw_12 *cdb;
10579
10580                 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
10581
10582                 *lba = scsi_4btoul(cdb->addr);
10583                 *len = scsi_4btoul(cdb->length);
10584                 break;
10585         }
10586         case WRITE_VERIFY_12: {
10587                 struct scsi_write_verify_12 *cdb;
10588
10589                 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
10590
10591                 *lba = scsi_4btoul(cdb->addr);
10592                 *len = scsi_4btoul(cdb->length);
10593                 break;
10594         }
10595         case READ_16:
10596         case WRITE_16: {
10597                 struct scsi_rw_16 *cdb;
10598
10599                 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
10600
10601                 *lba = scsi_8btou64(cdb->addr);
10602                 *len = scsi_4btoul(cdb->length);
10603                 break;
10604         }
10605         case WRITE_ATOMIC_16: {
10606                 struct scsi_write_atomic_16 *cdb;
10607
10608                 cdb = (struct scsi_write_atomic_16 *)io->scsiio.cdb;
10609
10610                 *lba = scsi_8btou64(cdb->addr);
10611                 *len = scsi_2btoul(cdb->length);
10612                 break;
10613         }
10614         case WRITE_VERIFY_16: {
10615                 struct scsi_write_verify_16 *cdb;
10616
10617                 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
10618
10619                 *lba = scsi_8btou64(cdb->addr);
10620                 *len = scsi_4btoul(cdb->length);
10621                 break;
10622         }
10623         case WRITE_SAME_10: {
10624                 struct scsi_write_same_10 *cdb;
10625
10626                 cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10627
10628                 *lba = scsi_4btoul(cdb->addr);
10629                 *len = scsi_2btoul(cdb->length);
10630                 break;
10631         }
10632         case WRITE_SAME_16: {
10633                 struct scsi_write_same_16 *cdb;
10634
10635                 cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10636
10637                 *lba = scsi_8btou64(cdb->addr);
10638                 *len = scsi_4btoul(cdb->length);
10639                 break;
10640         }
10641         case VERIFY_10: {
10642                 struct scsi_verify_10 *cdb;
10643
10644                 cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
10645
10646                 *lba = scsi_4btoul(cdb->addr);
10647                 *len = scsi_2btoul(cdb->length);
10648                 break;
10649         }
10650         case VERIFY_12: {
10651                 struct scsi_verify_12 *cdb;
10652
10653                 cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
10654
10655                 *lba = scsi_4btoul(cdb->addr);
10656                 *len = scsi_4btoul(cdb->length);
10657                 break;
10658         }
10659         case VERIFY_16: {
10660                 struct scsi_verify_16 *cdb;
10661
10662                 cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
10663
10664                 *lba = scsi_8btou64(cdb->addr);
10665                 *len = scsi_4btoul(cdb->length);
10666                 break;
10667         }
10668         case UNMAP: {
10669                 *lba = 0;
10670                 *len = UINT64_MAX;
10671                 break;
10672         }
10673         case SERVICE_ACTION_IN: {       /* GET LBA STATUS */
10674                 struct scsi_get_lba_status *cdb;
10675
10676                 cdb = (struct scsi_get_lba_status *)io->scsiio.cdb;
10677                 *lba = scsi_8btou64(cdb->addr);
10678                 *len = UINT32_MAX;
10679                 break;
10680         }
10681         default:
10682                 return (1);
10683                 break; /* NOTREACHED */
10684         }
10685
10686         return (0);
10687 }
10688
10689 static ctl_action
10690 ctl_extent_check_lba(uint64_t lba1, uint64_t len1, uint64_t lba2, uint64_t len2,
10691     bool seq)
10692 {
10693         uint64_t endlba1, endlba2;
10694
10695         endlba1 = lba1 + len1 - (seq ? 0 : 1);
10696         endlba2 = lba2 + len2 - 1;
10697
10698         if ((endlba1 < lba2) || (endlba2 < lba1))
10699                 return (CTL_ACTION_PASS);
10700         else
10701                 return (CTL_ACTION_BLOCK);
10702 }
10703
10704 static int
10705 ctl_extent_check_unmap(union ctl_io *io, uint64_t lba2, uint64_t len2)
10706 {
10707         struct ctl_ptr_len_flags *ptrlen;
10708         struct scsi_unmap_desc *buf, *end, *range;
10709         uint64_t lba;
10710         uint32_t len;
10711
10712         /* If not UNMAP -- go other way. */
10713         if (io->io_hdr.io_type != CTL_IO_SCSI ||
10714             io->scsiio.cdb[0] != UNMAP)
10715                 return (CTL_ACTION_ERROR);
10716
10717         /* If UNMAP without data -- block and wait for data. */
10718         ptrlen = (struct ctl_ptr_len_flags *)
10719             &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
10720         if ((io->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0 ||
10721             ptrlen->ptr == NULL)
10722                 return (CTL_ACTION_BLOCK);
10723
10724         /* UNMAP with data -- check for collision. */
10725         buf = (struct scsi_unmap_desc *)ptrlen->ptr;
10726         end = buf + ptrlen->len / sizeof(*buf);
10727         for (range = buf; range < end; range++) {
10728                 lba = scsi_8btou64(range->lba);
10729                 len = scsi_4btoul(range->length);
10730                 if ((lba < lba2 + len2) && (lba + len > lba2))
10731                         return (CTL_ACTION_BLOCK);
10732         }
10733         return (CTL_ACTION_PASS);
10734 }
10735
10736 static ctl_action
10737 ctl_extent_check(union ctl_io *io1, union ctl_io *io2, bool seq)
10738 {
10739         uint64_t lba1, lba2;
10740         uint64_t len1, len2;
10741         int retval;
10742
10743         if (ctl_get_lba_len(io2, &lba2, &len2) != 0)
10744                 return (CTL_ACTION_ERROR);
10745
10746         retval = ctl_extent_check_unmap(io1, lba2, len2);
10747         if (retval != CTL_ACTION_ERROR)
10748                 return (retval);
10749
10750         if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
10751                 return (CTL_ACTION_ERROR);
10752
10753         if (io1->io_hdr.flags & CTL_FLAG_SERSEQ_DONE)
10754                 seq = FALSE;
10755         return (ctl_extent_check_lba(lba1, len1, lba2, len2, seq));
10756 }
10757
10758 static ctl_action
10759 ctl_extent_check_seq(union ctl_io *io1, union ctl_io *io2)
10760 {
10761         uint64_t lba1, lba2;
10762         uint64_t len1, len2;
10763
10764         if (io1->io_hdr.flags & CTL_FLAG_SERSEQ_DONE)
10765                 return (CTL_ACTION_PASS);
10766         if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
10767                 return (CTL_ACTION_ERROR);
10768         if (ctl_get_lba_len(io2, &lba2, &len2) != 0)
10769                 return (CTL_ACTION_ERROR);
10770
10771         if (lba1 + len1 == lba2)
10772                 return (CTL_ACTION_BLOCK);
10773         return (CTL_ACTION_PASS);
10774 }
10775
10776 static ctl_action
10777 ctl_check_for_blockage(struct ctl_lun *lun, union ctl_io *pending_io,
10778     union ctl_io *ooa_io)
10779 {
10780         const struct ctl_cmd_entry *pending_entry, *ooa_entry;
10781         const ctl_serialize_action *serialize_row;
10782
10783         /*
10784          * The initiator attempted multiple untagged commands at the same
10785          * time.  Can't do that.
10786          */
10787         if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10788          && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10789          && ((pending_io->io_hdr.nexus.targ_port ==
10790               ooa_io->io_hdr.nexus.targ_port)
10791           && (pending_io->io_hdr.nexus.initid ==
10792               ooa_io->io_hdr.nexus.initid))
10793          && ((ooa_io->io_hdr.flags & (CTL_FLAG_ABORT |
10794               CTL_FLAG_STATUS_SENT)) == 0))
10795                 return (CTL_ACTION_OVERLAP);
10796
10797         /*
10798          * The initiator attempted to send multiple tagged commands with
10799          * the same ID.  (It's fine if different initiators have the same
10800          * tag ID.)
10801          *
10802          * Even if all of those conditions are true, we don't kill the I/O
10803          * if the command ahead of us has been aborted.  We won't end up
10804          * sending it to the FETD, and it's perfectly legal to resend a
10805          * command with the same tag number as long as the previous
10806          * instance of this tag number has been aborted somehow.
10807          */
10808         if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10809          && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10810          && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
10811          && ((pending_io->io_hdr.nexus.targ_port ==
10812               ooa_io->io_hdr.nexus.targ_port)
10813           && (pending_io->io_hdr.nexus.initid ==
10814               ooa_io->io_hdr.nexus.initid))
10815          && ((ooa_io->io_hdr.flags & (CTL_FLAG_ABORT |
10816               CTL_FLAG_STATUS_SENT)) == 0))
10817                 return (CTL_ACTION_OVERLAP_TAG);
10818
10819         /*
10820          * If we get a head of queue tag, SAM-3 says that we should
10821          * immediately execute it.
10822          *
10823          * What happens if this command would normally block for some other
10824          * reason?  e.g. a request sense with a head of queue tag
10825          * immediately after a write.  Normally that would block, but this
10826          * will result in its getting executed immediately...
10827          *
10828          * We currently return "pass" instead of "skip", so we'll end up
10829          * going through the rest of the queue to check for overlapped tags.
10830          *
10831          * XXX KDM check for other types of blockage first??
10832          */
10833         if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10834                 return (CTL_ACTION_PASS);
10835
10836         /*
10837          * Ordered tags have to block until all items ahead of them
10838          * have completed.  If we get called with an ordered tag, we always
10839          * block, if something else is ahead of us in the queue.
10840          */
10841         if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
10842                 return (CTL_ACTION_BLOCK);
10843
10844         /*
10845          * Simple tags get blocked until all head of queue and ordered tags
10846          * ahead of them have completed.  I'm lumping untagged commands in
10847          * with simple tags here.  XXX KDM is that the right thing to do?
10848          */
10849         if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10850           || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
10851          && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10852           || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
10853                 return (CTL_ACTION_BLOCK);
10854
10855         pending_entry = ctl_get_cmd_entry(&pending_io->scsiio, NULL);
10856         ooa_entry = ctl_get_cmd_entry(&ooa_io->scsiio, NULL);
10857
10858         serialize_row = ctl_serialize_table[ooa_entry->seridx];
10859
10860         switch (serialize_row[pending_entry->seridx]) {
10861         case CTL_SER_BLOCK:
10862                 return (CTL_ACTION_BLOCK);
10863         case CTL_SER_EXTENT:
10864                 return (ctl_extent_check(ooa_io, pending_io,
10865                     (lun->be_lun && lun->be_lun->serseq == CTL_LUN_SERSEQ_ON)));
10866         case CTL_SER_EXTENTOPT:
10867                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT].queue_flags
10868                     & SCP_QUEUE_ALG_MASK) != SCP_QUEUE_ALG_UNRESTRICTED)
10869                         return (ctl_extent_check(ooa_io, pending_io,
10870                             (lun->be_lun &&
10871                              lun->be_lun->serseq == CTL_LUN_SERSEQ_ON)));
10872                 return (CTL_ACTION_PASS);
10873         case CTL_SER_EXTENTSEQ:
10874                 if (lun->be_lun && lun->be_lun->serseq != CTL_LUN_SERSEQ_OFF)
10875                         return (ctl_extent_check_seq(ooa_io, pending_io));
10876                 return (CTL_ACTION_PASS);
10877         case CTL_SER_PASS:
10878                 return (CTL_ACTION_PASS);
10879         case CTL_SER_BLOCKOPT:
10880                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT].queue_flags
10881                     & SCP_QUEUE_ALG_MASK) != SCP_QUEUE_ALG_UNRESTRICTED)
10882                         return (CTL_ACTION_BLOCK);
10883                 return (CTL_ACTION_PASS);
10884         case CTL_SER_SKIP:
10885                 return (CTL_ACTION_SKIP);
10886         default:
10887                 panic("invalid serialization value %d",
10888                       serialize_row[pending_entry->seridx]);
10889         }
10890
10891         return (CTL_ACTION_ERROR);
10892 }
10893
10894 /*
10895  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
10896  * Assumptions:
10897  * - pending_io is generally either incoming, or on the blocked queue
10898  * - starting I/O is the I/O we want to start the check with.
10899  */
10900 static ctl_action
10901 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
10902               union ctl_io *starting_io)
10903 {
10904         union ctl_io *ooa_io;
10905         ctl_action action;
10906
10907         mtx_assert(&lun->lun_lock, MA_OWNED);
10908
10909         /*
10910          * Run back along the OOA queue, starting with the current
10911          * blocked I/O and going through every I/O before it on the
10912          * queue.  If starting_io is NULL, we'll just end up returning
10913          * CTL_ACTION_PASS.
10914          */
10915         for (ooa_io = starting_io; ooa_io != NULL;
10916              ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
10917              ooa_links)){
10918
10919                 /*
10920                  * This routine just checks to see whether
10921                  * cur_blocked is blocked by ooa_io, which is ahead
10922                  * of it in the queue.  It doesn't queue/dequeue
10923                  * cur_blocked.
10924                  */
10925                 action = ctl_check_for_blockage(lun, pending_io, ooa_io);
10926                 switch (action) {
10927                 case CTL_ACTION_BLOCK:
10928                 case CTL_ACTION_OVERLAP:
10929                 case CTL_ACTION_OVERLAP_TAG:
10930                 case CTL_ACTION_SKIP:
10931                 case CTL_ACTION_ERROR:
10932                         return (action);
10933                         break; /* NOTREACHED */
10934                 case CTL_ACTION_PASS:
10935                         break;
10936                 default:
10937                         panic("invalid action %d", action);
10938                         break;  /* NOTREACHED */
10939                 }
10940         }
10941
10942         return (CTL_ACTION_PASS);
10943 }
10944
10945 /*
10946  * Assumptions:
10947  * - An I/O has just completed, and has been removed from the per-LUN OOA
10948  *   queue, so some items on the blocked queue may now be unblocked.
10949  */
10950 static int
10951 ctl_check_blocked(struct ctl_lun *lun)
10952 {
10953         struct ctl_softc *softc = lun->ctl_softc;
10954         union ctl_io *cur_blocked, *next_blocked;
10955
10956         mtx_assert(&lun->lun_lock, MA_OWNED);
10957
10958         /*
10959          * Run forward from the head of the blocked queue, checking each
10960          * entry against the I/Os prior to it on the OOA queue to see if
10961          * there is still any blockage.
10962          *
10963          * We cannot use the TAILQ_FOREACH() macro, because it can't deal
10964          * with our removing a variable on it while it is traversing the
10965          * list.
10966          */
10967         for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
10968              cur_blocked != NULL; cur_blocked = next_blocked) {
10969                 union ctl_io *prev_ooa;
10970                 ctl_action action;
10971
10972                 next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
10973                                                           blocked_links);
10974
10975                 prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
10976                                                       ctl_ooaq, ooa_links);
10977
10978                 /*
10979                  * If cur_blocked happens to be the first item in the OOA
10980                  * queue now, prev_ooa will be NULL, and the action
10981                  * returned will just be CTL_ACTION_PASS.
10982                  */
10983                 action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
10984
10985                 switch (action) {
10986                 case CTL_ACTION_BLOCK:
10987                         /* Nothing to do here, still blocked */
10988                         break;
10989                 case CTL_ACTION_OVERLAP:
10990                 case CTL_ACTION_OVERLAP_TAG:
10991                         /*
10992                          * This shouldn't happen!  In theory we've already
10993                          * checked this command for overlap...
10994                          */
10995                         break;
10996                 case CTL_ACTION_PASS:
10997                 case CTL_ACTION_SKIP: {
10998                         const struct ctl_cmd_entry *entry;
10999
11000                         /*
11001                          * The skip case shouldn't happen, this transaction
11002                          * should have never made it onto the blocked queue.
11003                          */
11004                         /*
11005                          * This I/O is no longer blocked, we can remove it
11006                          * from the blocked queue.  Since this is a TAILQ
11007                          * (doubly linked list), we can do O(1) removals
11008                          * from any place on the list.
11009                          */
11010                         TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
11011                                      blocked_links);
11012                         cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
11013
11014                         if ((softc->ha_mode != CTL_HA_MODE_XFER) &&
11015                             (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)){
11016                                 /*
11017                                  * Need to send IO back to original side to
11018                                  * run
11019                                  */
11020                                 union ctl_ha_msg msg_info;
11021
11022                                 cur_blocked->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11023                                 msg_info.hdr.original_sc =
11024                                         cur_blocked->io_hdr.original_sc;
11025                                 msg_info.hdr.serializing_sc = cur_blocked;
11026                                 msg_info.hdr.msg_type = CTL_MSG_R2R;
11027                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11028                                     sizeof(msg_info.hdr), M_NOWAIT);
11029                                 break;
11030                         }
11031                         entry = ctl_get_cmd_entry(&cur_blocked->scsiio, NULL);
11032
11033                         /*
11034                          * Check this I/O for LUN state changes that may
11035                          * have happened while this command was blocked.
11036                          * The LUN state may have been changed by a command
11037                          * ahead of us in the queue, so we need to re-check
11038                          * for any states that can be caused by SCSI
11039                          * commands.
11040                          */
11041                         if (ctl_scsiio_lun_check(lun, entry,
11042                                                  &cur_blocked->scsiio) == 0) {
11043                                 cur_blocked->io_hdr.flags |=
11044                                                       CTL_FLAG_IS_WAS_ON_RTR;
11045                                 ctl_enqueue_rtr(cur_blocked);
11046                         } else
11047                                 ctl_done(cur_blocked);
11048                         break;
11049                 }
11050                 default:
11051                         /*
11052                          * This probably shouldn't happen -- we shouldn't
11053                          * get CTL_ACTION_ERROR, or anything else.
11054                          */
11055                         break;
11056                 }
11057         }
11058
11059         return (CTL_RETVAL_COMPLETE);
11060 }
11061
11062 /*
11063  * This routine (with one exception) checks LUN flags that can be set by
11064  * commands ahead of us in the OOA queue.  These flags have to be checked
11065  * when a command initially comes in, and when we pull a command off the
11066  * blocked queue and are preparing to execute it.  The reason we have to
11067  * check these flags for commands on the blocked queue is that the LUN
11068  * state may have been changed by a command ahead of us while we're on the
11069  * blocked queue.
11070  *
11071  * Ordering is somewhat important with these checks, so please pay
11072  * careful attention to the placement of any new checks.
11073  */
11074 static int
11075 ctl_scsiio_lun_check(struct ctl_lun *lun,
11076     const struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
11077 {
11078         struct ctl_softc *softc = lun->ctl_softc;
11079         int retval;
11080         uint32_t residx;
11081
11082         retval = 0;
11083
11084         mtx_assert(&lun->lun_lock, MA_OWNED);
11085
11086         /*
11087          * If this shelf is a secondary shelf controller, we may have to
11088          * reject some commands disallowed by HA mode and link state.
11089          */
11090         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
11091                 if (softc->ha_link == CTL_HA_LINK_OFFLINE &&
11092                     (entry->flags & CTL_CMD_FLAG_OK_ON_UNAVAIL) == 0) {
11093                         ctl_set_lun_unavail(ctsio);
11094                         retval = 1;
11095                         goto bailout;
11096                 }
11097                 if ((lun->flags & CTL_LUN_PEER_SC_PRIMARY) == 0 &&
11098                     (entry->flags & CTL_CMD_FLAG_OK_ON_UNAVAIL) == 0) {
11099                         ctl_set_lun_transit(ctsio);
11100                         retval = 1;
11101                         goto bailout;
11102                 }
11103                 if (softc->ha_mode == CTL_HA_MODE_ACT_STBY &&
11104                     (entry->flags & CTL_CMD_FLAG_OK_ON_STANDBY) == 0) {
11105                         ctl_set_lun_standby(ctsio);
11106                         retval = 1;
11107                         goto bailout;
11108                 }
11109
11110                 /* The rest of checks are only done on executing side */
11111                 if (softc->ha_mode == CTL_HA_MODE_XFER)
11112                         goto bailout;
11113         }
11114
11115         if (entry->pattern & CTL_LUN_PAT_WRITE) {
11116                 if (lun->be_lun &&
11117                     lun->be_lun->flags & CTL_LUN_FLAG_READONLY) {
11118                         ctl_set_hw_write_protected(ctsio);
11119                         retval = 1;
11120                         goto bailout;
11121                 }
11122                 if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT]
11123                     .eca_and_aen & SCP_SWP) != 0) {
11124                         ctl_set_sense(ctsio, /*current_error*/ 1,
11125                             /*sense_key*/ SSD_KEY_DATA_PROTECT,
11126                             /*asc*/ 0x27, /*ascq*/ 0x02, SSD_ELEM_NONE);
11127                         retval = 1;
11128                         goto bailout;
11129                 }
11130         }
11131
11132         /*
11133          * Check for a reservation conflict.  If this command isn't allowed
11134          * even on reserved LUNs, and if this initiator isn't the one who
11135          * reserved us, reject the command with a reservation conflict.
11136          */
11137         residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11138         if ((lun->flags & CTL_LUN_RESERVED)
11139          && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
11140                 if (lun->res_idx != residx) {
11141                         ctl_set_reservation_conflict(ctsio);
11142                         retval = 1;
11143                         goto bailout;
11144                 }
11145         }
11146
11147         if ((lun->flags & CTL_LUN_PR_RESERVED) == 0 ||
11148             (entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV)) {
11149                 /* No reservation or command is allowed. */;
11150         } else if ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_WRESV) &&
11151             (lun->res_type == SPR_TYPE_WR_EX ||
11152              lun->res_type == SPR_TYPE_WR_EX_RO ||
11153              lun->res_type == SPR_TYPE_WR_EX_AR)) {
11154                 /* The command is allowed for Write Exclusive resv. */;
11155         } else {
11156                 /*
11157                  * if we aren't registered or it's a res holder type
11158                  * reservation and this isn't the res holder then set a
11159                  * conflict.
11160                  */
11161                 if (ctl_get_prkey(lun, residx) == 0
11162                  || (residx != lun->pr_res_idx && lun->res_type < 4)) {
11163                         ctl_set_reservation_conflict(ctsio);
11164                         retval = 1;
11165                         goto bailout;
11166                 }
11167         }
11168
11169         if ((lun->flags & CTL_LUN_OFFLINE)
11170          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STANDBY) == 0)) {
11171                 ctl_set_lun_not_ready(ctsio);
11172                 retval = 1;
11173                 goto bailout;
11174         }
11175
11176         if ((lun->flags & CTL_LUN_STOPPED)
11177          && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
11178                 /* "Logical unit not ready, initializing cmd. required" */
11179                 ctl_set_lun_stopped(ctsio);
11180                 retval = 1;
11181                 goto bailout;
11182         }
11183
11184         if ((lun->flags & CTL_LUN_INOPERABLE)
11185          && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
11186                 /* "Medium format corrupted" */
11187                 ctl_set_medium_format_corrupted(ctsio);
11188                 retval = 1;
11189                 goto bailout;
11190         }
11191
11192 bailout:
11193         return (retval);
11194 }
11195
11196 static void
11197 ctl_failover_io(union ctl_io *io, int have_lock)
11198 {
11199         ctl_set_busy(&io->scsiio);
11200         ctl_done(io);
11201 }
11202
11203 static void
11204 ctl_failover_lun(union ctl_io *rio)
11205 {
11206         struct ctl_softc *softc = control_softc;
11207         struct ctl_lun *lun;
11208         struct ctl_io_hdr *io, *next_io;
11209         uint32_t targ_lun;
11210
11211         targ_lun = rio->io_hdr.nexus.targ_mapped_lun;
11212         CTL_DEBUG_PRINT(("FAILOVER for lun %ju\n", targ_lun));
11213
11214         /* Find and lock the LUN. */
11215         mtx_lock(&softc->ctl_lock);
11216         if ((targ_lun < CTL_MAX_LUNS) &&
11217             ((lun = softc->ctl_luns[targ_lun]) != NULL)) {
11218                 mtx_lock(&lun->lun_lock);
11219                 mtx_unlock(&softc->ctl_lock);
11220                 if (lun->flags & CTL_LUN_DISABLED) {
11221                         mtx_unlock(&lun->lun_lock);
11222                         return;
11223                 }
11224         } else {
11225                 mtx_unlock(&softc->ctl_lock);
11226                 return;
11227         }
11228
11229         if (softc->ha_mode == CTL_HA_MODE_XFER) {
11230                 TAILQ_FOREACH_SAFE(io, &lun->ooa_queue, ooa_links, next_io) {
11231                         /* We are master */
11232                         if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11233                                 if (io->flags & CTL_FLAG_IO_ACTIVE) {
11234                                         io->flags |= CTL_FLAG_ABORT;
11235                                         io->flags |= CTL_FLAG_FAILOVER;
11236                                 } else { /* This can be only due to DATAMOVE */
11237                                         io->msg_type = CTL_MSG_DATAMOVE_DONE;
11238                                         io->flags &= ~CTL_FLAG_DMA_INPROG;
11239                                         io->flags |= CTL_FLAG_IO_ACTIVE;
11240                                         io->port_status = 31340;
11241                                         ctl_enqueue_isc((union ctl_io *)io);
11242                                 }
11243                         }
11244                         /* We are slave */
11245                         if (io->flags & CTL_FLAG_SENT_2OTHER_SC) {
11246                                 io->flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11247                                 if (io->flags & CTL_FLAG_IO_ACTIVE) {
11248                                         io->flags |= CTL_FLAG_FAILOVER;
11249                                 } else {
11250                                         ctl_set_busy(&((union ctl_io *)io)->
11251                                             scsiio);
11252                                         ctl_done((union ctl_io *)io);
11253                                 }
11254                         }
11255                 }
11256         } else { /* SERIALIZE modes */
11257                 TAILQ_FOREACH_SAFE(io, &lun->blocked_queue, blocked_links,
11258                     next_io) {
11259                         /* We are master */
11260                         if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11261                                 TAILQ_REMOVE(&lun->blocked_queue, io,
11262                                     blocked_links);
11263                                 io->flags &= ~CTL_FLAG_BLOCKED;
11264                                 TAILQ_REMOVE(&lun->ooa_queue, io, ooa_links);
11265                                 ctl_free_io((union ctl_io *)io);
11266                         }
11267                 }
11268                 TAILQ_FOREACH_SAFE(io, &lun->ooa_queue, ooa_links, next_io) {
11269                         /* We are master */
11270                         if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11271                                 TAILQ_REMOVE(&lun->ooa_queue, io, ooa_links);
11272                                 ctl_free_io((union ctl_io *)io);
11273                         }
11274                         /* We are slave */
11275                         if (io->flags & CTL_FLAG_SENT_2OTHER_SC) {
11276                                 io->flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11277                                 if (!(io->flags & CTL_FLAG_IO_ACTIVE)) {
11278                                         ctl_set_busy(&((union ctl_io *)io)->
11279                                             scsiio);
11280                                         ctl_done((union ctl_io *)io);
11281                                 }
11282                         }
11283                 }
11284                 ctl_check_blocked(lun);
11285         }
11286         mtx_unlock(&lun->lun_lock);
11287 }
11288
11289 static int
11290 ctl_scsiio_precheck(struct ctl_softc *softc, struct ctl_scsiio *ctsio)
11291 {
11292         struct ctl_lun *lun;
11293         const struct ctl_cmd_entry *entry;
11294         uint32_t initidx, targ_lun;
11295         int retval;
11296
11297         retval = 0;
11298
11299         lun = NULL;
11300
11301         targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
11302         if ((targ_lun < CTL_MAX_LUNS)
11303          && ((lun = softc->ctl_luns[targ_lun]) != NULL)) {
11304                 /*
11305                  * If the LUN is invalid, pretend that it doesn't exist.
11306                  * It will go away as soon as all pending I/O has been
11307                  * completed.
11308                  */
11309                 mtx_lock(&lun->lun_lock);
11310                 if (lun->flags & CTL_LUN_DISABLED) {
11311                         mtx_unlock(&lun->lun_lock);
11312                         lun = NULL;
11313                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11314                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11315                 } else {
11316                         ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
11317                         ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
11318                                 lun->be_lun;
11319
11320                         /*
11321                          * Every I/O goes into the OOA queue for a
11322                          * particular LUN, and stays there until completion.
11323                          */
11324 #ifdef CTL_TIME_IO
11325                         if (TAILQ_EMPTY(&lun->ooa_queue)) {
11326                                 lun->idle_time += getsbinuptime() -
11327                                     lun->last_busy;
11328                         }
11329 #endif
11330                         TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr,
11331                             ooa_links);
11332                 }
11333         } else {
11334                 ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11335                 ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11336         }
11337
11338         /* Get command entry and return error if it is unsuppotyed. */
11339         entry = ctl_validate_command(ctsio);
11340         if (entry == NULL) {
11341                 if (lun)
11342                         mtx_unlock(&lun->lun_lock);
11343                 return (retval);
11344         }
11345
11346         ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
11347         ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
11348
11349         /*
11350          * Check to see whether we can send this command to LUNs that don't
11351          * exist.  This should pretty much only be the case for inquiry
11352          * and request sense.  Further checks, below, really require having
11353          * a LUN, so we can't really check the command anymore.  Just put
11354          * it on the rtr queue.
11355          */
11356         if (lun == NULL) {
11357                 if (entry->flags & CTL_CMD_FLAG_OK_ON_NO_LUN) {
11358                         ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11359                         ctl_enqueue_rtr((union ctl_io *)ctsio);
11360                         return (retval);
11361                 }
11362
11363                 ctl_set_unsupported_lun(ctsio);
11364                 ctl_done((union ctl_io *)ctsio);
11365                 CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
11366                 return (retval);
11367         } else {
11368                 /*
11369                  * Make sure we support this particular command on this LUN.
11370                  * e.g., we don't support writes to the control LUN.
11371                  */
11372                 if (!ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
11373                         mtx_unlock(&lun->lun_lock);
11374                         ctl_set_invalid_opcode(ctsio);
11375                         ctl_done((union ctl_io *)ctsio);
11376                         return (retval);
11377                 }
11378         }
11379
11380         initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11381
11382 #ifdef CTL_WITH_CA
11383         /*
11384          * If we've got a request sense, it'll clear the contingent
11385          * allegiance condition.  Otherwise, if we have a CA condition for
11386          * this initiator, clear it, because it sent down a command other
11387          * than request sense.
11388          */
11389         if ((ctsio->cdb[0] != REQUEST_SENSE)
11390          && (ctl_is_set(lun->have_ca, initidx)))
11391                 ctl_clear_mask(lun->have_ca, initidx);
11392 #endif
11393
11394         /*
11395          * If the command has this flag set, it handles its own unit
11396          * attention reporting, we shouldn't do anything.  Otherwise we
11397          * check for any pending unit attentions, and send them back to the
11398          * initiator.  We only do this when a command initially comes in,
11399          * not when we pull it off the blocked queue.
11400          *
11401          * According to SAM-3, section 5.3.2, the order that things get
11402          * presented back to the host is basically unit attentions caused
11403          * by some sort of reset event, busy status, reservation conflicts
11404          * or task set full, and finally any other status.
11405          *
11406          * One issue here is that some of the unit attentions we report
11407          * don't fall into the "reset" category (e.g. "reported luns data
11408          * has changed").  So reporting it here, before the reservation
11409          * check, may be technically wrong.  I guess the only thing to do
11410          * would be to check for and report the reset events here, and then
11411          * check for the other unit attention types after we check for a
11412          * reservation conflict.
11413          *
11414          * XXX KDM need to fix this
11415          */
11416         if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
11417                 ctl_ua_type ua_type;
11418
11419                 ua_type = ctl_build_ua(lun, initidx, &ctsio->sense_data,
11420                     SSD_TYPE_NONE);
11421                 if (ua_type != CTL_UA_NONE) {
11422                         mtx_unlock(&lun->lun_lock);
11423                         ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
11424                         ctsio->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
11425                         ctsio->sense_len = SSD_FULL_SIZE;
11426                         ctl_done((union ctl_io *)ctsio);
11427                         return (retval);
11428                 }
11429         }
11430
11431
11432         if (ctl_scsiio_lun_check(lun, entry, ctsio) != 0) {
11433                 mtx_unlock(&lun->lun_lock);
11434                 ctl_done((union ctl_io *)ctsio);
11435                 return (retval);
11436         }
11437
11438         /*
11439          * XXX CHD this is where we want to send IO to other side if
11440          * this LUN is secondary on this SC. We will need to make a copy
11441          * of the IO and flag the IO on this side as SENT_2OTHER and the flag
11442          * the copy we send as FROM_OTHER.
11443          * We also need to stuff the address of the original IO so we can
11444          * find it easily. Something similar will need be done on the other
11445          * side so when we are done we can find the copy.
11446          */
11447         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
11448             (lun->flags & CTL_LUN_PEER_SC_PRIMARY) != 0 &&
11449             (entry->flags & CTL_CMD_FLAG_RUN_HERE) == 0) {
11450                 union ctl_ha_msg msg_info;
11451                 int isc_retval;
11452
11453                 ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11454                 ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11455                 mtx_unlock(&lun->lun_lock);
11456
11457                 msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
11458                 msg_info.hdr.original_sc = (union ctl_io *)ctsio;
11459                 msg_info.hdr.serializing_sc = NULL;
11460                 msg_info.hdr.nexus = ctsio->io_hdr.nexus;
11461                 msg_info.scsi.tag_num = ctsio->tag_num;
11462                 msg_info.scsi.tag_type = ctsio->tag_type;
11463                 msg_info.scsi.cdb_len = ctsio->cdb_len;
11464                 memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
11465
11466                 if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11467                     sizeof(msg_info.scsi) - sizeof(msg_info.scsi.sense_data),
11468                     M_WAITOK)) > CTL_HA_STATUS_SUCCESS) {
11469                         ctl_set_busy(ctsio);
11470                         ctl_done((union ctl_io *)ctsio);
11471                         return (retval);
11472                 }
11473                 return (retval);
11474         }
11475
11476         switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
11477                               (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
11478                               ctl_ooaq, ooa_links))) {
11479         case CTL_ACTION_BLOCK:
11480                 ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
11481                 TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
11482                                   blocked_links);
11483                 mtx_unlock(&lun->lun_lock);
11484                 return (retval);
11485         case CTL_ACTION_PASS:
11486         case CTL_ACTION_SKIP:
11487                 ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11488                 mtx_unlock(&lun->lun_lock);
11489                 ctl_enqueue_rtr((union ctl_io *)ctsio);
11490                 break;
11491         case CTL_ACTION_OVERLAP:
11492                 mtx_unlock(&lun->lun_lock);
11493                 ctl_set_overlapped_cmd(ctsio);
11494                 ctl_done((union ctl_io *)ctsio);
11495                 break;
11496         case CTL_ACTION_OVERLAP_TAG:
11497                 mtx_unlock(&lun->lun_lock);
11498                 ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
11499                 ctl_done((union ctl_io *)ctsio);
11500                 break;
11501         case CTL_ACTION_ERROR:
11502         default:
11503                 mtx_unlock(&lun->lun_lock);
11504                 ctl_set_internal_failure(ctsio,
11505                                          /*sks_valid*/ 0,
11506                                          /*retry_count*/ 0);
11507                 ctl_done((union ctl_io *)ctsio);
11508                 break;
11509         }
11510         return (retval);
11511 }
11512
11513 const struct ctl_cmd_entry *
11514 ctl_get_cmd_entry(struct ctl_scsiio *ctsio, int *sa)
11515 {
11516         const struct ctl_cmd_entry *entry;
11517         int service_action;
11518
11519         entry = &ctl_cmd_table[ctsio->cdb[0]];
11520         if (sa)
11521                 *sa = ((entry->flags & CTL_CMD_FLAG_SA5) != 0);
11522         if (entry->flags & CTL_CMD_FLAG_SA5) {
11523                 service_action = ctsio->cdb[1] & SERVICE_ACTION_MASK;
11524                 entry = &((const struct ctl_cmd_entry *)
11525                     entry->execute)[service_action];
11526         }
11527         return (entry);
11528 }
11529
11530 const struct ctl_cmd_entry *
11531 ctl_validate_command(struct ctl_scsiio *ctsio)
11532 {
11533         const struct ctl_cmd_entry *entry;
11534         int i, sa;
11535         uint8_t diff;
11536
11537         entry = ctl_get_cmd_entry(ctsio, &sa);
11538         if (entry->execute == NULL) {
11539                 if (sa)
11540                         ctl_set_invalid_field(ctsio,
11541                                               /*sks_valid*/ 1,
11542                                               /*command*/ 1,
11543                                               /*field*/ 1,
11544                                               /*bit_valid*/ 1,
11545                                               /*bit*/ 4);
11546                 else
11547                         ctl_set_invalid_opcode(ctsio);
11548                 ctl_done((union ctl_io *)ctsio);
11549                 return (NULL);
11550         }
11551         KASSERT(entry->length > 0,
11552             ("Not defined length for command 0x%02x/0x%02x",
11553              ctsio->cdb[0], ctsio->cdb[1]));
11554         for (i = 1; i < entry->length; i++) {
11555                 diff = ctsio->cdb[i] & ~entry->usage[i - 1];
11556                 if (diff == 0)
11557                         continue;
11558                 ctl_set_invalid_field(ctsio,
11559                                       /*sks_valid*/ 1,
11560                                       /*command*/ 1,
11561                                       /*field*/ i,
11562                                       /*bit_valid*/ 1,
11563                                       /*bit*/ fls(diff) - 1);
11564                 ctl_done((union ctl_io *)ctsio);
11565                 return (NULL);
11566         }
11567         return (entry);
11568 }
11569
11570 static int
11571 ctl_cmd_applicable(uint8_t lun_type, const struct ctl_cmd_entry *entry)
11572 {
11573
11574         switch (lun_type) {
11575         case T_PROCESSOR:
11576                 if ((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0)
11577                         return (0);
11578                 break;
11579         case T_DIRECT:
11580                 if ((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0)
11581                         return (0);
11582                 break;
11583         default:
11584                 return (0);
11585         }
11586         return (1);
11587 }
11588
11589 static int
11590 ctl_scsiio(struct ctl_scsiio *ctsio)
11591 {
11592         int retval;
11593         const struct ctl_cmd_entry *entry;
11594
11595         retval = CTL_RETVAL_COMPLETE;
11596
11597         CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
11598
11599         entry = ctl_get_cmd_entry(ctsio, NULL);
11600
11601         /*
11602          * If this I/O has been aborted, just send it straight to
11603          * ctl_done() without executing it.
11604          */
11605         if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
11606                 ctl_done((union ctl_io *)ctsio);
11607                 goto bailout;
11608         }
11609
11610         /*
11611          * All the checks should have been handled by ctl_scsiio_precheck().
11612          * We should be clear now to just execute the I/O.
11613          */
11614         retval = entry->execute(ctsio);
11615
11616 bailout:
11617         return (retval);
11618 }
11619
11620 /*
11621  * Since we only implement one target right now, a bus reset simply resets
11622  * our single target.
11623  */
11624 static int
11625 ctl_bus_reset(struct ctl_softc *softc, union ctl_io *io)
11626 {
11627         return(ctl_target_reset(softc, io, CTL_UA_BUS_RESET));
11628 }
11629
11630 static int
11631 ctl_target_reset(struct ctl_softc *softc, union ctl_io *io,
11632                  ctl_ua_type ua_type)
11633 {
11634         struct ctl_port *port;
11635         struct ctl_lun *lun;
11636         int retval;
11637
11638         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11639                 union ctl_ha_msg msg_info;
11640
11641                 msg_info.hdr.nexus = io->io_hdr.nexus;
11642                 if (ua_type==CTL_UA_TARG_RESET)
11643                         msg_info.task.task_action = CTL_TASK_TARGET_RESET;
11644                 else
11645                         msg_info.task.task_action = CTL_TASK_BUS_RESET;
11646                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11647                 msg_info.hdr.original_sc = NULL;
11648                 msg_info.hdr.serializing_sc = NULL;
11649                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11650                     sizeof(msg_info.task), M_WAITOK);
11651         }
11652         retval = 0;
11653
11654         mtx_lock(&softc->ctl_lock);
11655         port = ctl_io_port(&io->io_hdr);
11656         STAILQ_FOREACH(lun, &softc->lun_list, links) {
11657                 if (port != NULL &&
11658                     ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
11659                         continue;
11660                 retval += ctl_do_lun_reset(lun, io, ua_type);
11661         }
11662         mtx_unlock(&softc->ctl_lock);
11663         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11664         return (retval);
11665 }
11666
11667 /*
11668  * The LUN should always be set.  The I/O is optional, and is used to
11669  * distinguish between I/Os sent by this initiator, and by other
11670  * initiators.  We set unit attention for initiators other than this one.
11671  * SAM-3 is vague on this point.  It does say that a unit attention should
11672  * be established for other initiators when a LUN is reset (see section
11673  * 5.7.3), but it doesn't specifically say that the unit attention should
11674  * be established for this particular initiator when a LUN is reset.  Here
11675  * is the relevant text, from SAM-3 rev 8:
11676  *
11677  * 5.7.2 When a SCSI initiator port aborts its own tasks
11678  *
11679  * When a SCSI initiator port causes its own task(s) to be aborted, no
11680  * notification that the task(s) have been aborted shall be returned to
11681  * the SCSI initiator port other than the completion response for the
11682  * command or task management function action that caused the task(s) to
11683  * be aborted and notification(s) associated with related effects of the
11684  * action (e.g., a reset unit attention condition).
11685  *
11686  * XXX KDM for now, we're setting unit attention for all initiators.
11687  */
11688 static int
11689 ctl_do_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
11690 {
11691         union ctl_io *xio;
11692 #if 0
11693         uint32_t initidx;
11694 #endif
11695 #ifdef CTL_WITH_CA
11696         int i;
11697 #endif
11698
11699         mtx_lock(&lun->lun_lock);
11700         /*
11701          * Run through the OOA queue and abort each I/O.
11702          */
11703         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11704              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11705                 xio->io_hdr.flags |= CTL_FLAG_ABORT | CTL_FLAG_ABORT_STATUS;
11706         }
11707
11708         /*
11709          * This version sets unit attention for every
11710          */
11711 #if 0
11712         initidx = ctl_get_initindex(&io->io_hdr.nexus);
11713         ctl_est_ua_all(lun, initidx, ua_type);
11714 #else
11715         ctl_est_ua_all(lun, -1, ua_type);
11716 #endif
11717
11718         /*
11719          * A reset (any kind, really) clears reservations established with
11720          * RESERVE/RELEASE.  It does not clear reservations established
11721          * with PERSISTENT RESERVE OUT, but we don't support that at the
11722          * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
11723          * reservations made with the RESERVE/RELEASE commands, because
11724          * those commands are obsolete in SPC-3.
11725          */
11726         lun->flags &= ~CTL_LUN_RESERVED;
11727
11728 #ifdef CTL_WITH_CA
11729         for (i = 0; i < CTL_MAX_INITIATORS; i++)
11730                 ctl_clear_mask(lun->have_ca, i);
11731 #endif
11732         mtx_unlock(&lun->lun_lock);
11733
11734         return (0);
11735 }
11736
11737 static int
11738 ctl_lun_reset(struct ctl_softc *softc, union ctl_io *io)
11739 {
11740         struct ctl_lun *lun;
11741         uint32_t targ_lun;
11742         int retval;
11743
11744         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11745         mtx_lock(&softc->ctl_lock);
11746         if ((targ_lun >= CTL_MAX_LUNS) ||
11747             (lun = softc->ctl_luns[targ_lun]) == NULL) {
11748                 mtx_unlock(&softc->ctl_lock);
11749                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11750                 return (1);
11751         }
11752         retval = ctl_do_lun_reset(lun, io, CTL_UA_LUN_RESET);
11753         mtx_unlock(&softc->ctl_lock);
11754         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11755
11756         if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0) {
11757                 union ctl_ha_msg msg_info;
11758
11759                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11760                 msg_info.hdr.nexus = io->io_hdr.nexus;
11761                 msg_info.task.task_action = CTL_TASK_LUN_RESET;
11762                 msg_info.hdr.original_sc = NULL;
11763                 msg_info.hdr.serializing_sc = NULL;
11764                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11765                     sizeof(msg_info.task), M_WAITOK);
11766         }
11767         return (retval);
11768 }
11769
11770 static void
11771 ctl_abort_tasks_lun(struct ctl_lun *lun, uint32_t targ_port, uint32_t init_id,
11772     int other_sc)
11773 {
11774         union ctl_io *xio;
11775
11776         mtx_assert(&lun->lun_lock, MA_OWNED);
11777
11778         /*
11779          * Run through the OOA queue and attempt to find the given I/O.
11780          * The target port, initiator ID, tag type and tag number have to
11781          * match the values that we got from the initiator.  If we have an
11782          * untagged command to abort, simply abort the first untagged command
11783          * we come to.  We only allow one untagged command at a time of course.
11784          */
11785         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11786              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11787
11788                 if ((targ_port == UINT32_MAX ||
11789                      targ_port == xio->io_hdr.nexus.targ_port) &&
11790                     (init_id == UINT32_MAX ||
11791                      init_id == xio->io_hdr.nexus.initid)) {
11792                         if (targ_port != xio->io_hdr.nexus.targ_port ||
11793                             init_id != xio->io_hdr.nexus.initid)
11794                                 xio->io_hdr.flags |= CTL_FLAG_ABORT_STATUS;
11795                         xio->io_hdr.flags |= CTL_FLAG_ABORT;
11796                         if (!other_sc && !(lun->flags & CTL_LUN_PRIMARY_SC)) {
11797                                 union ctl_ha_msg msg_info;
11798
11799                                 msg_info.hdr.nexus = xio->io_hdr.nexus;
11800                                 msg_info.task.task_action = CTL_TASK_ABORT_TASK;
11801                                 msg_info.task.tag_num = xio->scsiio.tag_num;
11802                                 msg_info.task.tag_type = xio->scsiio.tag_type;
11803                                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11804                                 msg_info.hdr.original_sc = NULL;
11805                                 msg_info.hdr.serializing_sc = NULL;
11806                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11807                                     sizeof(msg_info.task), M_NOWAIT);
11808                         }
11809                 }
11810         }
11811 }
11812
11813 static int
11814 ctl_abort_task_set(union ctl_io *io)
11815 {
11816         struct ctl_softc *softc = control_softc;
11817         struct ctl_lun *lun;
11818         uint32_t targ_lun;
11819
11820         /*
11821          * Look up the LUN.
11822          */
11823         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11824         mtx_lock(&softc->ctl_lock);
11825         if ((targ_lun >= CTL_MAX_LUNS) ||
11826             (lun = softc->ctl_luns[targ_lun]) == NULL) {
11827                 mtx_unlock(&softc->ctl_lock);
11828                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11829                 return (1);
11830         }
11831
11832         mtx_lock(&lun->lun_lock);
11833         mtx_unlock(&softc->ctl_lock);
11834         if (io->taskio.task_action == CTL_TASK_ABORT_TASK_SET) {
11835                 ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
11836                     io->io_hdr.nexus.initid,
11837                     (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
11838         } else { /* CTL_TASK_CLEAR_TASK_SET */
11839                 ctl_abort_tasks_lun(lun, UINT32_MAX, UINT32_MAX,
11840                     (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
11841         }
11842         mtx_unlock(&lun->lun_lock);
11843         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11844         return (0);
11845 }
11846
11847 static int
11848 ctl_i_t_nexus_reset(union ctl_io *io)
11849 {
11850         struct ctl_softc *softc = control_softc;
11851         struct ctl_lun *lun;
11852         uint32_t initidx;
11853
11854         if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11855                 union ctl_ha_msg msg_info;
11856
11857                 msg_info.hdr.nexus = io->io_hdr.nexus;
11858                 msg_info.task.task_action = CTL_TASK_I_T_NEXUS_RESET;
11859                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11860                 msg_info.hdr.original_sc = NULL;
11861                 msg_info.hdr.serializing_sc = NULL;
11862                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11863                     sizeof(msg_info.task), M_WAITOK);
11864         }
11865
11866         initidx = ctl_get_initindex(&io->io_hdr.nexus);
11867         mtx_lock(&softc->ctl_lock);
11868         STAILQ_FOREACH(lun, &softc->lun_list, links) {
11869                 mtx_lock(&lun->lun_lock);
11870                 ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
11871                     io->io_hdr.nexus.initid, 1);
11872 #ifdef CTL_WITH_CA
11873                 ctl_clear_mask(lun->have_ca, initidx);
11874 #endif
11875                 if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == initidx))
11876                         lun->flags &= ~CTL_LUN_RESERVED;
11877                 ctl_est_ua(lun, initidx, CTL_UA_I_T_NEXUS_LOSS);
11878                 mtx_unlock(&lun->lun_lock);
11879         }
11880         mtx_unlock(&softc->ctl_lock);
11881         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11882         return (0);
11883 }
11884
11885 static int
11886 ctl_abort_task(union ctl_io *io)
11887 {
11888         union ctl_io *xio;
11889         struct ctl_lun *lun;
11890         struct ctl_softc *softc;
11891 #if 0
11892         struct sbuf sb;
11893         char printbuf[128];
11894 #endif
11895         int found;
11896         uint32_t targ_lun;
11897
11898         softc = control_softc;
11899         found = 0;
11900
11901         /*
11902          * Look up the LUN.
11903          */
11904         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11905         mtx_lock(&softc->ctl_lock);
11906         if ((targ_lun >= CTL_MAX_LUNS) ||
11907             (lun = softc->ctl_luns[targ_lun]) == NULL) {
11908                 mtx_unlock(&softc->ctl_lock);
11909                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11910                 return (1);
11911         }
11912
11913 #if 0
11914         printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
11915                lun->lun, io->taskio.tag_num, io->taskio.tag_type);
11916 #endif
11917
11918         mtx_lock(&lun->lun_lock);
11919         mtx_unlock(&softc->ctl_lock);
11920         /*
11921          * Run through the OOA queue and attempt to find the given I/O.
11922          * The target port, initiator ID, tag type and tag number have to
11923          * match the values that we got from the initiator.  If we have an
11924          * untagged command to abort, simply abort the first untagged command
11925          * we come to.  We only allow one untagged command at a time of course.
11926          */
11927         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11928              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11929 #if 0
11930                 sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
11931
11932                 sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
11933                             lun->lun, xio->scsiio.tag_num,
11934                             xio->scsiio.tag_type,
11935                             (xio->io_hdr.blocked_links.tqe_prev
11936                             == NULL) ? "" : " BLOCKED",
11937                             (xio->io_hdr.flags &
11938                             CTL_FLAG_DMA_INPROG) ? " DMA" : "",
11939                             (xio->io_hdr.flags &
11940                             CTL_FLAG_ABORT) ? " ABORT" : "",
11941                             (xio->io_hdr.flags &
11942                             CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
11943                 ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
11944                 sbuf_finish(&sb);
11945                 printf("%s\n", sbuf_data(&sb));
11946 #endif
11947
11948                 if ((xio->io_hdr.nexus.targ_port != io->io_hdr.nexus.targ_port)
11949                  || (xio->io_hdr.nexus.initid != io->io_hdr.nexus.initid)
11950                  || (xio->io_hdr.flags & CTL_FLAG_ABORT))
11951                         continue;
11952
11953                 /*
11954                  * If the abort says that the task is untagged, the
11955                  * task in the queue must be untagged.  Otherwise,
11956                  * we just check to see whether the tag numbers
11957                  * match.  This is because the QLogic firmware
11958                  * doesn't pass back the tag type in an abort
11959                  * request.
11960                  */
11961 #if 0
11962                 if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
11963                   && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
11964                  || (xio->scsiio.tag_num == io->taskio.tag_num))
11965 #endif
11966                 /*
11967                  * XXX KDM we've got problems with FC, because it
11968                  * doesn't send down a tag type with aborts.  So we
11969                  * can only really go by the tag number...
11970                  * This may cause problems with parallel SCSI.
11971                  * Need to figure that out!!
11972                  */
11973                 if (xio->scsiio.tag_num == io->taskio.tag_num) {
11974                         xio->io_hdr.flags |= CTL_FLAG_ABORT;
11975                         found = 1;
11976                         if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0 &&
11977                             !(lun->flags & CTL_LUN_PRIMARY_SC)) {
11978                                 union ctl_ha_msg msg_info;
11979
11980                                 msg_info.hdr.nexus = io->io_hdr.nexus;
11981                                 msg_info.task.task_action = CTL_TASK_ABORT_TASK;
11982                                 msg_info.task.tag_num = io->taskio.tag_num;
11983                                 msg_info.task.tag_type = io->taskio.tag_type;
11984                                 msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11985                                 msg_info.hdr.original_sc = NULL;
11986                                 msg_info.hdr.serializing_sc = NULL;
11987 #if 0
11988                                 printf("Sent Abort to other side\n");
11989 #endif
11990                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
11991                                     sizeof(msg_info.task), M_NOWAIT);
11992                         }
11993 #if 0
11994                         printf("ctl_abort_task: found I/O to abort\n");
11995 #endif
11996                 }
11997         }
11998         mtx_unlock(&lun->lun_lock);
11999
12000         if (found == 0) {
12001                 /*
12002                  * This isn't really an error.  It's entirely possible for
12003                  * the abort and command completion to cross on the wire.
12004                  * This is more of an informative/diagnostic error.
12005                  */
12006 #if 0
12007                 printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
12008                        "%u:%u:%u tag %d type %d\n",
12009                        io->io_hdr.nexus.initid,
12010                        io->io_hdr.nexus.targ_port,
12011                        io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
12012                        io->taskio.tag_type);
12013 #endif
12014         }
12015         io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
12016         return (0);
12017 }
12018
12019 static int
12020 ctl_query_task(union ctl_io *io, int task_set)
12021 {
12022         union ctl_io *xio;
12023         struct ctl_lun *lun;
12024         struct ctl_softc *softc;
12025         int found = 0;
12026         uint32_t targ_lun;
12027
12028         softc = control_softc;
12029         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12030         mtx_lock(&softc->ctl_lock);
12031         if ((targ_lun >= CTL_MAX_LUNS) ||
12032             (lun = softc->ctl_luns[targ_lun]) == NULL) {
12033                 mtx_unlock(&softc->ctl_lock);
12034                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
12035                 return (1);
12036         }
12037         mtx_lock(&lun->lun_lock);
12038         mtx_unlock(&softc->ctl_lock);
12039         for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
12040              xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
12041
12042                 if ((xio->io_hdr.nexus.targ_port != io->io_hdr.nexus.targ_port)
12043                  || (xio->io_hdr.nexus.initid != io->io_hdr.nexus.initid)
12044                  || (xio->io_hdr.flags & CTL_FLAG_ABORT))
12045                         continue;
12046
12047                 if (task_set || xio->scsiio.tag_num == io->taskio.tag_num) {
12048                         found = 1;
12049                         break;
12050                 }
12051         }
12052         mtx_unlock(&lun->lun_lock);
12053         if (found)
12054                 io->taskio.task_status = CTL_TASK_FUNCTION_SUCCEEDED;
12055         else
12056                 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
12057         return (0);
12058 }
12059
12060 static int
12061 ctl_query_async_event(union ctl_io *io)
12062 {
12063         struct ctl_lun *lun;
12064         struct ctl_softc *softc;
12065         ctl_ua_type ua;
12066         uint32_t targ_lun, initidx;
12067
12068         softc = control_softc;
12069         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12070         mtx_lock(&softc->ctl_lock);
12071         if ((targ_lun >= CTL_MAX_LUNS) ||
12072             (lun = softc->ctl_luns[targ_lun]) == NULL) {
12073                 mtx_unlock(&softc->ctl_lock);
12074                 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
12075                 return (1);
12076         }
12077         mtx_lock(&lun->lun_lock);
12078         mtx_unlock(&softc->ctl_lock);
12079         initidx = ctl_get_initindex(&io->io_hdr.nexus);
12080         ua = ctl_build_qae(lun, initidx, io->taskio.task_resp);
12081         mtx_unlock(&lun->lun_lock);
12082         if (ua != CTL_UA_NONE)
12083                 io->taskio.task_status = CTL_TASK_FUNCTION_SUCCEEDED;
12084         else
12085                 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
12086         return (0);
12087 }
12088
12089 static void
12090 ctl_run_task(union ctl_io *io)
12091 {
12092         struct ctl_softc *softc = control_softc;
12093         int retval = 1;
12094
12095         CTL_DEBUG_PRINT(("ctl_run_task\n"));
12096         KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
12097             ("ctl_run_task: Unextected io_type %d\n", io->io_hdr.io_type));
12098         io->taskio.task_status = CTL_TASK_FUNCTION_NOT_SUPPORTED;
12099         bzero(io->taskio.task_resp, sizeof(io->taskio.task_resp));
12100         switch (io->taskio.task_action) {
12101         case CTL_TASK_ABORT_TASK:
12102                 retval = ctl_abort_task(io);
12103                 break;
12104         case CTL_TASK_ABORT_TASK_SET:
12105         case CTL_TASK_CLEAR_TASK_SET:
12106                 retval = ctl_abort_task_set(io);
12107                 break;
12108         case CTL_TASK_CLEAR_ACA:
12109                 break;
12110         case CTL_TASK_I_T_NEXUS_RESET:
12111                 retval = ctl_i_t_nexus_reset(io);
12112                 break;
12113         case CTL_TASK_LUN_RESET:
12114                 retval = ctl_lun_reset(softc, io);
12115                 break;
12116         case CTL_TASK_TARGET_RESET:
12117                 retval = ctl_target_reset(softc, io, CTL_UA_TARG_RESET);
12118                 break;
12119         case CTL_TASK_BUS_RESET:
12120                 retval = ctl_bus_reset(softc, io);
12121                 break;
12122         case CTL_TASK_PORT_LOGIN:
12123                 break;
12124         case CTL_TASK_PORT_LOGOUT:
12125                 break;
12126         case CTL_TASK_QUERY_TASK:
12127                 retval = ctl_query_task(io, 0);
12128                 break;
12129         case CTL_TASK_QUERY_TASK_SET:
12130                 retval = ctl_query_task(io, 1);
12131                 break;
12132         case CTL_TASK_QUERY_ASYNC_EVENT:
12133                 retval = ctl_query_async_event(io);
12134                 break;
12135         default:
12136                 printf("%s: got unknown task management event %d\n",
12137                        __func__, io->taskio.task_action);
12138                 break;
12139         }
12140         if (retval == 0)
12141                 io->io_hdr.status = CTL_SUCCESS;
12142         else
12143                 io->io_hdr.status = CTL_ERROR;
12144         ctl_done(io);
12145 }
12146
12147 /*
12148  * For HA operation.  Handle commands that come in from the other
12149  * controller.
12150  */
12151 static void
12152 ctl_handle_isc(union ctl_io *io)
12153 {
12154         int free_io;
12155         struct ctl_lun *lun;
12156         struct ctl_softc *softc = control_softc;
12157         uint32_t targ_lun;
12158
12159         targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12160         lun = softc->ctl_luns[targ_lun];
12161
12162         switch (io->io_hdr.msg_type) {
12163         case CTL_MSG_SERIALIZE:
12164                 free_io = ctl_serialize_other_sc_cmd(&io->scsiio);
12165                 break;
12166         case CTL_MSG_R2R: {
12167                 const struct ctl_cmd_entry *entry;
12168
12169                 /*
12170                  * This is only used in SER_ONLY mode.
12171                  */
12172                 free_io = 0;
12173                 entry = ctl_get_cmd_entry(&io->scsiio, NULL);
12174                 mtx_lock(&lun->lun_lock);
12175                 if (ctl_scsiio_lun_check(lun,
12176                     entry, (struct ctl_scsiio *)io) != 0) {
12177                         mtx_unlock(&lun->lun_lock);
12178                         ctl_done(io);
12179                         break;
12180                 }
12181                 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
12182                 mtx_unlock(&lun->lun_lock);
12183                 ctl_enqueue_rtr(io);
12184                 break;
12185         }
12186         case CTL_MSG_FINISH_IO:
12187                 if (softc->ha_mode == CTL_HA_MODE_XFER) {
12188                         free_io = 0;
12189                         ctl_done(io);
12190                 } else {
12191                         free_io = 1;
12192                         mtx_lock(&lun->lun_lock);
12193                         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
12194                                      ooa_links);
12195                         ctl_check_blocked(lun);
12196                         mtx_unlock(&lun->lun_lock);
12197                 }
12198                 break;
12199         case CTL_MSG_PERS_ACTION:
12200                 ctl_hndl_per_res_out_on_other_sc(
12201                         (union ctl_ha_msg *)&io->presio.pr_msg);
12202                 free_io = 1;
12203                 break;
12204         case CTL_MSG_BAD_JUJU:
12205                 free_io = 0;
12206                 ctl_done(io);
12207                 break;
12208         case CTL_MSG_DATAMOVE:
12209                 /* Only used in XFER mode */
12210                 free_io = 0;
12211                 ctl_datamove_remote(io);
12212                 break;
12213         case CTL_MSG_DATAMOVE_DONE:
12214                 /* Only used in XFER mode */
12215                 free_io = 0;
12216                 io->scsiio.be_move_done(io);
12217                 break;
12218         case CTL_MSG_FAILOVER:
12219                 ctl_failover_lun(io);
12220                 free_io = 1;
12221                 break;
12222         default:
12223                 free_io = 1;
12224                 printf("%s: Invalid message type %d\n",
12225                        __func__, io->io_hdr.msg_type);
12226                 break;
12227         }
12228         if (free_io)
12229                 ctl_free_io(io);
12230
12231 }
12232
12233
12234 /*
12235  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
12236  * there is no match.
12237  */
12238 static ctl_lun_error_pattern
12239 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
12240 {
12241         const struct ctl_cmd_entry *entry;
12242         ctl_lun_error_pattern filtered_pattern, pattern;
12243
12244         pattern = desc->error_pattern;
12245
12246         /*
12247          * XXX KDM we need more data passed into this function to match a
12248          * custom pattern, and we actually need to implement custom pattern
12249          * matching.
12250          */
12251         if (pattern & CTL_LUN_PAT_CMD)
12252                 return (CTL_LUN_PAT_CMD);
12253
12254         if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
12255                 return (CTL_LUN_PAT_ANY);
12256
12257         entry = ctl_get_cmd_entry(ctsio, NULL);
12258
12259         filtered_pattern = entry->pattern & pattern;
12260
12261         /*
12262          * If the user requested specific flags in the pattern (e.g.
12263          * CTL_LUN_PAT_RANGE), make sure the command supports all of those
12264          * flags.
12265          *
12266          * If the user did not specify any flags, it doesn't matter whether
12267          * or not the command supports the flags.
12268          */
12269         if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
12270              (pattern & ~CTL_LUN_PAT_MASK))
12271                 return (CTL_LUN_PAT_NONE);
12272
12273         /*
12274          * If the user asked for a range check, see if the requested LBA
12275          * range overlaps with this command's LBA range.
12276          */
12277         if (filtered_pattern & CTL_LUN_PAT_RANGE) {
12278                 uint64_t lba1;
12279                 uint64_t len1;
12280                 ctl_action action;
12281                 int retval;
12282
12283                 retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
12284                 if (retval != 0)
12285                         return (CTL_LUN_PAT_NONE);
12286
12287                 action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
12288                                               desc->lba_range.len, FALSE);
12289                 /*
12290                  * A "pass" means that the LBA ranges don't overlap, so
12291                  * this doesn't match the user's range criteria.
12292                  */
12293                 if (action == CTL_ACTION_PASS)
12294                         return (CTL_LUN_PAT_NONE);
12295         }
12296
12297         return (filtered_pattern);
12298 }
12299
12300 static void
12301 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
12302 {
12303         struct ctl_error_desc *desc, *desc2;
12304
12305         mtx_assert(&lun->lun_lock, MA_OWNED);
12306
12307         STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
12308                 ctl_lun_error_pattern pattern;
12309                 /*
12310                  * Check to see whether this particular command matches
12311                  * the pattern in the descriptor.
12312                  */
12313                 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
12314                 if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
12315                         continue;
12316
12317                 switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
12318                 case CTL_LUN_INJ_ABORTED:
12319                         ctl_set_aborted(&io->scsiio);
12320                         break;
12321                 case CTL_LUN_INJ_MEDIUM_ERR:
12322                         ctl_set_medium_error(&io->scsiio,
12323                             (io->io_hdr.flags & CTL_FLAG_DATA_MASK) !=
12324                              CTL_FLAG_DATA_OUT);
12325                         break;
12326                 case CTL_LUN_INJ_UA:
12327                         /* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
12328                          * OCCURRED */
12329                         ctl_set_ua(&io->scsiio, 0x29, 0x00);
12330                         break;
12331                 case CTL_LUN_INJ_CUSTOM:
12332                         /*
12333                          * We're assuming the user knows what he is doing.
12334                          * Just copy the sense information without doing
12335                          * checks.
12336                          */
12337                         bcopy(&desc->custom_sense, &io->scsiio.sense_data,
12338                               MIN(sizeof(desc->custom_sense),
12339                                   sizeof(io->scsiio.sense_data)));
12340                         io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
12341                         io->scsiio.sense_len = SSD_FULL_SIZE;
12342                         io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
12343                         break;
12344                 case CTL_LUN_INJ_NONE:
12345                 default:
12346                         /*
12347                          * If this is an error injection type we don't know
12348                          * about, clear the continuous flag (if it is set)
12349                          * so it will get deleted below.
12350                          */
12351                         desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
12352                         break;
12353                 }
12354                 /*
12355                  * By default, each error injection action is a one-shot
12356                  */
12357                 if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
12358                         continue;
12359
12360                 STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
12361
12362                 free(desc, M_CTL);
12363         }
12364 }
12365
12366 #ifdef CTL_IO_DELAY
12367 static void
12368 ctl_datamove_timer_wakeup(void *arg)
12369 {
12370         union ctl_io *io;
12371
12372         io = (union ctl_io *)arg;
12373
12374         ctl_datamove(io);
12375 }
12376 #endif /* CTL_IO_DELAY */
12377
12378 void
12379 ctl_datamove(union ctl_io *io)
12380 {
12381         struct ctl_lun *lun;
12382         void (*fe_datamove)(union ctl_io *io);
12383
12384         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
12385
12386         CTL_DEBUG_PRINT(("ctl_datamove\n"));
12387
12388         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12389 #ifdef CTL_TIME_IO
12390         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12391                 char str[256];
12392                 char path_str[64];
12393                 struct sbuf sb;
12394
12395                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12396                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12397
12398                 sbuf_cat(&sb, path_str);
12399                 switch (io->io_hdr.io_type) {
12400                 case CTL_IO_SCSI:
12401                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12402                         sbuf_printf(&sb, "\n");
12403                         sbuf_cat(&sb, path_str);
12404                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12405                                     io->scsiio.tag_num, io->scsiio.tag_type);
12406                         break;
12407                 case CTL_IO_TASK:
12408                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12409                                     "Tag Type: %d\n", io->taskio.task_action,
12410                                     io->taskio.tag_num, io->taskio.tag_type);
12411                         break;
12412                 default:
12413                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12414                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12415                         break;
12416                 }
12417                 sbuf_cat(&sb, path_str);
12418                 sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
12419                             (intmax_t)time_uptime - io->io_hdr.start_time);
12420                 sbuf_finish(&sb);
12421                 printf("%s", sbuf_data(&sb));
12422         }
12423 #endif /* CTL_TIME_IO */
12424
12425 #ifdef CTL_IO_DELAY
12426         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12427                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12428         } else {
12429                 if ((lun != NULL)
12430                  && (lun->delay_info.datamove_delay > 0)) {
12431
12432                         callout_init(&io->io_hdr.delay_callout, /*mpsafe*/ 1);
12433                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12434                         callout_reset(&io->io_hdr.delay_callout,
12435                                       lun->delay_info.datamove_delay * hz,
12436                                       ctl_datamove_timer_wakeup, io);
12437                         if (lun->delay_info.datamove_type ==
12438                             CTL_DELAY_TYPE_ONESHOT)
12439                                 lun->delay_info.datamove_delay = 0;
12440                         return;
12441                 }
12442         }
12443 #endif
12444
12445         /*
12446          * This command has been aborted.  Set the port status, so we fail
12447          * the data move.
12448          */
12449         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12450                 printf("ctl_datamove: tag 0x%04x on (%u:%u:%u) aborted\n",
12451                        io->scsiio.tag_num, io->io_hdr.nexus.initid,
12452                        io->io_hdr.nexus.targ_port,
12453                        io->io_hdr.nexus.targ_lun);
12454                 io->io_hdr.port_status = 31337;
12455                 /*
12456                  * Note that the backend, in this case, will get the
12457                  * callback in its context.  In other cases it may get
12458                  * called in the frontend's interrupt thread context.
12459                  */
12460                 io->scsiio.be_move_done(io);
12461                 return;
12462         }
12463
12464         /* Don't confuse frontend with zero length data move. */
12465         if (io->scsiio.kern_data_len == 0) {
12466                 io->scsiio.be_move_done(io);
12467                 return;
12468         }
12469
12470         fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
12471         fe_datamove(io);
12472 }
12473
12474 static void
12475 ctl_send_datamove_done(union ctl_io *io, int have_lock)
12476 {
12477         union ctl_ha_msg msg;
12478 #ifdef CTL_TIME_IO
12479         struct bintime cur_bt;
12480 #endif
12481
12482         memset(&msg, 0, sizeof(msg));
12483         msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
12484         msg.hdr.original_sc = io;
12485         msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
12486         msg.hdr.nexus = io->io_hdr.nexus;
12487         msg.hdr.status = io->io_hdr.status;
12488         msg.scsi.tag_num = io->scsiio.tag_num;
12489         msg.scsi.tag_type = io->scsiio.tag_type;
12490         msg.scsi.scsi_status = io->scsiio.scsi_status;
12491         memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12492                io->scsiio.sense_len);
12493         msg.scsi.sense_len = io->scsiio.sense_len;
12494         msg.scsi.sense_residual = io->scsiio.sense_residual;
12495         msg.scsi.fetd_status = io->io_hdr.port_status;
12496         msg.scsi.residual = io->scsiio.residual;
12497         io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12498         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12499                 ctl_failover_io(io, /*have_lock*/ have_lock);
12500                 return;
12501         }
12502         ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12503             sizeof(msg.scsi) - sizeof(msg.scsi.sense_data) +
12504             msg.scsi.sense_len, M_WAITOK);
12505
12506 #ifdef CTL_TIME_IO
12507         getbinuptime(&cur_bt);
12508         bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
12509         bintime_add(&io->io_hdr.dma_bt, &cur_bt);
12510 #endif
12511         io->io_hdr.num_dmas++;
12512 }
12513
12514 /*
12515  * The DMA to the remote side is done, now we need to tell the other side
12516  * we're done so it can continue with its data movement.
12517  */
12518 static void
12519 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
12520 {
12521         union ctl_io *io;
12522         int i;
12523
12524         io = rq->context;
12525
12526         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12527                 printf("%s: ISC DMA write failed with error %d", __func__,
12528                        rq->ret);
12529                 ctl_set_internal_failure(&io->scsiio,
12530                                          /*sks_valid*/ 1,
12531                                          /*retry_count*/ rq->ret);
12532         }
12533
12534         ctl_dt_req_free(rq);
12535
12536         for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12537                 free(io->io_hdr.local_sglist[i].addr, M_CTL);
12538         free(io->io_hdr.remote_sglist, M_CTL);
12539         io->io_hdr.remote_sglist = NULL;
12540         io->io_hdr.local_sglist = NULL;
12541
12542         /*
12543          * The data is in local and remote memory, so now we need to send
12544          * status (good or back) back to the other side.
12545          */
12546         ctl_send_datamove_done(io, /*have_lock*/ 0);
12547 }
12548
12549 /*
12550  * We've moved the data from the host/controller into local memory.  Now we
12551  * need to push it over to the remote controller's memory.
12552  */
12553 static int
12554 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
12555 {
12556         int retval;
12557
12558         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
12559                                           ctl_datamove_remote_write_cb);
12560         return (retval);
12561 }
12562
12563 static void
12564 ctl_datamove_remote_write(union ctl_io *io)
12565 {
12566         int retval;
12567         void (*fe_datamove)(union ctl_io *io);
12568
12569         /*
12570          * - Get the data from the host/HBA into local memory.
12571          * - DMA memory from the local controller to the remote controller.
12572          * - Send status back to the remote controller.
12573          */
12574
12575         retval = ctl_datamove_remote_sgl_setup(io);
12576         if (retval != 0)
12577                 return;
12578
12579         /* Switch the pointer over so the FETD knows what to do */
12580         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12581
12582         /*
12583          * Use a custom move done callback, since we need to send completion
12584          * back to the other controller, not to the backend on this side.
12585          */
12586         io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
12587
12588         fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
12589         fe_datamove(io);
12590 }
12591
12592 static int
12593 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
12594 {
12595 #if 0
12596         char str[256];
12597         char path_str[64];
12598         struct sbuf sb;
12599 #endif
12600         int i;
12601
12602         for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12603                 free(io->io_hdr.local_sglist[i].addr, M_CTL);
12604         free(io->io_hdr.remote_sglist, M_CTL);
12605         io->io_hdr.remote_sglist = NULL;
12606         io->io_hdr.local_sglist = NULL;
12607
12608 #if 0
12609         scsi_path_string(io, path_str, sizeof(path_str));
12610         sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12611         sbuf_cat(&sb, path_str);
12612         scsi_command_string(&io->scsiio, NULL, &sb);
12613         sbuf_printf(&sb, "\n");
12614         sbuf_cat(&sb, path_str);
12615         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12616                     io->scsiio.tag_num, io->scsiio.tag_type);
12617         sbuf_cat(&sb, path_str);
12618         sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
12619                     io->io_hdr.flags, io->io_hdr.status);
12620         sbuf_finish(&sb);
12621         printk("%s", sbuf_data(&sb));
12622 #endif
12623
12624
12625         /*
12626          * The read is done, now we need to send status (good or bad) back
12627          * to the other side.
12628          */
12629         ctl_send_datamove_done(io, /*have_lock*/ 0);
12630
12631         return (0);
12632 }
12633
12634 static void
12635 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
12636 {
12637         union ctl_io *io;
12638         void (*fe_datamove)(union ctl_io *io);
12639
12640         io = rq->context;
12641
12642         if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12643                 printf("%s: ISC DMA read failed with error %d\n", __func__,
12644                        rq->ret);
12645                 ctl_set_internal_failure(&io->scsiio,
12646                                          /*sks_valid*/ 1,
12647                                          /*retry_count*/ rq->ret);
12648         }
12649
12650         ctl_dt_req_free(rq);
12651
12652         /* Switch the pointer over so the FETD knows what to do */
12653         io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12654
12655         /*
12656          * Use a custom move done callback, since we need to send completion
12657          * back to the other controller, not to the backend on this side.
12658          */
12659         io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
12660
12661         /* XXX KDM add checks like the ones in ctl_datamove? */
12662
12663         fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
12664         fe_datamove(io);
12665 }
12666
12667 static int
12668 ctl_datamove_remote_sgl_setup(union ctl_io *io)
12669 {
12670         struct ctl_sg_entry *local_sglist;
12671         uint32_t len_to_go;
12672         int retval;
12673         int i;
12674
12675         retval = 0;
12676         local_sglist = io->io_hdr.local_sglist;
12677         len_to_go = io->scsiio.kern_data_len;
12678
12679         /*
12680          * The difficult thing here is that the size of the various
12681          * S/G segments may be different than the size from the
12682          * remote controller.  That'll make it harder when DMAing
12683          * the data back to the other side.
12684          */
12685         for (i = 0; len_to_go > 0; i++) {
12686                 local_sglist[i].len = MIN(len_to_go, CTL_HA_DATAMOVE_SEGMENT);
12687                 local_sglist[i].addr =
12688                     malloc(local_sglist[i].len, M_CTL, M_WAITOK);
12689
12690                 len_to_go -= local_sglist[i].len;
12691         }
12692         /*
12693          * Reset the number of S/G entries accordingly.  The original
12694          * number of S/G entries is available in rem_sg_entries.
12695          */
12696         io->scsiio.kern_sg_entries = i;
12697
12698 #if 0
12699         printf("%s: kern_sg_entries = %d\n", __func__,
12700                io->scsiio.kern_sg_entries);
12701         for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12702                 printf("%s: sg[%d] = %p, %lu\n", __func__, i,
12703                        local_sglist[i].addr, local_sglist[i].len);
12704 #endif
12705
12706         return (retval);
12707 }
12708
12709 static int
12710 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
12711                          ctl_ha_dt_cb callback)
12712 {
12713         struct ctl_ha_dt_req *rq;
12714         struct ctl_sg_entry *remote_sglist, *local_sglist;
12715         uint32_t local_used, remote_used, total_used;
12716         int i, j, isc_ret;
12717
12718         rq = ctl_dt_req_alloc();
12719
12720         /*
12721          * If we failed to allocate the request, and if the DMA didn't fail
12722          * anyway, set busy status.  This is just a resource allocation
12723          * failure.
12724          */
12725         if ((rq == NULL)
12726          && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
12727              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS))
12728                 ctl_set_busy(&io->scsiio);
12729
12730         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
12731             (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) {
12732
12733                 if (rq != NULL)
12734                         ctl_dt_req_free(rq);
12735
12736                 /*
12737                  * The data move failed.  We need to return status back
12738                  * to the other controller.  No point in trying to DMA
12739                  * data to the remote controller.
12740                  */
12741
12742                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12743
12744                 return (1);
12745         }
12746
12747         local_sglist = io->io_hdr.local_sglist;
12748         remote_sglist = io->io_hdr.remote_sglist;
12749         local_used = 0;
12750         remote_used = 0;
12751         total_used = 0;
12752
12753         /*
12754          * Pull/push the data over the wire from/to the other controller.
12755          * This takes into account the possibility that the local and
12756          * remote sglists may not be identical in terms of the size of
12757          * the elements and the number of elements.
12758          *
12759          * One fundamental assumption here is that the length allocated for
12760          * both the local and remote sglists is identical.  Otherwise, we've
12761          * essentially got a coding error of some sort.
12762          */
12763         isc_ret = CTL_HA_STATUS_SUCCESS;
12764         for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
12765                 uint32_t cur_len;
12766                 uint8_t *tmp_ptr;
12767
12768                 rq->command = command;
12769                 rq->context = io;
12770
12771                 /*
12772                  * Both pointers should be aligned.  But it is possible
12773                  * that the allocation length is not.  They should both
12774                  * also have enough slack left over at the end, though,
12775                  * to round up to the next 8 byte boundary.
12776                  */
12777                 cur_len = MIN(local_sglist[i].len - local_used,
12778                               remote_sglist[j].len - remote_used);
12779                 rq->size = cur_len;
12780
12781                 tmp_ptr = (uint8_t *)local_sglist[i].addr;
12782                 tmp_ptr += local_used;
12783
12784 #if 0
12785                 /* Use physical addresses when talking to ISC hardware */
12786                 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
12787                         /* XXX KDM use busdma */
12788                         rq->local = vtophys(tmp_ptr);
12789                 } else
12790                         rq->local = tmp_ptr;
12791 #else
12792                 KASSERT((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0,
12793                     ("HA does not support BUS_ADDR"));
12794                 rq->local = tmp_ptr;
12795 #endif
12796
12797                 tmp_ptr = (uint8_t *)remote_sglist[j].addr;
12798                 tmp_ptr += remote_used;
12799                 rq->remote = tmp_ptr;
12800
12801                 rq->callback = NULL;
12802
12803                 local_used += cur_len;
12804                 if (local_used >= local_sglist[i].len) {
12805                         i++;
12806                         local_used = 0;
12807                 }
12808
12809                 remote_used += cur_len;
12810                 if (remote_used >= remote_sglist[j].len) {
12811                         j++;
12812                         remote_used = 0;
12813                 }
12814                 total_used += cur_len;
12815
12816                 if (total_used >= io->scsiio.kern_data_len)
12817                         rq->callback = callback;
12818
12819 #if 0
12820                 printf("%s: %s: local %p remote %p size %d\n", __func__,
12821                        (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
12822                        rq->local, rq->remote, rq->size);
12823 #endif
12824
12825                 isc_ret = ctl_dt_single(rq);
12826                 if (isc_ret > CTL_HA_STATUS_SUCCESS)
12827                         break;
12828         }
12829         if (isc_ret != CTL_HA_STATUS_WAIT) {
12830                 rq->ret = isc_ret;
12831                 callback(rq);
12832         }
12833
12834         return (0);
12835 }
12836
12837 static void
12838 ctl_datamove_remote_read(union ctl_io *io)
12839 {
12840         int retval;
12841         int i;
12842
12843         /*
12844          * This will send an error to the other controller in the case of a
12845          * failure.
12846          */
12847         retval = ctl_datamove_remote_sgl_setup(io);
12848         if (retval != 0)
12849                 return;
12850
12851         retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
12852                                           ctl_datamove_remote_read_cb);
12853         if (retval != 0) {
12854                 /*
12855                  * Make sure we free memory if there was an error..  The
12856                  * ctl_datamove_remote_xfer() function will send the
12857                  * datamove done message, or call the callback with an
12858                  * error if there is a problem.
12859                  */
12860                 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12861                         free(io->io_hdr.local_sglist[i].addr, M_CTL);
12862                 free(io->io_hdr.remote_sglist, M_CTL);
12863                 io->io_hdr.remote_sglist = NULL;
12864                 io->io_hdr.local_sglist = NULL;
12865         }
12866 }
12867
12868 /*
12869  * Process a datamove request from the other controller.  This is used for
12870  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
12871  * first.  Once that is complete, the data gets DMAed into the remote
12872  * controller's memory.  For reads, we DMA from the remote controller's
12873  * memory into our memory first, and then move it out to the FETD.
12874  */
12875 static void
12876 ctl_datamove_remote(union ctl_io *io)
12877 {
12878
12879         mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
12880
12881         if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12882                 ctl_failover_io(io, /*have_lock*/ 0);
12883                 return;
12884         }
12885
12886         /*
12887          * Note that we look for an aborted I/O here, but don't do some of
12888          * the other checks that ctl_datamove() normally does.
12889          * We don't need to run the datamove delay code, since that should
12890          * have been done if need be on the other controller.
12891          */
12892         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12893                 printf("%s: tag 0x%04x on (%u:%u:%u) aborted\n", __func__,
12894                        io->scsiio.tag_num, io->io_hdr.nexus.initid,
12895                        io->io_hdr.nexus.targ_port,
12896                        io->io_hdr.nexus.targ_lun);
12897                 io->io_hdr.port_status = 31338;
12898                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12899                 return;
12900         }
12901
12902         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
12903                 ctl_datamove_remote_write(io);
12904         else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
12905                 ctl_datamove_remote_read(io);
12906         else {
12907                 io->io_hdr.port_status = 31339;
12908                 ctl_send_datamove_done(io, /*have_lock*/ 0);
12909         }
12910 }
12911
12912 static void
12913 ctl_process_done(union ctl_io *io)
12914 {
12915         struct ctl_lun *lun;
12916         struct ctl_softc *softc = control_softc;
12917         void (*fe_done)(union ctl_io *io);
12918         union ctl_ha_msg msg;
12919         uint32_t targ_port = io->io_hdr.nexus.targ_port;
12920
12921         CTL_DEBUG_PRINT(("ctl_process_done\n"));
12922         fe_done = softc->ctl_ports[targ_port]->fe_done;
12923
12924 #ifdef CTL_TIME_IO
12925         if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12926                 char str[256];
12927                 char path_str[64];
12928                 struct sbuf sb;
12929
12930                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12931                 sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12932
12933                 sbuf_cat(&sb, path_str);
12934                 switch (io->io_hdr.io_type) {
12935                 case CTL_IO_SCSI:
12936                         ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12937                         sbuf_printf(&sb, "\n");
12938                         sbuf_cat(&sb, path_str);
12939                         sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12940                                     io->scsiio.tag_num, io->scsiio.tag_type);
12941                         break;
12942                 case CTL_IO_TASK:
12943                         sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12944                                     "Tag Type: %d\n", io->taskio.task_action,
12945                                     io->taskio.tag_num, io->taskio.tag_type);
12946                         break;
12947                 default:
12948                         printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12949                         panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12950                         break;
12951                 }
12952                 sbuf_cat(&sb, path_str);
12953                 sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
12954                             (intmax_t)time_uptime - io->io_hdr.start_time);
12955                 sbuf_finish(&sb);
12956                 printf("%s", sbuf_data(&sb));
12957         }
12958 #endif /* CTL_TIME_IO */
12959
12960         switch (io->io_hdr.io_type) {
12961         case CTL_IO_SCSI:
12962                 break;
12963         case CTL_IO_TASK:
12964                 if (ctl_debug & CTL_DEBUG_INFO)
12965                         ctl_io_error_print(io, NULL);
12966                 fe_done(io);
12967                 return;
12968         default:
12969                 panic("ctl_process_done: invalid io type %d\n",
12970                       io->io_hdr.io_type);
12971                 break; /* NOTREACHED */
12972         }
12973
12974         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12975         if (lun == NULL) {
12976                 CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
12977                                  io->io_hdr.nexus.targ_mapped_lun));
12978                 goto bailout;
12979         }
12980
12981         mtx_lock(&lun->lun_lock);
12982
12983         /*
12984          * Check to see if we have any errors to inject here.  We only
12985          * inject errors for commands that don't already have errors set.
12986          */
12987         if ((STAILQ_FIRST(&lun->error_list) != NULL) &&
12988             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) &&
12989             ((io->io_hdr.flags & CTL_FLAG_STATUS_SENT) == 0))
12990                 ctl_inject_error(lun, io);
12991
12992         /*
12993          * XXX KDM how do we treat commands that aren't completed
12994          * successfully?
12995          *
12996          * XXX KDM should we also track I/O latency?
12997          */
12998         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
12999             io->io_hdr.io_type == CTL_IO_SCSI) {
13000 #ifdef CTL_TIME_IO
13001                 struct bintime cur_bt;
13002 #endif
13003                 int type;
13004
13005                 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13006                     CTL_FLAG_DATA_IN)
13007                         type = CTL_STATS_READ;
13008                 else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13009                     CTL_FLAG_DATA_OUT)
13010                         type = CTL_STATS_WRITE;
13011                 else
13012                         type = CTL_STATS_NO_IO;
13013
13014                 lun->stats.ports[targ_port].bytes[type] +=
13015                     io->scsiio.kern_total_len;
13016                 lun->stats.ports[targ_port].operations[type]++;
13017 #ifdef CTL_TIME_IO
13018                 bintime_add(&lun->stats.ports[targ_port].dma_time[type],
13019                    &io->io_hdr.dma_bt);
13020                 getbinuptime(&cur_bt);
13021                 bintime_sub(&cur_bt, &io->io_hdr.start_bt);
13022                 bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
13023 #endif
13024                 lun->stats.ports[targ_port].num_dmas[type] +=
13025                     io->io_hdr.num_dmas;
13026         }
13027
13028         /*
13029          * Remove this from the OOA queue.
13030          */
13031         TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
13032 #ifdef CTL_TIME_IO
13033         if (TAILQ_EMPTY(&lun->ooa_queue))
13034                 lun->last_busy = getsbinuptime();
13035 #endif
13036
13037         /*
13038          * Run through the blocked queue on this LUN and see if anything
13039          * has become unblocked, now that this transaction is done.
13040          */
13041         ctl_check_blocked(lun);
13042
13043         /*
13044          * If the LUN has been invalidated, free it if there is nothing
13045          * left on its OOA queue.
13046          */
13047         if ((lun->flags & CTL_LUN_INVALID)
13048          && TAILQ_EMPTY(&lun->ooa_queue)) {
13049                 mtx_unlock(&lun->lun_lock);
13050                 mtx_lock(&softc->ctl_lock);
13051                 ctl_free_lun(lun);
13052                 mtx_unlock(&softc->ctl_lock);
13053         } else
13054                 mtx_unlock(&lun->lun_lock);
13055
13056 bailout:
13057
13058         /*
13059          * If this command has been aborted, make sure we set the status
13060          * properly.  The FETD is responsible for freeing the I/O and doing
13061          * whatever it needs to do to clean up its state.
13062          */
13063         if (io->io_hdr.flags & CTL_FLAG_ABORT)
13064                 ctl_set_task_aborted(&io->scsiio);
13065
13066         /*
13067          * If enabled, print command error status.
13068          */
13069         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS &&
13070             (ctl_debug & CTL_DEBUG_INFO) != 0)
13071                 ctl_io_error_print(io, NULL);
13072
13073         /*
13074          * Tell the FETD or the other shelf controller we're done with this
13075          * command.  Note that only SCSI commands get to this point.  Task
13076          * management commands are completed above.
13077          */
13078         if ((softc->ha_mode != CTL_HA_MODE_XFER) &&
13079             (io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)) {
13080                 memset(&msg, 0, sizeof(msg));
13081                 msg.hdr.msg_type = CTL_MSG_FINISH_IO;
13082                 msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
13083                 msg.hdr.nexus = io->io_hdr.nexus;
13084                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13085                     sizeof(msg.scsi) - sizeof(msg.scsi.sense_data),
13086                     M_WAITOK);
13087         }
13088
13089         fe_done(io);
13090 }
13091
13092 #ifdef CTL_WITH_CA
13093 /*
13094  * Front end should call this if it doesn't do autosense.  When the request
13095  * sense comes back in from the initiator, we'll dequeue this and send it.
13096  */
13097 int
13098 ctl_queue_sense(union ctl_io *io)
13099 {
13100         struct ctl_lun *lun;
13101         struct ctl_port *port;
13102         struct ctl_softc *softc;
13103         uint32_t initidx, targ_lun;
13104
13105         softc = control_softc;
13106
13107         CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
13108
13109         /*
13110          * LUN lookup will likely move to the ctl_work_thread() once we
13111          * have our new queueing infrastructure (that doesn't put things on
13112          * a per-LUN queue initially).  That is so that we can handle
13113          * things like an INQUIRY to a LUN that we don't have enabled.  We
13114          * can't deal with that right now.
13115          */
13116         mtx_lock(&softc->ctl_lock);
13117
13118         /*
13119          * If we don't have a LUN for this, just toss the sense
13120          * information.
13121          */
13122         port = ctl_io_port(&ctsio->io_hdr);
13123         targ_lun = ctl_lun_map_from_port(port, io->io_hdr.nexus.targ_lun);
13124         if ((targ_lun < CTL_MAX_LUNS)
13125          && (softc->ctl_luns[targ_lun] != NULL))
13126                 lun = softc->ctl_luns[targ_lun];
13127         else
13128                 goto bailout;
13129
13130         initidx = ctl_get_initindex(&io->io_hdr.nexus);
13131
13132         mtx_lock(&lun->lun_lock);
13133         /*
13134          * Already have CA set for this LUN...toss the sense information.
13135          */
13136         if (ctl_is_set(lun->have_ca, initidx)) {
13137                 mtx_unlock(&lun->lun_lock);
13138                 goto bailout;
13139         }
13140
13141         memcpy(&lun->pending_sense[initidx], &io->scsiio.sense_data,
13142                MIN(sizeof(lun->pending_sense[initidx]),
13143                sizeof(io->scsiio.sense_data)));
13144         ctl_set_mask(lun->have_ca, initidx);
13145         mtx_unlock(&lun->lun_lock);
13146
13147 bailout:
13148         mtx_unlock(&softc->ctl_lock);
13149
13150         ctl_free_io(io);
13151
13152         return (CTL_RETVAL_COMPLETE);
13153 }
13154 #endif
13155
13156 /*
13157  * Primary command inlet from frontend ports.  All SCSI and task I/O
13158  * requests must go through this function.
13159  */
13160 int
13161 ctl_queue(union ctl_io *io)
13162 {
13163         struct ctl_port *port;
13164
13165         CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
13166
13167 #ifdef CTL_TIME_IO
13168         io->io_hdr.start_time = time_uptime;
13169         getbinuptime(&io->io_hdr.start_bt);
13170 #endif /* CTL_TIME_IO */
13171
13172         /* Map FE-specific LUN ID into global one. */
13173         port = ctl_io_port(&io->io_hdr);
13174         io->io_hdr.nexus.targ_mapped_lun =
13175             ctl_lun_map_from_port(port, io->io_hdr.nexus.targ_lun);
13176
13177         switch (io->io_hdr.io_type) {
13178         case CTL_IO_SCSI:
13179         case CTL_IO_TASK:
13180                 if (ctl_debug & CTL_DEBUG_CDB)
13181                         ctl_io_print(io);
13182                 ctl_enqueue_incoming(io);
13183                 break;
13184         default:
13185                 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
13186                 return (EINVAL);
13187         }
13188
13189         return (CTL_RETVAL_COMPLETE);
13190 }
13191
13192 #ifdef CTL_IO_DELAY
13193 static void
13194 ctl_done_timer_wakeup(void *arg)
13195 {
13196         union ctl_io *io;
13197
13198         io = (union ctl_io *)arg;
13199         ctl_done(io);
13200 }
13201 #endif /* CTL_IO_DELAY */
13202
13203 void
13204 ctl_serseq_done(union ctl_io *io)
13205 {
13206         struct ctl_lun *lun;
13207
13208         lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13209         if (lun->be_lun == NULL ||
13210             lun->be_lun->serseq == CTL_LUN_SERSEQ_OFF)
13211                 return;
13212         mtx_lock(&lun->lun_lock);
13213         io->io_hdr.flags |= CTL_FLAG_SERSEQ_DONE;
13214         ctl_check_blocked(lun);
13215         mtx_unlock(&lun->lun_lock);
13216 }
13217
13218 void
13219 ctl_done(union ctl_io *io)
13220 {
13221
13222         /*
13223          * Enable this to catch duplicate completion issues.
13224          */
13225 #if 0
13226         if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
13227                 printf("%s: type %d msg %d cdb %x iptl: "
13228                        "%u:%u:%u tag 0x%04x "
13229                        "flag %#x status %x\n",
13230                         __func__,
13231                         io->io_hdr.io_type,
13232                         io->io_hdr.msg_type,
13233                         io->scsiio.cdb[0],
13234                         io->io_hdr.nexus.initid,
13235                         io->io_hdr.nexus.targ_port,
13236                         io->io_hdr.nexus.targ_lun,
13237                         (io->io_hdr.io_type ==
13238                         CTL_IO_TASK) ?
13239                         io->taskio.tag_num :
13240                         io->scsiio.tag_num,
13241                         io->io_hdr.flags,
13242                         io->io_hdr.status);
13243         } else
13244                 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
13245 #endif
13246
13247         /*
13248          * This is an internal copy of an I/O, and should not go through
13249          * the normal done processing logic.
13250          */
13251         if (io->io_hdr.flags & CTL_FLAG_INT_COPY)
13252                 return;
13253
13254 #ifdef CTL_IO_DELAY
13255         if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
13256                 struct ctl_lun *lun;
13257
13258                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13259
13260                 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
13261         } else {
13262                 struct ctl_lun *lun;
13263
13264                 lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13265
13266                 if ((lun != NULL)
13267                  && (lun->delay_info.done_delay > 0)) {
13268
13269                         callout_init(&io->io_hdr.delay_callout, /*mpsafe*/ 1);
13270                         io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
13271                         callout_reset(&io->io_hdr.delay_callout,
13272                                       lun->delay_info.done_delay * hz,
13273                                       ctl_done_timer_wakeup, io);
13274                         if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
13275                                 lun->delay_info.done_delay = 0;
13276                         return;
13277                 }
13278         }
13279 #endif /* CTL_IO_DELAY */
13280
13281         ctl_enqueue_done(io);
13282 }
13283
13284 static void
13285 ctl_work_thread(void *arg)
13286 {
13287         struct ctl_thread *thr = (struct ctl_thread *)arg;
13288         struct ctl_softc *softc = thr->ctl_softc;
13289         union ctl_io *io;
13290         int retval;
13291
13292         CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
13293
13294         for (;;) {
13295                 /*
13296                  * We handle the queues in this order:
13297                  * - ISC
13298                  * - done queue (to free up resources, unblock other commands)
13299                  * - RtR queue
13300                  * - incoming queue
13301                  *
13302                  * If those queues are empty, we break out of the loop and
13303                  * go to sleep.
13304                  */
13305                 mtx_lock(&thr->queue_lock);
13306                 io = (union ctl_io *)STAILQ_FIRST(&thr->isc_queue);
13307                 if (io != NULL) {
13308                         STAILQ_REMOVE_HEAD(&thr->isc_queue, links);
13309                         mtx_unlock(&thr->queue_lock);
13310                         ctl_handle_isc(io);
13311                         continue;
13312                 }
13313                 io = (union ctl_io *)STAILQ_FIRST(&thr->done_queue);
13314                 if (io != NULL) {
13315                         STAILQ_REMOVE_HEAD(&thr->done_queue, links);
13316                         /* clear any blocked commands, call fe_done */
13317                         mtx_unlock(&thr->queue_lock);
13318                         ctl_process_done(io);
13319                         continue;
13320                 }
13321                 io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
13322                 if (io != NULL) {
13323                         STAILQ_REMOVE_HEAD(&thr->incoming_queue, links);
13324                         mtx_unlock(&thr->queue_lock);
13325                         if (io->io_hdr.io_type == CTL_IO_TASK)
13326                                 ctl_run_task(io);
13327                         else
13328                                 ctl_scsiio_precheck(softc, &io->scsiio);
13329                         continue;
13330                 }
13331                 io = (union ctl_io *)STAILQ_FIRST(&thr->rtr_queue);
13332                 if (io != NULL) {
13333                         STAILQ_REMOVE_HEAD(&thr->rtr_queue, links);
13334                         mtx_unlock(&thr->queue_lock);
13335                         retval = ctl_scsiio(&io->scsiio);
13336                         if (retval != CTL_RETVAL_COMPLETE)
13337                                 CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
13338                         continue;
13339                 }
13340
13341                 /* Sleep until we have something to do. */
13342                 mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0);
13343         }
13344 }
13345
13346 static void
13347 ctl_lun_thread(void *arg)
13348 {
13349         struct ctl_softc *softc = (struct ctl_softc *)arg;
13350         struct ctl_be_lun *be_lun;
13351
13352         CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
13353
13354         for (;;) {
13355                 mtx_lock(&softc->ctl_lock);
13356                 be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
13357                 if (be_lun != NULL) {
13358                         STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
13359                         mtx_unlock(&softc->ctl_lock);
13360                         ctl_create_lun(be_lun);
13361                         continue;
13362                 }
13363
13364                 /* Sleep until we have something to do. */
13365                 mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock,
13366                     PDROP | PRIBIO, "-", 0);
13367         }
13368 }
13369
13370 static void
13371 ctl_thresh_thread(void *arg)
13372 {
13373         struct ctl_softc *softc = (struct ctl_softc *)arg;
13374         struct ctl_lun *lun;
13375         struct scsi_da_rw_recovery_page *rwpage;
13376         struct ctl_logical_block_provisioning_page *page;
13377         const char *attr;
13378         union ctl_ha_msg msg;
13379         uint64_t thres, val;
13380         int i, e, set;
13381
13382         CTL_DEBUG_PRINT(("ctl_thresh_thread starting\n"));
13383
13384         for (;;) {
13385                 mtx_lock(&softc->ctl_lock);
13386                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
13387                         if ((lun->flags & CTL_LUN_DISABLED) ||
13388                             (lun->flags & CTL_LUN_OFFLINE) ||
13389                             lun->backend->lun_attr == NULL)
13390                                 continue;
13391                         if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
13392                             softc->ha_mode == CTL_HA_MODE_XFER)
13393                                 continue;
13394                         rwpage = &lun->mode_pages.rw_er_page[CTL_PAGE_CURRENT];
13395                         if ((rwpage->byte8 & SMS_RWER_LBPERE) == 0)
13396                                 continue;
13397                         e = 0;
13398                         page = &lun->mode_pages.lbp_page[CTL_PAGE_CURRENT];
13399                         for (i = 0; i < CTL_NUM_LBP_THRESH; i++) {
13400                                 if ((page->descr[i].flags & SLBPPD_ENABLED) == 0)
13401                                         continue;
13402                                 thres = scsi_4btoul(page->descr[i].count);
13403                                 thres <<= CTL_LBP_EXPONENT;
13404                                 switch (page->descr[i].resource) {
13405                                 case 0x01:
13406                                         attr = "blocksavail";
13407                                         break;
13408                                 case 0x02:
13409                                         attr = "blocksused";
13410                                         break;
13411                                 case 0xf1:
13412                                         attr = "poolblocksavail";
13413                                         break;
13414                                 case 0xf2:
13415                                         attr = "poolblocksused";
13416                                         break;
13417                                 default:
13418                                         continue;
13419                                 }
13420                                 mtx_unlock(&softc->ctl_lock); // XXX
13421                                 val = lun->backend->lun_attr(
13422                                     lun->be_lun->be_lun, attr);
13423                                 mtx_lock(&softc->ctl_lock);
13424                                 if (val == UINT64_MAX)
13425                                         continue;
13426                                 if ((page->descr[i].flags & SLBPPD_ARMING_MASK)
13427                                     == SLBPPD_ARMING_INC)
13428                                         e = (val >= thres);
13429                                 else
13430                                         e = (val <= thres);
13431                                 if (e)
13432                                         break;
13433                         }
13434                         mtx_lock(&lun->lun_lock);
13435                         if (e) {
13436                                 scsi_u64to8b((uint8_t *)&page->descr[i] -
13437                                     (uint8_t *)page, lun->ua_tpt_info);
13438                                 if (lun->lasttpt == 0 ||
13439                                     time_uptime - lun->lasttpt >= CTL_LBP_UA_PERIOD) {
13440                                         lun->lasttpt = time_uptime;
13441                                         ctl_est_ua_all(lun, -1, CTL_UA_THIN_PROV_THRES);
13442                                         set = 1;
13443                                 } else
13444                                         set = 0;
13445                         } else {
13446                                 lun->lasttpt = 0;
13447                                 ctl_clr_ua_all(lun, -1, CTL_UA_THIN_PROV_THRES);
13448                                 set = -1;
13449                         }
13450                         mtx_unlock(&lun->lun_lock);
13451                         if (set != 0 &&
13452                             lun->ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
13453                                 /* Send msg to other side. */
13454                                 bzero(&msg.ua, sizeof(msg.ua));
13455                                 msg.hdr.msg_type = CTL_MSG_UA;
13456                                 msg.hdr.nexus.initid = -1;
13457                                 msg.hdr.nexus.targ_port = -1;
13458                                 msg.hdr.nexus.targ_lun = lun->lun;
13459                                 msg.hdr.nexus.targ_mapped_lun = lun->lun;
13460                                 msg.ua.ua_all = 1;
13461                                 msg.ua.ua_set = (set > 0);
13462                                 msg.ua.ua_type = CTL_UA_THIN_PROV_THRES;
13463                                 memcpy(msg.ua.ua_info, lun->ua_tpt_info, 8);
13464                                 mtx_unlock(&softc->ctl_lock); // XXX
13465                                 ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13466                                     sizeof(msg.ua), M_WAITOK);
13467                                 mtx_lock(&softc->ctl_lock);
13468                         }
13469                 }
13470                 mtx_unlock(&softc->ctl_lock);
13471                 pause("-", CTL_LBP_PERIOD * hz);
13472         }
13473 }
13474
13475 static void
13476 ctl_enqueue_incoming(union ctl_io *io)
13477 {
13478         struct ctl_softc *softc = control_softc;
13479         struct ctl_thread *thr;
13480         u_int idx;
13481
13482         idx = (io->io_hdr.nexus.targ_port * 127 +
13483                io->io_hdr.nexus.initid) % worker_threads;
13484         thr = &softc->threads[idx];
13485         mtx_lock(&thr->queue_lock);
13486         STAILQ_INSERT_TAIL(&thr->incoming_queue, &io->io_hdr, links);
13487         mtx_unlock(&thr->queue_lock);
13488         wakeup(thr);
13489 }
13490
13491 static void
13492 ctl_enqueue_rtr(union ctl_io *io)
13493 {
13494         struct ctl_softc *softc = control_softc;
13495         struct ctl_thread *thr;
13496
13497         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13498         mtx_lock(&thr->queue_lock);
13499         STAILQ_INSERT_TAIL(&thr->rtr_queue, &io->io_hdr, links);
13500         mtx_unlock(&thr->queue_lock);
13501         wakeup(thr);
13502 }
13503
13504 static void
13505 ctl_enqueue_done(union ctl_io *io)
13506 {
13507         struct ctl_softc *softc = control_softc;
13508         struct ctl_thread *thr;
13509
13510         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13511         mtx_lock(&thr->queue_lock);
13512         STAILQ_INSERT_TAIL(&thr->done_queue, &io->io_hdr, links);
13513         mtx_unlock(&thr->queue_lock);
13514         wakeup(thr);
13515 }
13516
13517 static void
13518 ctl_enqueue_isc(union ctl_io *io)
13519 {
13520         struct ctl_softc *softc = control_softc;
13521         struct ctl_thread *thr;
13522
13523         thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13524         mtx_lock(&thr->queue_lock);
13525         STAILQ_INSERT_TAIL(&thr->isc_queue, &io->io_hdr, links);
13526         mtx_unlock(&thr->queue_lock);
13527         wakeup(thr);
13528 }
13529
13530 /*
13531  *  vim: ts=8
13532  */