sys/cam/ctl/ctl_tpc.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2014 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/kernel.h>
  35 #include <sys/types.h>
  36 #include <sys/lock.h>
  37 #include <sys/module.h>
  38 #include <sys/mutex.h>
  39 #include <sys/condvar.h>
  40 #include <sys/malloc.h>
  41 #include <sys/conf.h>
  42 #include <sys/queue.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/nv.h>
  45 #include <sys/dnv.h>
  46 #include <machine/atomic.h>
  47
  48 #include <cam/cam.h>
  49 #include <cam/scsi/scsi_all.h>
  50 #include <cam/scsi/scsi_da.h>
  51 #include <cam/ctl/ctl_io.h>
  52 #include <cam/ctl/ctl.h>
  53 #include <cam/ctl/ctl_frontend.h>
  54 #include <cam/ctl/ctl_util.h>
  55 #include <cam/ctl/ctl_backend.h>
  56 #include <cam/ctl/ctl_ioctl.h>
  57 #include <cam/ctl/ctl_ha.h>
  58 #include <cam/ctl/ctl_private.h>
  59 #include <cam/ctl/ctl_debug.h>
  60 #include <cam/ctl/ctl_scsi_all.h>
  61 #include <cam/ctl/ctl_tpc.h>
  62 #include <cam/ctl/ctl_error.h>
  63
  64 #define TPC_MAX_CSCDS   64
  65 #define TPC_MAX_SEGS    64
  66 #define TPC_MAX_SEG     0
  67 #define TPC_MAX_LIST    8192
  68 #define TPC_MAX_INLINE  0
  69 #define TPC_MAX_LISTS   255
  70 #define TPC_MAX_IO_SIZE (1024 * 1024)
  71 #define TPC_MAX_IOCHUNK_SIZE    (TPC_MAX_IO_SIZE * 16)
  72 #define TPC_MIN_TOKEN_TIMEOUT   1
  73 #define TPC_DFL_TOKEN_TIMEOUT   60
  74 #define TPC_MAX_TOKEN_TIMEOUT   600
  75
  76 MALLOC_DEFINE(M_CTL_TPC, "ctltpc", "CTL TPC");
  77
  78 typedef enum {
  79         TPC_ERR_RETRY           = 0x000,
  80         TPC_ERR_FAIL            = 0x001,
  81         TPC_ERR_MASK            = 0x0ff,
  82         TPC_ERR_NO_DECREMENT    = 0x100
  83 } tpc_error_action;
  84
  85 struct tpc_list;
  86 TAILQ_HEAD(runl, tpc_io);
  87 struct tpc_io {
  88         union ctl_io            *io;
  89         uint8_t                  target;
  90         uint32_t                 cscd;
  91         uint64_t                 lun;
  92         uint8_t                 *buf;
  93         struct tpc_list         *list;
  94         struct runl              run;
  95         TAILQ_ENTRY(tpc_io)      rlinks;
  96         TAILQ_ENTRY(tpc_io)      links;
  97 };
  98
  99 struct tpc_token {
 100         uint8_t                  token[512];
 101         uint64_t                 lun;
 102         uint32_t                 blocksize;
 103         uint8_t                 *params;
 104         struct scsi_range_desc  *range;
 105         int                      nrange;
 106         int                      active;
 107         time_t                   last_active;
 108         uint32_t                 timeout;
 109         TAILQ_ENTRY(tpc_token)   links;
 110 };
 111
 112 struct tpc_list {
 113         uint8_t                  service_action;
 114         int                      init_port;
 115         uint32_t                 init_idx;
 116         uint32_t                 list_id;
 117         uint8_t                  flags;
 118         uint8_t                 *params;
 119         struct scsi_ec_cscd     *cscd;
 120         struct scsi_ec_segment  *seg[TPC_MAX_SEGS];
 121         uint8_t                 *inl;
 122         int                      ncscd;
 123         int                      nseg;
 124         int                      leninl;
 125         struct tpc_token        *token;
 126         struct scsi_range_desc  *range;
 127         int                      nrange;
 128         off_t                    offset_into_rod;
 129
 130         int                      curseg;
 131         off_t                    cursectors;
 132         off_t                    curbytes;
 133         int                      curops;
 134         int                      stage;
 135         off_t                    segsectors;
 136         off_t                    segbytes;
 137         int                      tbdio;
 138         int                      error;
 139         int                      abort;
 140         int                      completed;
 141         time_t                   last_active;
 142         TAILQ_HEAD(, tpc_io)     allio;
 143         struct scsi_sense_data   fwd_sense_data;
 144         uint8_t                  fwd_sense_len;
 145         uint8_t                  fwd_scsi_status;
 146         uint8_t                  fwd_target;
 147         uint16_t                 fwd_cscd;
 148         struct scsi_sense_data   sense_data;
 149         uint8_t                  sense_len;
 150         uint8_t                  scsi_status;
 151         struct ctl_scsiio       *ctsio;
 152         struct ctl_lun          *lun;
 153         int                      res_token_valid;
 154         uint8_t                  res_token[512];
 155         TAILQ_ENTRY(tpc_list)    links;
 156 };
 157
 158 static void
 159 tpc_timeout(void *arg)
 160 {
 161         struct ctl_softc *softc = arg;
 162         struct ctl_lun *lun;
 163         struct tpc_token *token, *ttoken;
 164         struct tpc_list *list, *tlist;
 165
 166         /* Free completed lists with expired timeout. */
 167         STAILQ_FOREACH(lun, &softc->lun_list, links) {
 168                 mtx_lock(&lun->lun_lock);
 169                 TAILQ_FOREACH_SAFE(list, &lun->tpc_lists, links, tlist) {
 170                         if (!list->completed || time_uptime < list->last_active +
 171                             TPC_DFL_TOKEN_TIMEOUT)
 172                                 continue;
 173                         TAILQ_REMOVE(&lun->tpc_lists, list, links);
 174                         free(list, M_CTL);
 175                 }
 176                 mtx_unlock(&lun->lun_lock);
 177         }
 178
 179         /* Free inactive ROD tokens with expired timeout. */
 180         mtx_lock(&softc->tpc_lock);
 181         TAILQ_FOREACH_SAFE(token, &softc->tpc_tokens, links, ttoken) {
 182                 if (token->active ||
 183                     time_uptime < token->last_active + token->timeout + 1)
 184                         continue;
 185                 TAILQ_REMOVE(&softc->tpc_tokens, token, links);
 186                 free(token->params, M_CTL);
 187                 free(token, M_CTL);
 188         }
 189         mtx_unlock(&softc->tpc_lock);
 190         callout_schedule(&softc->tpc_timeout, hz);
 191 }
 192
 193 void
 194 ctl_tpc_init(struct ctl_softc *softc)
 195 {
 196
 197         mtx_init(&softc->tpc_lock, "CTL TPC mutex", NULL, MTX_DEF);
 198         TAILQ_INIT(&softc->tpc_tokens);
 199         callout_init_mtx(&softc->tpc_timeout, &softc->ctl_lock, 0);
 200         callout_reset(&softc->tpc_timeout, hz, tpc_timeout, softc);
 201 }
 202
 203 void
 204 ctl_tpc_shutdown(struct ctl_softc *softc)
 205 {
 206         struct tpc_token *token;
 207
 208         callout_drain(&softc->tpc_timeout);
 209
 210         /* Free ROD tokens. */
 211         mtx_lock(&softc->tpc_lock);
 212         while ((token = TAILQ_FIRST(&softc->tpc_tokens)) != NULL) {
 213                 TAILQ_REMOVE(&softc->tpc_tokens, token, links);
 214                 free(token->params, M_CTL);
 215                 free(token, M_CTL);
 216         }
 217         mtx_unlock(&softc->tpc_lock);
 218         mtx_destroy(&softc->tpc_lock);
 219 }
 220
 221 void
 222 ctl_tpc_lun_init(struct ctl_lun *lun)
 223 {
 224
 225         TAILQ_INIT(&lun->tpc_lists);
 226 }
 227
 228 void
 229 ctl_tpc_lun_clear(struct ctl_lun *lun, uint32_t initidx)
 230 {
 231         struct tpc_list *list, *tlist;
 232
 233         TAILQ_FOREACH_SAFE(list, &lun->tpc_lists, links, tlist) {
 234                 if (initidx != -1 && list->init_idx != initidx)
 235                         continue;
 236                 if (!list->completed)
 237                         continue;
 238                 TAILQ_REMOVE(&lun->tpc_lists, list, links);
 239                 free(list, M_CTL);
 240         }
 241 }
 242
 243 void
 244 ctl_tpc_lun_shutdown(struct ctl_lun *lun)
 245 {
 246         struct ctl_softc *softc = lun->ctl_softc;
 247         struct tpc_list *list;
 248         struct tpc_token *token, *ttoken;
 249
 250         /* Free lists for this LUN. */
 251         while ((list = TAILQ_FIRST(&lun->tpc_lists)) != NULL) {
 252                 TAILQ_REMOVE(&lun->tpc_lists, list, links);
 253                 KASSERT(list->completed,
 254                     ("Not completed TPC (%p) on shutdown", list));
 255                 free(list, M_CTL);
 256         }
 257
 258         /* Free ROD tokens for this LUN. */
 259         mtx_lock(&softc->tpc_lock);
 260         TAILQ_FOREACH_SAFE(token, &softc->tpc_tokens, links, ttoken) {
 261                 if (token->lun != lun->lun || token->active)
 262                         continue;
 263                 TAILQ_REMOVE(&softc->tpc_tokens, token, links);
 264                 free(token->params, M_CTL);
 265                 free(token, M_CTL);
 266         }
 267         mtx_unlock(&softc->tpc_lock);
 268 }
 269
 270 int
 271 ctl_inquiry_evpd_tpc(struct ctl_scsiio *ctsio, int alloc_len)
 272 {
 273         struct ctl_lun *lun = CTL_LUN(ctsio);
 274         struct scsi_vpd_tpc *tpc_ptr;
 275         struct scsi_vpd_tpc_descriptor *d_ptr;
 276         struct scsi_vpd_tpc_descriptor_bdrl *bdrl_ptr;
 277         struct scsi_vpd_tpc_descriptor_sc *sc_ptr;
 278         struct scsi_vpd_tpc_descriptor_sc_descr *scd_ptr;
 279         struct scsi_vpd_tpc_descriptor_pd *pd_ptr;
 280         struct scsi_vpd_tpc_descriptor_sd *sd_ptr;
 281         struct scsi_vpd_tpc_descriptor_sdid *sdid_ptr;
 282         struct scsi_vpd_tpc_descriptor_rtf *rtf_ptr;
 283         struct scsi_vpd_tpc_descriptor_rtf_block *rtfb_ptr;
 284         struct scsi_vpd_tpc_descriptor_srt *srt_ptr;
 285         struct scsi_vpd_tpc_descriptor_srtd *srtd_ptr;
 286         struct scsi_vpd_tpc_descriptor_gco *gco_ptr;
 287         int data_len;
 288
 289         data_len = sizeof(struct scsi_vpd_tpc) +
 290             sizeof(struct scsi_vpd_tpc_descriptor_bdrl) +
 291             roundup2(sizeof(struct scsi_vpd_tpc_descriptor_sc) +
 292              2 * sizeof(struct scsi_vpd_tpc_descriptor_sc_descr) + 11, 4) +
 293             sizeof(struct scsi_vpd_tpc_descriptor_pd) +
 294             roundup2(sizeof(struct scsi_vpd_tpc_descriptor_sd) + 4, 4) +
 295             roundup2(sizeof(struct scsi_vpd_tpc_descriptor_sdid) + 2, 4) +
 296             sizeof(struct scsi_vpd_tpc_descriptor_rtf) +
 297              sizeof(struct scsi_vpd_tpc_descriptor_rtf_block) +
 298             sizeof(struct scsi_vpd_tpc_descriptor_srt) +
 299              2*sizeof(struct scsi_vpd_tpc_descriptor_srtd) +
 300             sizeof(struct scsi_vpd_tpc_descriptor_gco);
 301
 302         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
 303         tpc_ptr = (struct scsi_vpd_tpc *)ctsio->kern_data_ptr;
 304         ctsio->kern_rel_offset = 0;
 305         ctsio->kern_sg_entries = 0;
 306         ctsio->kern_data_len = min(data_len, alloc_len);
 307         ctsio->kern_total_len = ctsio->kern_data_len;
 308
 309         /*
 310          * The control device is always connected.  The disk device, on the
 311          * other hand, may not be online all the time.
 312          */
 313         if (lun != NULL)
 314                 tpc_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
 315                                      lun->be_lun->lun_type;
 316         else
 317                 tpc_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
 318         tpc_ptr->page_code = SVPD_SCSI_TPC;
 319         scsi_ulto2b(data_len - 4, tpc_ptr->page_length);
 320
 321         /* Block Device ROD Limits */
 322         d_ptr = (struct scsi_vpd_tpc_descriptor *)&tpc_ptr->descr[0];
 323         bdrl_ptr = (struct scsi_vpd_tpc_descriptor_bdrl *)d_ptr;
 324         scsi_ulto2b(SVPD_TPC_BDRL, bdrl_ptr->desc_type);
 325         scsi_ulto2b(sizeof(*bdrl_ptr) - 4, bdrl_ptr->desc_length);
 326         scsi_ulto2b(TPC_MAX_SEGS, bdrl_ptr->maximum_ranges);
 327         scsi_ulto4b(TPC_MAX_TOKEN_TIMEOUT,
 328             bdrl_ptr->maximum_inactivity_timeout);
 329         scsi_ulto4b(TPC_DFL_TOKEN_TIMEOUT,
 330             bdrl_ptr->default_inactivity_timeout);
 331         scsi_u64to8b(0, bdrl_ptr->maximum_token_transfer_size);
 332         scsi_u64to8b(0, bdrl_ptr->optimal_transfer_count);
 333
 334         /* Supported commands */
 335         d_ptr = (struct scsi_vpd_tpc_descriptor *)
 336             (&d_ptr->parameters[0] + scsi_2btoul(d_ptr->desc_length));
 337         sc_ptr = (struct scsi_vpd_tpc_descriptor_sc *)d_ptr;
 338         scsi_ulto2b(SVPD_TPC_SC, sc_ptr->desc_type);
 339         sc_ptr->list_length = 2 * sizeof(*scd_ptr) + 11;
 340         scsi_ulto2b(roundup2(1 + sc_ptr->list_length, 4), sc_ptr->desc_length);
 341         scd_ptr = &sc_ptr->descr[0];
 342         scd_ptr->opcode = EXTENDED_COPY;
 343         scd_ptr->sa_length = 5;
 344         scd_ptr->supported_service_actions[0] = EC_EC_LID1;
 345         scd_ptr->supported_service_actions[1] = EC_EC_LID4;
 346         scd_ptr->supported_service_actions[2] = EC_PT;
 347         scd_ptr->supported_service_actions[3] = EC_WUT;
 348         scd_ptr->supported_service_actions[4] = EC_COA;
 349         scd_ptr = (struct scsi_vpd_tpc_descriptor_sc_descr *)
 350             &scd_ptr->supported_service_actions[scd_ptr->sa_length];
 351         scd_ptr->opcode = RECEIVE_COPY_STATUS;
 352         scd_ptr->sa_length = 6;
 353         scd_ptr->supported_service_actions[0] = RCS_RCS_LID1;
 354         scd_ptr->supported_service_actions[1] = RCS_RCFD;
 355         scd_ptr->supported_service_actions[2] = RCS_RCS_LID4;
 356         scd_ptr->supported_service_actions[3] = RCS_RCOP;
 357         scd_ptr->supported_service_actions[4] = RCS_RRTI;
 358         scd_ptr->supported_service_actions[5] = RCS_RART;
 359
 360         /* Parameter data. */
 361         d_ptr = (struct scsi_vpd_tpc_descriptor *)
 362             (&d_ptr->parameters[0] + scsi_2btoul(d_ptr->desc_length));
 363         pd_ptr = (struct scsi_vpd_tpc_descriptor_pd *)d_ptr;
 364         scsi_ulto2b(SVPD_TPC_PD, pd_ptr->desc_type);
 365         scsi_ulto2b(sizeof(*pd_ptr) - 4, pd_ptr->desc_length);
 366         scsi_ulto2b(TPC_MAX_CSCDS, pd_ptr->maximum_cscd_descriptor_count);
 367         scsi_ulto2b(TPC_MAX_SEGS, pd_ptr->maximum_segment_descriptor_count);
 368         scsi_ulto4b(TPC_MAX_LIST, pd_ptr->maximum_descriptor_list_length);
 369         scsi_ulto4b(TPC_MAX_INLINE, pd_ptr->maximum_inline_data_length);
 370
 371         /* Supported Descriptors */
 372         d_ptr = (struct scsi_vpd_tpc_descriptor *)
 373             (&d_ptr->parameters[0] + scsi_2btoul(d_ptr->desc_length));
 374         sd_ptr = (struct scsi_vpd_tpc_descriptor_sd *)d_ptr;
 375         scsi_ulto2b(SVPD_TPC_SD, sd_ptr->desc_type);
 376         scsi_ulto2b(roundup2(sizeof(*sd_ptr) - 4 + 4, 4), sd_ptr->desc_length);
 377         sd_ptr->list_length = 4;
 378         sd_ptr->supported_descriptor_codes[0] = EC_SEG_B2B;
 379         sd_ptr->supported_descriptor_codes[1] = EC_SEG_VERIFY;
 380         sd_ptr->supported_descriptor_codes[2] = EC_SEG_REGISTER_KEY;
 381         sd_ptr->supported_descriptor_codes[3] = EC_CSCD_ID;
 382
 383         /* Supported CSCD Descriptor IDs */
 384         d_ptr = (struct scsi_vpd_tpc_descriptor *)
 385             (&d_ptr->parameters[0] + scsi_2btoul(d_ptr->desc_length));
 386         sdid_ptr = (struct scsi_vpd_tpc_descriptor_sdid *)d_ptr;
 387         scsi_ulto2b(SVPD_TPC_SDID, sdid_ptr->desc_type);
 388         scsi_ulto2b(roundup2(sizeof(*sdid_ptr) - 4 + 2, 4), sdid_ptr->desc_length);
 389         scsi_ulto2b(2, sdid_ptr->list_length);
 390         scsi_ulto2b(0xffff, &sdid_ptr->supported_descriptor_ids[0]);
 391
 392         /* ROD Token Features */
 393         d_ptr = (struct scsi_vpd_tpc_descriptor *)
 394             (&d_ptr->parameters[0] + scsi_2btoul(d_ptr->desc_length));
 395         rtf_ptr = (struct scsi_vpd_tpc_descriptor_rtf *)d_ptr;
 396         scsi_ulto2b(SVPD_TPC_RTF, rtf_ptr->desc_type);
 397         scsi_ulto2b(sizeof(*rtf_ptr) - 4 + sizeof(*rtfb_ptr), rtf_ptr->desc_length);
 398         rtf_ptr->remote_tokens = 0;
 399         scsi_ulto4b(TPC_MIN_TOKEN_TIMEOUT, rtf_ptr->minimum_token_lifetime);
 400         scsi_ulto4b(UINT32_MAX, rtf_ptr->maximum_token_lifetime);
 401         scsi_ulto4b(TPC_MAX_TOKEN_TIMEOUT,
 402             rtf_ptr->maximum_token_inactivity_timeout);
 403         scsi_ulto2b(sizeof(*rtfb_ptr), rtf_ptr->type_specific_features_length);
 404         rtfb_ptr = (struct scsi_vpd_tpc_descriptor_rtf_block *)
 405             &rtf_ptr->type_specific_features;
 406         rtfb_ptr->type_format = SVPD_TPC_RTF_BLOCK;
 407         scsi_ulto2b(sizeof(*rtfb_ptr) - 4, rtfb_ptr->desc_length);
 408         scsi_ulto2b(0, rtfb_ptr->optimal_length_granularity);
 409         scsi_u64to8b(0, rtfb_ptr->maximum_bytes);
 410         scsi_u64to8b(0, rtfb_ptr->optimal_bytes);
 411         scsi_u64to8b(UINT64_MAX, rtfb_ptr->optimal_bytes_to_token_per_segment);
 412         scsi_u64to8b(TPC_MAX_IOCHUNK_SIZE,
 413             rtfb_ptr->optimal_bytes_from_token_per_segment);
 414
 415         /* Supported ROD Tokens */
 416         d_ptr = (struct scsi_vpd_tpc_descriptor *)
 417             (&d_ptr->parameters[0] + scsi_2btoul(d_ptr->desc_length));
 418         srt_ptr = (struct scsi_vpd_tpc_descriptor_srt *)d_ptr;
 419         scsi_ulto2b(SVPD_TPC_SRT, srt_ptr->desc_type);
 420         scsi_ulto2b(sizeof(*srt_ptr) - 4 + 2*sizeof(*srtd_ptr), srt_ptr->desc_length);
 421         scsi_ulto2b(2*sizeof(*srtd_ptr), srt_ptr->rod_type_descriptors_length);
 422         srtd_ptr = (struct scsi_vpd_tpc_descriptor_srtd *)
 423             &srt_ptr->rod_type_descriptors;
 424         scsi_ulto4b(ROD_TYPE_AUR, srtd_ptr->rod_type);
 425         srtd_ptr->flags = SVPD_TPC_SRTD_TIN | SVPD_TPC_SRTD_TOUT;
 426         scsi_ulto2b(0, srtd_ptr->preference_indicator);
 427         srtd_ptr++;
 428         scsi_ulto4b(ROD_TYPE_BLOCK_ZERO, srtd_ptr->rod_type);
 429         srtd_ptr->flags = SVPD_TPC_SRTD_TIN;
 430         scsi_ulto2b(0, srtd_ptr->preference_indicator);
 431
 432         /* General Copy Operations */
 433         d_ptr = (struct scsi_vpd_tpc_descriptor *)
 434             (&d_ptr->parameters[0] + scsi_2btoul(d_ptr->desc_length));
 435         gco_ptr = (struct scsi_vpd_tpc_descriptor_gco *)d_ptr;
 436         scsi_ulto2b(SVPD_TPC_GCO, gco_ptr->desc_type);
 437         scsi_ulto2b(sizeof(*gco_ptr) - 4, gco_ptr->desc_length);
 438         scsi_ulto4b(TPC_MAX_LISTS, gco_ptr->total_concurrent_copies);
 439         scsi_ulto4b(TPC_MAX_LISTS, gco_ptr->maximum_identified_concurrent_copies);
 440         scsi_ulto4b(TPC_MAX_SEG, gco_ptr->maximum_segment_length);
 441         gco_ptr->data_segment_granularity = 0;
 442         gco_ptr->inline_data_granularity = 0;
 443
 444         ctl_set_success(ctsio);
 445         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
 446         ctsio->be_move_done = ctl_config_move_done;
 447         ctl_datamove((union ctl_io *)ctsio);
 448
 449         return (CTL_RETVAL_COMPLETE);
 450 }
 451
 452 int
 453 ctl_receive_copy_operating_parameters(struct ctl_scsiio *ctsio)
 454 {
 455         struct scsi_receive_copy_operating_parameters *cdb;
 456         struct scsi_receive_copy_operating_parameters_data *data;
 457         int retval;
 458         int alloc_len, total_len;
 459
 460         CTL_DEBUG_PRINT(("ctl_report_supported_tmf\n"));
 461
 462         cdb = (struct scsi_receive_copy_operating_parameters *)ctsio->cdb;
 463
 464         retval = CTL_RETVAL_COMPLETE;
 465
 466         total_len = sizeof(*data) + 4;
 467         alloc_len = scsi_4btoul(cdb->length);
 468
 469         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
 470         ctsio->kern_sg_entries = 0;
 471         ctsio->kern_rel_offset = 0;
 472         ctsio->kern_data_len = min(total_len, alloc_len);
 473         ctsio->kern_total_len = ctsio->kern_data_len;
 474
 475         data = (struct scsi_receive_copy_operating_parameters_data *)ctsio->kern_data_ptr;
 476         scsi_ulto4b(sizeof(*data) - 4 + 4, data->length);
 477         data->snlid = RCOP_SNLID;
 478         scsi_ulto2b(TPC_MAX_CSCDS, data->maximum_cscd_descriptor_count);
 479         scsi_ulto2b(TPC_MAX_SEGS, data->maximum_segment_descriptor_count);
 480         scsi_ulto4b(TPC_MAX_LIST, data->maximum_descriptor_list_length);
 481         scsi_ulto4b(TPC_MAX_SEG, data->maximum_segment_length);
 482         scsi_ulto4b(TPC_MAX_INLINE, data->maximum_inline_data_length);
 483         scsi_ulto4b(0, data->held_data_limit);
 484         scsi_ulto4b(0, data->maximum_stream_device_transfer_size);
 485         scsi_ulto2b(TPC_MAX_LISTS, data->total_concurrent_copies);
 486         data->maximum_concurrent_copies = TPC_MAX_LISTS;
 487         data->data_segment_granularity = 0;
 488         data->inline_data_granularity = 0;
 489         data->held_data_granularity = 0;
 490         data->implemented_descriptor_list_length = 4;
 491         data->list_of_implemented_descriptor_type_codes[0] = EC_SEG_B2B;
 492         data->list_of_implemented_descriptor_type_codes[1] = EC_SEG_VERIFY;
 493         data->list_of_implemented_descriptor_type_codes[2] = EC_SEG_REGISTER_KEY;
 494         data->list_of_implemented_descriptor_type_codes[3] = EC_CSCD_ID;
 495
 496         ctl_set_success(ctsio);
 497         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
 498         ctsio->be_move_done = ctl_config_move_done;
 499         ctl_datamove((union ctl_io *)ctsio);
 500         return (retval);
 501 }
 502
 503 static struct tpc_list *
 504 tpc_find_list(struct ctl_lun *lun, uint32_t list_id, uint32_t init_idx)
 505 {
 506         struct tpc_list *list;
 507
 508         mtx_assert(&lun->lun_lock, MA_OWNED);
 509         TAILQ_FOREACH(list, &lun->tpc_lists, links) {
 510                 if ((list->flags & EC_LIST_ID_USAGE_MASK) !=
 511                      EC_LIST_ID_USAGE_NONE && list->list_id == list_id &&
 512                     list->init_idx == init_idx)
 513                         break;
 514         }
 515         return (list);
 516 }
 517
 518 int
 519 ctl_receive_copy_status_lid1(struct ctl_scsiio *ctsio)
 520 {
 521         struct ctl_lun *lun = CTL_LUN(ctsio);
 522         struct scsi_receive_copy_status_lid1 *cdb;
 523         struct scsi_receive_copy_status_lid1_data *data;
 524         struct tpc_list *list;
 525         struct tpc_list list_copy;
 526         int retval;
 527         int alloc_len, total_len;
 528         uint32_t list_id;
 529
 530         CTL_DEBUG_PRINT(("ctl_receive_copy_status_lid1\n"));
 531
 532         cdb = (struct scsi_receive_copy_status_lid1 *)ctsio->cdb;
 533         retval = CTL_RETVAL_COMPLETE;
 534
 535         list_id = cdb->list_identifier;
 536         mtx_lock(&lun->lun_lock);
 537         list = tpc_find_list(lun, list_id,
 538             ctl_get_initindex(&ctsio->io_hdr.nexus));
 539         if (list == NULL) {
 540                 mtx_unlock(&lun->lun_lock);
 541                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
 542                     /*command*/ 1, /*field*/ 2, /*bit_valid*/ 0,
 543                     /*bit*/ 0);
 544                 ctl_done((union ctl_io *)ctsio);
 545                 return (retval);
 546         }
 547         list_copy = *list;
 548         if (list->completed) {
 549                 TAILQ_REMOVE(&lun->tpc_lists, list, links);
 550                 free(list, M_CTL);
 551         }
 552         mtx_unlock(&lun->lun_lock);
 553
 554         total_len = sizeof(*data);
 555         alloc_len = scsi_4btoul(cdb->length);
 556
 557         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
 558         ctsio->kern_sg_entries = 0;
 559         ctsio->kern_rel_offset = 0;
 560         ctsio->kern_data_len = min(total_len, alloc_len);
 561         ctsio->kern_total_len = ctsio->kern_data_len;
 562
 563         data = (struct scsi_receive_copy_status_lid1_data *)ctsio->kern_data_ptr;
 564         scsi_ulto4b(sizeof(*data) - 4, data->available_data);
 565         if (list_copy.completed) {
 566                 if (list_copy.error || list_copy.abort)
 567                         data->copy_command_status = RCS_CCS_ERROR;
 568                 else
 569                         data->copy_command_status = RCS_CCS_COMPLETED;
 570         } else
 571                 data->copy_command_status = RCS_CCS_INPROG;
 572         scsi_ulto2b(list_copy.curseg, data->segments_processed);
 573         if (list_copy.curbytes <= UINT32_MAX) {
 574                 data->transfer_count_units = RCS_TC_BYTES;
 575                 scsi_ulto4b(list_copy.curbytes, data->transfer_count);
 576         } else {
 577                 data->transfer_count_units = RCS_TC_MBYTES;
 578                 scsi_ulto4b(list_copy.curbytes >> 20, data->transfer_count);
 579         }
 580
 581         ctl_set_success(ctsio);
 582         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
 583         ctsio->be_move_done = ctl_config_move_done;
 584         ctl_datamove((union ctl_io *)ctsio);
 585         return (retval);
 586 }
 587
 588 int
 589 ctl_receive_copy_failure_details(struct ctl_scsiio *ctsio)
 590 {
 591         struct ctl_lun *lun = CTL_LUN(ctsio);
 592         struct scsi_receive_copy_failure_details *cdb;
 593         struct scsi_receive_copy_failure_details_data *data;
 594         struct tpc_list *list;
 595         struct tpc_list list_copy;
 596         int retval;
 597         int alloc_len, total_len;
 598         uint32_t list_id;
 599
 600         CTL_DEBUG_PRINT(("ctl_receive_copy_failure_details\n"));
 601
 602         cdb = (struct scsi_receive_copy_failure_details *)ctsio->cdb;
 603         retval = CTL_RETVAL_COMPLETE;
 604
 605         list_id = cdb->list_identifier;
 606         mtx_lock(&lun->lun_lock);
 607         list = tpc_find_list(lun, list_id,
 608             ctl_get_initindex(&ctsio->io_hdr.nexus));
 609         if (list == NULL || !list->completed) {
 610                 mtx_unlock(&lun->lun_lock);
 611                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
 612                     /*command*/ 1, /*field*/ 2, /*bit_valid*/ 0,
 613                     /*bit*/ 0);
 614                 ctl_done((union ctl_io *)ctsio);
 615                 return (retval);
 616         }
 617         list_copy = *list;
 618         TAILQ_REMOVE(&lun->tpc_lists, list, links);
 619         free(list, M_CTL);
 620         mtx_unlock(&lun->lun_lock);
 621
 622         total_len = sizeof(*data) + list_copy.sense_len;
 623         alloc_len = scsi_4btoul(cdb->length);
 624
 625         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
 626         ctsio->kern_sg_entries = 0;
 627         ctsio->kern_rel_offset = 0;
 628         ctsio->kern_data_len = min(total_len, alloc_len);
 629         ctsio->kern_total_len = ctsio->kern_data_len;
 630
 631         data = (struct scsi_receive_copy_failure_details_data *)ctsio->kern_data_ptr;
 632         if (list_copy.completed && (list_copy.error || list_copy.abort)) {
 633                 scsi_ulto4b(sizeof(*data) - 4 + list_copy.sense_len,
 634                     data->available_data);
 635                 data->copy_command_status = RCS_CCS_ERROR;
 636         } else
 637                 scsi_ulto4b(0, data->available_data);
 638         scsi_ulto2b(list_copy.sense_len, data->sense_data_length);
 639         memcpy(data->sense_data, &list_copy.sense_data, list_copy.sense_len);
 640
 641         ctl_set_success(ctsio);
 642         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
 643         ctsio->be_move_done = ctl_config_move_done;
 644         ctl_datamove((union ctl_io *)ctsio);
 645         return (retval);
 646 }
 647
 648 int
 649 ctl_receive_copy_status_lid4(struct ctl_scsiio *ctsio)
 650 {
 651         struct ctl_lun *lun = CTL_LUN(ctsio);
 652         struct scsi_receive_copy_status_lid4 *cdb;
 653         struct scsi_receive_copy_status_lid4_data *data;
 654         struct tpc_list *list;
 655         struct tpc_list list_copy;
 656         int retval;
 657         int alloc_len, total_len;
 658         uint32_t list_id;
 659
 660         CTL_DEBUG_PRINT(("ctl_receive_copy_status_lid4\n"));
 661
 662         cdb = (struct scsi_receive_copy_status_lid4 *)ctsio->cdb;
 663         retval = CTL_RETVAL_COMPLETE;
 664
 665         list_id = scsi_4btoul(cdb->list_identifier);
 666         mtx_lock(&lun->lun_lock);
 667         list = tpc_find_list(lun, list_id,
 668             ctl_get_initindex(&ctsio->io_hdr.nexus));
 669         if (list == NULL) {
 670                 mtx_unlock(&lun->lun_lock);
 671                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
 672                     /*command*/ 1, /*field*/ 2, /*bit_valid*/ 0,
 673                     /*bit*/ 0);
 674                 ctl_done((union ctl_io *)ctsio);
 675                 return (retval);
 676         }
 677         list_copy = *list;
 678         if (list->completed) {
 679                 TAILQ_REMOVE(&lun->tpc_lists, list, links);
 680                 free(list, M_CTL);
 681         }
 682         mtx_unlock(&lun->lun_lock);
 683
 684         total_len = sizeof(*data) + list_copy.sense_len;
 685         alloc_len = scsi_4btoul(cdb->length);
 686
 687         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
 688         ctsio->kern_sg_entries = 0;
 689         ctsio->kern_rel_offset = 0;
 690         ctsio->kern_data_len = min(total_len, alloc_len);
 691         ctsio->kern_total_len = ctsio->kern_data_len;
 692
 693         data = (struct scsi_receive_copy_status_lid4_data *)ctsio->kern_data_ptr;
 694         scsi_ulto4b(sizeof(*data) - 4 + list_copy.sense_len,
 695             data->available_data);
 696         data->response_to_service_action = list_copy.service_action;
 697         if (list_copy.completed) {
 698                 if (list_copy.error)
 699                         data->copy_command_status = RCS_CCS_ERROR;
 700                 else if (list_copy.abort)
 701                         data->copy_command_status = RCS_CCS_ABORTED;
 702                 else
 703                         data->copy_command_status = RCS_CCS_COMPLETED;
 704         } else
 705                 data->copy_command_status = RCS_CCS_INPROG_FG;
 706         scsi_ulto2b(list_copy.curops, data->operation_counter);
 707         scsi_ulto4b(UINT32_MAX, data->estimated_status_update_delay);
 708         data->transfer_count_units = RCS_TC_BYTES;
 709         scsi_u64to8b(list_copy.curbytes, data->transfer_count);
 710         scsi_ulto2b(list_copy.curseg, data->segments_processed);
 711         data->length_of_the_sense_data_field = list_copy.sense_len;
 712         data->sense_data_length = list_copy.sense_len;
 713         memcpy(data->sense_data, &list_copy.sense_data, list_copy.sense_len);
 714
 715         ctl_set_success(ctsio);
 716         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
 717         ctsio->be_move_done = ctl_config_move_done;
 718         ctl_datamove((union ctl_io *)ctsio);
 719         return (retval);
 720 }
 721
 722 int
 723 ctl_copy_operation_abort(struct ctl_scsiio *ctsio)
 724 {
 725         struct ctl_lun *lun = CTL_LUN(ctsio);
 726         struct scsi_copy_operation_abort *cdb;
 727         struct tpc_list *list;
 728         int retval;
 729         uint32_t list_id;
 730
 731         CTL_DEBUG_PRINT(("ctl_copy_operation_abort\n"));
 732
 733         cdb = (struct scsi_copy_operation_abort *)ctsio->cdb;
 734         retval = CTL_RETVAL_COMPLETE;
 735
 736         list_id = scsi_4btoul(cdb->list_identifier);
 737         mtx_lock(&lun->lun_lock);
 738         list = tpc_find_list(lun, list_id,
 739             ctl_get_initindex(&ctsio->io_hdr.nexus));
 740         if (list == NULL) {
 741                 mtx_unlock(&lun->lun_lock);
 742                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
 743                     /*command*/ 1, /*field*/ 2, /*bit_valid*/ 0,
 744                     /*bit*/ 0);
 745                 ctl_done((union ctl_io *)ctsio);
 746                 return (retval);
 747         }
 748         list->abort = 1;
 749         mtx_unlock(&lun->lun_lock);
 750
 751         ctl_set_success(ctsio);
 752         ctl_done((union ctl_io *)ctsio);
 753         return (retval);
 754 }
 755
 756 static uint64_t
 757 tpc_resolve(struct tpc_list *list, uint16_t idx, uint32_t *ss,
 758     uint32_t *pb, uint32_t *pbo)
 759 {
 760
 761         if (idx == 0xffff) {
 762                 if (ss && list->lun->be_lun)
 763                         *ss = list->lun->be_lun->blocksize;
 764                 if (pb && list->lun->be_lun)
 765                         *pb = list->lun->be_lun->blocksize <<
 766                             list->lun->be_lun->pblockexp;
 767                 if (pbo && list->lun->be_lun)
 768                         *pbo = list->lun->be_lun->blocksize *
 769                             list->lun->be_lun->pblockoff;
 770                 return (list->lun->lun);
 771         }
 772         if (idx >= list->ncscd)
 773                 return (UINT64_MAX);
 774         return (tpcl_resolve(list->lun->ctl_softc,
 775             list->init_port, &list->cscd[idx], ss, pb, pbo));
 776 }
 777
 778 static void
 779 tpc_set_io_error_sense(struct tpc_list *list)
 780 {
 781         int flen;
 782         uint8_t csi[4];
 783         uint8_t sks[3];
 784         uint8_t fbuf[4 + 64];
 785
 786         scsi_ulto4b(list->curseg, csi);
 787         if (list->fwd_cscd <= 0x07ff) {
 788                 sks[0] = SSD_SKS_SEGMENT_VALID;
 789                 scsi_ulto2b((uint8_t *)&list->cscd[list->fwd_cscd] -
 790                     list->params, &sks[1]);
 791         } else
 792                 sks[0] = 0;
 793         if (list->fwd_scsi_status) {
 794                 fbuf[0] = 0x0c;
 795                 fbuf[2] = list->fwd_target;
 796                 flen = list->fwd_sense_len;
 797                 if (flen > 64) {
 798                         flen = 64;
 799                         fbuf[2] |= SSD_FORWARDED_FSDT;
 800                 }
 801                 fbuf[1] = 2 + flen;
 802                 fbuf[3] = list->fwd_scsi_status;
 803                 bcopy(&list->fwd_sense_data, &fbuf[4], flen);
 804                 flen += 4;
 805         } else
 806                 flen = 0;
 807         ctl_set_sense(list->ctsio, /*current_error*/ 1,
 808             /*sense_key*/ SSD_KEY_COPY_ABORTED,
 809             /*asc*/ 0x0d, /*ascq*/ 0x01,
 810             SSD_ELEM_COMMAND, sizeof(csi), csi,
 811             sks[0] ? SSD_ELEM_SKS : SSD_ELEM_SKIP, sizeof(sks), sks,
 812             flen ? SSD_ELEM_DESC : SSD_ELEM_SKIP, flen, fbuf,
 813             SSD_ELEM_NONE);
 814 }
 815
 816 static int
 817 tpc_process_b2b(struct tpc_list *list)
 818 {
 819         struct scsi_ec_segment_b2b *seg;
 820         struct scsi_ec_cscd_dtsp *sdstp, *ddstp;
 821         struct tpc_io *tior, *tiow;
 822         struct runl run;
 823         uint64_t sl, dl;
 824         off_t srclba, dstlba, numbytes, donebytes, roundbytes;
 825         int numlba;
 826         uint32_t srcblock, dstblock, pb, pbo, adj;
 827         uint16_t scscd, dcscd;
 828         uint8_t csi[4];
 829
 830         scsi_ulto4b(list->curseg, csi);
 831         if (list->stage == 1) {
 832                 while ((tior = TAILQ_FIRST(&list->allio)) != NULL) {
 833                         TAILQ_REMOVE(&list->allio, tior, links);
 834                         ctl_free_io(tior->io);
 835                         free(tior->buf, M_CTL);
 836                         free(tior, M_CTL);
 837                 }
 838                 if (list->abort) {
 839                         ctl_set_task_aborted(list->ctsio);
 840                         return (CTL_RETVAL_ERROR);
 841                 } else if (list->error) {
 842                         tpc_set_io_error_sense(list);
 843                         return (CTL_RETVAL_ERROR);
 844                 }
 845                 list->cursectors += list->segsectors;
 846                 list->curbytes += list->segbytes;
 847                 return (CTL_RETVAL_COMPLETE);
 848         }
 849
 850         TAILQ_INIT(&list->allio);
 851         seg = (struct scsi_ec_segment_b2b *)list->seg[list->curseg];
 852         scscd = scsi_2btoul(seg->src_cscd);
 853         dcscd = scsi_2btoul(seg->dst_cscd);
 854         sl = tpc_resolve(list, scscd, &srcblock, NULL, NULL);
 855         dl = tpc_resolve(list, dcscd, &dstblock, &pb, &pbo);
 856         if (sl == UINT64_MAX || dl == UINT64_MAX) {
 857                 ctl_set_sense(list->ctsio, /*current_error*/ 1,
 858                     /*sense_key*/ SSD_KEY_COPY_ABORTED,
 859                     /*asc*/ 0x08, /*ascq*/ 0x04,
 860                     SSD_ELEM_COMMAND, sizeof(csi), csi,
 861                     SSD_ELEM_NONE);
 862                 return (CTL_RETVAL_ERROR);
 863         }
 864         if (pbo > 0)
 865                 pbo = pb - pbo;
 866         sdstp = &list->cscd[scscd].dtsp;
 867         if (scsi_3btoul(sdstp->block_length) != 0)
 868                 srcblock = scsi_3btoul(sdstp->block_length);
 869         ddstp = &list->cscd[dcscd].dtsp;
 870         if (scsi_3btoul(ddstp->block_length) != 0)
 871                 dstblock = scsi_3btoul(ddstp->block_length);
 872         numlba = scsi_2btoul(seg->number_of_blocks);
 873         if (seg->flags & EC_SEG_DC)
 874                 numbytes = (off_t)numlba * dstblock;
 875         else
 876                 numbytes = (off_t)numlba * srcblock;
 877         srclba = scsi_8btou64(seg->src_lba);
 878         dstlba = scsi_8btou64(seg->dst_lba);
 879
 880 //      printf("Copy %ju bytes from %ju @ %ju to %ju @ %ju\n",
 881 //          (uintmax_t)numbytes, sl, scsi_8btou64(seg->src_lba),
 882 //          dl, scsi_8btou64(seg->dst_lba));
 883
 884         if (numbytes == 0)
 885                 return (CTL_RETVAL_COMPLETE);
 886
 887         if (numbytes % srcblock != 0 || numbytes % dstblock != 0) {
 888                 ctl_set_sense(list->ctsio, /*current_error*/ 1,
 889                     /*sense_key*/ SSD_KEY_COPY_ABORTED,
 890                     /*asc*/ 0x26, /*ascq*/ 0x0A,
 891                     SSD_ELEM_COMMAND, sizeof(csi), csi,
 892                     SSD_ELEM_NONE);
 893                 return (CTL_RETVAL_ERROR);
 894         }
 895
 896         list->segbytes = numbytes;
 897         list->segsectors = numbytes / dstblock;
 898         donebytes = 0;
 899         TAILQ_INIT(&run);
 900         list->tbdio = 0;
 901         while (donebytes < numbytes) {
 902                 roundbytes = numbytes - donebytes;
 903                 if (roundbytes > TPC_MAX_IO_SIZE) {
 904                         roundbytes = TPC_MAX_IO_SIZE;
 905                         roundbytes -= roundbytes % dstblock;
 906                         if (pb > dstblock) {
 907                                 adj = (dstlba * dstblock + roundbytes - pbo) % pb;
 908                                 if (roundbytes > adj)
 909                                         roundbytes -= adj;
 910                         }
 911                 }
 912
 913                 tior = malloc(sizeof(*tior), M_CTL, M_WAITOK | M_ZERO);
 914                 TAILQ_INIT(&tior->run);
 915                 tior->buf = malloc(roundbytes, M_CTL, M_WAITOK);
 916                 tior->list = list;
 917                 TAILQ_INSERT_TAIL(&list->allio, tior, links);
 918                 tior->io = tpcl_alloc_io();
 919                 ctl_scsi_read_write(tior->io,
 920                                     /*data_ptr*/ tior->buf,
 921                                     /*data_len*/ roundbytes,
 922                                     /*read_op*/ 1,
 923                                     /*byte2*/ 0,
 924                                     /*minimum_cdb_size*/ 0,
 925                                     /*lba*/ srclba,
 926                                     /*num_blocks*/ roundbytes / srcblock,
 927                                     /*tag_type*/ CTL_TAG_SIMPLE,
 928                                     /*control*/ 0);
 929                 tior->io->io_hdr.retries = 3;
 930                 tior->target = SSD_FORWARDED_SDS_EXSRC;
 931                 tior->cscd = scscd;
 932                 tior->lun = sl;
 933                 tior->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tior;
 934
 935                 tiow = malloc(sizeof(*tior), M_CTL, M_WAITOK | M_ZERO);
 936                 TAILQ_INIT(&tiow->run);
 937                 tiow->list = list;
 938                 TAILQ_INSERT_TAIL(&list->allio, tiow, links);
 939                 tiow->io = tpcl_alloc_io();
 940                 ctl_scsi_read_write(tiow->io,
 941                                     /*data_ptr*/ tior->buf,
 942                                     /*data_len*/ roundbytes,
 943                                     /*read_op*/ 0,
 944                                     /*byte2*/ 0,
 945                                     /*minimum_cdb_size*/ 0,
 946                                     /*lba*/ dstlba,
 947                                     /*num_blocks*/ roundbytes / dstblock,
 948                                     /*tag_type*/ CTL_TAG_SIMPLE,
 949                                     /*control*/ 0);
 950                 tiow->io->io_hdr.retries = 3;
 951                 tiow->target = SSD_FORWARDED_SDS_EXDST;
 952                 tiow->cscd = dcscd;
 953                 tiow->lun = dl;
 954                 tiow->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tiow;
 955
 956                 TAILQ_INSERT_TAIL(&tior->run, tiow, rlinks);
 957                 TAILQ_INSERT_TAIL(&run, tior, rlinks);
 958                 list->tbdio++;
 959                 donebytes += roundbytes;
 960                 srclba += roundbytes / srcblock;
 961                 dstlba += roundbytes / dstblock;
 962         }
 963
 964         while ((tior = TAILQ_FIRST(&run)) != NULL) {
 965                 TAILQ_REMOVE(&run, tior, rlinks);
 966                 if (tpcl_queue(tior->io, tior->lun) != CTL_RETVAL_COMPLETE)
 967                         panic("tpcl_queue() error");
 968         }
 969
 970         list->stage++;
 971         return (CTL_RETVAL_QUEUED);
 972 }
 973
 974 static int
 975 tpc_process_verify(struct tpc_list *list)
 976 {
 977         struct scsi_ec_segment_verify *seg;
 978         struct tpc_io *tio;
 979         uint64_t sl;
 980         uint16_t cscd;
 981         uint8_t csi[4];
 982
 983         scsi_ulto4b(list->curseg, csi);
 984         if (list->stage == 1) {
 985                 while ((tio = TAILQ_FIRST(&list->allio)) != NULL) {
 986                         TAILQ_REMOVE(&list->allio, tio, links);
 987                         ctl_free_io(tio->io);
 988                         free(tio, M_CTL);
 989                 }
 990                 if (list->abort) {
 991                         ctl_set_task_aborted(list->ctsio);
 992                         return (CTL_RETVAL_ERROR);
 993                 } else if (list->error) {
 994                         tpc_set_io_error_sense(list);
 995                         return (CTL_RETVAL_ERROR);
 996                 } else
 997                         return (CTL_RETVAL_COMPLETE);
 998         }
 999
1000         TAILQ_INIT(&list->allio);
1001         seg = (struct scsi_ec_segment_verify *)list->seg[list->curseg];
1002         cscd = scsi_2btoul(seg->src_cscd);
1003         sl = tpc_resolve(list, cscd, NULL, NULL, NULL);
1004         if (sl == UINT64_MAX) {
1005                 ctl_set_sense(list->ctsio, /*current_error*/ 1,
1006                     /*sense_key*/ SSD_KEY_COPY_ABORTED,
1007                     /*asc*/ 0x08, /*ascq*/ 0x04,
1008                     SSD_ELEM_COMMAND, sizeof(csi), csi,
1009                     SSD_ELEM_NONE);
1010                 return (CTL_RETVAL_ERROR);
1011         }
1012
1013 //      printf("Verify %ju\n", sl);
1014
1015         if ((seg->tur & 0x01) == 0)
1016                 return (CTL_RETVAL_COMPLETE);
1017
1018         list->tbdio = 1;
1019         tio = malloc(sizeof(*tio), M_CTL, M_WAITOK | M_ZERO);
1020         TAILQ_INIT(&tio->run);
1021         tio->list = list;
1022         TAILQ_INSERT_TAIL(&list->allio, tio, links);
1023         tio->io = tpcl_alloc_io();
1024         ctl_scsi_tur(tio->io, /*tag_type*/ CTL_TAG_SIMPLE, /*control*/ 0);
1025         tio->io->io_hdr.retries = 3;
1026         tio->target = SSD_FORWARDED_SDS_EXSRC;
1027         tio->cscd = cscd;
1028         tio->lun = sl;
1029         tio->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tio;
1030         list->stage++;
1031         if (tpcl_queue(tio->io, tio->lun) != CTL_RETVAL_COMPLETE)
1032                 panic("tpcl_queue() error");
1033         return (CTL_RETVAL_QUEUED);
1034 }
1035
1036 static int
1037 tpc_process_register_key(struct tpc_list *list)
1038 {
1039         struct scsi_ec_segment_register_key *seg;
1040         struct tpc_io *tio;
1041         uint64_t dl;
1042         int datalen;
1043         uint16_t cscd;
1044         uint8_t csi[4];
1045
1046         scsi_ulto4b(list->curseg, csi);
1047         if (list->stage == 1) {
1048                 while ((tio = TAILQ_FIRST(&list->allio)) != NULL) {
1049                         TAILQ_REMOVE(&list->allio, tio, links);
1050                         ctl_free_io(tio->io);
1051                         free(tio->buf, M_CTL);
1052                         free(tio, M_CTL);
1053                 }
1054                 if (list->abort) {
1055                         ctl_set_task_aborted(list->ctsio);
1056                         return (CTL_RETVAL_ERROR);
1057                 } else if (list->error) {
1058                         tpc_set_io_error_sense(list);
1059                         return (CTL_RETVAL_ERROR);
1060                 } else
1061                         return (CTL_RETVAL_COMPLETE);
1062         }
1063
1064         TAILQ_INIT(&list->allio);
1065         seg = (struct scsi_ec_segment_register_key *)list->seg[list->curseg];
1066         cscd = scsi_2btoul(seg->dst_cscd);
1067         dl = tpc_resolve(list, cscd, NULL, NULL, NULL);
1068         if (dl == UINT64_MAX) {
1069                 ctl_set_sense(list->ctsio, /*current_error*/ 1,
1070                     /*sense_key*/ SSD_KEY_COPY_ABORTED,
1071                     /*asc*/ 0x08, /*ascq*/ 0x04,
1072                     SSD_ELEM_COMMAND, sizeof(csi), csi,
1073                     SSD_ELEM_NONE);
1074                 return (CTL_RETVAL_ERROR);
1075         }
1076
1077 //      printf("Register Key %ju\n", dl);
1078
1079         list->tbdio = 1;
1080         tio = malloc(sizeof(*tio), M_CTL, M_WAITOK | M_ZERO);
1081         TAILQ_INIT(&tio->run);
1082         tio->list = list;
1083         TAILQ_INSERT_TAIL(&list->allio, tio, links);
1084         tio->io = tpcl_alloc_io();
1085         datalen = sizeof(struct scsi_per_res_out_parms);
1086         tio->buf = malloc(datalen, M_CTL, M_WAITOK);
1087         ctl_scsi_persistent_res_out(tio->io,
1088             tio->buf, datalen, SPRO_REGISTER, -1,
1089             scsi_8btou64(seg->res_key), scsi_8btou64(seg->sa_res_key),
1090             /*tag_type*/ CTL_TAG_SIMPLE, /*control*/ 0);
1091         tio->io->io_hdr.retries = 3;
1092         tio->target = SSD_FORWARDED_SDS_EXDST;
1093         tio->cscd = cscd;
1094         tio->lun = dl;
1095         tio->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tio;
1096         list->stage++;
1097         if (tpcl_queue(tio->io, tio->lun) != CTL_RETVAL_COMPLETE)
1098                 panic("tpcl_queue() error");
1099         return (CTL_RETVAL_QUEUED);
1100 }
1101
1102 static off_t
1103 tpc_ranges_length(struct scsi_range_desc *range, int nrange)
1104 {
1105         off_t length = 0;
1106         int r;
1107
1108         for (r = 0; r < nrange; r++)
1109                 length += scsi_4btoul(range[r].length);
1110         return (length);
1111 }
1112
1113 static int
1114 tpc_check_ranges_l(struct scsi_range_desc *range, int nrange, uint64_t maxlba,
1115     uint64_t *lba)
1116 {
1117         uint64_t b1;
1118         uint32_t l1;
1119         int i;
1120
1121         for (i = 0; i < nrange; i++) {
1122                 b1 = scsi_8btou64(range[i].lba);
1123                 l1 = scsi_4btoul(range[i].length);
1124                 if (b1 + l1 < b1 || b1 + l1 > maxlba + 1) {
1125                         *lba = MAX(b1, maxlba + 1);
1126                         return (-1);
1127                 }
1128         }
1129         return (0);
1130 }
1131
1132 static int
1133 tpc_check_ranges_x(struct scsi_range_desc *range, int nrange)
1134 {
1135         uint64_t b1, b2;
1136         uint32_t l1, l2;
1137         int i, j;
1138
1139         for (i = 0; i < nrange - 1; i++) {
1140                 b1 = scsi_8btou64(range[i].lba);
1141                 l1 = scsi_4btoul(range[i].length);
1142                 for (j = i + 1; j < nrange; j++) {
1143                         b2 = scsi_8btou64(range[j].lba);
1144                         l2 = scsi_4btoul(range[j].length);
1145                         if (b1 + l1 > b2 && b2 + l2 > b1)
1146                                 return (-1);
1147                 }
1148         }
1149         return (0);
1150 }
1151
1152 static int
1153 tpc_skip_ranges(struct scsi_range_desc *range, int nrange, off_t skip,
1154     int *srange, off_t *soffset)
1155 {
1156         off_t off;
1157         int r;
1158
1159         r = 0;
1160         off = 0;
1161         while (r < nrange) {
1162                 if (skip - off < scsi_4btoul(range[r].length)) {
1163                         *srange = r;
1164                         *soffset = skip - off;
1165                         return (0);
1166                 }
1167                 off += scsi_4btoul(range[r].length);
1168                 r++;
1169         }
1170         return (-1);
1171 }
1172
1173 static int
1174 tpc_process_wut(struct tpc_list *list)
1175 {
1176         struct tpc_io *tio, *tior, *tiow;
1177         struct runl run;
1178         int drange, srange;
1179         off_t doffset, soffset;
1180         off_t srclba, dstlba, numbytes, donebytes, roundbytes;
1181         uint32_t srcblock, dstblock, pb, pbo, adj;
1182
1183         if (list->stage > 0) {
1184                 /* Cleanup after previous rounds. */
1185                 while ((tio = TAILQ_FIRST(&list->allio)) != NULL) {
1186                         TAILQ_REMOVE(&list->allio, tio, links);
1187                         ctl_free_io(tio->io);
1188                         free(tio->buf, M_CTL);
1189                         free(tio, M_CTL);
1190                 }
1191                 if (list->abort) {
1192                         ctl_set_task_aborted(list->ctsio);
1193                         return (CTL_RETVAL_ERROR);
1194                 } else if (list->error) {
1195                         if (list->fwd_scsi_status) {
1196                                 list->ctsio->io_hdr.status =
1197                                     CTL_SCSI_ERROR | CTL_AUTOSENSE;
1198                                 list->ctsio->scsi_status = list->fwd_scsi_status;
1199                                 list->ctsio->sense_data = list->fwd_sense_data;
1200                                 list->ctsio->sense_len = list->fwd_sense_len;
1201                         } else {
1202                                 ctl_set_invalid_field(list->ctsio,
1203                                     /*sks_valid*/ 0, /*command*/ 0,
1204                                     /*field*/ 0, /*bit_valid*/ 0, /*bit*/ 0);
1205                         }
1206                         return (CTL_RETVAL_ERROR);
1207                 }
1208                 list->cursectors += list->segsectors;
1209                 list->curbytes += list->segbytes;
1210         }
1211
1212         /* Check where we are on destination ranges list. */
1213         if (tpc_skip_ranges(list->range, list->nrange, list->cursectors,
1214             &drange, &doffset) != 0)
1215                 return (CTL_RETVAL_COMPLETE);
1216         dstblock = list->lun->be_lun->blocksize;
1217         pb = dstblock << list->lun->be_lun->pblockexp;
1218         if (list->lun->be_lun->pblockoff > 0)
1219                 pbo = pb - dstblock * list->lun->be_lun->pblockoff;
1220         else
1221                 pbo = 0;
1222
1223         /* Check where we are on source ranges list. */
1224         srcblock = list->token->blocksize;
1225         if (tpc_skip_ranges(list->token->range, list->token->nrange,
1226             list->offset_into_rod + list->cursectors * dstblock / srcblock,
1227             &srange, &soffset) != 0) {
1228                 ctl_set_invalid_field(list->ctsio, /*sks_valid*/ 0,
1229                     /*command*/ 0, /*field*/ 0, /*bit_valid*/ 0, /*bit*/ 0);
1230                 return (CTL_RETVAL_ERROR);
1231         }
1232
1233         srclba = scsi_8btou64(list->token->range[srange].lba) + soffset;
1234         dstlba = scsi_8btou64(list->range[drange].lba) + doffset;
1235         numbytes = srcblock *
1236             (scsi_4btoul(list->token->range[srange].length) - soffset);
1237         numbytes = omin(numbytes, dstblock *
1238             (scsi_4btoul(list->range[drange].length) - doffset));
1239         if (numbytes > TPC_MAX_IOCHUNK_SIZE) {
1240                 numbytes = TPC_MAX_IOCHUNK_SIZE;
1241                 numbytes -= numbytes % dstblock;
1242                 if (pb > dstblock) {
1243                         adj = (dstlba * dstblock + numbytes - pbo) % pb;
1244                         if (numbytes > adj)
1245                                 numbytes -= adj;
1246                 }
1247         }
1248
1249         if (numbytes % srcblock != 0 || numbytes % dstblock != 0) {
1250                 ctl_set_invalid_field(list->ctsio, /*sks_valid*/ 0,
1251                     /*command*/ 0, /*field*/ 0, /*bit_valid*/ 0, /*bit*/ 0);
1252                 return (CTL_RETVAL_ERROR);
1253         }
1254
1255         list->segbytes = numbytes;
1256         list->segsectors = numbytes / dstblock;
1257 //printf("Copy chunk of %ju sectors from %ju to %ju\n", list->segsectors,
1258 //    srclba, dstlba);
1259         donebytes = 0;
1260         TAILQ_INIT(&run);
1261         list->tbdio = 0;
1262         TAILQ_INIT(&list->allio);
1263         while (donebytes < numbytes) {
1264                 roundbytes = numbytes - donebytes;
1265                 if (roundbytes > TPC_MAX_IO_SIZE) {
1266                         roundbytes = TPC_MAX_IO_SIZE;
1267                         roundbytes -= roundbytes % dstblock;
1268                         if (pb > dstblock) {
1269                                 adj = (dstlba * dstblock + roundbytes - pbo) % pb;
1270                                 if (roundbytes > adj)
1271                                         roundbytes -= adj;
1272                         }
1273                 }
1274
1275                 tior = malloc(sizeof(*tior), M_CTL, M_WAITOK | M_ZERO);
1276                 TAILQ_INIT(&tior->run);
1277                 tior->buf = malloc(roundbytes, M_CTL, M_WAITOK);
1278                 tior->list = list;
1279                 TAILQ_INSERT_TAIL(&list->allio, tior, links);
1280                 tior->io = tpcl_alloc_io();
1281                 ctl_scsi_read_write(tior->io,
1282                                     /*data_ptr*/ tior->buf,
1283                                     /*data_len*/ roundbytes,
1284                                     /*read_op*/ 1,
1285                                     /*byte2*/ 0,
1286                                     /*minimum_cdb_size*/ 0,
1287                                     /*lba*/ srclba,
1288                                     /*num_blocks*/ roundbytes / srcblock,
1289                                     /*tag_type*/ CTL_TAG_SIMPLE,
1290                                     /*control*/ 0);
1291                 tior->io->io_hdr.retries = 3;
1292                 tior->lun = list->token->lun;
1293                 tior->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tior;
1294
1295                 tiow = malloc(sizeof(*tiow), M_CTL, M_WAITOK | M_ZERO);
1296                 TAILQ_INIT(&tiow->run);
1297                 tiow->list = list;
1298                 TAILQ_INSERT_TAIL(&list->allio, tiow, links);
1299                 tiow->io = tpcl_alloc_io();
1300                 ctl_scsi_read_write(tiow->io,
1301                                     /*data_ptr*/ tior->buf,
1302                                     /*data_len*/ roundbytes,
1303                                     /*read_op*/ 0,
1304                                     /*byte2*/ 0,
1305                                     /*minimum_cdb_size*/ 0,
1306                                     /*lba*/ dstlba,
1307                                     /*num_blocks*/ roundbytes / dstblock,
1308                                     /*tag_type*/ CTL_TAG_SIMPLE,
1309                                     /*control*/ 0);
1310                 tiow->io->io_hdr.retries = 3;
1311                 tiow->lun = list->lun->lun;
1312                 tiow->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tiow;
1313
1314                 TAILQ_INSERT_TAIL(&tior->run, tiow, rlinks);
1315                 TAILQ_INSERT_TAIL(&run, tior, rlinks);
1316                 list->tbdio++;
1317                 donebytes += roundbytes;
1318                 srclba += roundbytes / srcblock;
1319                 dstlba += roundbytes / dstblock;
1320         }
1321
1322         while ((tior = TAILQ_FIRST(&run)) != NULL) {
1323                 TAILQ_REMOVE(&run, tior, rlinks);
1324                 if (tpcl_queue(tior->io, tior->lun) != CTL_RETVAL_COMPLETE)
1325                         panic("tpcl_queue() error");
1326         }
1327
1328         list->stage++;
1329         return (CTL_RETVAL_QUEUED);
1330 }
1331
1332 static int
1333 tpc_process_zero_wut(struct tpc_list *list)
1334 {
1335         struct tpc_io *tio, *tiow;
1336         struct runl run, *prun;
1337         int r;
1338         uint32_t dstblock, len;
1339
1340         if (list->stage > 0) {
1341 complete:
1342                 /* Cleanup after previous rounds. */
1343                 while ((tio = TAILQ_FIRST(&list->allio)) != NULL) {
1344                         TAILQ_REMOVE(&list->allio, tio, links);
1345                         ctl_free_io(tio->io);
1346                         free(tio, M_CTL);
1347                 }
1348                 if (list->abort) {
1349                         ctl_set_task_aborted(list->ctsio);
1350                         return (CTL_RETVAL_ERROR);
1351                 } else if (list->error) {
1352                         if (list->fwd_scsi_status) {
1353                                 list->ctsio->io_hdr.status =
1354                                     CTL_SCSI_ERROR | CTL_AUTOSENSE;
1355                                 list->ctsio->scsi_status = list->fwd_scsi_status;
1356                                 list->ctsio->sense_data = list->fwd_sense_data;
1357                                 list->ctsio->sense_len = list->fwd_sense_len;
1358                         } else {
1359                                 ctl_set_invalid_field(list->ctsio,
1360                                     /*sks_valid*/ 0, /*command*/ 0,
1361                                     /*field*/ 0, /*bit_valid*/ 0, /*bit*/ 0);
1362                         }
1363                         return (CTL_RETVAL_ERROR);
1364                 }
1365                 list->cursectors += list->segsectors;
1366                 list->curbytes += list->segbytes;
1367                 return (CTL_RETVAL_COMPLETE);
1368         }
1369
1370         dstblock = list->lun->be_lun->blocksize;
1371         TAILQ_INIT(&run);
1372         prun = &run;
1373         list->tbdio = 1;
1374         TAILQ_INIT(&list->allio);
1375         list->segsectors = 0;
1376         for (r = 0; r < list->nrange; r++) {
1377                 len = scsi_4btoul(list->range[r].length);
1378                 if (len == 0)
1379                         continue;
1380
1381                 tiow = malloc(sizeof(*tiow), M_CTL, M_WAITOK | M_ZERO);
1382                 TAILQ_INIT(&tiow->run);
1383                 tiow->list = list;
1384                 TAILQ_INSERT_TAIL(&list->allio, tiow, links);
1385                 tiow->io = tpcl_alloc_io();
1386                 ctl_scsi_write_same(tiow->io,
1387                                     /*data_ptr*/ NULL,
1388                                     /*data_len*/ 0,
1389                                     /*byte2*/ SWS_NDOB,
1390                                     /*lba*/ scsi_8btou64(list->range[r].lba),
1391                                     /*num_blocks*/ len,
1392                                     /*tag_type*/ CTL_TAG_SIMPLE,
1393                                     /*control*/ 0);
1394                 tiow->io->io_hdr.retries = 3;
1395                 tiow->lun = list->lun->lun;
1396                 tiow->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tiow;
1397
1398                 TAILQ_INSERT_TAIL(prun, tiow, rlinks);
1399                 prun = &tiow->run;
1400                 list->segsectors += len;
1401         }
1402         list->segbytes = list->segsectors * dstblock;
1403
1404         if (TAILQ_EMPTY(&run))
1405                 goto complete;
1406
1407         while ((tiow = TAILQ_FIRST(&run)) != NULL) {
1408                 TAILQ_REMOVE(&run, tiow, rlinks);
1409                 if (tpcl_queue(tiow->io, tiow->lun) != CTL_RETVAL_COMPLETE)
1410                         panic("tpcl_queue() error");
1411         }
1412
1413         list->stage++;
1414         return (CTL_RETVAL_QUEUED);
1415 }
1416
1417 static void
1418 tpc_process(struct tpc_list *list)
1419 {
1420         struct ctl_lun *lun = list->lun;
1421         struct ctl_softc *softc = lun->ctl_softc;
1422         struct scsi_ec_segment *seg;
1423         struct ctl_scsiio *ctsio = list->ctsio;
1424         int retval = CTL_RETVAL_COMPLETE;
1425         uint8_t csi[4];
1426
1427         if (list->service_action == EC_WUT) {
1428                 if (list->token != NULL)
1429                         retval = tpc_process_wut(list);
1430                 else
1431                         retval = tpc_process_zero_wut(list);
1432                 if (retval == CTL_RETVAL_QUEUED)
1433                         return;
1434                 if (retval == CTL_RETVAL_ERROR) {
1435                         list->error = 1;
1436                         goto done;
1437                 }
1438         } else {
1439 //printf("ZZZ %d cscd, %d segs\n", list->ncscd, list->nseg);
1440                 while (list->curseg < list->nseg) {
1441                         seg = list->seg[list->curseg];
1442                         switch (seg->type_code) {
1443                         case EC_SEG_B2B:
1444                                 retval = tpc_process_b2b(list);
1445                                 break;
1446                         case EC_SEG_VERIFY:
1447                                 retval = tpc_process_verify(list);
1448                                 break;
1449                         case EC_SEG_REGISTER_KEY:
1450                                 retval = tpc_process_register_key(list);
1451                                 break;
1452                         default:
1453                                 scsi_ulto4b(list->curseg, csi);
1454                                 ctl_set_sense(ctsio, /*current_error*/ 1,
1455                                     /*sense_key*/ SSD_KEY_COPY_ABORTED,
1456                                     /*asc*/ 0x26, /*ascq*/ 0x09,
1457                                     SSD_ELEM_COMMAND, sizeof(csi), csi,
1458                                     SSD_ELEM_NONE);
1459                                 goto done;
1460                         }
1461                         if (retval == CTL_RETVAL_QUEUED)
1462                                 return;
1463                         if (retval == CTL_RETVAL_ERROR) {
1464                                 list->error = 1;
1465                                 goto done;
1466                         }
1467                         list->curseg++;
1468                         list->stage = 0;
1469                 }
1470         }
1471
1472         ctl_set_success(ctsio);
1473
1474 done:
1475 //printf("ZZZ done\n");
1476         free(list->params, M_CTL);
1477         list->params = NULL;
1478         if (list->token) {
1479                 mtx_lock(&softc->tpc_lock);
1480                 if (--list->token->active == 0)
1481                         list->token->last_active = time_uptime;
1482                 mtx_unlock(&softc->tpc_lock);
1483                 list->token = NULL;
1484         }
1485         mtx_lock(&lun->lun_lock);
1486         if ((list->flags & EC_LIST_ID_USAGE_MASK) == EC_LIST_ID_USAGE_NONE) {
1487                 TAILQ_REMOVE(&lun->tpc_lists, list, links);
1488                 free(list, M_CTL);
1489         } else {
1490                 list->completed = 1;
1491                 list->last_active = time_uptime;
1492                 list->sense_data = ctsio->sense_data;
1493                 list->sense_len = ctsio->sense_len;
1494                 list->scsi_status = ctsio->scsi_status;
1495         }
1496         mtx_unlock(&lun->lun_lock);
1497
1498         ctl_done((union ctl_io *)ctsio);
1499 }
1500
1501 /*
1502  * For any sort of check condition, busy, etc., we just retry.  We do not
1503  * decrement the retry count for unit attention type errors.  These are
1504  * normal, and we want to save the retry count for "real" errors.  Otherwise,
1505  * we could end up with situations where a command will succeed in some
1506  * situations and fail in others, depending on whether a unit attention is
1507  * pending.  Also, some of our error recovery actions, most notably the
1508  * LUN reset action, will cause a unit attention.
1509  *
1510  * We can add more detail here later if necessary.
1511  */
1512 static tpc_error_action
1513 tpc_checkcond_parse(union ctl_io *io)
1514 {
1515         tpc_error_action error_action;
1516         int error_code, sense_key, asc, ascq;
1517
1518         /*
1519          * Default to retrying the command.
1520          */
1521         error_action = TPC_ERR_RETRY;
1522
1523         scsi_extract_sense_len(&io->scsiio.sense_data,
1524                                io->scsiio.sense_len,
1525                                &error_code,
1526                                &sense_key,
1527                                &asc,
1528                                &ascq,
1529                                /*show_errors*/ 1);
1530
1531         switch (error_code) {
1532         case SSD_DEFERRED_ERROR:
1533         case SSD_DESC_DEFERRED_ERROR:
1534                 error_action |= TPC_ERR_NO_DECREMENT;
1535                 break;
1536         case SSD_CURRENT_ERROR:
1537         case SSD_DESC_CURRENT_ERROR:
1538         default:
1539                 switch (sense_key) {
1540                 case SSD_KEY_UNIT_ATTENTION:
1541                         error_action |= TPC_ERR_NO_DECREMENT;
1542                         break;
1543                 case SSD_KEY_HARDWARE_ERROR:
1544                         /*
1545                          * This is our generic "something bad happened"
1546                          * error code.  It often isn't recoverable.
1547                          */
1548                         if ((asc == 0x44) && (ascq == 0x00))
1549                                 error_action = TPC_ERR_FAIL;
1550                         break;
1551                 case SSD_KEY_NOT_READY:
1552                         /*
1553                          * If the LUN is powered down, there likely isn't
1554                          * much point in retrying right now.
1555                          */
1556                         if ((asc == 0x04) && (ascq == 0x02))
1557                                 error_action = TPC_ERR_FAIL;
1558                         /*
1559                          * If the LUN is offline, there probably isn't much
1560                          * point in retrying, either.
1561                          */
1562                         if ((asc == 0x04) && (ascq == 0x03))
1563                                 error_action = TPC_ERR_FAIL;
1564                         break;
1565                 }
1566         }
1567         return (error_action);
1568 }
1569
1570 static tpc_error_action
1571 tpc_error_parse(union ctl_io *io)
1572 {
1573         tpc_error_action error_action = TPC_ERR_RETRY;
1574
1575         switch (io->io_hdr.io_type) {
1576         case CTL_IO_SCSI:
1577                 switch (io->io_hdr.status & CTL_STATUS_MASK) {
1578                 case CTL_SCSI_ERROR:
1579                         switch (io->scsiio.scsi_status) {
1580                         case SCSI_STATUS_CHECK_COND:
1581                                 error_action = tpc_checkcond_parse(io);
1582                                 break;
1583                         default:
1584                                 break;
1585                         }
1586                         break;
1587                 default:
1588                         break;
1589                 }
1590                 break;
1591         case CTL_IO_TASK:
1592                 break;
1593         default:
1594                 panic("%s: invalid ctl_io type %d\n", __func__,
1595                       io->io_hdr.io_type);
1596                 break;
1597         }
1598         return (error_action);
1599 }
1600
1601 void
1602 tpc_done(union ctl_io *io)
1603 {
1604         struct tpc_io *tio, *tior;
1605
1606         /*
1607          * Very minimal retry logic.  We basically retry if we got an error
1608          * back, and the retry count is greater than 0.  If we ever want
1609          * more sophisticated initiator type behavior, the CAM error
1610          * recovery code in ../common might be helpful.
1611          */
1612         tio = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1613         if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
1614          && (io->io_hdr.retries > 0)) {
1615                 ctl_io_status old_status;
1616                 tpc_error_action error_action;
1617
1618                 error_action = tpc_error_parse(io);
1619                 switch (error_action & TPC_ERR_MASK) {
1620                 case TPC_ERR_FAIL:
1621                         break;
1622                 case TPC_ERR_RETRY:
1623                 default:
1624                         if ((error_action & TPC_ERR_NO_DECREMENT) == 0)
1625                                 io->io_hdr.retries--;
1626                         old_status = io->io_hdr.status;
1627                         io->io_hdr.status = CTL_STATUS_NONE;
1628                         io->io_hdr.flags &= ~CTL_FLAG_ABORT;
1629                         io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
1630                         if (tpcl_queue(io, tio->lun) != CTL_RETVAL_COMPLETE) {
1631                                 printf("%s: error returned from ctl_queue()!\n",
1632                                        __func__);
1633                                 io->io_hdr.status = old_status;
1634                         } else
1635                                 return;
1636                 }
1637         }
1638
1639         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) {
1640                 tio->list->error = 1;
1641                 if (io->io_hdr.io_type == CTL_IO_SCSI &&
1642                     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR) {
1643                         tio->list->fwd_scsi_status = io->scsiio.scsi_status;
1644                         tio->list->fwd_sense_data = io->scsiio.sense_data;
1645                         tio->list->fwd_sense_len = io->scsiio.sense_len;
1646                         tio->list->fwd_target = tio->target;
1647                         tio->list->fwd_cscd = tio->cscd;
1648                 }
1649         } else
1650                 atomic_add_int(&tio->list->curops, 1);
1651         if (!tio->list->error && !tio->list->abort) {
1652                 while ((tior = TAILQ_FIRST(&tio->run)) != NULL) {
1653                         TAILQ_REMOVE(&tio->run, tior, rlinks);
1654                         atomic_add_int(&tio->list->tbdio, 1);
1655                         if (tpcl_queue(tior->io, tior->lun) != CTL_RETVAL_COMPLETE)
1656                                 panic("tpcl_queue() error");
1657                 }
1658         }
1659         if (atomic_fetchadd_int(&tio->list->tbdio, -1) == 1)
1660                 tpc_process(tio->list);
1661 }
1662
1663 int
1664 ctl_extended_copy_lid1(struct ctl_scsiio *ctsio)
1665 {
1666         struct ctl_lun *lun = CTL_LUN(ctsio);
1667         struct scsi_extended_copy *cdb;
1668         struct scsi_extended_copy_lid1_data *data;
1669         struct scsi_ec_cscd *cscd;
1670         struct scsi_ec_segment *seg;
1671         struct tpc_list *list, *tlist;
1672         uint8_t *ptr;
1673         const char *value;
1674         int len, off, lencscd, lenseg, leninl, nseg;
1675
1676         CTL_DEBUG_PRINT(("ctl_extended_copy_lid1\n"));
1677
1678         cdb = (struct scsi_extended_copy *)ctsio->cdb;
1679         len = scsi_4btoul(cdb->length);
1680
1681         if (len == 0) {
1682                 ctl_set_success(ctsio);
1683                 goto done;
1684         }
1685         if (len < sizeof(struct scsi_extended_copy_lid1_data) ||
1686             len > sizeof(struct scsi_extended_copy_lid1_data) +
1687             TPC_MAX_LIST + TPC_MAX_INLINE) {
1688                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 1,
1689                     /*field*/ 9, /*bit_valid*/ 0, /*bit*/ 0);
1690                 goto done;
1691         }
1692
1693         /*
1694          * If we've got a kernel request that hasn't been malloced yet,
1695          * malloc it and tell the caller the data buffer is here.
1696          */
1697         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
1698                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);
1699                 ctsio->kern_data_len = len;
1700                 ctsio->kern_total_len = len;
1701                 ctsio->kern_rel_offset = 0;
1702                 ctsio->kern_sg_entries = 0;
1703                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
1704                 ctsio->be_move_done = ctl_config_move_done;
1705                 ctl_datamove((union ctl_io *)ctsio);
1706
1707                 return (CTL_RETVAL_COMPLETE);
1708         }
1709
1710         data = (struct scsi_extended_copy_lid1_data *)ctsio->kern_data_ptr;
1711         lencscd = scsi_2btoul(data->cscd_list_length);
1712         lenseg = scsi_4btoul(data->segment_list_length);
1713         leninl = scsi_4btoul(data->inline_data_length);
1714         if (lencscd > TPC_MAX_CSCDS * sizeof(struct scsi_ec_cscd)) {
1715                 ctl_set_sense(ctsio, /*current_error*/ 1,
1716                     /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1717                     /*asc*/ 0x26, /*ascq*/ 0x06, SSD_ELEM_NONE);
1718                 goto done;
1719         }
1720         if (lenseg > TPC_MAX_SEGS * sizeof(struct scsi_ec_segment)) {
1721                 ctl_set_sense(ctsio, /*current_error*/ 1,
1722                     /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1723                     /*asc*/ 0x26, /*ascq*/ 0x08, SSD_ELEM_NONE);
1724                 goto done;
1725         }
1726         if (lencscd + lenseg > TPC_MAX_LIST ||
1727             leninl > TPC_MAX_INLINE ||
1728             len < sizeof(struct scsi_extended_copy_lid1_data) +
1729              lencscd + lenseg + leninl) {
1730                 ctl_set_param_len_error(ctsio);
1731                 goto done;
1732         }
1733
1734         list = malloc(sizeof(struct tpc_list), M_CTL, M_WAITOK | M_ZERO);
1735         list->service_action = cdb->service_action;
1736         value = dnvlist_get_string(lun->be_lun->options, "insecure_tpc", NULL);
1737         if (value != NULL && strcmp(value, "on") == 0)
1738                 list->init_port = -1;
1739         else
1740                 list->init_port = ctsio->io_hdr.nexus.targ_port;
1741         list->init_idx = ctl_get_initindex(&ctsio->io_hdr.nexus);
1742         list->list_id = data->list_identifier;
1743         list->flags = data->flags;
1744         list->params = ctsio->kern_data_ptr;
1745         list->cscd = (struct scsi_ec_cscd *)&data->data[0];
1746         ptr = &data->data[0];
1747         for (off = 0; off < lencscd; off += sizeof(struct scsi_ec_cscd)) {
1748                 cscd = (struct scsi_ec_cscd *)(ptr + off);
1749                 if (cscd->type_code != EC_CSCD_ID) {
1750                         free(list, M_CTL);
1751                         ctl_set_sense(ctsio, /*current_error*/ 1,
1752                             /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1753                             /*asc*/ 0x26, /*ascq*/ 0x07, SSD_ELEM_NONE);
1754                         goto done;
1755                 }
1756         }
1757         ptr = &data->data[lencscd];
1758         for (nseg = 0, off = 0; off < lenseg; nseg++) {
1759                 if (nseg >= TPC_MAX_SEGS) {
1760                         free(list, M_CTL);
1761                         ctl_set_sense(ctsio, /*current_error*/ 1,
1762                             /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1763                             /*asc*/ 0x26, /*ascq*/ 0x08, SSD_ELEM_NONE);
1764                         goto done;
1765                 }
1766                 seg = (struct scsi_ec_segment *)(ptr + off);
1767                 if (seg->type_code != EC_SEG_B2B &&
1768                     seg->type_code != EC_SEG_VERIFY &&
1769                     seg->type_code != EC_SEG_REGISTER_KEY) {
1770                         free(list, M_CTL);
1771                         ctl_set_sense(ctsio, /*current_error*/ 1,
1772                             /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1773                             /*asc*/ 0x26, /*ascq*/ 0x09, SSD_ELEM_NONE);
1774                         goto done;
1775                 }
1776                 list->seg[nseg] = seg;
1777                 off += sizeof(struct scsi_ec_segment) +
1778                     scsi_2btoul(seg->descr_length);
1779         }
1780         list->inl = &data->data[lencscd + lenseg];
1781         list->ncscd = lencscd / sizeof(struct scsi_ec_cscd);
1782         list->nseg = nseg;
1783         list->leninl = leninl;
1784         list->ctsio = ctsio;
1785         list->lun = lun;
1786         mtx_lock(&lun->lun_lock);
1787         if ((list->flags & EC_LIST_ID_USAGE_MASK) != EC_LIST_ID_USAGE_NONE) {
1788                 tlist = tpc_find_list(lun, list->list_id, list->init_idx);
1789                 if (tlist != NULL && !tlist->completed) {
1790                         mtx_unlock(&lun->lun_lock);
1791                         free(list, M_CTL);
1792                         ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
1793                             /*command*/ 0, /*field*/ 0, /*bit_valid*/ 0,
1794                             /*bit*/ 0);
1795                         goto done;
1796                 }
1797                 if (tlist != NULL) {
1798                         TAILQ_REMOVE(&lun->tpc_lists, tlist, links);
1799                         free(tlist, M_CTL);
1800                 }
1801         }
1802         TAILQ_INSERT_TAIL(&lun->tpc_lists, list, links);
1803         mtx_unlock(&lun->lun_lock);
1804
1805         tpc_process(list);
1806         return (CTL_RETVAL_COMPLETE);
1807
1808 done:
1809         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
1810                 free(ctsio->kern_data_ptr, M_CTL);
1811                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
1812         }
1813         ctl_done((union ctl_io *)ctsio);
1814         return (CTL_RETVAL_COMPLETE);
1815 }
1816
1817 int
1818 ctl_extended_copy_lid4(struct ctl_scsiio *ctsio)
1819 {
1820         struct ctl_lun *lun = CTL_LUN(ctsio);
1821         struct scsi_extended_copy *cdb;
1822         struct scsi_extended_copy_lid4_data *data;
1823         struct scsi_ec_cscd *cscd;
1824         struct scsi_ec_segment *seg;
1825         struct tpc_list *list, *tlist;
1826         uint8_t *ptr;
1827         const char *value;
1828         int len, off, lencscd, lenseg, leninl, nseg;
1829
1830         CTL_DEBUG_PRINT(("ctl_extended_copy_lid4\n"));
1831
1832         cdb = (struct scsi_extended_copy *)ctsio->cdb;
1833         len = scsi_4btoul(cdb->length);
1834
1835         if (len == 0) {
1836                 ctl_set_success(ctsio);
1837                 goto done;
1838         }
1839         if (len < sizeof(struct scsi_extended_copy_lid4_data) ||
1840             len > sizeof(struct scsi_extended_copy_lid4_data) +
1841             TPC_MAX_LIST + TPC_MAX_INLINE) {
1842                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 1,
1843                     /*field*/ 9, /*bit_valid*/ 0, /*bit*/ 0);
1844                 goto done;
1845         }
1846
1847         /*
1848          * If we've got a kernel request that hasn't been malloced yet,
1849          * malloc it and tell the caller the data buffer is here.
1850          */
1851         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
1852                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);
1853                 ctsio->kern_data_len = len;
1854                 ctsio->kern_total_len = len;
1855                 ctsio->kern_rel_offset = 0;
1856                 ctsio->kern_sg_entries = 0;
1857                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
1858                 ctsio->be_move_done = ctl_config_move_done;
1859                 ctl_datamove((union ctl_io *)ctsio);
1860
1861                 return (CTL_RETVAL_COMPLETE);
1862         }
1863
1864         data = (struct scsi_extended_copy_lid4_data *)ctsio->kern_data_ptr;
1865         lencscd = scsi_2btoul(data->cscd_list_length);
1866         lenseg = scsi_2btoul(data->segment_list_length);
1867         leninl = scsi_2btoul(data->inline_data_length);
1868         if (lencscd > TPC_MAX_CSCDS * sizeof(struct scsi_ec_cscd)) {
1869                 ctl_set_sense(ctsio, /*current_error*/ 1,
1870                     /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1871                     /*asc*/ 0x26, /*ascq*/ 0x06, SSD_ELEM_NONE);
1872                 goto done;
1873         }
1874         if (lenseg > TPC_MAX_SEGS * sizeof(struct scsi_ec_segment)) {
1875                 ctl_set_sense(ctsio, /*current_error*/ 1,
1876                     /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1877                     /*asc*/ 0x26, /*ascq*/ 0x08, SSD_ELEM_NONE);
1878                 goto done;
1879         }
1880         if (lencscd + lenseg > TPC_MAX_LIST ||
1881             leninl > TPC_MAX_INLINE ||
1882             len < sizeof(struct scsi_extended_copy_lid1_data) +
1883              lencscd + lenseg + leninl) {
1884                 ctl_set_param_len_error(ctsio);
1885                 goto done;
1886         }
1887
1888         list = malloc(sizeof(struct tpc_list), M_CTL, M_WAITOK | M_ZERO);
1889         list->service_action = cdb->service_action;
1890         value = dnvlist_get_string(lun->be_lun->options, "insecure_tpc", NULL);
1891         if (value != NULL && strcmp(value, "on") == 0)
1892                 list->init_port = -1;
1893         else
1894                 list->init_port = ctsio->io_hdr.nexus.targ_port;
1895         list->init_idx = ctl_get_initindex(&ctsio->io_hdr.nexus);
1896         list->list_id = scsi_4btoul(data->list_identifier);
1897         list->flags = data->flags;
1898         list->params = ctsio->kern_data_ptr;
1899         list->cscd = (struct scsi_ec_cscd *)&data->data[0];
1900         ptr = &data->data[0];
1901         for (off = 0; off < lencscd; off += sizeof(struct scsi_ec_cscd)) {
1902                 cscd = (struct scsi_ec_cscd *)(ptr + off);
1903                 if (cscd->type_code != EC_CSCD_ID) {
1904                         free(list, M_CTL);
1905                         ctl_set_sense(ctsio, /*current_error*/ 1,
1906                             /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1907                             /*asc*/ 0x26, /*ascq*/ 0x07, SSD_ELEM_NONE);
1908                         goto done;
1909                 }
1910         }
1911         ptr = &data->data[lencscd];
1912         for (nseg = 0, off = 0; off < lenseg; nseg++) {
1913                 if (nseg >= TPC_MAX_SEGS) {
1914                         free(list, M_CTL);
1915                         ctl_set_sense(ctsio, /*current_error*/ 1,
1916                             /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1917                             /*asc*/ 0x26, /*ascq*/ 0x08, SSD_ELEM_NONE);
1918                         goto done;
1919                 }
1920                 seg = (struct scsi_ec_segment *)(ptr + off);
1921                 if (seg->type_code != EC_SEG_B2B &&
1922                     seg->type_code != EC_SEG_VERIFY &&
1923                     seg->type_code != EC_SEG_REGISTER_KEY) {
1924                         free(list, M_CTL);
1925                         ctl_set_sense(ctsio, /*current_error*/ 1,
1926                             /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1927                             /*asc*/ 0x26, /*ascq*/ 0x09, SSD_ELEM_NONE);
1928                         goto done;
1929                 }
1930                 list->seg[nseg] = seg;
1931                 off += sizeof(struct scsi_ec_segment) +
1932                     scsi_2btoul(seg->descr_length);
1933         }
1934         list->inl = &data->data[lencscd + lenseg];
1935         list->ncscd = lencscd / sizeof(struct scsi_ec_cscd);
1936         list->nseg = nseg;
1937         list->leninl = leninl;
1938         list->ctsio = ctsio;
1939         list->lun = lun;
1940         mtx_lock(&lun->lun_lock);
1941         if ((list->flags & EC_LIST_ID_USAGE_MASK) != EC_LIST_ID_USAGE_NONE) {
1942                 tlist = tpc_find_list(lun, list->list_id, list->init_idx);
1943                 if (tlist != NULL && !tlist->completed) {
1944                         mtx_unlock(&lun->lun_lock);
1945                         free(list, M_CTL);
1946                         ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
1947                             /*command*/ 0, /*field*/ 0, /*bit_valid*/ 0,
1948                             /*bit*/ 0);
1949                         goto done;
1950                 }
1951                 if (tlist != NULL) {
1952                         TAILQ_REMOVE(&lun->tpc_lists, tlist, links);
1953                         free(tlist, M_CTL);
1954                 }
1955         }
1956         TAILQ_INSERT_TAIL(&lun->tpc_lists, list, links);
1957         mtx_unlock(&lun->lun_lock);
1958
1959         tpc_process(list);
1960         return (CTL_RETVAL_COMPLETE);
1961
1962 done:
1963         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
1964                 free(ctsio->kern_data_ptr, M_CTL);
1965                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
1966         }
1967         ctl_done((union ctl_io *)ctsio);
1968         return (CTL_RETVAL_COMPLETE);
1969 }
1970
1971 static void
1972 tpc_create_token(struct ctl_lun *lun, struct ctl_port *port, off_t len,
1973     struct scsi_token *token)
1974 {
1975         static int id = 0;
1976         struct scsi_vpd_id_descriptor *idd = NULL;
1977         struct scsi_ec_cscd_id *cscd;
1978         struct scsi_read_capacity_data_long *dtsd;
1979         int targid_len;
1980
1981         scsi_ulto4b(ROD_TYPE_AUR, token->type);
1982         scsi_ulto2b(0x01f8, token->length);
1983         scsi_u64to8b(atomic_fetchadd_int(&id, 1), &token->body[0]);
1984         if (lun->lun_devid)
1985                 idd = scsi_get_devid_desc((struct scsi_vpd_id_descriptor *)
1986                     lun->lun_devid->data, lun->lun_devid->len,
1987                     scsi_devid_is_lun_naa);
1988         if (idd == NULL && lun->lun_devid)
1989                 idd = scsi_get_devid_desc((struct scsi_vpd_id_descriptor *)
1990                     lun->lun_devid->data, lun->lun_devid->len,
1991                     scsi_devid_is_lun_eui64);
1992         if (idd != NULL) {
1993                 cscd = (struct scsi_ec_cscd_id *)&token->body[8];
1994                 cscd->type_code = EC_CSCD_ID;
1995                 cscd->luidt_pdt = T_DIRECT;
1996                 memcpy(&cscd->codeset, idd, 4 + idd->length);
1997                 scsi_ulto3b(lun->be_lun->blocksize, cscd->dtsp.block_length);
1998         }
1999         scsi_u64to8b(0, &token->body[40]); /* XXX: Should be 128bit value. */
2000         scsi_u64to8b(len, &token->body[48]);
2001
2002         /* ROD token device type specific data (RC16 without first field) */
2003         dtsd = (struct scsi_read_capacity_data_long *)&token->body[88 - 8];
2004         scsi_ulto4b(lun->be_lun->blocksize, dtsd->length);
2005         dtsd->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
2006         scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, dtsd->lalba_lbp);
2007         if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
2008                 dtsd->lalba_lbp[0] |= SRC16_LBPME | SRC16_LBPRZ;
2009
2010         if (port->target_devid) {
2011                 targid_len = port->target_devid->len;
2012                 memcpy(&token->body[120], port->target_devid->data, targid_len);
2013         } else
2014                 targid_len = 32;
2015         arc4rand(&token->body[120 + targid_len], 384 - targid_len, 0);
2016 };
2017
2018 int
2019 ctl_populate_token(struct ctl_scsiio *ctsio)
2020 {
2021         struct ctl_softc *softc = CTL_SOFTC(ctsio);
2022         struct ctl_port *port = CTL_PORT(ctsio);
2023         struct ctl_lun *lun = CTL_LUN(ctsio);
2024         struct scsi_populate_token *cdb;
2025         struct scsi_populate_token_data *data;
2026         struct tpc_list *list, *tlist;
2027         struct tpc_token *token;
2028         uint64_t lba;
2029         int len, lendata, lendesc;
2030
2031         CTL_DEBUG_PRINT(("ctl_populate_token\n"));
2032
2033         cdb = (struct scsi_populate_token *)ctsio->cdb;
2034         len = scsi_4btoul(cdb->length);
2035
2036         if (len < sizeof(struct scsi_populate_token_data) ||
2037             len > sizeof(struct scsi_populate_token_data) +
2038              TPC_MAX_SEGS * sizeof(struct scsi_range_desc)) {
2039                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 1,
2040                     /*field*/ 9, /*bit_valid*/ 0, /*bit*/ 0);
2041                 goto done;
2042         }
2043
2044         /*
2045          * If we've got a kernel request that hasn't been malloced yet,
2046          * malloc it and tell the caller the data buffer is here.
2047          */
2048         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
2049                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);
2050                 ctsio->kern_data_len = len;
2051                 ctsio->kern_total_len = len;
2052                 ctsio->kern_rel_offset = 0;
2053                 ctsio->kern_sg_entries = 0;
2054                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
2055                 ctsio->be_move_done = ctl_config_move_done;
2056                 ctl_datamove((union ctl_io *)ctsio);
2057
2058                 return (CTL_RETVAL_COMPLETE);
2059         }
2060
2061         data = (struct scsi_populate_token_data *)ctsio->kern_data_ptr;
2062         lendata = scsi_2btoul(data->length);
2063         if (lendata < sizeof(struct scsi_populate_token_data) - 2 +
2064             sizeof(struct scsi_range_desc)) {
2065                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 0,
2066                     /*field*/ 0, /*bit_valid*/ 0, /*bit*/ 0);
2067                 goto done;
2068         }
2069         lendesc = scsi_2btoul(data->range_descriptor_length);
2070         if (lendesc < sizeof(struct scsi_range_desc) ||
2071             len < sizeof(struct scsi_populate_token_data) + lendesc ||
2072             lendata < sizeof(struct scsi_populate_token_data) - 2 + lendesc) {
2073                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 0,
2074                     /*field*/ 14, /*bit_valid*/ 0, /*bit*/ 0);
2075                 goto done;
2076         }
2077 /*
2078         printf("PT(list=%u) flags=%x to=%d rt=%x len=%x\n",
2079             scsi_4btoul(cdb->list_identifier),
2080             data->flags, scsi_4btoul(data->inactivity_timeout),
2081             scsi_4btoul(data->rod_type),
2082             scsi_2btoul(data->range_descriptor_length));
2083 */
2084
2085         /* Validate INACTIVITY TIMEOUT field */
2086         if (scsi_4btoul(data->inactivity_timeout) > TPC_MAX_TOKEN_TIMEOUT) {
2087                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
2088                     /*command*/ 0, /*field*/ 4, /*bit_valid*/ 0,
2089                     /*bit*/ 0);
2090                 goto done;
2091         }
2092
2093         /* Validate ROD TYPE field */
2094         if ((data->flags & EC_PT_RTV) &&
2095             scsi_4btoul(data->rod_type) != ROD_TYPE_AUR) {
2096                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 0,
2097                     /*field*/ 8, /*bit_valid*/ 0, /*bit*/ 0);
2098                 goto done;
2099         }
2100
2101         /* Validate list of ranges */
2102         if (tpc_check_ranges_l(&data->desc[0],
2103             scsi_2btoul(data->range_descriptor_length) /
2104             sizeof(struct scsi_range_desc),
2105             lun->be_lun->maxlba, &lba) != 0) {
2106                 ctl_set_lba_out_of_range(ctsio, lba);
2107                 goto done;
2108         }
2109         if (tpc_check_ranges_x(&data->desc[0],
2110             scsi_2btoul(data->range_descriptor_length) /
2111             sizeof(struct scsi_range_desc)) != 0) {
2112                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 0,
2113                     /*command*/ 0, /*field*/ 0, /*bit_valid*/ 0,
2114                     /*bit*/ 0);
2115                 goto done;
2116         }
2117
2118         list = malloc(sizeof(struct tpc_list), M_CTL, M_WAITOK | M_ZERO);
2119         list->service_action = cdb->service_action;
2120         list->init_port = ctsio->io_hdr.nexus.targ_port;
2121         list->init_idx = ctl_get_initindex(&ctsio->io_hdr.nexus);
2122         list->list_id = scsi_4btoul(cdb->list_identifier);
2123         list->flags = data->flags;
2124         list->ctsio = ctsio;
2125         list->lun = lun;
2126         mtx_lock(&lun->lun_lock);
2127         tlist = tpc_find_list(lun, list->list_id, list->init_idx);
2128         if (tlist != NULL && !tlist->completed) {
2129                 mtx_unlock(&lun->lun_lock);
2130                 free(list, M_CTL);
2131                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
2132                     /*command*/ 0, /*field*/ 0, /*bit_valid*/ 0,
2133                     /*bit*/ 0);
2134                 goto done;
2135         }
2136         if (tlist != NULL) {
2137                 TAILQ_REMOVE(&lun->tpc_lists, tlist, links);
2138                 free(tlist, M_CTL);
2139         }
2140         TAILQ_INSERT_TAIL(&lun->tpc_lists, list, links);
2141         mtx_unlock(&lun->lun_lock);
2142
2143         token = malloc(sizeof(*token), M_CTL, M_WAITOK | M_ZERO);
2144         token->lun = lun->lun;
2145         token->blocksize = lun->be_lun->blocksize;
2146         token->params = ctsio->kern_data_ptr;
2147         token->range = &data->desc[0];
2148         token->nrange = scsi_2btoul(data->range_descriptor_length) /
2149             sizeof(struct scsi_range_desc);
2150         list->cursectors = tpc_ranges_length(token->range, token->nrange);
2151         list->curbytes = (off_t)list->cursectors * lun->be_lun->blocksize;
2152         tpc_create_token(lun, port, list->curbytes,
2153             (struct scsi_token *)token->token);
2154         token->active = 0;
2155         token->last_active = time_uptime;
2156         token->timeout = scsi_4btoul(data->inactivity_timeout);
2157         if (token->timeout == 0)
2158                 token->timeout = TPC_DFL_TOKEN_TIMEOUT;
2159         else if (token->timeout < TPC_MIN_TOKEN_TIMEOUT)
2160                 token->timeout = TPC_MIN_TOKEN_TIMEOUT;
2161         memcpy(list->res_token, token->token, sizeof(list->res_token));
2162         list->res_token_valid = 1;
2163         list->curseg = 0;
2164         list->completed = 1;
2165         list->last_active = time_uptime;
2166         mtx_lock(&softc->tpc_lock);
2167         TAILQ_INSERT_TAIL(&softc->tpc_tokens, token, links);
2168         mtx_unlock(&softc->tpc_lock);
2169         ctl_set_success(ctsio);
2170         ctl_done((union ctl_io *)ctsio);
2171         return (CTL_RETVAL_COMPLETE);
2172
2173 done:
2174         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
2175                 free(ctsio->kern_data_ptr, M_CTL);
2176                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
2177         }
2178         ctl_done((union ctl_io *)ctsio);
2179         return (CTL_RETVAL_COMPLETE);
2180 }
2181
2182 int
2183 ctl_write_using_token(struct ctl_scsiio *ctsio)
2184 {
2185         struct ctl_softc *softc = CTL_SOFTC(ctsio);
2186         struct ctl_lun *lun = CTL_LUN(ctsio);
2187         struct scsi_write_using_token *cdb;
2188         struct scsi_write_using_token_data *data;
2189         struct tpc_list *list, *tlist;
2190         struct tpc_token *token;
2191         uint64_t lba;
2192         int len, lendata, lendesc;
2193
2194         CTL_DEBUG_PRINT(("ctl_write_using_token\n"));
2195
2196         cdb = (struct scsi_write_using_token *)ctsio->cdb;
2197         len = scsi_4btoul(cdb->length);
2198
2199         if (len < sizeof(struct scsi_write_using_token_data) ||
2200             len > sizeof(struct scsi_write_using_token_data) +
2201              TPC_MAX_SEGS * sizeof(struct scsi_range_desc)) {
2202                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 1,
2203                     /*field*/ 9, /*bit_valid*/ 0, /*bit*/ 0);
2204                 goto done;
2205         }
2206
2207         /*
2208          * If we've got a kernel request that hasn't been malloced yet,
2209          * malloc it and tell the caller the data buffer is here.
2210          */
2211         if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
2212                 ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);
2213                 ctsio->kern_data_len = len;
2214                 ctsio->kern_total_len = len;
2215                 ctsio->kern_rel_offset = 0;
2216                 ctsio->kern_sg_entries = 0;
2217                 ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
2218                 ctsio->be_move_done = ctl_config_move_done;
2219                 ctl_datamove((union ctl_io *)ctsio);
2220
2221                 return (CTL_RETVAL_COMPLETE);
2222         }
2223
2224         data = (struct scsi_write_using_token_data *)ctsio->kern_data_ptr;
2225         lendata = scsi_2btoul(data->length);
2226         if (lendata < sizeof(struct scsi_write_using_token_data) - 2 +
2227             sizeof(struct scsi_range_desc)) {
2228                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 0,
2229                     /*field*/ 0, /*bit_valid*/ 0, /*bit*/ 0);
2230                 goto done;
2231         }
2232         lendesc = scsi_2btoul(data->range_descriptor_length);
2233         if (lendesc < sizeof(struct scsi_range_desc) ||
2234             len < sizeof(struct scsi_write_using_token_data) + lendesc ||
2235             lendata < sizeof(struct scsi_write_using_token_data) - 2 + lendesc) {
2236                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 0,
2237                     /*field*/ 534, /*bit_valid*/ 0, /*bit*/ 0);
2238                 goto done;
2239         }
2240 /*
2241         printf("WUT(list=%u) flags=%x off=%ju len=%x\n",
2242             scsi_4btoul(cdb->list_identifier),
2243             data->flags, scsi_8btou64(data->offset_into_rod),
2244             scsi_2btoul(data->range_descriptor_length));
2245 */
2246
2247         /* Validate list of ranges */
2248         if (tpc_check_ranges_l(&data->desc[0],
2249             scsi_2btoul(data->range_descriptor_length) /
2250             sizeof(struct scsi_range_desc),
2251             lun->be_lun->maxlba, &lba) != 0) {
2252                 ctl_set_lba_out_of_range(ctsio, lba);
2253                 goto done;
2254         }
2255         if (tpc_check_ranges_x(&data->desc[0],
2256             scsi_2btoul(data->range_descriptor_length) /
2257             sizeof(struct scsi_range_desc)) != 0) {
2258                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 0,
2259                     /*command*/ 0, /*field*/ 0, /*bit_valid*/ 0,
2260                     /*bit*/ 0);
2261                 goto done;
2262         }
2263
2264         list = malloc(sizeof(struct tpc_list), M_CTL, M_WAITOK | M_ZERO);
2265         list->service_action = cdb->service_action;
2266         list->init_port = ctsio->io_hdr.nexus.targ_port;
2267         list->init_idx = ctl_get_initindex(&ctsio->io_hdr.nexus);
2268         list->list_id = scsi_4btoul(cdb->list_identifier);
2269         list->flags = data->flags;
2270         list->params = ctsio->kern_data_ptr;
2271         list->range = &data->desc[0];
2272         list->nrange = scsi_2btoul(data->range_descriptor_length) /
2273             sizeof(struct scsi_range_desc);
2274         list->offset_into_rod = scsi_8btou64(data->offset_into_rod);
2275         list->ctsio = ctsio;
2276         list->lun = lun;
2277         mtx_lock(&lun->lun_lock);
2278         tlist = tpc_find_list(lun, list->list_id, list->init_idx);
2279         if (tlist != NULL && !tlist->completed) {
2280                 mtx_unlock(&lun->lun_lock);
2281                 free(list, M_CTL);
2282                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
2283                     /*command*/ 0, /*field*/ 0, /*bit_valid*/ 0,
2284                     /*bit*/ 0);
2285                 goto done;
2286         }
2287         if (tlist != NULL) {
2288                 TAILQ_REMOVE(&lun->tpc_lists, tlist, links);
2289                 free(tlist, M_CTL);
2290         }
2291         TAILQ_INSERT_TAIL(&lun->tpc_lists, list, links);
2292         mtx_unlock(&lun->lun_lock);
2293
2294         /* Block device zero ROD token -> no token. */
2295         if (scsi_4btoul(data->rod_token) == ROD_TYPE_BLOCK_ZERO) {
2296                 tpc_process(list);
2297                 return (CTL_RETVAL_COMPLETE);
2298         }
2299
2300         mtx_lock(&softc->tpc_lock);
2301         TAILQ_FOREACH(token, &softc->tpc_tokens, links) {
2302                 if (memcmp(token->token, data->rod_token,
2303                     sizeof(data->rod_token)) == 0)
2304                         break;
2305         }
2306         if (token != NULL) {
2307                 token->active++;
2308                 list->token = token;
2309                 if (data->flags & EC_WUT_DEL_TKN)
2310                         token->timeout = 0;
2311         }
2312         mtx_unlock(&softc->tpc_lock);
2313         if (token == NULL) {
2314                 mtx_lock(&lun->lun_lock);
2315                 TAILQ_REMOVE(&lun->tpc_lists, list, links);
2316                 mtx_unlock(&lun->lun_lock);
2317                 free(list, M_CTL);
2318                 ctl_set_sense(ctsio, /*current_error*/ 1,
2319                     /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
2320                     /*asc*/ 0x23, /*ascq*/ 0x04, SSD_ELEM_NONE);
2321                 goto done;
2322         }
2323
2324         tpc_process(list);
2325         return (CTL_RETVAL_COMPLETE);
2326
2327 done:
2328         if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
2329                 free(ctsio->kern_data_ptr, M_CTL);
2330                 ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
2331         }
2332         ctl_done((union ctl_io *)ctsio);
2333         return (CTL_RETVAL_COMPLETE);
2334 }
2335
2336 int
2337 ctl_receive_rod_token_information(struct ctl_scsiio *ctsio)
2338 {
2339         struct ctl_lun *lun = CTL_LUN(ctsio);
2340         struct scsi_receive_rod_token_information *cdb;
2341         struct scsi_receive_copy_status_lid4_data *data;
2342         struct tpc_list *list;
2343         struct tpc_list list_copy;
2344         uint8_t *ptr;
2345         int retval;
2346         int alloc_len, total_len, token_len;
2347         uint32_t list_id;
2348
2349         CTL_DEBUG_PRINT(("ctl_receive_rod_token_information\n"));
2350
2351         cdb = (struct scsi_receive_rod_token_information *)ctsio->cdb;
2352         retval = CTL_RETVAL_COMPLETE;
2353
2354         list_id = scsi_4btoul(cdb->list_identifier);
2355         mtx_lock(&lun->lun_lock);
2356         list = tpc_find_list(lun, list_id,
2357             ctl_get_initindex(&ctsio->io_hdr.nexus));
2358         if (list == NULL) {
2359                 mtx_unlock(&lun->lun_lock);
2360                 ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
2361                     /*command*/ 1, /*field*/ 2, /*bit_valid*/ 0,
2362                     /*bit*/ 0);
2363                 ctl_done((union ctl_io *)ctsio);
2364                 return (retval);
2365         }
2366         list_copy = *list;
2367         if (list->completed) {
2368                 TAILQ_REMOVE(&lun->tpc_lists, list, links);
2369                 free(list, M_CTL);
2370         }
2371         mtx_unlock(&lun->lun_lock);
2372
2373         token_len = list_copy.res_token_valid ? 2 + sizeof(list_copy.res_token) : 0;
2374         total_len = sizeof(*data) + list_copy.sense_len + 4 + token_len;
2375         alloc_len = scsi_4btoul(cdb->length);
2376
2377         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
2378         ctsio->kern_sg_entries = 0;
2379         ctsio->kern_rel_offset = 0;
2380         ctsio->kern_data_len = min(total_len, alloc_len);
2381         ctsio->kern_total_len = ctsio->kern_data_len;
2382
2383         data = (struct scsi_receive_copy_status_lid4_data *)ctsio->kern_data_ptr;
2384         scsi_ulto4b(sizeof(*data) - 4 + list_copy.sense_len +
2385             4 + token_len, data->available_data);
2386         data->response_to_service_action = list_copy.service_action;
2387         if (list_copy.completed) {
2388                 if (list_copy.error)
2389                         data->copy_command_status = RCS_CCS_ERROR;
2390                 else if (list_copy.abort)
2391                         data->copy_command_status = RCS_CCS_ABORTED;
2392                 else
2393                         data->copy_command_status = RCS_CCS_COMPLETED;
2394         } else
2395                 data->copy_command_status = RCS_CCS_INPROG_FG;
2396         scsi_ulto2b(list_copy.curops, data->operation_counter);
2397         scsi_ulto4b(UINT32_MAX, data->estimated_status_update_delay);
2398         data->transfer_count_units = RCS_TC_LBAS;
2399         scsi_u64to8b(list_copy.cursectors, data->transfer_count);
2400         scsi_ulto2b(list_copy.curseg, data->segments_processed);
2401         data->length_of_the_sense_data_field = list_copy.sense_len;
2402         data->sense_data_length = list_copy.sense_len;
2403         memcpy(data->sense_data, &list_copy.sense_data, list_copy.sense_len);
2404
2405         ptr = &data->sense_data[data->length_of_the_sense_data_field];
2406         scsi_ulto4b(token_len, &ptr[0]);
2407         if (list_copy.res_token_valid) {
2408                 scsi_ulto2b(0, &ptr[4]);
2409                 memcpy(&ptr[6], list_copy.res_token, sizeof(list_copy.res_token));
2410         }
2411 /*
2412         printf("RRTI(list=%u) valid=%d\n",
2413             scsi_4btoul(cdb->list_identifier), list_copy.res_token_valid);
2414 */
2415         ctl_set_success(ctsio);
2416         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
2417         ctsio->be_move_done = ctl_config_move_done;
2418         ctl_datamove((union ctl_io *)ctsio);
2419         return (retval);
2420 }
2421
2422 int
2423 ctl_report_all_rod_tokens(struct ctl_scsiio *ctsio)
2424 {
2425         struct ctl_softc *softc = CTL_SOFTC(ctsio);
2426         struct scsi_report_all_rod_tokens *cdb;
2427         struct scsi_report_all_rod_tokens_data *data;
2428         struct tpc_token *token;
2429         int retval;
2430         int alloc_len, total_len, tokens, i;
2431
2432         CTL_DEBUG_PRINT(("ctl_receive_rod_token_information\n"));
2433
2434         cdb = (struct scsi_report_all_rod_tokens *)ctsio->cdb;
2435         retval = CTL_RETVAL_COMPLETE;
2436
2437         tokens = 0;
2438         mtx_lock(&softc->tpc_lock);
2439         TAILQ_FOREACH(token, &softc->tpc_tokens, links)
2440                 tokens++;
2441         mtx_unlock(&softc->tpc_lock);
2442         if (tokens > 512)
2443                 tokens = 512;
2444
2445         total_len = sizeof(*data) + tokens * 96;
2446         alloc_len = scsi_4btoul(cdb->length);
2447
2448         ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
2449         ctsio->kern_sg_entries = 0;
2450         ctsio->kern_rel_offset = 0;
2451         ctsio->kern_data_len = min(total_len, alloc_len);
2452         ctsio->kern_total_len = ctsio->kern_data_len;
2453
2454         data = (struct scsi_report_all_rod_tokens_data *)ctsio->kern_data_ptr;
2455         i = 0;
2456         mtx_lock(&softc->tpc_lock);
2457         TAILQ_FOREACH(token, &softc->tpc_tokens, links) {
2458                 if (i >= tokens)
2459                         break;
2460                 memcpy(&data->rod_management_token_list[i * 96],
2461                     token->token, 96);
2462                 i++;
2463         }
2464         mtx_unlock(&softc->tpc_lock);
2465         scsi_ulto4b(sizeof(*data) - 4 + i * 96, data->available_data);
2466 /*
2467         printf("RART tokens=%d\n", i);
2468 */
2469         ctl_set_success(ctsio);
2470         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
2471         ctsio->be_move_done = ctl_config_move_done;
2472         ctl_datamove((union ctl_io *)ctsio);
2473         return (retval);
2474 }
2475