sys/powerpc/pseries/phyp_vscsi.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright 2013 Nathan Whitehorn
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/kernel.h>
  35 #include <sys/malloc.h>
  36 #include <sys/module.h>
  37 #include <sys/selinfo.h>
  38 #include <sys/bus.h>
  39 #include <sys/conf.h>
  40 #include <sys/eventhandler.h>
  41 #include <sys/rman.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/bio.h>
  44 #include <sys/ioccom.h>
  45 #include <sys/uio.h>
  46 #include <sys/proc.h>
  47 #include <sys/signalvar.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/endian.h>
  50 #include <sys/vmem.h>
  51
  52 #include <cam/cam.h>
  53 #include <cam/cam_ccb.h>
  54 #include <cam/cam_debug.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/cam_sim.h>
  57 #include <cam/cam_xpt_periph.h>
  58 #include <cam/cam_xpt_sim.h>
  59 #include <cam/scsi/scsi_all.h>
  60 #include <cam/scsi/scsi_message.h>
  61
  62 #include <dev/ofw/openfirm.h>
  63 #include <dev/ofw/ofw_bus.h>
  64 #include <dev/ofw/ofw_bus_subr.h>
  65
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68
  69 #include <powerpc/pseries/phyp-hvcall.h>
  70
  71 struct vscsi_softc;
  72
  73 /* VSCSI CRQ format from table 260 of PAPR spec 2.4 (page 760) */
  74 struct vscsi_crq {
  75         uint8_t valid;
  76         uint8_t format;
  77         uint8_t reserved;
  78         uint8_t status;
  79         uint16_t timeout;
  80         uint16_t iu_length;
  81         uint64_t iu_data;
  82 };
  83
  84 struct vscsi_xfer {
  85         TAILQ_ENTRY(vscsi_xfer) queue;
  86         struct vscsi_softc *sc;
  87         union ccb *ccb;
  88         bus_dmamap_t dmamap;
  89         uint64_t tag;
  90
  91         vmem_addr_t srp_iu_offset;
  92         vmem_size_t srp_iu_size;
  93 };
  94
  95 TAILQ_HEAD(vscsi_xferq, vscsi_xfer);
  96
  97 struct vscsi_softc {
  98         device_t        dev;
  99         struct cam_devq *devq;
 100         struct cam_sim  *sim;
 101         struct cam_path *path;
 102         struct mtx io_lock;
 103
 104         cell_t          unit;
 105         int             bus_initialized;
 106         int             bus_logged_in;
 107         int             max_transactions;
 108
 109         int             irqid;
 110         struct resource *irq;
 111         void            *irq_cookie;
 112
 113         bus_dma_tag_t   crq_tag;
 114         struct vscsi_crq *crq_queue;
 115         int             n_crqs, cur_crq;
 116         bus_dmamap_t    crq_map;
 117         bus_addr_t      crq_phys;
 118
 119         vmem_t          *srp_iu_arena;
 120         void            *srp_iu_queue;
 121         bus_addr_t      srp_iu_phys;
 122
 123         bus_dma_tag_t   data_tag;
 124
 125         struct vscsi_xfer loginxp;
 126         struct vscsi_xfer *xfer;
 127         struct vscsi_xferq active_xferq;
 128         struct vscsi_xferq free_xferq;
 129 };
 130
 131 struct srp_login {
 132         uint8_t type;
 133         uint8_t reserved[7];
 134         uint64_t tag;
 135         uint64_t max_cmd_length;
 136         uint32_t reserved2;
 137         uint16_t buffer_formats;
 138         uint8_t flags;
 139         uint8_t reserved3[5];
 140         uint8_t initiator_port_id[16];
 141         uint8_t target_port_id[16];
 142 } __packed;
 143
 144 struct srp_login_rsp {
 145         uint8_t type;
 146         uint8_t reserved[3];
 147         uint32_t request_limit_delta;
 148         uint8_t tag;
 149         uint32_t max_i_to_t_len;
 150         uint32_t max_t_to_i_len;
 151         uint16_t buffer_formats;
 152         uint8_t flags;
 153         /* Some reserved bits follow */
 154 } __packed;
 155
 156 struct srp_cmd {
 157         uint8_t type;
 158         uint8_t flags1;
 159         uint8_t reserved[3];
 160         uint8_t formats;
 161         uint8_t out_buffer_count;
 162         uint8_t in_buffer_count;
 163         uint64_t tag;
 164         uint32_t reserved2;
 165         uint64_t lun;
 166         uint8_t reserved3[3];
 167         uint8_t additional_cdb;
 168         uint8_t cdb[16];
 169         uint8_t data_payload[0];
 170 } __packed;
 171
 172 struct srp_rsp {
 173         uint8_t type;
 174         uint8_t reserved[3];
 175         uint32_t request_limit_delta;
 176         uint64_t tag;
 177         uint16_t reserved2;
 178         uint8_t flags;
 179         uint8_t status;
 180         uint32_t data_out_resid;
 181         uint32_t data_in_resid;
 182         uint32_t sense_data_len;
 183         uint32_t response_data_len;
 184         uint8_t data_payload[0];
 185 } __packed;
 186
 187 struct srp_tsk_mgmt {
 188         uint8_t type;
 189         uint8_t reserved[7];
 190         uint64_t tag;
 191         uint32_t reserved2;
 192         uint64_t lun;
 193         uint8_t reserved3[2];
 194         uint8_t function;
 195         uint8_t reserved4;
 196         uint64_t manage_tag;
 197         uint64_t reserved5;
 198 } __packed;
 199
 200 /* Message code type */
 201 #define SRP_LOGIN_REQ   0x00
 202 #define SRP_TSK_MGMT    0x01
 203 #define SRP_CMD         0x02
 204 #define SRP_I_LOGOUT    0x03
 205
 206 #define SRP_LOGIN_RSP   0xC0
 207 #define SRP_RSP         0xC1
 208 #define SRP_LOGIN_REJ   0xC2
 209
 210 #define SRP_T_LOGOUT    0x80
 211 #define SRP_CRED_REQ    0x81
 212 #define SRP_AER_REQ     0x82
 213
 214 #define SRP_CRED_RSP    0x41
 215 #define SRP_AER_RSP     0x41
 216
 217 /* Flags for srp_rsp flags field */
 218 #define SRP_RSPVALID    0x01
 219 #define SRP_SNSVALID    0x02
 220 #define SRP_DOOVER      0x04
 221 #define SRP_DOUNDER     0x08
 222 #define SRP_DIOVER      0x10
 223 #define SRP_DIUNDER     0x20
 224
 225 #define MAD_SUCESS                      0x00
 226 #define MAD_NOT_SUPPORTED               0xf1
 227 #define MAD_FAILED                      0xf7
 228
 229 #define MAD_EMPTY_IU                    0x01
 230 #define MAD_ERROR_LOGGING_REQUEST       0x02
 231 #define MAD_ADAPTER_INFO_REQUEST        0x03
 232 #define MAD_CAPABILITIES_EXCHANGE       0x05
 233 #define MAD_PHYS_ADAP_INFO_REQUEST      0x06
 234 #define MAD_TAPE_PASSTHROUGH_REQUEST    0x07
 235 #define MAD_ENABLE_FAST_FAIL            0x08
 236
 237 static int      vscsi_probe(device_t);
 238 static int      vscsi_attach(device_t);
 239 static int      vscsi_detach(device_t);
 240 static void     vscsi_cam_action(struct cam_sim *, union ccb *);
 241 static void     vscsi_cam_poll(struct cam_sim *);
 242 static void     vscsi_intr(void *arg);
 243 static void     vscsi_check_response_queue(struct vscsi_softc *sc);
 244 static void     vscsi_setup_bus(struct vscsi_softc *sc);
 245
 246 static void     vscsi_srp_login(struct vscsi_softc *sc);
 247 static void     vscsi_crq_load_cb(void *, bus_dma_segment_t *, int, int);
 248 static void     vscsi_scsi_command(void *xxp, bus_dma_segment_t *segs,
 249                     int nsegs, int err);
 250 static void     vscsi_task_management(struct vscsi_softc *sc, union ccb *ccb);
 251 static void     vscsi_srp_response(struct vscsi_xfer *, struct vscsi_crq *);
 252
 253 static devclass_t       vscsi_devclass;
 254 static device_method_t  vscsi_methods[] = {
 255         DEVMETHOD(device_probe,         vscsi_probe),
 256         DEVMETHOD(device_attach,        vscsi_attach),
 257         DEVMETHOD(device_detach,        vscsi_detach),
 258
 259         DEVMETHOD_END
 260 };
 261 static driver_t vscsi_driver = {
 262         "vscsi",
 263         vscsi_methods,
 264         sizeof(struct vscsi_softc)
 265 };
 266 DRIVER_MODULE(vscsi, vdevice, vscsi_driver, vscsi_devclass, 0, 0);
 267 MALLOC_DEFINE(M_VSCSI, "vscsi", "CAM device queue for VSCSI");
 268
 269 static int
 270 vscsi_probe(device_t dev)
 271 {
 272
 273         if (!ofw_bus_is_compatible(dev, "IBM,v-scsi"))
 274                 return (ENXIO);
 275
 276         device_set_desc(dev, "POWER Hypervisor Virtual SCSI Bus");
 277         return (0);
 278 }
 279
 280 static int
 281 vscsi_attach(device_t dev)
 282 {
 283         struct vscsi_softc *sc;
 284         struct vscsi_xfer *xp;
 285         int error, i;
 286
 287         sc = device_get_softc(dev);
 288         if (sc == NULL)
 289                 return (EINVAL);
 290
 291         sc->dev = dev;
 292         mtx_init(&sc->io_lock, "vscsi", NULL, MTX_DEF);
 293
 294         /* Get properties */
 295         OF_getencprop(ofw_bus_get_node(dev), "reg", &sc->unit,
 296             sizeof(sc->unit));
 297
 298         /* Setup interrupt */
 299         sc->irqid = 0;
 300         sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irqid,
 301             RF_ACTIVE);
 302
 303         if (!sc->irq) {
 304                 device_printf(dev, "Could not allocate IRQ\n");
 305                 mtx_destroy(&sc->io_lock);
 306                 return (ENXIO);
 307         }
 308
 309         bus_setup_intr(dev, sc->irq, INTR_TYPE_CAM | INTR_MPSAFE |
 310             INTR_ENTROPY, NULL, vscsi_intr, sc, &sc->irq_cookie);
 311
 312         /* Data DMA */
 313         error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
 314             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
 315             256, BUS_SPACE_MAXSIZE_32BIT, 0, busdma_lock_mutex, &sc->io_lock,
 316             &sc->data_tag);
 317
 318         TAILQ_INIT(&sc->active_xferq);
 319         TAILQ_INIT(&sc->free_xferq);
 320
 321         /* First XFER for login data */
 322         sc->loginxp.sc = sc;
 323         bus_dmamap_create(sc->data_tag, 0, &sc->loginxp.dmamap);
 324         TAILQ_INSERT_TAIL(&sc->free_xferq, &sc->loginxp, queue);
 325
 326         /* CRQ area */
 327         error = bus_dma_tag_create(bus_get_dma_tag(dev), PAGE_SIZE, 0,
 328             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 8*PAGE_SIZE,
 329             1, BUS_SPACE_MAXSIZE, 0, NULL, NULL, &sc->crq_tag);
 330         error = bus_dmamem_alloc(sc->crq_tag, (void **)&sc->crq_queue,
 331             BUS_DMA_WAITOK | BUS_DMA_ZERO, &sc->crq_map);
 332         sc->crq_phys = 0;
 333         sc->n_crqs = 0;
 334         error = bus_dmamap_load(sc->crq_tag, sc->crq_map, sc->crq_queue,
 335             8*PAGE_SIZE, vscsi_crq_load_cb, sc, 0);
 336
 337         mtx_lock(&sc->io_lock);
 338         vscsi_setup_bus(sc);
 339         sc->xfer = malloc(sizeof(sc->xfer[0])*sc->max_transactions, M_VSCSI,
 340             M_NOWAIT);
 341         for (i = 0; i < sc->max_transactions; i++) {
 342                 xp = &sc->xfer[i];
 343                 xp->sc = sc;
 344
 345                 error = bus_dmamap_create(sc->data_tag, 0, &xp->dmamap);
 346                 if (error) {
 347                         device_printf(dev, "Could not create DMA map (%d)\n",
 348                             error);
 349                         break;
 350                 }
 351
 352                 TAILQ_INSERT_TAIL(&sc->free_xferq, xp, queue);
 353         }
 354         mtx_unlock(&sc->io_lock);
 355
 356         /* Allocate CAM bits */
 357         if ((sc->devq = cam_simq_alloc(sc->max_transactions)) == NULL)
 358                 return (ENOMEM);
 359
 360         sc->sim = cam_sim_alloc(vscsi_cam_action, vscsi_cam_poll, "vscsi", sc,
 361                                 device_get_unit(dev), &sc->io_lock,
 362                                 sc->max_transactions, sc->max_transactions,
 363                                 sc->devq);
 364         if (sc->sim == NULL) {
 365                 cam_simq_free(sc->devq);
 366                 sc->devq = NULL;
 367                 device_printf(dev, "CAM SIM attach failed\n");
 368                 return (EINVAL);
 369         }
 370
 371         mtx_lock(&sc->io_lock);
 372         if (xpt_bus_register(sc->sim, dev, 0) != 0) {
 373                 device_printf(dev, "XPT bus registration failed\n");
 374                 cam_sim_free(sc->sim, FALSE);
 375                 sc->sim = NULL;
 376                 cam_simq_free(sc->devq);
 377                 sc->devq = NULL;
 378                 mtx_unlock(&sc->io_lock);
 379                 return (EINVAL);
 380         }
 381         mtx_unlock(&sc->io_lock);
 382
 383         return (0);
 384 }
 385
 386 static int
 387 vscsi_detach(device_t dev)
 388 {
 389         struct vscsi_softc *sc;
 390
 391         sc = device_get_softc(dev);
 392         if (sc == NULL)
 393                 return (EINVAL);
 394
 395         if (sc->sim != NULL) {
 396                 mtx_lock(&sc->io_lock);
 397                 xpt_bus_deregister(cam_sim_path(sc->sim));
 398                 cam_sim_free(sc->sim, FALSE);
 399                 sc->sim = NULL;
 400                 mtx_unlock(&sc->io_lock);
 401         }
 402
 403         if (sc->devq != NULL) {
 404                 cam_simq_free(sc->devq);
 405                 sc->devq = NULL;
 406         }
 407
 408         mtx_destroy(&sc->io_lock);
 409
 410         return (0);
 411 }
 412
 413 static void
 414 vscsi_cam_action(struct cam_sim *sim, union ccb *ccb)
 415 {
 416         struct vscsi_softc *sc = cam_sim_softc(sim);
 417
 418         mtx_assert(&sc->io_lock, MA_OWNED);
 419
 420         switch (ccb->ccb_h.func_code) {
 421         case XPT_PATH_INQ:
 422         {
 423                 struct ccb_pathinq *cpi = &ccb->cpi;
 424
 425                 cpi->version_num = 1;
 426                 cpi->hba_inquiry = PI_TAG_ABLE;
 427                 cpi->hba_misc = PIM_EXTLUNS;
 428                 cpi->target_sprt = 0;
 429                 cpi->hba_eng_cnt = 0;
 430                 cpi->max_target = 0;
 431                 cpi->max_lun = 0;
 432                 cpi->initiator_id = ~0;
 433                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 434                 strlcpy(cpi->hba_vid, "IBM", HBA_IDLEN);
 435                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 436                 cpi->unit_number = cam_sim_unit(sim);
 437                 cpi->bus_id = cam_sim_bus(sim);
 438                 cpi->base_transfer_speed = 150000;
 439                 cpi->transport = XPORT_SRP;
 440                 cpi->transport_version = 0;
 441                 cpi->protocol = PROTO_SCSI;
 442                 cpi->protocol_version = SCSI_REV_SPC4;
 443                 cpi->ccb_h.status = CAM_REQ_CMP;
 444                 break;
 445         }
 446         case XPT_RESET_BUS:
 447                 ccb->ccb_h.status = CAM_REQ_CMP;
 448                 break;
 449         case XPT_RESET_DEV:
 450                 ccb->ccb_h.status = CAM_REQ_INPROG;
 451                 vscsi_task_management(sc, ccb);
 452                 return;
 453         case XPT_GET_TRAN_SETTINGS:
 454                 ccb->cts.protocol = PROTO_SCSI;
 455                 ccb->cts.protocol_version = SCSI_REV_SPC4;
 456                 ccb->cts.transport = XPORT_SRP;
 457                 ccb->cts.transport_version = 0;
 458                 ccb->cts.proto_specific.valid = 0;
 459                 ccb->cts.xport_specific.valid = 0;
 460                 ccb->ccb_h.status = CAM_REQ_CMP;
 461                 break;
 462         case XPT_SET_TRAN_SETTINGS:
 463                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
 464                 break;
 465         case XPT_SCSI_IO:
 466         {
 467                 struct vscsi_xfer *xp;
 468
 469                 ccb->ccb_h.status = CAM_REQ_INPROG;
 470
 471                 xp = TAILQ_FIRST(&sc->free_xferq);
 472                 if (xp == NULL)
 473                         panic("SCSI queue flooded");
 474                 xp->ccb = ccb;
 475                 TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 476                 TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 477                 bus_dmamap_load_ccb(sc->data_tag, xp->dmamap,
 478                     ccb, vscsi_scsi_command, xp, 0);
 479
 480                 return;
 481         }
 482         default:
 483                 ccb->ccb_h.status = CAM_REQ_INVALID;
 484                 break;
 485         }
 486
 487         xpt_done(ccb);
 488         return;
 489 }
 490
 491 static void
 492 vscsi_srp_login(struct vscsi_softc *sc)
 493 {
 494         struct vscsi_xfer *xp;
 495         struct srp_login *login;
 496         struct vscsi_crq crq;
 497         int err;
 498
 499         mtx_assert(&sc->io_lock, MA_OWNED);
 500
 501         xp = TAILQ_FIRST(&sc->free_xferq);
 502         if (xp == NULL)
 503                 panic("SCSI queue flooded");
 504         xp->ccb = NULL;
 505         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 506         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 507
 508         /* Set up command */
 509         xp->srp_iu_size = crq.iu_length = 64;
 510         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 511             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 512         if (err)
 513                 panic("Error during VMEM allocation (%d)", err);
 514
 515         login = (struct srp_login *)((uint8_t *)xp->sc->srp_iu_queue +
 516             (uintptr_t)xp->srp_iu_offset);
 517         bzero(login, xp->srp_iu_size);
 518         login->type = SRP_LOGIN_REQ;
 519         login->tag = (uint64_t)(xp);
 520         login->max_cmd_length = htobe64(256);
 521         login->buffer_formats = htobe16(0x1 | 0x2); /* Direct and indirect */
 522         login->flags = 0;
 523
 524         /* Create CRQ entry */
 525         crq.valid = 0x80;
 526         crq.format = 0x01;
 527         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 528         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 529
 530         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 531             ((uint64_t *)(&crq))[1]);
 532         if (err != 0)
 533                 panic("CRQ send failure (%d)", err);
 534 }
 535
 536 static void
 537 vscsi_task_management(struct vscsi_softc *sc, union ccb *ccb)
 538 {
 539         struct srp_tsk_mgmt *cmd;
 540         struct vscsi_xfer *xp;
 541         struct vscsi_crq crq;
 542         int err;
 543
 544         mtx_assert(&sc->io_lock, MA_OWNED);
 545
 546         xp = TAILQ_FIRST(&sc->free_xferq);
 547         if (xp == NULL)
 548                 panic("SCSI queue flooded");
 549         xp->ccb = ccb;
 550         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 551         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 552
 553         xp->srp_iu_size = crq.iu_length = sizeof(*cmd);
 554         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 555             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 556         if (err)
 557                 panic("Error during VMEM allocation (%d)", err);
 558
 559         cmd = (struct srp_tsk_mgmt *)((uint8_t *)xp->sc->srp_iu_queue +
 560             (uintptr_t)xp->srp_iu_offset);
 561         bzero(cmd, xp->srp_iu_size);
 562         cmd->type = SRP_TSK_MGMT;
 563         cmd->tag = (uint64_t)xp;
 564         cmd->lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
 565
 566         switch (ccb->ccb_h.func_code) {
 567         case XPT_RESET_DEV:
 568                 cmd->function = 0x08;
 569                 break;
 570         default:
 571                 panic("Unimplemented code %d", ccb->ccb_h.func_code);
 572                 break;
 573         }
 574
 575         bus_dmamap_sync(xp->sc->crq_tag, xp->sc->crq_map, BUS_DMASYNC_PREWRITE);
 576
 577         /* Create CRQ entry */
 578         crq.valid = 0x80;
 579         crq.format = 0x01;
 580         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 581
 582         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 583             ((uint64_t *)(&crq))[1]);
 584         if (err != 0)
 585                 panic("CRQ send failure (%d)", err);
 586 }
 587
 588 static void
 589 vscsi_scsi_command(void *xxp, bus_dma_segment_t *segs, int nsegs, int err)
 590 {
 591         struct vscsi_xfer *xp = xxp;
 592         uint8_t *cdb;
 593         union ccb *ccb = xp->ccb;
 594         struct srp_cmd *cmd;
 595         uint64_t chunk_addr;
 596         uint32_t chunk_size;
 597         int desc_start, i;
 598         struct vscsi_crq crq;
 599
 600         KASSERT(err == 0, ("DMA error %d\n", err));
 601
 602         mtx_assert(&xp->sc->io_lock, MA_OWNED);
 603
 604         cdb = (ccb->ccb_h.flags & CAM_CDB_POINTER) ?
 605             ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes;
 606
 607         /* Command format from Table 20, page 37 of SRP spec */
 608         crq.iu_length = 48 + ((nsegs > 1) ? 20 : 16) +
 609             ((ccb->csio.cdb_len > 16) ? (ccb->csio.cdb_len - 16) : 0);
 610         xp->srp_iu_size = crq.iu_length;
 611         if (nsegs > 1)
 612                 xp->srp_iu_size += nsegs*16;
 613         xp->srp_iu_size = roundup(xp->srp_iu_size, 16);
 614         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 615             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 616         if (err)
 617                 panic("Error during VMEM allocation (%d)", err);
 618
 619         cmd = (struct srp_cmd *)((uint8_t *)xp->sc->srp_iu_queue +
 620             (uintptr_t)xp->srp_iu_offset);
 621         bzero(cmd, xp->srp_iu_size);
 622         cmd->type = SRP_CMD;
 623         if (ccb->csio.cdb_len > 16)
 624                 cmd->additional_cdb = (ccb->csio.cdb_len - 16) << 2;
 625         memcpy(cmd->cdb, cdb, ccb->csio.cdb_len);
 626
 627         cmd->tag = (uint64_t)(xp); /* Let the responder find this again */
 628         cmd->lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
 629
 630         if (nsegs > 1) {
 631                 /* Use indirect descriptors */
 632                 switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 633                 case CAM_DIR_OUT:
 634                         cmd->formats = (2 << 4);
 635                         break;
 636                 case CAM_DIR_IN:
 637                         cmd->formats = 2;
 638                         break;
 639                 default:
 640                         panic("Does not support bidirectional commands (%d)",
 641                             ccb->ccb_h.flags & CAM_DIR_MASK);
 642                         break;
 643                 }
 644
 645                 desc_start = ((ccb->csio.cdb_len > 16) ?
 646                     ccb->csio.cdb_len - 16 : 0);
 647                 chunk_addr = xp->sc->srp_iu_phys + xp->srp_iu_offset + 20 +
 648                     desc_start + sizeof(*cmd);
 649                 chunk_size = 16*nsegs;
 650                 memcpy(&cmd->data_payload[desc_start], &chunk_addr, 8);
 651                 memcpy(&cmd->data_payload[desc_start+12], &chunk_size, 4);
 652                 chunk_size = 0;
 653                 for (i = 0; i < nsegs; i++)
 654                         chunk_size += segs[i].ds_len;
 655                 memcpy(&cmd->data_payload[desc_start+16], &chunk_size, 4);
 656                 desc_start += 20;
 657                 for (i = 0; i < nsegs; i++) {
 658                         chunk_addr = segs[i].ds_addr;
 659                         chunk_size = segs[i].ds_len;
 660
 661                         memcpy(&cmd->data_payload[desc_start + 16*i],
 662                             &chunk_addr, 8);
 663                         /* Set handle tag to 0 */
 664                         memcpy(&cmd->data_payload[desc_start + 16*i + 12],
 665                             &chunk_size, 4);
 666                 }
 667         } else if (nsegs == 1) {
 668                 switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 669                 case CAM_DIR_OUT:
 670                         cmd->formats = (1 << 4);
 671                         break;
 672                 case CAM_DIR_IN:
 673                         cmd->formats = 1;
 674                         break;
 675                 default:
 676                         panic("Does not support bidirectional commands (%d)",
 677                             ccb->ccb_h.flags & CAM_DIR_MASK);
 678                         break;
 679                 }
 680
 681                 /*
 682                  * Memory descriptor:
 683                  * 8 byte address
 684                  * 4 byte handle
 685                  * 4 byte length
 686                  */
 687
 688                 chunk_addr = segs[0].ds_addr;
 689                 chunk_size = segs[0].ds_len;
 690                 desc_start = ((ccb->csio.cdb_len > 16) ?
 691                     ccb->csio.cdb_len - 16 : 0);
 692
 693                 memcpy(&cmd->data_payload[desc_start], &chunk_addr, 8);
 694                 /* Set handle tag to 0 */
 695                 memcpy(&cmd->data_payload[desc_start+12], &chunk_size, 4);
 696                 KASSERT(xp->srp_iu_size >= 48 + ((ccb->csio.cdb_len > 16) ?
 697                     ccb->csio.cdb_len : 16), ("SRP IU command length"));
 698         } else {
 699                 cmd->formats = 0;
 700         }
 701         bus_dmamap_sync(xp->sc->crq_tag, xp->sc->crq_map, BUS_DMASYNC_PREWRITE);
 702
 703         /* Create CRQ entry */
 704         crq.valid = 0x80;
 705         crq.format = 0x01;
 706         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 707
 708         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 709             ((uint64_t *)(&crq))[1]);
 710         if (err != 0)
 711                 panic("CRQ send failure (%d)", err);
 712 }
 713
 714 static void
 715 vscsi_crq_load_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int err)
 716 {
 717         struct vscsi_softc *sc = xsc;
 718
 719         sc->crq_phys = segs[0].ds_addr;
 720         sc->n_crqs = PAGE_SIZE/sizeof(struct vscsi_crq);
 721
 722         sc->srp_iu_queue = (uint8_t *)(sc->crq_queue);
 723         sc->srp_iu_phys = segs[0].ds_addr;
 724         sc->srp_iu_arena = vmem_create("VSCSI SRP IU", PAGE_SIZE,
 725             segs[0].ds_len - PAGE_SIZE, 16, 0, M_BESTFIT | M_NOWAIT);
 726 }
 727
 728 static void
 729 vscsi_setup_bus(struct vscsi_softc *sc)
 730 {
 731         struct vscsi_crq crq;
 732         struct vscsi_xfer *xp;
 733         int error;
 734
 735         struct {
 736                 uint32_t type;
 737                 uint16_t status;
 738                 uint16_t length;
 739                 uint64_t tag;
 740                 uint64_t buffer;
 741                 struct {
 742                         char srp_version[8];
 743                         char partition_name[96];
 744                         uint32_t partition_number;
 745                         uint32_t mad_version;
 746                         uint32_t os_type;
 747                         uint32_t port_max_txu[8];
 748                 } payload;
 749         } mad_adapter_info;
 750
 751         bzero(&crq, sizeof(crq));
 752
 753         /* Init message */
 754         crq.valid = 0xc0;
 755         crq.format = 0x01;
 756
 757         do {
 758                 error = phyp_hcall(H_FREE_CRQ, sc->unit);
 759         } while (error == H_BUSY);
 760
 761         /* See initialization sequence page 757 */
 762         bzero(sc->crq_queue, sc->n_crqs*sizeof(sc->crq_queue[0]));
 763         sc->cur_crq = 0;
 764         sc->bus_initialized = 0;
 765         sc->bus_logged_in = 0;
 766         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 767         error = phyp_hcall(H_REG_CRQ, sc->unit, sc->crq_phys,
 768             sc->n_crqs*sizeof(sc->crq_queue[0]));
 769         KASSERT(error == 0, ("CRQ registration success"));
 770
 771         error = phyp_hcall(H_SEND_CRQ, sc->unit, ((uint64_t *)(&crq))[0],
 772             ((uint64_t *)(&crq))[1]);
 773         if (error != 0)
 774                 panic("CRQ setup failure (%d)", error);
 775
 776         while (sc->bus_initialized == 0)
 777                 vscsi_check_response_queue(sc);
 778
 779         /* Send MAD adapter info */
 780         mad_adapter_info.type = MAD_ADAPTER_INFO_REQUEST;
 781         mad_adapter_info.status = 0;
 782         mad_adapter_info.length = sizeof(mad_adapter_info.payload);
 783
 784         strcpy(mad_adapter_info.payload.srp_version, "16.a");
 785         strcpy(mad_adapter_info.payload.partition_name, "UNKNOWN");
 786         mad_adapter_info.payload.partition_number = -1;
 787         mad_adapter_info.payload.mad_version = 1;
 788         mad_adapter_info.payload.os_type = 2; /* Claim we are Linux */
 789         mad_adapter_info.payload.port_max_txu[0] = 0;
 790         /* If this fails, we get the defaults above */
 791         OF_getprop(OF_finddevice("/"), "ibm,partition-name",
 792             mad_adapter_info.payload.partition_name,
 793             sizeof(mad_adapter_info.payload.partition_name));
 794         OF_getprop(OF_finddevice("/"), "ibm,partition-no",
 795             &mad_adapter_info.payload.partition_number,
 796             sizeof(mad_adapter_info.payload.partition_number));
 797
 798         xp = TAILQ_FIRST(&sc->free_xferq);
 799         xp->ccb = NULL;
 800         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 801         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 802         xp->srp_iu_size = crq.iu_length = sizeof(mad_adapter_info);
 803         vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 804             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 805         mad_adapter_info.buffer = xp->sc->srp_iu_phys + xp->srp_iu_offset + 24;
 806         mad_adapter_info.tag = (uint64_t)xp;
 807         memcpy((uint8_t *)xp->sc->srp_iu_queue + (uintptr_t)xp->srp_iu_offset,
 808                 &mad_adapter_info, sizeof(mad_adapter_info));
 809         crq.valid = 0x80;
 810         crq.format = 0x02;
 811         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 812         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 813         phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 814             ((uint64_t *)(&crq))[1]);
 815
 816         while (TAILQ_EMPTY(&sc->free_xferq))
 817                 vscsi_check_response_queue(sc);
 818
 819         /* Send SRP login */
 820         vscsi_srp_login(sc);
 821         while (sc->bus_logged_in == 0)
 822                 vscsi_check_response_queue(sc);
 823
 824         error = phyp_hcall(H_VIO_SIGNAL, sc->unit, 1); /* Enable interrupts */
 825 }
 826
 827 static void
 828 vscsi_intr(void *xsc)
 829 {
 830         struct vscsi_softc *sc = xsc;
 831
 832         mtx_lock(&sc->io_lock);
 833         vscsi_check_response_queue(sc);
 834         mtx_unlock(&sc->io_lock);
 835 }
 836
 837 static void
 838 vscsi_srp_response(struct vscsi_xfer *xp, struct vscsi_crq *crq)
 839 {
 840         union ccb *ccb = xp->ccb;
 841         struct vscsi_softc *sc = xp->sc;
 842         struct srp_rsp *rsp;
 843         uint32_t sense_len;
 844
 845         /* SRP response packet in original request */
 846         rsp = (struct srp_rsp *)((uint8_t *)sc->srp_iu_queue +
 847             (uintptr_t)xp->srp_iu_offset);
 848         ccb->csio.scsi_status = rsp->status;
 849         if (ccb->csio.scsi_status == SCSI_STATUS_OK)
 850                 ccb->ccb_h.status = CAM_REQ_CMP;
 851         else
 852                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
 853 #ifdef NOTYET
 854         /* Collect fast fail codes */
 855         if (crq->status != 0)
 856                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 857 #endif
 858
 859         if (ccb->ccb_h.status != CAM_REQ_CMP) {
 860                 ccb->ccb_h.status |= CAM_DEV_QFRZN;
 861                 xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
 862         }
 863
 864         if (!(rsp->flags & SRP_RSPVALID))
 865                 rsp->response_data_len = 0;
 866         if (!(rsp->flags & SRP_SNSVALID))
 867                 rsp->sense_data_len = 0;
 868         if (!(rsp->flags & (SRP_DOOVER | SRP_DOUNDER)))
 869                 rsp->data_out_resid = 0;
 870         if (!(rsp->flags & (SRP_DIOVER | SRP_DIUNDER)))
 871                 rsp->data_in_resid = 0;
 872
 873         if (rsp->flags & SRP_SNSVALID) {
 874                 bzero(&ccb->csio.sense_data, sizeof(struct scsi_sense_data));
 875                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 876                 sense_len = min(be32toh(rsp->sense_data_len),
 877                     ccb->csio.sense_len);
 878                 memcpy(&ccb->csio.sense_data,
 879                     &rsp->data_payload[be32toh(rsp->response_data_len)],
 880                     sense_len);
 881                 ccb->csio.sense_resid = ccb->csio.sense_len -
 882                     be32toh(rsp->sense_data_len);
 883         }
 884
 885         switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 886         case CAM_DIR_OUT:
 887                 ccb->csio.resid = rsp->data_out_resid;
 888                 break;
 889         case CAM_DIR_IN:
 890                 ccb->csio.resid = rsp->data_in_resid;
 891                 break;
 892         }
 893
 894         bus_dmamap_sync(sc->data_tag, xp->dmamap, BUS_DMASYNC_POSTREAD);
 895         bus_dmamap_unload(sc->data_tag, xp->dmamap);
 896         xpt_done(ccb);
 897         xp->ccb = NULL;
 898 }
 899
 900 static void
 901 vscsi_login_response(struct vscsi_xfer *xp, struct vscsi_crq *crq)
 902 {
 903         struct vscsi_softc *sc = xp->sc;
 904         struct srp_login_rsp *rsp;
 905
 906         /* SRP response packet in original request */
 907         rsp = (struct srp_login_rsp *)((uint8_t *)sc->srp_iu_queue +
 908             (uintptr_t)xp->srp_iu_offset);
 909         KASSERT(be16toh(rsp->buffer_formats) & 0x3, ("Both direct and indirect "
 910             "buffers supported"));
 911
 912         sc->max_transactions = be32toh(rsp->request_limit_delta);
 913         device_printf(sc->dev, "Queue depth %d commands\n",
 914             sc->max_transactions);
 915         sc->bus_logged_in = 1;
 916 }
 917
 918 static void
 919 vscsi_cam_poll(struct cam_sim *sim)
 920 {
 921         struct vscsi_softc *sc = cam_sim_softc(sim);
 922
 923         vscsi_check_response_queue(sc);
 924 }
 925
 926 static void
 927 vscsi_check_response_queue(struct vscsi_softc *sc)
 928 {
 929         struct vscsi_crq *crq;
 930         struct vscsi_xfer *xp;
 931         int code;
 932
 933         mtx_assert(&sc->io_lock, MA_OWNED);
 934
 935         while (sc->crq_queue[sc->cur_crq].valid != 0) {
 936                 /* The hypercalls at both ends of this are not optimal */
 937                 phyp_hcall(H_VIO_SIGNAL, sc->unit, 0);
 938                 bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_POSTREAD);
 939
 940                 crq = &sc->crq_queue[sc->cur_crq];
 941
 942                 switch (crq->valid) {
 943                 case 0xc0:
 944                         if (crq->format == 0x02)
 945                                 sc->bus_initialized = 1;
 946                         break;
 947                 case 0x80:
 948                         /* IU data is set to tag pointer (the XP) */
 949                         xp = (struct vscsi_xfer *)crq->iu_data;
 950
 951                         switch (crq->format) {
 952                         case 0x01:
 953                                 code = *((uint8_t *)sc->srp_iu_queue +
 954                                     (uintptr_t)xp->srp_iu_offset);
 955                                 switch (code) {
 956                                 case SRP_RSP:
 957                                         vscsi_srp_response(xp, crq);
 958                                         break;
 959                                 case SRP_LOGIN_RSP:
 960                                         vscsi_login_response(xp, crq);
 961                                         break;
 962                                 default:
 963                                         device_printf(sc->dev, "Unknown SRP "
 964                                             "response code %d\n", code);
 965                                         break;
 966                                 }
 967                                 break;
 968                         case 0x02:
 969                                 /* Ignore management datagrams */
 970                                 break;
 971                         default:
 972                                 panic("Unknown CRQ format %d\n", crq->format);
 973                                 break;
 974                         }
 975                         vmem_free(sc->srp_iu_arena, xp->srp_iu_offset,
 976                             xp->srp_iu_size);
 977                         TAILQ_REMOVE(&sc->active_xferq, xp, queue);
 978                         TAILQ_INSERT_TAIL(&sc->free_xferq, xp, queue);
 979                         break;
 980                 default:
 981                         device_printf(sc->dev,
 982                             "Unknown CRQ message type %d\n", crq->valid);
 983                         break;
 984                 }
 985
 986                 crq->valid = 0;
 987                 sc->cur_crq = (sc->cur_crq + 1) % sc->n_crqs;
 988
 989                 bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 990                 phyp_hcall(H_VIO_SIGNAL, sc->unit, 1);
 991         }
 992 }