sys/powerpc/pseries/phyp_vscsi.c

   1 /*-
   2  * Copyright 2013 Nathan Whitehorn
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/kernel.h>
  33 #include <sys/malloc.h>
  34 #include <sys/module.h>
  35 #include <sys/selinfo.h>
  36 #include <sys/bus.h>
  37 #include <sys/conf.h>
  38 #include <sys/eventhandler.h>
  39 #include <sys/rman.h>
  40 #include <sys/bus_dma.h>
  41 #include <sys/bio.h>
  42 #include <sys/ioccom.h>
  43 #include <sys/uio.h>
  44 #include <sys/proc.h>
  45 #include <sys/signalvar.h>
  46 #include <sys/sysctl.h>
  47 #include <sys/endian.h>
  48 #include <sys/vmem.h>
  49
  50 #include <cam/cam.h>
  51 #include <cam/cam_ccb.h>
  52 #include <cam/cam_debug.h>
  53 #include <cam/cam_periph.h>
  54 #include <cam/cam_sim.h>
  55 #include <cam/cam_xpt_periph.h>
  56 #include <cam/cam_xpt_sim.h>
  57 #include <cam/scsi/scsi_all.h>
  58 #include <cam/scsi/scsi_message.h>
  59
  60 #include <dev/ofw/openfirm.h>
  61 #include <dev/ofw/ofw_bus.h>
  62 #include <dev/ofw/ofw_bus_subr.h>
  63
  64 #include <machine/bus.h>
  65 #include <machine/resource.h>
  66
  67 #include <powerpc/pseries/phyp-hvcall.h>
  68
  69 struct vscsi_softc;
  70
  71 /* VSCSI CRQ format from table 260 of PAPR spec 2.4 (page 760) */
  72 struct vscsi_crq {
  73         uint8_t valid;
  74         uint8_t format;
  75         uint8_t reserved;
  76         uint8_t status;
  77         uint16_t timeout;
  78         uint16_t iu_length;
  79         uint64_t iu_data;
  80 };
  81
  82 struct vscsi_xfer {
  83         TAILQ_ENTRY(vscsi_xfer) queue;
  84         struct vscsi_softc *sc;
  85         union ccb *ccb;
  86         bus_dmamap_t dmamap;
  87         uint64_t tag;
  88
  89         vmem_addr_t srp_iu_offset;
  90         vmem_size_t srp_iu_size;
  91 };
  92
  93 TAILQ_HEAD(vscsi_xferq, vscsi_xfer);
  94
  95 struct vscsi_softc {
  96         device_t        dev;
  97         struct cam_devq *devq;
  98         struct cam_sim  *sim;
  99         struct cam_path *path;
 100         struct mtx io_lock;
 101
 102         cell_t          unit;
 103         int             bus_initialized;
 104         int             bus_logged_in;
 105         int             max_transactions;
 106
 107         int             irqid;
 108         struct resource *irq;
 109         void            *irq_cookie;
 110
 111         bus_dma_tag_t   crq_tag;
 112         struct vscsi_crq *crq_queue;
 113         int             n_crqs, cur_crq;
 114         bus_dmamap_t    crq_map;
 115         bus_addr_t      crq_phys;
 116
 117         vmem_t          *srp_iu_arena;
 118         void            *srp_iu_queue;
 119         bus_addr_t      srp_iu_phys;
 120
 121         bus_dma_tag_t   data_tag;
 122
 123         struct vscsi_xfer loginxp;
 124         struct vscsi_xfer *xfer;
 125         struct vscsi_xferq active_xferq;
 126         struct vscsi_xferq free_xferq;
 127 };
 128
 129 struct srp_login {
 130         uint8_t type;
 131         uint8_t reserved[7];
 132         uint64_t tag;
 133         uint64_t max_cmd_length;
 134         uint32_t reserved2;
 135         uint16_t buffer_formats;
 136         uint8_t flags;
 137         uint8_t reserved3[5];
 138         uint8_t initiator_port_id[16];
 139         uint8_t target_port_id[16];
 140 } __packed;
 141
 142 struct srp_login_rsp {
 143         uint8_t type;
 144         uint8_t reserved[3];
 145         uint32_t request_limit_delta;
 146         uint8_t tag;
 147         uint32_t max_i_to_t_len;
 148         uint32_t max_t_to_i_len;
 149         uint16_t buffer_formats;
 150         uint8_t flags;
 151         /* Some reserved bits follow */
 152 } __packed;
 153
 154 struct srp_cmd {
 155         uint8_t type;
 156         uint8_t flags1;
 157         uint8_t reserved[3];
 158         uint8_t formats;
 159         uint8_t out_buffer_count;
 160         uint8_t in_buffer_count;
 161         uint64_t tag;
 162         uint32_t reserved2;
 163         uint64_t lun;
 164         uint8_t reserved3[3];
 165         uint8_t additional_cdb;
 166         uint8_t cdb[16];
 167         uint8_t data_payload[0];
 168 } __packed;
 169
 170 struct srp_rsp {
 171         uint8_t type;
 172         uint8_t reserved[3];
 173         uint32_t request_limit_delta;
 174         uint64_t tag;
 175         uint16_t reserved2;
 176         uint8_t flags;
 177         uint8_t status;
 178         uint32_t data_out_resid;
 179         uint32_t data_in_resid;
 180         uint32_t sense_data_len;
 181         uint32_t response_data_len;
 182         uint8_t data_payload[0];
 183 } __packed;
 184
 185 struct srp_tsk_mgmt {
 186         uint8_t type;
 187         uint8_t reserved[7];
 188         uint64_t tag;
 189         uint32_t reserved2;
 190         uint64_t lun;
 191         uint8_t reserved3[2];
 192         uint8_t function;
 193         uint8_t reserved4;
 194         uint64_t manage_tag;
 195         uint64_t reserved5;
 196 } __packed;
 197
 198 /* Message code type */
 199 #define SRP_LOGIN_REQ   0x00
 200 #define SRP_TSK_MGMT    0x01
 201 #define SRP_CMD         0x02
 202 #define SRP_I_LOGOUT    0x03
 203
 204 #define SRP_LOGIN_RSP   0xC0
 205 #define SRP_RSP         0xC1
 206 #define SRP_LOGIN_REJ   0xC2
 207
 208 #define SRP_T_LOGOUT    0x80
 209 #define SRP_CRED_REQ    0x81
 210 #define SRP_AER_REQ     0x82
 211
 212 #define SRP_CRED_RSP    0x41
 213 #define SRP_AER_RSP     0x41
 214
 215 /* Flags for srp_rsp flags field */
 216 #define SRP_RSPVALID    0x01
 217 #define SRP_SNSVALID    0x02
 218 #define SRP_DOOVER      0x04
 219 #define SRP_DOUNDER     0x08
 220 #define SRP_DIOVER      0x10
 221 #define SRP_DIUNDER     0x20
 222
 223 #define MAD_SUCESS                      0x00
 224 #define MAD_NOT_SUPPORTED               0xf1
 225 #define MAD_FAILED                      0xf7
 226
 227 #define MAD_EMPTY_IU                    0x01
 228 #define MAD_ERROR_LOGGING_REQUEST       0x02
 229 #define MAD_ADAPTER_INFO_REQUEST        0x03
 230 #define MAD_CAPABILITIES_EXCHANGE       0x05
 231 #define MAD_PHYS_ADAP_INFO_REQUEST      0x06
 232 #define MAD_TAPE_PASSTHROUGH_REQUEST    0x07
 233 #define MAD_ENABLE_FAST_FAIL            0x08
 234
 235 static int      vscsi_probe(device_t);
 236 static int      vscsi_attach(device_t);
 237 static int      vscsi_detach(device_t);
 238 static void     vscsi_cam_action(struct cam_sim *, union ccb *);
 239 static void     vscsi_cam_poll(struct cam_sim *);
 240 static void     vscsi_intr(void *arg);
 241 static void     vscsi_check_response_queue(struct vscsi_softc *sc);
 242 static void     vscsi_setup_bus(struct vscsi_softc *sc);
 243
 244 static void     vscsi_srp_login(struct vscsi_softc *sc);
 245 static void     vscsi_crq_load_cb(void *, bus_dma_segment_t *, int, int);
 246 static void     vscsi_scsi_command(void *xxp, bus_dma_segment_t *segs,
 247                     int nsegs, int err);
 248 static void     vscsi_task_management(struct vscsi_softc *sc, union ccb *ccb);
 249 static void     vscsi_srp_response(struct vscsi_xfer *, struct vscsi_crq *);
 250
 251 static devclass_t       vscsi_devclass;
 252 static device_method_t  vscsi_methods[] = {
 253         DEVMETHOD(device_probe,         vscsi_probe),
 254         DEVMETHOD(device_attach,        vscsi_attach),
 255         DEVMETHOD(device_detach,        vscsi_detach),
 256
 257         DEVMETHOD_END
 258 };
 259 static driver_t vscsi_driver = {
 260         "vscsi",
 261         vscsi_methods,
 262         sizeof(struct vscsi_softc)
 263 };
 264 DRIVER_MODULE(vscsi, vdevice, vscsi_driver, vscsi_devclass, 0, 0);
 265 MALLOC_DEFINE(M_VSCSI, "vscsi", "CAM device queue for VSCSI");
 266
 267 static int
 268 vscsi_probe(device_t dev)
 269 {
 270
 271         if (!ofw_bus_is_compatible(dev, "IBM,v-scsi"))
 272                 return (ENXIO);
 273
 274         device_set_desc(dev, "POWER Hypervisor Virtual SCSI Bus");
 275         return (0);
 276 }
 277
 278 static int
 279 vscsi_attach(device_t dev)
 280 {
 281         struct vscsi_softc *sc;
 282         struct vscsi_xfer *xp;
 283         int error, i;
 284
 285         sc = device_get_softc(dev);
 286         if (sc == NULL)
 287                 return (EINVAL);
 288
 289         sc->dev = dev;
 290         mtx_init(&sc->io_lock, "vscsi", NULL, MTX_DEF);
 291
 292         /* Get properties */
 293         OF_getencprop(ofw_bus_get_node(dev), "reg", &sc->unit,
 294             sizeof(sc->unit));
 295
 296         /* Setup interrupt */
 297         sc->irqid = 0;
 298         sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irqid,
 299             RF_ACTIVE);
 300
 301         if (!sc->irq) {
 302                 device_printf(dev, "Could not allocate IRQ\n");
 303                 mtx_destroy(&sc->io_lock);
 304                 return (ENXIO);
 305         }
 306
 307         bus_setup_intr(dev, sc->irq, INTR_TYPE_CAM | INTR_MPSAFE |
 308             INTR_ENTROPY, NULL, vscsi_intr, sc, &sc->irq_cookie);
 309
 310         /* Data DMA */
 311         error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
 312             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
 313             256, BUS_SPACE_MAXSIZE_32BIT, 0, busdma_lock_mutex, &sc->io_lock,
 314             &sc->data_tag);
 315
 316         TAILQ_INIT(&sc->active_xferq);
 317         TAILQ_INIT(&sc->free_xferq);
 318
 319         /* First XFER for login data */
 320         sc->loginxp.sc = sc;
 321         bus_dmamap_create(sc->data_tag, 0, &sc->loginxp.dmamap);
 322         TAILQ_INSERT_TAIL(&sc->free_xferq, &sc->loginxp, queue);
 323
 324         /* CRQ area */
 325         error = bus_dma_tag_create(bus_get_dma_tag(dev), PAGE_SIZE, 0,
 326             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 8*PAGE_SIZE,
 327             1, BUS_SPACE_MAXSIZE, 0, NULL, NULL, &sc->crq_tag);
 328         error = bus_dmamem_alloc(sc->crq_tag, (void **)&sc->crq_queue,
 329             BUS_DMA_WAITOK | BUS_DMA_ZERO, &sc->crq_map);
 330         sc->crq_phys = 0;
 331         sc->n_crqs = 0;
 332         error = bus_dmamap_load(sc->crq_tag, sc->crq_map, sc->crq_queue,
 333             8*PAGE_SIZE, vscsi_crq_load_cb, sc, 0);
 334
 335         mtx_lock(&sc->io_lock);
 336         vscsi_setup_bus(sc);
 337         sc->xfer = malloc(sizeof(sc->xfer[0])*sc->max_transactions, M_VSCSI,
 338             M_NOWAIT);
 339         for (i = 0; i < sc->max_transactions; i++) {
 340                 xp = &sc->xfer[i];
 341                 xp->sc = sc;
 342
 343                 error = bus_dmamap_create(sc->data_tag, 0, &xp->dmamap);
 344                 if (error) {
 345                         device_printf(dev, "Could not create DMA map (%d)\n",
 346                             error);
 347                         break;
 348                 }
 349
 350                 TAILQ_INSERT_TAIL(&sc->free_xferq, xp, queue);
 351         }
 352         mtx_unlock(&sc->io_lock);
 353
 354         /* Allocate CAM bits */
 355         if ((sc->devq = cam_simq_alloc(sc->max_transactions)) == NULL)
 356                 return (ENOMEM);
 357
 358         sc->sim = cam_sim_alloc(vscsi_cam_action, vscsi_cam_poll, "vscsi", sc,
 359                                 device_get_unit(dev), &sc->io_lock,
 360                                 sc->max_transactions, sc->max_transactions,
 361                                 sc->devq);
 362         if (sc->sim == NULL) {
 363                 cam_simq_free(sc->devq);
 364                 sc->devq = NULL;
 365                 device_printf(dev, "CAM SIM attach failed\n");
 366                 return (EINVAL);
 367         }
 368
 369
 370         mtx_lock(&sc->io_lock);
 371         if (xpt_bus_register(sc->sim, dev, 0) != 0) {
 372                 device_printf(dev, "XPT bus registration failed\n");
 373                 cam_sim_free(sc->sim, FALSE);
 374                 sc->sim = NULL;
 375                 cam_simq_free(sc->devq);
 376                 sc->devq = NULL;
 377                 mtx_unlock(&sc->io_lock);
 378                 return (EINVAL);
 379         }
 380         mtx_unlock(&sc->io_lock);
 381
 382         return (0);
 383 }
 384
 385 static int
 386 vscsi_detach(device_t dev)
 387 {
 388         struct vscsi_softc *sc;
 389
 390         sc = device_get_softc(dev);
 391         if (sc == NULL)
 392                 return (EINVAL);
 393
 394         if (sc->sim != NULL) {
 395                 mtx_lock(&sc->io_lock);
 396                 xpt_bus_deregister(cam_sim_path(sc->sim));
 397                 cam_sim_free(sc->sim, FALSE);
 398                 sc->sim = NULL;
 399                 mtx_unlock(&sc->io_lock);
 400         }
 401
 402         if (sc->devq != NULL) {
 403                 cam_simq_free(sc->devq);
 404                 sc->devq = NULL;
 405         }
 406
 407         mtx_destroy(&sc->io_lock);
 408
 409         return (0);
 410 }
 411
 412 static void
 413 vscsi_cam_action(struct cam_sim *sim, union ccb *ccb)
 414 {
 415         struct vscsi_softc *sc = cam_sim_softc(sim);
 416
 417         mtx_assert(&sc->io_lock, MA_OWNED);
 418
 419         switch (ccb->ccb_h.func_code) {
 420         case XPT_PATH_INQ:
 421         {
 422                 struct ccb_pathinq *cpi = &ccb->cpi;
 423
 424                 cpi->version_num = 1;
 425                 cpi->hba_inquiry = PI_TAG_ABLE;
 426                 cpi->hba_misc = PIM_EXTLUNS;
 427                 cpi->target_sprt = 0;
 428                 cpi->hba_eng_cnt = 0;
 429                 cpi->max_target = 0;
 430                 cpi->max_lun = 0;
 431                 cpi->initiator_id = ~0;
 432                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 433                 strncpy(cpi->hba_vid, "IBM", HBA_IDLEN);
 434                 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 435                 cpi->unit_number = cam_sim_unit(sim);
 436                 cpi->bus_id = cam_sim_bus(sim);
 437                 cpi->base_transfer_speed = 150000;
 438                 cpi->transport = XPORT_SRP;
 439                 cpi->transport_version = 0;
 440                 cpi->protocol = PROTO_SCSI;
 441                 cpi->protocol_version = SCSI_REV_SPC4;
 442                 cpi->ccb_h.status = CAM_REQ_CMP;
 443                 break;
 444         }
 445         case XPT_RESET_BUS:
 446                 ccb->ccb_h.status = CAM_REQ_CMP;
 447                 break;
 448         case XPT_RESET_DEV:
 449                 ccb->ccb_h.status = CAM_REQ_INPROG;
 450                 vscsi_task_management(sc, ccb);
 451                 return;
 452         case XPT_GET_TRAN_SETTINGS:
 453                 ccb->cts.protocol = PROTO_SCSI;
 454                 ccb->cts.protocol_version = SCSI_REV_SPC4;
 455                 ccb->cts.transport = XPORT_SRP;
 456                 ccb->cts.transport_version = 0;
 457                 ccb->cts.proto_specific.valid = 0;
 458                 ccb->cts.xport_specific.valid = 0;
 459                 ccb->ccb_h.status = CAM_REQ_CMP;
 460                 break;
 461         case XPT_SET_TRAN_SETTINGS:
 462                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
 463                 break;
 464         case XPT_SCSI_IO:
 465         {
 466                 struct vscsi_xfer *xp;
 467
 468                 ccb->ccb_h.status = CAM_REQ_INPROG;
 469
 470                 xp = TAILQ_FIRST(&sc->free_xferq);
 471                 if (xp == NULL)
 472                         panic("SCSI queue flooded");
 473                 xp->ccb = ccb;
 474                 TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 475                 TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 476                 bus_dmamap_load_ccb(sc->data_tag, xp->dmamap,
 477                     ccb, vscsi_scsi_command, xp, 0);
 478
 479                 return;
 480         }
 481         default:
 482                 ccb->ccb_h.status = CAM_REQ_INVALID;
 483                 break;
 484         }
 485
 486         xpt_done(ccb);
 487         return;
 488 }
 489
 490 static void
 491 vscsi_srp_login(struct vscsi_softc *sc)
 492 {
 493         struct vscsi_xfer *xp;
 494         struct srp_login *login;
 495         struct vscsi_crq crq;
 496         int err;
 497
 498         mtx_assert(&sc->io_lock, MA_OWNED);
 499
 500         xp = TAILQ_FIRST(&sc->free_xferq);
 501         if (xp == NULL)
 502                 panic("SCSI queue flooded");
 503         xp->ccb = NULL;
 504         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 505         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 506
 507         /* Set up command */
 508         xp->srp_iu_size = crq.iu_length = 64;
 509         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 510             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 511         if (err)
 512                 panic("Error during VMEM allocation (%d)", err);
 513
 514         login = (struct srp_login *)((uint8_t *)xp->sc->srp_iu_queue +
 515             (uintptr_t)xp->srp_iu_offset);
 516         bzero(login, xp->srp_iu_size);
 517         login->type = SRP_LOGIN_REQ;
 518         login->tag = (uint64_t)(xp);
 519         login->max_cmd_length = htobe64(256);
 520         login->buffer_formats = htobe16(0x1 | 0x2); /* Direct and indirect */
 521         login->flags = 0;
 522
 523         /* Create CRQ entry */
 524         crq.valid = 0x80;
 525         crq.format = 0x01;
 526         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 527         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 528
 529         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 530             ((uint64_t *)(&crq))[1]);
 531         if (err != 0)
 532                 panic("CRQ send failure (%d)", err);
 533 }
 534
 535 static void
 536 vscsi_task_management(struct vscsi_softc *sc, union ccb *ccb)
 537 {
 538         struct srp_tsk_mgmt *cmd;
 539         struct vscsi_xfer *xp;
 540         struct vscsi_crq crq;
 541         int err;
 542
 543         mtx_assert(&sc->io_lock, MA_OWNED);
 544
 545         xp = TAILQ_FIRST(&sc->free_xferq);
 546         if (xp == NULL)
 547                 panic("SCSI queue flooded");
 548         xp->ccb = ccb;
 549         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 550         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 551
 552         xp->srp_iu_size = crq.iu_length = sizeof(*cmd);
 553         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 554             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 555         if (err)
 556                 panic("Error during VMEM allocation (%d)", err);
 557
 558         cmd = (struct srp_tsk_mgmt *)((uint8_t *)xp->sc->srp_iu_queue +
 559             (uintptr_t)xp->srp_iu_offset);
 560         bzero(cmd, xp->srp_iu_size);
 561         cmd->type = SRP_TSK_MGMT;
 562         cmd->tag = (uint64_t)xp;
 563         cmd->lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
 564
 565         switch (ccb->ccb_h.func_code) {
 566         case XPT_RESET_DEV:
 567                 cmd->function = 0x08;
 568                 break;
 569         default:
 570                 panic("Unimplemented code %d", ccb->ccb_h.func_code);
 571                 break;
 572         }
 573
 574         bus_dmamap_sync(xp->sc->crq_tag, xp->sc->crq_map, BUS_DMASYNC_PREWRITE);
 575
 576         /* Create CRQ entry */
 577         crq.valid = 0x80;
 578         crq.format = 0x01;
 579         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 580
 581         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 582             ((uint64_t *)(&crq))[1]);
 583         if (err != 0)
 584                 panic("CRQ send failure (%d)", err);
 585 }
 586
 587 static void
 588 vscsi_scsi_command(void *xxp, bus_dma_segment_t *segs, int nsegs, int err)
 589 {
 590         struct vscsi_xfer *xp = xxp;
 591         uint8_t *cdb;
 592         union ccb *ccb = xp->ccb;
 593         struct srp_cmd *cmd;
 594         uint64_t chunk_addr;
 595         uint32_t chunk_size;
 596         int desc_start, i;
 597         struct vscsi_crq crq;
 598
 599         KASSERT(err == 0, ("DMA error %d\n", err));
 600
 601         mtx_assert(&xp->sc->io_lock, MA_OWNED);
 602
 603         cdb = (ccb->ccb_h.flags & CAM_CDB_POINTER) ?
 604             ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes;
 605
 606         /* Command format from Table 20, page 37 of SRP spec */
 607         crq.iu_length = 48 + ((nsegs > 1) ? 20 : 16) +
 608             ((ccb->csio.cdb_len > 16) ? (ccb->csio.cdb_len - 16) : 0);
 609         xp->srp_iu_size = crq.iu_length;
 610         if (nsegs > 1)
 611                 xp->srp_iu_size += nsegs*16;
 612         xp->srp_iu_size = roundup(xp->srp_iu_size, 16);
 613         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 614             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 615         if (err)
 616                 panic("Error during VMEM allocation (%d)", err);
 617
 618         cmd = (struct srp_cmd *)((uint8_t *)xp->sc->srp_iu_queue +
 619             (uintptr_t)xp->srp_iu_offset);
 620         bzero(cmd, xp->srp_iu_size);
 621         cmd->type = SRP_CMD;
 622         if (ccb->csio.cdb_len > 16)
 623                 cmd->additional_cdb = (ccb->csio.cdb_len - 16) << 2;
 624         memcpy(cmd->cdb, cdb, ccb->csio.cdb_len);
 625
 626         cmd->tag = (uint64_t)(xp); /* Let the responder find this again */
 627         cmd->lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
 628
 629         if (nsegs > 1) {
 630                 /* Use indirect descriptors */
 631                 switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 632                 case CAM_DIR_OUT:
 633                         cmd->formats = (2 << 4);
 634                         break;
 635                 case CAM_DIR_IN:
 636                         cmd->formats = 2;
 637                         break;
 638                 default:
 639                         panic("Does not support bidirectional commands (%d)",
 640                             ccb->ccb_h.flags & CAM_DIR_MASK);
 641                         break;
 642                 }
 643
 644                 desc_start = ((ccb->csio.cdb_len > 16) ?
 645                     ccb->csio.cdb_len - 16 : 0);
 646                 chunk_addr = xp->sc->srp_iu_phys + xp->srp_iu_offset + 20 +
 647                     desc_start + sizeof(*cmd);
 648                 chunk_size = 16*nsegs;
 649                 memcpy(&cmd->data_payload[desc_start], &chunk_addr, 8);
 650                 memcpy(&cmd->data_payload[desc_start+12], &chunk_size, 4);
 651                 chunk_size = 0;
 652                 for (i = 0; i < nsegs; i++)
 653                         chunk_size += segs[i].ds_len;
 654                 memcpy(&cmd->data_payload[desc_start+16], &chunk_size, 4);
 655                 desc_start += 20;
 656                 for (i = 0; i < nsegs; i++) {
 657                         chunk_addr = segs[i].ds_addr;
 658                         chunk_size = segs[i].ds_len;
 659
 660                         memcpy(&cmd->data_payload[desc_start + 16*i],
 661                             &chunk_addr, 8);
 662                         /* Set handle tag to 0 */
 663                         memcpy(&cmd->data_payload[desc_start + 16*i + 12],
 664                             &chunk_size, 4);
 665                 }
 666         } else if (nsegs == 1) {
 667                 switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 668                 case CAM_DIR_OUT:
 669                         cmd->formats = (1 << 4);
 670                         break;
 671                 case CAM_DIR_IN:
 672                         cmd->formats = 1;
 673                         break;
 674                 default:
 675                         panic("Does not support bidirectional commands (%d)",
 676                             ccb->ccb_h.flags & CAM_DIR_MASK);
 677                         break;
 678                 }
 679
 680                 /*
 681                  * Memory descriptor:
 682                  * 8 byte address
 683                  * 4 byte handle
 684                  * 4 byte length
 685                  */
 686
 687                 chunk_addr = segs[0].ds_addr;
 688                 chunk_size = segs[0].ds_len;
 689                 desc_start = ((ccb->csio.cdb_len > 16) ?
 690                     ccb->csio.cdb_len - 16 : 0);
 691
 692                 memcpy(&cmd->data_payload[desc_start], &chunk_addr, 8);
 693                 /* Set handle tag to 0 */
 694                 memcpy(&cmd->data_payload[desc_start+12], &chunk_size, 4);
 695                 KASSERT(xp->srp_iu_size >= 48 + ((ccb->csio.cdb_len > 16) ?
 696                     ccb->csio.cdb_len : 16), ("SRP IU command length"));
 697         } else {
 698                 cmd->formats = 0;
 699         }
 700         bus_dmamap_sync(xp->sc->crq_tag, xp->sc->crq_map, BUS_DMASYNC_PREWRITE);
 701
 702         /* Create CRQ entry */
 703         crq.valid = 0x80;
 704         crq.format = 0x01;
 705         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 706
 707         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 708             ((uint64_t *)(&crq))[1]);
 709         if (err != 0)
 710                 panic("CRQ send failure (%d)", err);
 711 }
 712
 713 static void
 714 vscsi_crq_load_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int err)
 715 {
 716         struct vscsi_softc *sc = xsc;
 717
 718         sc->crq_phys = segs[0].ds_addr;
 719         sc->n_crqs = PAGE_SIZE/sizeof(struct vscsi_crq);
 720
 721         sc->srp_iu_queue = (uint8_t *)(sc->crq_queue);
 722         sc->srp_iu_phys = segs[0].ds_addr;
 723         sc->srp_iu_arena = vmem_create("VSCSI SRP IU", PAGE_SIZE,
 724             segs[0].ds_len - PAGE_SIZE, 16, 0, M_BESTFIT | M_NOWAIT);
 725 }
 726
 727 static void
 728 vscsi_setup_bus(struct vscsi_softc *sc)
 729 {
 730         struct vscsi_crq crq;
 731         struct vscsi_xfer *xp;
 732         int error;
 733
 734         struct {
 735                 uint32_t type;
 736                 uint16_t status;
 737                 uint16_t length;
 738                 uint64_t tag;
 739                 uint64_t buffer;
 740                 struct {
 741                         char srp_version[8];
 742                         char partition_name[96];
 743                         uint32_t partition_number;
 744                         uint32_t mad_version;
 745                         uint32_t os_type;
 746                         uint32_t port_max_txu[8];
 747                 } payload;
 748         } mad_adapter_info;
 749
 750         bzero(&crq, sizeof(crq));
 751
 752         /* Init message */
 753         crq.valid = 0xc0;
 754         crq.format = 0x01;
 755
 756         do {
 757                 error = phyp_hcall(H_FREE_CRQ, sc->unit);
 758         } while (error == H_BUSY);
 759
 760         /* See initialization sequence page 757 */
 761         bzero(sc->crq_queue, sc->n_crqs*sizeof(sc->crq_queue[0]));
 762         sc->cur_crq = 0;
 763         sc->bus_initialized = 0;
 764         sc->bus_logged_in = 0;
 765         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 766         error = phyp_hcall(H_REG_CRQ, sc->unit, sc->crq_phys,
 767             sc->n_crqs*sizeof(sc->crq_queue[0]));
 768         KASSERT(error == 0, ("CRQ registration success"));
 769
 770         error = phyp_hcall(H_SEND_CRQ, sc->unit, ((uint64_t *)(&crq))[0],
 771             ((uint64_t *)(&crq))[1]);
 772         if (error != 0)
 773                 panic("CRQ setup failure (%d)", error);
 774
 775         while (sc->bus_initialized == 0)
 776                 vscsi_check_response_queue(sc);
 777
 778         /* Send MAD adapter info */
 779         mad_adapter_info.type = MAD_ADAPTER_INFO_REQUEST;
 780         mad_adapter_info.status = 0;
 781         mad_adapter_info.length = sizeof(mad_adapter_info.payload);
 782
 783         strcpy(mad_adapter_info.payload.srp_version, "16.a");
 784         strcpy(mad_adapter_info.payload.partition_name, "UNKNOWN");
 785         mad_adapter_info.payload.partition_number = -1;
 786         mad_adapter_info.payload.mad_version = 1;
 787         mad_adapter_info.payload.os_type = 2; /* Claim we are Linux */
 788         mad_adapter_info.payload.port_max_txu[0] = 0;
 789         /* If this fails, we get the defaults above */
 790         OF_getprop(OF_finddevice("/"), "ibm,partition-name",
 791             mad_adapter_info.payload.partition_name,
 792             sizeof(mad_adapter_info.payload.partition_name));
 793         OF_getprop(OF_finddevice("/"), "ibm,partition-no",
 794             &mad_adapter_info.payload.partition_number,
 795             sizeof(mad_adapter_info.payload.partition_number));
 796
 797         xp = TAILQ_FIRST(&sc->free_xferq);
 798         xp->ccb = NULL;
 799         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 800         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 801         xp->srp_iu_size = crq.iu_length = sizeof(mad_adapter_info);
 802         vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 803             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 804         mad_adapter_info.buffer = xp->sc->srp_iu_phys + xp->srp_iu_offset + 24;
 805         mad_adapter_info.tag = (uint64_t)xp;
 806         memcpy((uint8_t *)xp->sc->srp_iu_queue + (uintptr_t)xp->srp_iu_offset,
 807                 &mad_adapter_info, sizeof(mad_adapter_info));
 808         crq.valid = 0x80;
 809         crq.format = 0x02;
 810         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 811         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 812         phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 813             ((uint64_t *)(&crq))[1]);
 814
 815         while (TAILQ_EMPTY(&sc->free_xferq))
 816                 vscsi_check_response_queue(sc);
 817
 818         /* Send SRP login */
 819         vscsi_srp_login(sc);
 820         while (sc->bus_logged_in == 0)
 821                 vscsi_check_response_queue(sc);
 822
 823         error = phyp_hcall(H_VIO_SIGNAL, sc->unit, 1); /* Enable interrupts */
 824 }
 825
 826
 827 static void
 828 vscsi_intr(void *xsc)
 829 {
 830         struct vscsi_softc *sc = xsc;
 831
 832         mtx_lock(&sc->io_lock);
 833         vscsi_check_response_queue(sc);
 834         mtx_unlock(&sc->io_lock);
 835 }
 836
 837 static void
 838 vscsi_srp_response(struct vscsi_xfer *xp, struct vscsi_crq *crq)
 839 {
 840         union ccb *ccb = xp->ccb;
 841         struct vscsi_softc *sc = xp->sc;
 842         struct srp_rsp *rsp;
 843         uint32_t sense_len;
 844
 845         /* SRP response packet in original request */
 846         rsp = (struct srp_rsp *)((uint8_t *)sc->srp_iu_queue +
 847             (uintptr_t)xp->srp_iu_offset);
 848         ccb->csio.scsi_status = rsp->status;
 849         if (ccb->csio.scsi_status == SCSI_STATUS_OK)
 850                 ccb->ccb_h.status = CAM_REQ_CMP;
 851         else
 852                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
 853 #ifdef NOTYET
 854         /* Collect fast fail codes */
 855         if (crq->status != 0)
 856                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 857 #endif
 858
 859         if (ccb->ccb_h.status != CAM_REQ_CMP) {
 860                 ccb->ccb_h.status |= CAM_DEV_QFRZN;
 861                 xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
 862         }
 863
 864         if (!(rsp->flags & SRP_RSPVALID))
 865                 rsp->response_data_len = 0;
 866         if (!(rsp->flags & SRP_SNSVALID))
 867                 rsp->sense_data_len = 0;
 868         if (!(rsp->flags & (SRP_DOOVER | SRP_DOUNDER)))
 869                 rsp->data_out_resid = 0;
 870         if (!(rsp->flags & (SRP_DIOVER | SRP_DIUNDER)))
 871                 rsp->data_in_resid = 0;
 872
 873         if (rsp->flags & SRP_SNSVALID) {
 874                 bzero(&ccb->csio.sense_data, sizeof(struct scsi_sense_data));
 875                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 876                 sense_len = min(be32toh(rsp->sense_data_len),
 877                     ccb->csio.sense_len);
 878                 memcpy(&ccb->csio.sense_data,
 879                     &rsp->data_payload[be32toh(rsp->response_data_len)],
 880                     sense_len);
 881                 ccb->csio.sense_resid = ccb->csio.sense_len -
 882                     be32toh(rsp->sense_data_len);
 883         }
 884
 885         switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 886         case CAM_DIR_OUT:
 887                 ccb->csio.resid = rsp->data_out_resid;
 888                 break;
 889         case CAM_DIR_IN:
 890                 ccb->csio.resid = rsp->data_in_resid;
 891                 break;
 892         }
 893
 894         bus_dmamap_sync(sc->data_tag, xp->dmamap, BUS_DMASYNC_POSTREAD);
 895         bus_dmamap_unload(sc->data_tag, xp->dmamap);
 896         xpt_done(ccb);
 897         xp->ccb = NULL;
 898 }
 899
 900 static void
 901 vscsi_login_response(struct vscsi_xfer *xp, struct vscsi_crq *crq)
 902 {
 903         struct vscsi_softc *sc = xp->sc;
 904         struct srp_login_rsp *rsp;
 905
 906         /* SRP response packet in original request */
 907         rsp = (struct srp_login_rsp *)((uint8_t *)sc->srp_iu_queue +
 908             (uintptr_t)xp->srp_iu_offset);
 909         KASSERT(be16toh(rsp->buffer_formats) & 0x3, ("Both direct and indirect "
 910             "buffers supported"));
 911
 912         sc->max_transactions = be32toh(rsp->request_limit_delta);
 913         device_printf(sc->dev, "Queue depth %d commands\n",
 914             sc->max_transactions);
 915         sc->bus_logged_in = 1;
 916 }
 917
 918 static void
 919 vscsi_cam_poll(struct cam_sim *sim)
 920 {
 921         struct vscsi_softc *sc = cam_sim_softc(sim);
 922
 923         vscsi_check_response_queue(sc);
 924 }
 925
 926 static void
 927 vscsi_check_response_queue(struct vscsi_softc *sc)
 928 {
 929         struct vscsi_crq *crq;
 930         struct vscsi_xfer *xp;
 931         int code;
 932
 933         mtx_assert(&sc->io_lock, MA_OWNED);
 934
 935         while (sc->crq_queue[sc->cur_crq].valid != 0) {
 936                 /* The hypercalls at both ends of this are not optimal */
 937                 phyp_hcall(H_VIO_SIGNAL, sc->unit, 0);
 938                 bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_POSTREAD);
 939
 940                 crq = &sc->crq_queue[sc->cur_crq];
 941
 942                 switch (crq->valid) {
 943                 case 0xc0:
 944                         if (crq->format == 0x02)
 945                                 sc->bus_initialized = 1;
 946                         break;
 947                 case 0x80:
 948                         /* IU data is set to tag pointer (the XP) */
 949                         xp = (struct vscsi_xfer *)crq->iu_data;
 950
 951                         switch (crq->format) {
 952                         case 0x01:
 953                                 code = *((uint8_t *)sc->srp_iu_queue +
 954                                     (uintptr_t)xp->srp_iu_offset);
 955                                 switch (code) {
 956                                 case SRP_RSP:
 957                                         vscsi_srp_response(xp, crq);
 958                                         break;
 959                                 case SRP_LOGIN_RSP:
 960                                         vscsi_login_response(xp, crq);
 961                                         break;
 962                                 default:
 963                                         device_printf(sc->dev, "Unknown SRP "
 964                                             "response code %d\n", code);
 965                                         break;
 966                                 }
 967                                 break;
 968                         case 0x02:
 969                                 /* Ignore management datagrams */
 970                                 break;
 971                         default:
 972                                 panic("Unknown CRQ format %d\n", crq->format);
 973                                 break;
 974                         }
 975                         vmem_free(sc->srp_iu_arena, xp->srp_iu_offset,
 976                             xp->srp_iu_size);
 977                         TAILQ_REMOVE(&sc->active_xferq, xp, queue);
 978                         TAILQ_INSERT_TAIL(&sc->free_xferq, xp, queue);
 979                         break;
 980                 default:
 981                         device_printf(sc->dev,
 982                             "Unknown CRQ message type %d\n", crq->valid);
 983                         break;
 984                 }
 985
 986                 crq->valid = 0;
 987                 sc->cur_crq = (sc->cur_crq + 1) % sc->n_crqs;
 988
 989                 bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 990                 phyp_hcall(H_VIO_SIGNAL, sc->unit, 1);
 991         }
 992 }
 993