sys/powerpc/pseries/phyp_vscsi.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright 2013 Nathan Whitehorn
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/kernel.h>
  35 #include <sys/malloc.h>
  36 #include <sys/module.h>
  37 #include <sys/selinfo.h>
  38 #include <sys/bus.h>
  39 #include <sys/conf.h>
  40 #include <sys/eventhandler.h>
  41 #include <sys/rman.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/bio.h>
  44 #include <sys/ioccom.h>
  45 #include <sys/uio.h>
  46 #include <sys/proc.h>
  47 #include <sys/signalvar.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/endian.h>
  50 #include <sys/vmem.h>
  51
  52 #include <cam/cam.h>
  53 #include <cam/cam_ccb.h>
  54 #include <cam/cam_debug.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/cam_sim.h>
  57 #include <cam/cam_xpt_periph.h>
  58 #include <cam/cam_xpt_sim.h>
  59 #include <cam/scsi/scsi_all.h>
  60 #include <cam/scsi/scsi_message.h>
  61
  62 #include <dev/ofw/openfirm.h>
  63 #include <dev/ofw/ofw_bus.h>
  64 #include <dev/ofw/ofw_bus_subr.h>
  65
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68
  69 #include <powerpc/pseries/phyp-hvcall.h>
  70
  71 struct vscsi_softc;
  72
  73 /* VSCSI CRQ format from table 260 of PAPR spec 2.4 (page 760) */
  74 struct vscsi_crq {
  75         uint8_t valid;
  76         uint8_t format;
  77         uint8_t reserved;
  78         uint8_t status;
  79         uint16_t timeout;
  80         uint16_t iu_length;
  81         uint64_t iu_data;
  82 };
  83
  84 struct vscsi_xfer {
  85         TAILQ_ENTRY(vscsi_xfer) queue;
  86         struct vscsi_softc *sc;
  87         union ccb *ccb;
  88         bus_dmamap_t dmamap;
  89         uint64_t tag;
  90
  91         vmem_addr_t srp_iu_offset;
  92         vmem_size_t srp_iu_size;
  93 };
  94
  95 TAILQ_HEAD(vscsi_xferq, vscsi_xfer);
  96
  97 struct vscsi_softc {
  98         device_t        dev;
  99         struct cam_devq *devq;
 100         struct cam_sim  *sim;
 101         struct cam_path *path;
 102         struct mtx io_lock;
 103
 104         cell_t          unit;
 105         int             bus_initialized;
 106         int             bus_logged_in;
 107         int             max_transactions;
 108
 109         int             irqid;
 110         struct resource *irq;
 111         void            *irq_cookie;
 112
 113         bus_dma_tag_t   crq_tag;
 114         struct vscsi_crq *crq_queue;
 115         int             n_crqs, cur_crq;
 116         bus_dmamap_t    crq_map;
 117         bus_addr_t      crq_phys;
 118
 119         vmem_t          *srp_iu_arena;
 120         void            *srp_iu_queue;
 121         bus_addr_t      srp_iu_phys;
 122
 123         bus_dma_tag_t   data_tag;
 124
 125         struct vscsi_xfer loginxp;
 126         struct vscsi_xfer *xfer;
 127         struct vscsi_xferq active_xferq;
 128         struct vscsi_xferq free_xferq;
 129 };
 130
 131 struct srp_login {
 132         uint8_t type;
 133         uint8_t reserved[7];
 134         uint64_t tag;
 135         uint64_t max_cmd_length;
 136         uint32_t reserved2;
 137         uint16_t buffer_formats;
 138         uint8_t flags;
 139         uint8_t reserved3[5];
 140         uint8_t initiator_port_id[16];
 141         uint8_t target_port_id[16];
 142 } __packed;
 143
 144 struct srp_login_rsp {
 145         uint8_t type;
 146         uint8_t reserved[3];
 147         uint32_t request_limit_delta;
 148         uint8_t tag;
 149         uint32_t max_i_to_t_len;
 150         uint32_t max_t_to_i_len;
 151         uint16_t buffer_formats;
 152         uint8_t flags;
 153         /* Some reserved bits follow */
 154 } __packed;
 155
 156 struct srp_cmd {
 157         uint8_t type;
 158         uint8_t flags1;
 159         uint8_t reserved[3];
 160         uint8_t formats;
 161         uint8_t out_buffer_count;
 162         uint8_t in_buffer_count;
 163         uint64_t tag;
 164         uint32_t reserved2;
 165         uint64_t lun;
 166         uint8_t reserved3[3];
 167         uint8_t additional_cdb;
 168         uint8_t cdb[16];
 169         uint8_t data_payload[0];
 170 } __packed;
 171
 172 struct srp_rsp {
 173         uint8_t type;
 174         uint8_t reserved[3];
 175         uint32_t request_limit_delta;
 176         uint64_t tag;
 177         uint16_t reserved2;
 178         uint8_t flags;
 179         uint8_t status;
 180         uint32_t data_out_resid;
 181         uint32_t data_in_resid;
 182         uint32_t sense_data_len;
 183         uint32_t response_data_len;
 184         uint8_t data_payload[0];
 185 } __packed;
 186
 187 struct srp_tsk_mgmt {
 188         uint8_t type;
 189         uint8_t reserved[7];
 190         uint64_t tag;
 191         uint32_t reserved2;
 192         uint64_t lun;
 193         uint8_t reserved3[2];
 194         uint8_t function;
 195         uint8_t reserved4;
 196         uint64_t manage_tag;
 197         uint64_t reserved5;
 198 } __packed;
 199
 200 /* Message code type */
 201 #define SRP_LOGIN_REQ   0x00
 202 #define SRP_TSK_MGMT    0x01
 203 #define SRP_CMD         0x02
 204 #define SRP_I_LOGOUT    0x03
 205
 206 #define SRP_LOGIN_RSP   0xC0
 207 #define SRP_RSP         0xC1
 208 #define SRP_LOGIN_REJ   0xC2
 209
 210 #define SRP_T_LOGOUT    0x80
 211 #define SRP_CRED_REQ    0x81
 212 #define SRP_AER_REQ     0x82
 213
 214 #define SRP_CRED_RSP    0x41
 215 #define SRP_AER_RSP     0x41
 216
 217 /* Flags for srp_rsp flags field */
 218 #define SRP_RSPVALID    0x01
 219 #define SRP_SNSVALID    0x02
 220 #define SRP_DOOVER      0x04
 221 #define SRP_DOUNDER     0x08
 222 #define SRP_DIOVER      0x10
 223 #define SRP_DIUNDER     0x20
 224
 225 #define MAD_SUCESS                      0x00
 226 #define MAD_NOT_SUPPORTED               0xf1
 227 #define MAD_FAILED                      0xf7
 228
 229 #define MAD_EMPTY_IU                    0x01
 230 #define MAD_ERROR_LOGGING_REQUEST       0x02
 231 #define MAD_ADAPTER_INFO_REQUEST        0x03
 232 #define MAD_CAPABILITIES_EXCHANGE       0x05
 233 #define MAD_PHYS_ADAP_INFO_REQUEST      0x06
 234 #define MAD_TAPE_PASSTHROUGH_REQUEST    0x07
 235 #define MAD_ENABLE_FAST_FAIL            0x08
 236
 237 static int      vscsi_probe(device_t);
 238 static int      vscsi_attach(device_t);
 239 static int      vscsi_detach(device_t);
 240 static void     vscsi_cam_action(struct cam_sim *, union ccb *);
 241 static void     vscsi_cam_poll(struct cam_sim *);
 242 static void     vscsi_intr(void *arg);
 243 static void     vscsi_check_response_queue(struct vscsi_softc *sc);
 244 static void     vscsi_setup_bus(struct vscsi_softc *sc);
 245
 246 static void     vscsi_srp_login(struct vscsi_softc *sc);
 247 static void     vscsi_crq_load_cb(void *, bus_dma_segment_t *, int, int);
 248 static void     vscsi_scsi_command(void *xxp, bus_dma_segment_t *segs,
 249                     int nsegs, int err);
 250 static void     vscsi_task_management(struct vscsi_softc *sc, union ccb *ccb);
 251 static void     vscsi_srp_response(struct vscsi_xfer *, struct vscsi_crq *);
 252
 253 static devclass_t       vscsi_devclass;
 254 static device_method_t  vscsi_methods[] = {
 255         DEVMETHOD(device_probe,         vscsi_probe),
 256         DEVMETHOD(device_attach,        vscsi_attach),
 257         DEVMETHOD(device_detach,        vscsi_detach),
 258
 259         DEVMETHOD_END
 260 };
 261 static driver_t vscsi_driver = {
 262         "vscsi",
 263         vscsi_methods,
 264         sizeof(struct vscsi_softc)
 265 };
 266 DRIVER_MODULE(vscsi, vdevice, vscsi_driver, vscsi_devclass, 0, 0);
 267 MALLOC_DEFINE(M_VSCSI, "vscsi", "CAM device queue for VSCSI");
 268
 269 static int
 270 vscsi_probe(device_t dev)
 271 {
 272
 273         if (!ofw_bus_is_compatible(dev, "IBM,v-scsi"))
 274                 return (ENXIO);
 275
 276         device_set_desc(dev, "POWER Hypervisor Virtual SCSI Bus");
 277         return (0);
 278 }
 279
 280 static int
 281 vscsi_attach(device_t dev)
 282 {
 283         struct vscsi_softc *sc;
 284         struct vscsi_xfer *xp;
 285         int error, i;
 286
 287         sc = device_get_softc(dev);
 288         if (sc == NULL)
 289                 return (EINVAL);
 290
 291         sc->dev = dev;
 292         mtx_init(&sc->io_lock, "vscsi", NULL, MTX_DEF);
 293
 294         /* Get properties */
 295         OF_getencprop(ofw_bus_get_node(dev), "reg", &sc->unit,
 296             sizeof(sc->unit));
 297
 298         /* Setup interrupt */
 299         sc->irqid = 0;
 300         sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irqid,
 301             RF_ACTIVE);
 302
 303         if (!sc->irq) {
 304                 device_printf(dev, "Could not allocate IRQ\n");
 305                 mtx_destroy(&sc->io_lock);
 306                 return (ENXIO);
 307         }
 308
 309         bus_setup_intr(dev, sc->irq, INTR_TYPE_CAM | INTR_MPSAFE |
 310             INTR_ENTROPY, NULL, vscsi_intr, sc, &sc->irq_cookie);
 311
 312         /* Data DMA */
 313         error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
 314             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
 315             256, BUS_SPACE_MAXSIZE_32BIT, 0, busdma_lock_mutex, &sc->io_lock,
 316             &sc->data_tag);
 317
 318         TAILQ_INIT(&sc->active_xferq);
 319         TAILQ_INIT(&sc->free_xferq);
 320
 321         /* First XFER for login data */
 322         sc->loginxp.sc = sc;
 323         bus_dmamap_create(sc->data_tag, 0, &sc->loginxp.dmamap);
 324         TAILQ_INSERT_TAIL(&sc->free_xferq, &sc->loginxp, queue);
 325
 326         /* CRQ area */
 327         error = bus_dma_tag_create(bus_get_dma_tag(dev), PAGE_SIZE, 0,
 328             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 8*PAGE_SIZE,
 329             1, BUS_SPACE_MAXSIZE, 0, NULL, NULL, &sc->crq_tag);
 330         error = bus_dmamem_alloc(sc->crq_tag, (void **)&sc->crq_queue,
 331             BUS_DMA_WAITOK | BUS_DMA_ZERO, &sc->crq_map);
 332         sc->crq_phys = 0;
 333         sc->n_crqs = 0;
 334         error = bus_dmamap_load(sc->crq_tag, sc->crq_map, sc->crq_queue,
 335             8*PAGE_SIZE, vscsi_crq_load_cb, sc, 0);
 336
 337         mtx_lock(&sc->io_lock);
 338         vscsi_setup_bus(sc);
 339         sc->xfer = malloc(sizeof(sc->xfer[0])*sc->max_transactions, M_VSCSI,
 340             M_NOWAIT);
 341         for (i = 0; i < sc->max_transactions; i++) {
 342                 xp = &sc->xfer[i];
 343                 xp->sc = sc;
 344
 345                 error = bus_dmamap_create(sc->data_tag, 0, &xp->dmamap);
 346                 if (error) {
 347                         device_printf(dev, "Could not create DMA map (%d)\n",
 348                             error);
 349                         break;
 350                 }
 351
 352                 TAILQ_INSERT_TAIL(&sc->free_xferq, xp, queue);
 353         }
 354         mtx_unlock(&sc->io_lock);
 355
 356         /* Allocate CAM bits */
 357         if ((sc->devq = cam_simq_alloc(sc->max_transactions)) == NULL)
 358                 return (ENOMEM);
 359
 360         sc->sim = cam_sim_alloc(vscsi_cam_action, vscsi_cam_poll, "vscsi", sc,
 361                                 device_get_unit(dev), &sc->io_lock,
 362                                 sc->max_transactions, sc->max_transactions,
 363                                 sc->devq);
 364         if (sc->sim == NULL) {
 365                 cam_simq_free(sc->devq);
 366                 sc->devq = NULL;
 367                 device_printf(dev, "CAM SIM attach failed\n");
 368                 return (EINVAL);
 369         }
 370
 371
 372         mtx_lock(&sc->io_lock);
 373         if (xpt_bus_register(sc->sim, dev, 0) != 0) {
 374                 device_printf(dev, "XPT bus registration failed\n");
 375                 cam_sim_free(sc->sim, FALSE);
 376                 sc->sim = NULL;
 377                 cam_simq_free(sc->devq);
 378                 sc->devq = NULL;
 379                 mtx_unlock(&sc->io_lock);
 380                 return (EINVAL);
 381         }
 382         mtx_unlock(&sc->io_lock);
 383
 384         return (0);
 385 }
 386
 387 static int
 388 vscsi_detach(device_t dev)
 389 {
 390         struct vscsi_softc *sc;
 391
 392         sc = device_get_softc(dev);
 393         if (sc == NULL)
 394                 return (EINVAL);
 395
 396         if (sc->sim != NULL) {
 397                 mtx_lock(&sc->io_lock);
 398                 xpt_bus_deregister(cam_sim_path(sc->sim));
 399                 cam_sim_free(sc->sim, FALSE);
 400                 sc->sim = NULL;
 401                 mtx_unlock(&sc->io_lock);
 402         }
 403
 404         if (sc->devq != NULL) {
 405                 cam_simq_free(sc->devq);
 406                 sc->devq = NULL;
 407         }
 408
 409         mtx_destroy(&sc->io_lock);
 410
 411         return (0);
 412 }
 413
 414 static void
 415 vscsi_cam_action(struct cam_sim *sim, union ccb *ccb)
 416 {
 417         struct vscsi_softc *sc = cam_sim_softc(sim);
 418
 419         mtx_assert(&sc->io_lock, MA_OWNED);
 420
 421         switch (ccb->ccb_h.func_code) {
 422         case XPT_PATH_INQ:
 423         {
 424                 struct ccb_pathinq *cpi = &ccb->cpi;
 425
 426                 cpi->version_num = 1;
 427                 cpi->hba_inquiry = PI_TAG_ABLE;
 428                 cpi->hba_misc = PIM_EXTLUNS;
 429                 cpi->target_sprt = 0;
 430                 cpi->hba_eng_cnt = 0;
 431                 cpi->max_target = 0;
 432                 cpi->max_lun = 0;
 433                 cpi->initiator_id = ~0;
 434                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 435                 strlcpy(cpi->hba_vid, "IBM", HBA_IDLEN);
 436                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 437                 cpi->unit_number = cam_sim_unit(sim);
 438                 cpi->bus_id = cam_sim_bus(sim);
 439                 cpi->base_transfer_speed = 150000;
 440                 cpi->transport = XPORT_SRP;
 441                 cpi->transport_version = 0;
 442                 cpi->protocol = PROTO_SCSI;
 443                 cpi->protocol_version = SCSI_REV_SPC4;
 444                 cpi->ccb_h.status = CAM_REQ_CMP;
 445                 break;
 446         }
 447         case XPT_RESET_BUS:
 448                 ccb->ccb_h.status = CAM_REQ_CMP;
 449                 break;
 450         case XPT_RESET_DEV:
 451                 ccb->ccb_h.status = CAM_REQ_INPROG;
 452                 vscsi_task_management(sc, ccb);
 453                 return;
 454         case XPT_GET_TRAN_SETTINGS:
 455                 ccb->cts.protocol = PROTO_SCSI;
 456                 ccb->cts.protocol_version = SCSI_REV_SPC4;
 457                 ccb->cts.transport = XPORT_SRP;
 458                 ccb->cts.transport_version = 0;
 459                 ccb->cts.proto_specific.valid = 0;
 460                 ccb->cts.xport_specific.valid = 0;
 461                 ccb->ccb_h.status = CAM_REQ_CMP;
 462                 break;
 463         case XPT_SET_TRAN_SETTINGS:
 464                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
 465                 break;
 466         case XPT_SCSI_IO:
 467         {
 468                 struct vscsi_xfer *xp;
 469
 470                 ccb->ccb_h.status = CAM_REQ_INPROG;
 471
 472                 xp = TAILQ_FIRST(&sc->free_xferq);
 473                 if (xp == NULL)
 474                         panic("SCSI queue flooded");
 475                 xp->ccb = ccb;
 476                 TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 477                 TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 478                 bus_dmamap_load_ccb(sc->data_tag, xp->dmamap,
 479                     ccb, vscsi_scsi_command, xp, 0);
 480
 481                 return;
 482         }
 483         default:
 484                 ccb->ccb_h.status = CAM_REQ_INVALID;
 485                 break;
 486         }
 487
 488         xpt_done(ccb);
 489         return;
 490 }
 491
 492 static void
 493 vscsi_srp_login(struct vscsi_softc *sc)
 494 {
 495         struct vscsi_xfer *xp;
 496         struct srp_login *login;
 497         struct vscsi_crq crq;
 498         int err;
 499
 500         mtx_assert(&sc->io_lock, MA_OWNED);
 501
 502         xp = TAILQ_FIRST(&sc->free_xferq);
 503         if (xp == NULL)
 504                 panic("SCSI queue flooded");
 505         xp->ccb = NULL;
 506         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 507         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 508
 509         /* Set up command */
 510         xp->srp_iu_size = crq.iu_length = 64;
 511         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 512             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 513         if (err)
 514                 panic("Error during VMEM allocation (%d)", err);
 515
 516         login = (struct srp_login *)((uint8_t *)xp->sc->srp_iu_queue +
 517             (uintptr_t)xp->srp_iu_offset);
 518         bzero(login, xp->srp_iu_size);
 519         login->type = SRP_LOGIN_REQ;
 520         login->tag = (uint64_t)(xp);
 521         login->max_cmd_length = htobe64(256);
 522         login->buffer_formats = htobe16(0x1 | 0x2); /* Direct and indirect */
 523         login->flags = 0;
 524
 525         /* Create CRQ entry */
 526         crq.valid = 0x80;
 527         crq.format = 0x01;
 528         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 529         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 530
 531         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 532             ((uint64_t *)(&crq))[1]);
 533         if (err != 0)
 534                 panic("CRQ send failure (%d)", err);
 535 }
 536
 537 static void
 538 vscsi_task_management(struct vscsi_softc *sc, union ccb *ccb)
 539 {
 540         struct srp_tsk_mgmt *cmd;
 541         struct vscsi_xfer *xp;
 542         struct vscsi_crq crq;
 543         int err;
 544
 545         mtx_assert(&sc->io_lock, MA_OWNED);
 546
 547         xp = TAILQ_FIRST(&sc->free_xferq);
 548         if (xp == NULL)
 549                 panic("SCSI queue flooded");
 550         xp->ccb = ccb;
 551         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 552         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 553
 554         xp->srp_iu_size = crq.iu_length = sizeof(*cmd);
 555         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 556             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 557         if (err)
 558                 panic("Error during VMEM allocation (%d)", err);
 559
 560         cmd = (struct srp_tsk_mgmt *)((uint8_t *)xp->sc->srp_iu_queue +
 561             (uintptr_t)xp->srp_iu_offset);
 562         bzero(cmd, xp->srp_iu_size);
 563         cmd->type = SRP_TSK_MGMT;
 564         cmd->tag = (uint64_t)xp;
 565         cmd->lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
 566
 567         switch (ccb->ccb_h.func_code) {
 568         case XPT_RESET_DEV:
 569                 cmd->function = 0x08;
 570                 break;
 571         default:
 572                 panic("Unimplemented code %d", ccb->ccb_h.func_code);
 573                 break;
 574         }
 575
 576         bus_dmamap_sync(xp->sc->crq_tag, xp->sc->crq_map, BUS_DMASYNC_PREWRITE);
 577
 578         /* Create CRQ entry */
 579         crq.valid = 0x80;
 580         crq.format = 0x01;
 581         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 582
 583         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 584             ((uint64_t *)(&crq))[1]);
 585         if (err != 0)
 586                 panic("CRQ send failure (%d)", err);
 587 }
 588
 589 static void
 590 vscsi_scsi_command(void *xxp, bus_dma_segment_t *segs, int nsegs, int err)
 591 {
 592         struct vscsi_xfer *xp = xxp;
 593         uint8_t *cdb;
 594         union ccb *ccb = xp->ccb;
 595         struct srp_cmd *cmd;
 596         uint64_t chunk_addr;
 597         uint32_t chunk_size;
 598         int desc_start, i;
 599         struct vscsi_crq crq;
 600
 601         KASSERT(err == 0, ("DMA error %d\n", err));
 602
 603         mtx_assert(&xp->sc->io_lock, MA_OWNED);
 604
 605         cdb = (ccb->ccb_h.flags & CAM_CDB_POINTER) ?
 606             ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes;
 607
 608         /* Command format from Table 20, page 37 of SRP spec */
 609         crq.iu_length = 48 + ((nsegs > 1) ? 20 : 16) +
 610             ((ccb->csio.cdb_len > 16) ? (ccb->csio.cdb_len - 16) : 0);
 611         xp->srp_iu_size = crq.iu_length;
 612         if (nsegs > 1)
 613                 xp->srp_iu_size += nsegs*16;
 614         xp->srp_iu_size = roundup(xp->srp_iu_size, 16);
 615         err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 616             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 617         if (err)
 618                 panic("Error during VMEM allocation (%d)", err);
 619
 620         cmd = (struct srp_cmd *)((uint8_t *)xp->sc->srp_iu_queue +
 621             (uintptr_t)xp->srp_iu_offset);
 622         bzero(cmd, xp->srp_iu_size);
 623         cmd->type = SRP_CMD;
 624         if (ccb->csio.cdb_len > 16)
 625                 cmd->additional_cdb = (ccb->csio.cdb_len - 16) << 2;
 626         memcpy(cmd->cdb, cdb, ccb->csio.cdb_len);
 627
 628         cmd->tag = (uint64_t)(xp); /* Let the responder find this again */
 629         cmd->lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
 630
 631         if (nsegs > 1) {
 632                 /* Use indirect descriptors */
 633                 switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 634                 case CAM_DIR_OUT:
 635                         cmd->formats = (2 << 4);
 636                         break;
 637                 case CAM_DIR_IN:
 638                         cmd->formats = 2;
 639                         break;
 640                 default:
 641                         panic("Does not support bidirectional commands (%d)",
 642                             ccb->ccb_h.flags & CAM_DIR_MASK);
 643                         break;
 644                 }
 645
 646                 desc_start = ((ccb->csio.cdb_len > 16) ?
 647                     ccb->csio.cdb_len - 16 : 0);
 648                 chunk_addr = xp->sc->srp_iu_phys + xp->srp_iu_offset + 20 +
 649                     desc_start + sizeof(*cmd);
 650                 chunk_size = 16*nsegs;
 651                 memcpy(&cmd->data_payload[desc_start], &chunk_addr, 8);
 652                 memcpy(&cmd->data_payload[desc_start+12], &chunk_size, 4);
 653                 chunk_size = 0;
 654                 for (i = 0; i < nsegs; i++)
 655                         chunk_size += segs[i].ds_len;
 656                 memcpy(&cmd->data_payload[desc_start+16], &chunk_size, 4);
 657                 desc_start += 20;
 658                 for (i = 0; i < nsegs; i++) {
 659                         chunk_addr = segs[i].ds_addr;
 660                         chunk_size = segs[i].ds_len;
 661
 662                         memcpy(&cmd->data_payload[desc_start + 16*i],
 663                             &chunk_addr, 8);
 664                         /* Set handle tag to 0 */
 665                         memcpy(&cmd->data_payload[desc_start + 16*i + 12],
 666                             &chunk_size, 4);
 667                 }
 668         } else if (nsegs == 1) {
 669                 switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 670                 case CAM_DIR_OUT:
 671                         cmd->formats = (1 << 4);
 672                         break;
 673                 case CAM_DIR_IN:
 674                         cmd->formats = 1;
 675                         break;
 676                 default:
 677                         panic("Does not support bidirectional commands (%d)",
 678                             ccb->ccb_h.flags & CAM_DIR_MASK);
 679                         break;
 680                 }
 681
 682                 /*
 683                  * Memory descriptor:
 684                  * 8 byte address
 685                  * 4 byte handle
 686                  * 4 byte length
 687                  */
 688
 689                 chunk_addr = segs[0].ds_addr;
 690                 chunk_size = segs[0].ds_len;
 691                 desc_start = ((ccb->csio.cdb_len > 16) ?
 692                     ccb->csio.cdb_len - 16 : 0);
 693
 694                 memcpy(&cmd->data_payload[desc_start], &chunk_addr, 8);
 695                 /* Set handle tag to 0 */
 696                 memcpy(&cmd->data_payload[desc_start+12], &chunk_size, 4);
 697                 KASSERT(xp->srp_iu_size >= 48 + ((ccb->csio.cdb_len > 16) ?
 698                     ccb->csio.cdb_len : 16), ("SRP IU command length"));
 699         } else {
 700                 cmd->formats = 0;
 701         }
 702         bus_dmamap_sync(xp->sc->crq_tag, xp->sc->crq_map, BUS_DMASYNC_PREWRITE);
 703
 704         /* Create CRQ entry */
 705         crq.valid = 0x80;
 706         crq.format = 0x01;
 707         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 708
 709         err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 710             ((uint64_t *)(&crq))[1]);
 711         if (err != 0)
 712                 panic("CRQ send failure (%d)", err);
 713 }
 714
 715 static void
 716 vscsi_crq_load_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int err)
 717 {
 718         struct vscsi_softc *sc = xsc;
 719
 720         sc->crq_phys = segs[0].ds_addr;
 721         sc->n_crqs = PAGE_SIZE/sizeof(struct vscsi_crq);
 722
 723         sc->srp_iu_queue = (uint8_t *)(sc->crq_queue);
 724         sc->srp_iu_phys = segs[0].ds_addr;
 725         sc->srp_iu_arena = vmem_create("VSCSI SRP IU", PAGE_SIZE,
 726             segs[0].ds_len - PAGE_SIZE, 16, 0, M_BESTFIT | M_NOWAIT);
 727 }
 728
 729 static void
 730 vscsi_setup_bus(struct vscsi_softc *sc)
 731 {
 732         struct vscsi_crq crq;
 733         struct vscsi_xfer *xp;
 734         int error;
 735
 736         struct {
 737                 uint32_t type;
 738                 uint16_t status;
 739                 uint16_t length;
 740                 uint64_t tag;
 741                 uint64_t buffer;
 742                 struct {
 743                         char srp_version[8];
 744                         char partition_name[96];
 745                         uint32_t partition_number;
 746                         uint32_t mad_version;
 747                         uint32_t os_type;
 748                         uint32_t port_max_txu[8];
 749                 } payload;
 750         } mad_adapter_info;
 751
 752         bzero(&crq, sizeof(crq));
 753
 754         /* Init message */
 755         crq.valid = 0xc0;
 756         crq.format = 0x01;
 757
 758         do {
 759                 error = phyp_hcall(H_FREE_CRQ, sc->unit);
 760         } while (error == H_BUSY);
 761
 762         /* See initialization sequence page 757 */
 763         bzero(sc->crq_queue, sc->n_crqs*sizeof(sc->crq_queue[0]));
 764         sc->cur_crq = 0;
 765         sc->bus_initialized = 0;
 766         sc->bus_logged_in = 0;
 767         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 768         error = phyp_hcall(H_REG_CRQ, sc->unit, sc->crq_phys,
 769             sc->n_crqs*sizeof(sc->crq_queue[0]));
 770         KASSERT(error == 0, ("CRQ registration success"));
 771
 772         error = phyp_hcall(H_SEND_CRQ, sc->unit, ((uint64_t *)(&crq))[0],
 773             ((uint64_t *)(&crq))[1]);
 774         if (error != 0)
 775                 panic("CRQ setup failure (%d)", error);
 776
 777         while (sc->bus_initialized == 0)
 778                 vscsi_check_response_queue(sc);
 779
 780         /* Send MAD adapter info */
 781         mad_adapter_info.type = MAD_ADAPTER_INFO_REQUEST;
 782         mad_adapter_info.status = 0;
 783         mad_adapter_info.length = sizeof(mad_adapter_info.payload);
 784
 785         strcpy(mad_adapter_info.payload.srp_version, "16.a");
 786         strcpy(mad_adapter_info.payload.partition_name, "UNKNOWN");
 787         mad_adapter_info.payload.partition_number = -1;
 788         mad_adapter_info.payload.mad_version = 1;
 789         mad_adapter_info.payload.os_type = 2; /* Claim we are Linux */
 790         mad_adapter_info.payload.port_max_txu[0] = 0;
 791         /* If this fails, we get the defaults above */
 792         OF_getprop(OF_finddevice("/"), "ibm,partition-name",
 793             mad_adapter_info.payload.partition_name,
 794             sizeof(mad_adapter_info.payload.partition_name));
 795         OF_getprop(OF_finddevice("/"), "ibm,partition-no",
 796             &mad_adapter_info.payload.partition_number,
 797             sizeof(mad_adapter_info.payload.partition_number));
 798
 799         xp = TAILQ_FIRST(&sc->free_xferq);
 800         xp->ccb = NULL;
 801         TAILQ_REMOVE(&sc->free_xferq, xp, queue);
 802         TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
 803         xp->srp_iu_size = crq.iu_length = sizeof(mad_adapter_info);
 804         vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
 805             M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
 806         mad_adapter_info.buffer = xp->sc->srp_iu_phys + xp->srp_iu_offset + 24;
 807         mad_adapter_info.tag = (uint64_t)xp;
 808         memcpy((uint8_t *)xp->sc->srp_iu_queue + (uintptr_t)xp->srp_iu_offset,
 809                 &mad_adapter_info, sizeof(mad_adapter_info));
 810         crq.valid = 0x80;
 811         crq.format = 0x02;
 812         crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
 813         bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 814         phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
 815             ((uint64_t *)(&crq))[1]);
 816
 817         while (TAILQ_EMPTY(&sc->free_xferq))
 818                 vscsi_check_response_queue(sc);
 819
 820         /* Send SRP login */
 821         vscsi_srp_login(sc);
 822         while (sc->bus_logged_in == 0)
 823                 vscsi_check_response_queue(sc);
 824
 825         error = phyp_hcall(H_VIO_SIGNAL, sc->unit, 1); /* Enable interrupts */
 826 }
 827
 828
 829 static void
 830 vscsi_intr(void *xsc)
 831 {
 832         struct vscsi_softc *sc = xsc;
 833
 834         mtx_lock(&sc->io_lock);
 835         vscsi_check_response_queue(sc);
 836         mtx_unlock(&sc->io_lock);
 837 }
 838
 839 static void
 840 vscsi_srp_response(struct vscsi_xfer *xp, struct vscsi_crq *crq)
 841 {
 842         union ccb *ccb = xp->ccb;
 843         struct vscsi_softc *sc = xp->sc;
 844         struct srp_rsp *rsp;
 845         uint32_t sense_len;
 846
 847         /* SRP response packet in original request */
 848         rsp = (struct srp_rsp *)((uint8_t *)sc->srp_iu_queue +
 849             (uintptr_t)xp->srp_iu_offset);
 850         ccb->csio.scsi_status = rsp->status;
 851         if (ccb->csio.scsi_status == SCSI_STATUS_OK)
 852                 ccb->ccb_h.status = CAM_REQ_CMP;
 853         else
 854                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
 855 #ifdef NOTYET
 856         /* Collect fast fail codes */
 857         if (crq->status != 0)
 858                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 859 #endif
 860
 861         if (ccb->ccb_h.status != CAM_REQ_CMP) {
 862                 ccb->ccb_h.status |= CAM_DEV_QFRZN;
 863                 xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
 864         }
 865
 866         if (!(rsp->flags & SRP_RSPVALID))
 867                 rsp->response_data_len = 0;
 868         if (!(rsp->flags & SRP_SNSVALID))
 869                 rsp->sense_data_len = 0;
 870         if (!(rsp->flags & (SRP_DOOVER | SRP_DOUNDER)))
 871                 rsp->data_out_resid = 0;
 872         if (!(rsp->flags & (SRP_DIOVER | SRP_DIUNDER)))
 873                 rsp->data_in_resid = 0;
 874
 875         if (rsp->flags & SRP_SNSVALID) {
 876                 bzero(&ccb->csio.sense_data, sizeof(struct scsi_sense_data));
 877                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 878                 sense_len = min(be32toh(rsp->sense_data_len),
 879                     ccb->csio.sense_len);
 880                 memcpy(&ccb->csio.sense_data,
 881                     &rsp->data_payload[be32toh(rsp->response_data_len)],
 882                     sense_len);
 883                 ccb->csio.sense_resid = ccb->csio.sense_len -
 884                     be32toh(rsp->sense_data_len);
 885         }
 886
 887         switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
 888         case CAM_DIR_OUT:
 889                 ccb->csio.resid = rsp->data_out_resid;
 890                 break;
 891         case CAM_DIR_IN:
 892                 ccb->csio.resid = rsp->data_in_resid;
 893                 break;
 894         }
 895
 896         bus_dmamap_sync(sc->data_tag, xp->dmamap, BUS_DMASYNC_POSTREAD);
 897         bus_dmamap_unload(sc->data_tag, xp->dmamap);
 898         xpt_done(ccb);
 899         xp->ccb = NULL;
 900 }
 901
 902 static void
 903 vscsi_login_response(struct vscsi_xfer *xp, struct vscsi_crq *crq)
 904 {
 905         struct vscsi_softc *sc = xp->sc;
 906         struct srp_login_rsp *rsp;
 907
 908         /* SRP response packet in original request */
 909         rsp = (struct srp_login_rsp *)((uint8_t *)sc->srp_iu_queue +
 910             (uintptr_t)xp->srp_iu_offset);
 911         KASSERT(be16toh(rsp->buffer_formats) & 0x3, ("Both direct and indirect "
 912             "buffers supported"));
 913
 914         sc->max_transactions = be32toh(rsp->request_limit_delta);
 915         device_printf(sc->dev, "Queue depth %d commands\n",
 916             sc->max_transactions);
 917         sc->bus_logged_in = 1;
 918 }
 919
 920 static void
 921 vscsi_cam_poll(struct cam_sim *sim)
 922 {
 923         struct vscsi_softc *sc = cam_sim_softc(sim);
 924
 925         vscsi_check_response_queue(sc);
 926 }
 927
 928 static void
 929 vscsi_check_response_queue(struct vscsi_softc *sc)
 930 {
 931         struct vscsi_crq *crq;
 932         struct vscsi_xfer *xp;
 933         int code;
 934
 935         mtx_assert(&sc->io_lock, MA_OWNED);
 936
 937         while (sc->crq_queue[sc->cur_crq].valid != 0) {
 938                 /* The hypercalls at both ends of this are not optimal */
 939                 phyp_hcall(H_VIO_SIGNAL, sc->unit, 0);
 940                 bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_POSTREAD);
 941
 942                 crq = &sc->crq_queue[sc->cur_crq];
 943
 944                 switch (crq->valid) {
 945                 case 0xc0:
 946                         if (crq->format == 0x02)
 947                                 sc->bus_initialized = 1;
 948                         break;
 949                 case 0x80:
 950                         /* IU data is set to tag pointer (the XP) */
 951                         xp = (struct vscsi_xfer *)crq->iu_data;
 952
 953                         switch (crq->format) {
 954                         case 0x01:
 955                                 code = *((uint8_t *)sc->srp_iu_queue +
 956                                     (uintptr_t)xp->srp_iu_offset);
 957                                 switch (code) {
 958                                 case SRP_RSP:
 959                                         vscsi_srp_response(xp, crq);
 960                                         break;
 961                                 case SRP_LOGIN_RSP:
 962                                         vscsi_login_response(xp, crq);
 963                                         break;
 964                                 default:
 965                                         device_printf(sc->dev, "Unknown SRP "
 966                                             "response code %d\n", code);
 967                                         break;
 968                                 }
 969                                 break;
 970                         case 0x02:
 971                                 /* Ignore management datagrams */
 972                                 break;
 973                         default:
 974                                 panic("Unknown CRQ format %d\n", crq->format);
 975                                 break;
 976                         }
 977                         vmem_free(sc->srp_iu_arena, xp->srp_iu_offset,
 978                             xp->srp_iu_size);
 979                         TAILQ_REMOVE(&sc->active_xferq, xp, queue);
 980                         TAILQ_INSERT_TAIL(&sc->free_xferq, xp, queue);
 981                         break;
 982                 default:
 983                         device_printf(sc->dev,
 984                             "Unknown CRQ message type %d\n", crq->valid);
 985                         break;
 986                 }
 987
 988                 crq->valid = 0;
 989                 sc->cur_crq = (sc->cur_crq + 1) % sc->n_crqs;
 990
 991                 bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
 992                 phyp_hcall(H_VIO_SIGNAL, sc->unit, 1);
 993         }
 994 }
 995