sys/dev/aic7xxx/aic79xx.c

   1 /*
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002 Justin T. Gibbs.
   5  * Copyright (c) 2000-2002 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#102 $
  41  *
  42  * $FreeBSD$
  43  */
  44
  45 #ifdef __linux__
  46 #include "aic79xx_osm.h"
  47 #include "aic79xx_inline.h"
  48 #include "aicasm/aicasm_insformat.h"
  49 #else
  50 #include <dev/aic7xxx/aic79xx_osm.h>
  51 #include <dev/aic7xxx/aic79xx_inline.h>
  52 #include <dev/aic7xxx/aicasm/aicasm_insformat.h>
  53 #endif
  54
  55 /******************************** Globals *************************************/
  56 struct ahd_softc_tailq ahd_tailq = TAILQ_HEAD_INITIALIZER(ahd_tailq);
  57
  58 /***************************** Lookup Tables **********************************/
  59 char *ahd_chip_names[] =
  60 {
  61         "NONE",
  62         "aic7901",
  63         "aic7902",
  64         "aic7901A"
  65 };
  66 static const u_int num_chip_names = NUM_ELEMENTS(ahd_chip_names);
  67
  68 /*
  69  * Hardware error codes.
  70  */
  71 struct ahd_hard_error_entry {
  72         uint8_t errno;
  73         char *errmesg;
  74 };
  75
  76 static struct ahd_hard_error_entry ahd_hard_errors[] = {
  77         { DSCTMOUT,     "Discard Timer has timed out" },
  78         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  79         { SQPARERR,     "Sequencer Parity Error" },
  80         { DPARERR,      "Data-path Parity Error" },
  81         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  82         { CIOPARERR,    "CIOBUS Parity Error" },
  83 };
  84 static const u_int num_errors = NUM_ELEMENTS(ahd_hard_errors);
  85
  86 static struct ahd_phase_table_entry ahd_phase_table[] =
  87 {
  88         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  89         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  90         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  91         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  92         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
  93         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
  94         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
  95         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  96         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
  97         { 0,            MSG_NOOP,               "in unknown phase"      }
  98 };
  99
 100 /*
 101  * In most cases we only wish to itterate over real phases, so
 102  * exclude the last element from the count.
 103  */
 104 static const u_int num_phases = NUM_ELEMENTS(ahd_phase_table) - 1;
 105
 106 /* Our Sequencer Program */
 107 #include "aic79xx_seq.h"
 108
 109 /**************************** Function Declarations ***************************/
 110 static void             ahd_handle_transmission_error(struct ahd_softc *ahd);
 111 static void             ahd_handle_lqiphase_error(struct ahd_softc *ahd,
 112                                                   u_int lqistat1);
 113 static int              ahd_handle_pkt_busfree(struct ahd_softc *ahd,
 114                                                u_int busfreetime);
 115 static int              ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
 116 static void             ahd_handle_proto_violation(struct ahd_softc *ahd);
 117 static void             ahd_force_renegotiation(struct ahd_softc *ahd,
 118                                                 struct ahd_devinfo *devinfo);
 119
 120 static struct ahd_tmode_tstate*
 121                         ahd_alloc_tstate(struct ahd_softc *ahd,
 122                                          u_int scsi_id, char channel);
 123 #ifdef AHD_TARGET_MODE
 124 static void             ahd_free_tstate(struct ahd_softc *ahd,
 125                                         u_int scsi_id, char channel, int force);
 126 #endif
 127 static void             ahd_devlimited_syncrate(struct ahd_softc *ahd,
 128                                                 struct ahd_initiator_tinfo *,
 129                                                 u_int *period,
 130                                                 u_int *ppr_options,
 131                                                 role_t role);
 132 static void             ahd_update_neg_table(struct ahd_softc *ahd,
 133                                              struct ahd_devinfo *devinfo,
 134                                              struct ahd_transinfo *tinfo);
 135 static void             ahd_update_pending_scbs(struct ahd_softc *ahd);
 136 static void             ahd_fetch_devinfo(struct ahd_softc *ahd,
 137                                           struct ahd_devinfo *devinfo);
 138 static void             ahd_print_devinfo(struct ahd_softc *ahd,
 139                                           struct ahd_devinfo *devinfo);
 140 static void             ahd_scb_devinfo(struct ahd_softc *ahd,
 141                                         struct ahd_devinfo *devinfo,
 142                                         struct scb *scb);
 143 static void             ahd_setup_initiator_msgout(struct ahd_softc *ahd,
 144                                                    struct ahd_devinfo *devinfo,
 145                                                    struct scb *scb);
 146 static void             ahd_build_transfer_msg(struct ahd_softc *ahd,
 147                                                struct ahd_devinfo *devinfo);
 148 static void             ahd_construct_sdtr(struct ahd_softc *ahd,
 149                                            struct ahd_devinfo *devinfo,
 150                                            u_int period, u_int offset);
 151 static void             ahd_construct_wdtr(struct ahd_softc *ahd,
 152                                            struct ahd_devinfo *devinfo,
 153                                            u_int bus_width);
 154 static void             ahd_construct_ppr(struct ahd_softc *ahd,
 155                                           struct ahd_devinfo *devinfo,
 156                                           u_int period, u_int offset,
 157                                           u_int bus_width, u_int ppr_options);
 158 static void             ahd_clear_msg_state(struct ahd_softc *ahd);
 159 static void             ahd_handle_message_phase(struct ahd_softc *ahd);
 160 typedef enum {
 161         AHDMSG_1B,
 162         AHDMSG_2B,
 163         AHDMSG_EXT
 164 } ahd_msgtype;
 165 static int              ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
 166                                      u_int msgval, int full);
 167 static int              ahd_parse_msg(struct ahd_softc *ahd,
 168                                       struct ahd_devinfo *devinfo);
 169 static int              ahd_handle_msg_reject(struct ahd_softc *ahd,
 170                                               struct ahd_devinfo *devinfo);
 171 static void             ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
 172                                                 struct ahd_devinfo *devinfo);
 173 static void             ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
 174 static void             ahd_handle_devreset(struct ahd_softc *ahd,
 175                                             struct ahd_devinfo *devinfo,
 176                                             cam_status status, char *message,
 177                                             int verbose_level);
 178 #if AHD_TARGET_MODE
 179 static void             ahd_setup_target_msgin(struct ahd_softc *ahd,
 180                                                struct ahd_devinfo *devinfo,
 181                                                struct scb *scb);
 182 #endif
 183
 184 static bus_size_t       ahd_sglist_size(struct ahd_softc *ahd);
 185 static bus_size_t       ahd_sglist_allocsize(struct ahd_softc *ahd);
 186 static bus_dmamap_callback_t
 187                         ahd_dmamap_cb;
 188 static void             ahd_initialize_hscbs(struct ahd_softc *ahd);
 189 static int              ahd_init_scbdata(struct ahd_softc *ahd);
 190 static void             ahd_fini_scbdata(struct ahd_softc *ahd);
 191 static void             ahd_setup_iocell_workaround(struct ahd_softc *ahd);
 192 static void             ahd_iocell_first_selection(struct ahd_softc *ahd);
 193 static void             ahd_add_col_list(struct ahd_softc *ahd,
 194                                          struct scb *scb, u_int col_idx);
 195 static void             ahd_rem_col_list(struct ahd_softc *ahd,
 196                                          struct scb *scb);
 197 static void             ahd_chip_init(struct ahd_softc *ahd);
 198 static void             ahd_qinfifo_requeue(struct ahd_softc *ahd,
 199                                             struct scb *prev_scb,
 200                                             struct scb *scb);
 201 static int              ahd_qinfifo_count(struct ahd_softc *ahd);
 202 static int              ahd_search_scb_list(struct ahd_softc *ahd, int target,
 203                                             char channel, int lun, u_int tag,
 204                                             role_t role, uint32_t status,
 205                                             ahd_search_action action,
 206                                             u_int *list_head, u_int tid);
 207 static void             ahd_stitch_tid_list(struct ahd_softc *ahd,
 208                                             u_int tid_prev, u_int tid_cur,
 209                                             u_int tid_next);
 210 static void             ahd_add_scb_to_free_list(struct ahd_softc *ahd,
 211                                                  u_int scbid);
 212 static u_int            ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
 213                                      u_int prev, u_int next, u_int tid);
 214 static void             ahd_reset_current_bus(struct ahd_softc *ahd);
 215 static ahd_callback_t   ahd_reset_poll;
 216 #ifdef AHD_DUMP_SEQ
 217 static void             ahd_dumpseq(struct ahd_softc *ahd);
 218 #endif
 219 static void             ahd_loadseq(struct ahd_softc *ahd);
 220 static int              ahd_check_patch(struct ahd_softc *ahd,
 221                                         struct patch **start_patch,
 222                                         u_int start_instr, u_int *skip_addr);
 223 static u_int            ahd_resolve_seqaddr(struct ahd_softc *ahd,
 224                                             u_int address);
 225 static void             ahd_download_instr(struct ahd_softc *ahd,
 226                                            u_int instrptr, uint8_t *dconsts);
 227 #ifdef AHD_TARGET_MODE
 228 static void             ahd_queue_lstate_event(struct ahd_softc *ahd,
 229                                                struct ahd_tmode_lstate *lstate,
 230                                                u_int initiator_id,
 231                                                u_int event_type,
 232                                                u_int event_arg);
 233 static void             ahd_update_scsiid(struct ahd_softc *ahd,
 234                                           u_int targid_mask);
 235 static int              ahd_handle_target_cmd(struct ahd_softc *ahd,
 236                                               struct target_cmd *cmd);
 237 #endif
 238
 239 /******************************** Private Inlines *****************************/
 240 static __inline void    ahd_assert_atn(struct ahd_softc *ahd);
 241 static __inline int     ahd_currently_packetized(struct ahd_softc *ahd);
 242 static __inline int     ahd_set_active_fifo(struct ahd_softc *ahd);
 243
 244 static __inline void
 245 ahd_assert_atn(struct ahd_softc *ahd)
 246 {
 247         ahd_outb(ahd, SCSISIGO, ATNO);
 248 }
 249
 250 /*
 251  * Determine if the current connection has a packetized
 252  * agreement.  This does not necessarily mean that we
 253  * are currently in a packetized transfer.  We could
 254  * just as easily be sending or receiving a message.
 255  */
 256 static __inline int
 257 ahd_currently_packetized(struct ahd_softc *ahd)
 258 {
 259         ahd_mode_state   saved_modes;
 260         int              packetized;
 261
 262         saved_modes = ahd_save_modes(ahd);
 263         if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
 264                 /*
 265                  * The packetized bit refers to the last
 266                  * connection, not the current one.  Check
 267                  * for non-zero LQISTATE instead.
 268                  */
 269                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 270                 packetized = ahd_inb(ahd, LQISTATE) != 0;
 271         } else {
 272                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 273                 packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
 274         }
 275         ahd_restore_modes(ahd, saved_modes);
 276         return (packetized);
 277 }
 278
 279 static __inline int
 280 ahd_set_active_fifo(struct ahd_softc *ahd)
 281 {
 282         u_int active_fifo;
 283
 284         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 285         active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
 286 /* XXX This is a three possition switch in the B. */
 287         switch (active_fifo) {
 288         case 0:
 289         case 1:
 290                 ahd_set_modes(ahd, active_fifo, active_fifo);
 291                 return (1);
 292         default:
 293                 return (0);
 294         }
 295 }
 296
 297 /************************* Sequencer Execution Control ************************/
 298 /*
 299  * Restart the sequencer program from address zero
 300  */
 301 void
 302 ahd_restart(struct ahd_softc *ahd)
 303 {
 304
 305         ahd_pause(ahd);
 306
 307         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 308
 309         /* No more pending messages */
 310         ahd_clear_msg_state(ahd);
 311         ahd_outb(ahd, SCSISIGO, 0);             /* De-assert BSY */
 312         ahd_outb(ahd, MSG_OUT, MSG_NOOP);       /* No message to send */
 313         ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
 314         ahd_outb(ahd, SEQINTCTL, 0);
 315         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
 316         ahd_outb(ahd, SEQ_FLAGS, 0);
 317         ahd_outb(ahd, SAVED_SCSIID, 0xFF);
 318         ahd_outb(ahd, SAVED_LUN, 0xFF);
 319
 320         /*
 321          * Ensure that the sequencer's idea of TQINPOS
 322          * matches our own.  The sequencer increments TQINPOS
 323          * only after it sees a DMA complete and a reset could
 324          * occur before the increment leaving the kernel to believe
 325          * the command arrived but the sequencer to not.
 326          */
 327         ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
 328
 329         /* Always allow reselection */
 330         ahd_outb(ahd, SCSISEQ1,
 331                  ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 332         /* Ensure that no DMA operations are in progress */
 333         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 334         ahd_outb(ahd, SCBHCNT, 0);
 335         ahd_outb(ahd, CCSCBCTL, CCSCBRESET);
 336
 337         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
 338         ahd_unpause(ahd);
 339 }
 340
 341 void
 342 ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
 343 {
 344         ahd_mode_state   saved_modes;
 345
 346 #ifdef AHD_DEBUG
 347         if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
 348                 printf("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
 349 #endif
 350         saved_modes = ahd_save_modes(ahd);
 351         ahd_set_modes(ahd, fifo, fifo);
 352         ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
 353         if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 354                 ahd_outb(ahd, CCSGCTL, CCSGRESET);
 355         ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 356         ahd_outb(ahd, SG_STATE, 0);
 357         ahd_restore_modes(ahd, saved_modes);
 358 }
 359
 360 /************************* Input/Output Queues ********************************/
 361 void
 362 ahd_run_qoutfifo(struct ahd_softc *ahd)
 363 {
 364         struct scb *scb;
 365         u_int  scb_index;
 366
 367         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
 368         while ((ahd->qoutfifo[ahd->qoutfifonext]
 369              & QOUTFIFO_ENTRY_VALID_LE) == ahd->qoutfifonext_valid_tag) {
 370
 371                 scb_index = ahd_le16toh(ahd->qoutfifo[ahd->qoutfifonext]
 372                                       & ~QOUTFIFO_ENTRY_VALID_LE);
 373                 scb = ahd_lookup_scb(ahd, scb_index);
 374                 if (scb == NULL) {
 375                         printf("%s: WARNING no command for scb %d "
 376                                "(cmdcmplt)\nQOUTPOS = %d\n",
 377                                ahd_name(ahd), scb_index,
 378                                ahd->qoutfifonext);
 379                         ahd_dump_card_state(ahd);
 380                 } else
 381                         ahd_complete_scb(ahd, scb);
 382
 383                 ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
 384                 if (ahd->qoutfifonext == 0)
 385                         ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID_LE;
 386
 387         }
 388 }
 389
 390 /************************* Interrupt Handling *********************************/
 391 void
 392 ahd_handle_hwerrint(struct ahd_softc *ahd)
 393 {
 394         /*
 395          * Some catastrophic hardware error has occurred.
 396          * Print it for the user and disable the controller.
 397          */
 398         int i;
 399         int error;
 400
 401         error = ahd_inb(ahd, ERROR);
 402         for (i = 0; i < num_errors; i++) {
 403                 if ((error & ahd_hard_errors[i].errno) != 0)
 404                         printf("%s: hwerrint, %s\n",
 405                                ahd_name(ahd), ahd_hard_errors[i].errmesg);
 406         }
 407
 408         ahd_dump_card_state(ahd);
 409         panic("BRKADRINT");
 410
 411         /* Tell everyone that this HBA is no longer availible */
 412         ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 413                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 414                        CAM_NO_HBA);
 415
 416         /* Tell the system that this controller has gone away. */
 417         ahd_free(ahd);
 418 }
 419
 420 void
 421 ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
 422 {
 423         u_int seqintcode;
 424
 425         /*
 426          * Save the sequencer interrupt code and clear the SEQINT
 427          * bit. We will unpause the sequencer, if appropriate,
 428          * after servicing the request.
 429          */
 430         seqintcode = ahd_inb(ahd, SEQINTCODE);
 431         ahd_outb(ahd, CLRINT, CLRSEQINT);
 432         ahd_update_modes(ahd);
 433 #ifdef AHD_DEBUG
 434         if ((ahd_debug & AHD_SHOW_MISC) != 0)
 435                 printf("%s: Handle Seqint Called for code %d\n",
 436                        ahd_name(ahd), seqintcode);
 437 #endif
 438         switch (seqintcode) {
 439         case ENTERING_NONPACK:
 440         {
 441                 struct  scb *scb;
 442                 u_int   scbid;
 443
 444                 AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 445                                  ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 446                 scbid = ahd_get_scbptr(ahd);
 447                 scb = ahd_lookup_scb(ahd, scbid);
 448                 if (scb == NULL) {
 449                         /*
 450                          * Somehow need to know if this
 451                          * is from a selection or reselection.
 452                          * From that, we can termine target
 453                          * ID so we at least have an I_T nexus.
 454                          */
 455                 } else {
 456                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
 457                         ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
 458                         ahd_outb(ahd, SEQ_FLAGS, 0x0);
 459                 }
 460                 if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
 461                  && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
 462                         /*
 463                          * Phase change after read stream with
 464                          * CRC error with P0 asserted on last
 465                          * packet.
 466                          */
 467                         printf("Assuming LQIPHASE_NLQ with P0 assertion\n");
 468                 }
 469                 printf("Entering NONPACK\n");
 470                 break;
 471         }
 472         case INVALID_SEQINT:
 473                 printf("%s: Invalid Sequencer interrupt occurred.\n",
 474                        ahd_name(ahd));
 475                 ahd_dump_card_state(ahd);
 476                 printf("invalid seqint");
 477                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 478                 break;
 479         case STATUS_OVERRUN:
 480         {
 481                 printf("%s: Status Overrun", ahd_name(ahd));
 482                 ahd_dump_card_state(ahd);
 483                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 484                 break;
 485         }
 486         case CFG4ISTAT_INTR:
 487         {
 488                 struct  scb *scb;
 489                 u_int   scbid;
 490
 491                 ahd_update_modes(ahd);
 492                 scbid = ahd_get_scbptr(ahd);
 493                 scb = ahd_lookup_scb(ahd, scbid);
 494                 if (scb == NULL) {
 495                         ahd_dump_card_state(ahd);
 496                         printf("CFG4ISTAT: Free SCB %d referenced", scbid);
 497                         panic("For safety");
 498                 }
 499                 ahd_outq(ahd, HADDR, scb->sense_busaddr);
 500                 ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
 501                 ahd_outb(ahd, HCNT + 2, 0);
 502                 ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
 503                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
 504                 break;
 505         }
 506         case ILLEGAL_PHASE:
 507         {
 508                 u_int bus_phase;
 509
 510                 bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
 511                 printf("%s: ILLEGAL_PHASE 0x%x\n",
 512                        ahd_name(ahd), bus_phase);
 513
 514                 switch (bus_phase) {
 515                 case P_DATAOUT:
 516                 case P_DATAIN:
 517                 case P_DATAOUT_DT:
 518                 case P_DATAIN_DT:
 519                 case P_MESGOUT:
 520                 case P_STATUS:
 521                 case P_MESGIN:
 522                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 523                         printf("%s: Issued Bus Reset.\n", ahd_name(ahd));
 524                         break;
 525                 case P_COMMAND:
 526                 {
 527                         struct  ahd_devinfo devinfo;
 528                         struct  scb *scb;
 529                         struct  ahd_initiator_tinfo *targ_info;
 530                         struct  ahd_tmode_tstate *tstate;
 531                         struct  ahd_transinfo *tinfo;
 532                         u_int   scbid;
 533
 534                         /*
 535                          * If a target takes us into the command phase
 536                          * assume that it has been externally reset and
 537                          * has thus lost our previous packetized negotiation
 538                          * agreement.  Since we have not sent an identify
 539                          * message and may not have fully qualified the
 540                          * connection, we change our command to TUR, assert
 541                          * ATN and ABORT the task when we go to message in
 542                          * phase.  The OSM will see the REQUEUE_REQUEST
 543                          * status and retry the command.
 544                          */
 545                         scbid = ahd_get_scbptr(ahd);
 546                         scb = ahd_lookup_scb(ahd, scbid);
 547                         if (scb == NULL) {
 548                                 printf("Invalid phase with no valid SCB.  "
 549                                        "Resetting bus.\n");
 550                                 ahd_reset_channel(ahd, 'A',
 551                                                   /*Initiate Reset*/TRUE);
 552                                 break;
 553                         }
 554                         ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
 555                                             SCB_GET_TARGET(ahd, scb),
 556                                             SCB_GET_LUN(scb),
 557                                             SCB_GET_CHANNEL(ahd, scb),
 558                                             ROLE_INITIATOR);
 559                         targ_info = ahd_fetch_transinfo(ahd,
 560                                                         devinfo.channel,
 561                                                         devinfo.our_scsiid,
 562                                                         devinfo.target,
 563                                                         &tstate);
 564                         tinfo = &targ_info->curr;
 565                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
 566                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
 567                                       /*paused*/TRUE);
 568                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
 569                                          /*offset*/0, /*ppr_options*/0,
 570                                          AHD_TRANS_ACTIVE, /*paused*/TRUE);
 571                         ahd_outb(ahd, SCB_CDB_STORE, 0);
 572                         ahd_outb(ahd, SCB_CDB_STORE+1, 0);
 573                         ahd_outb(ahd, SCB_CDB_STORE+2, 0);
 574                         ahd_outb(ahd, SCB_CDB_STORE+3, 0);
 575                         ahd_outb(ahd, SCB_CDB_STORE+4, 0);
 576                         ahd_outb(ahd, SCB_CDB_STORE+5, 0);
 577                         ahd_outb(ahd, SCB_CDB_LEN, 6);
 578                         scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
 579                         scb->hscb->control |= MK_MESSAGE;
 580                         ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
 581                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
 582                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
 583                         /*
 584                          * The lun is 0, regardless of the SCB's lun
 585                          * as we have not sent an identify message.
 586                          */
 587                         ahd_outb(ahd, SAVED_LUN, 0);
 588                         ahd_outb(ahd, SEQ_FLAGS, 0);
 589                         ahd_assert_atn(ahd);
 590                         scb->flags &= ~(SCB_PACKETIZED);
 591                         scb->flags |= SCB_ABORT|SCB_CMDPHASE_ABORT;
 592                         ahd_freeze_devq(ahd, scb);
 593                         ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
 594                         ahd_freeze_scb(scb);
 595
 596                         /*
 597                          * Allow the sequencer to continue with
 598                          * non-pack processing.
 599                          */
 600                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 601                         ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
 602                         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
 603                                 ahd_outb(ahd, CLRLQOINT1, 0);
 604                         }
 605                         printf("Continuing non-pack processing...\n");
 606                         break;
 607                 }
 608                 }
 609                 break;
 610         }
 611         case CFG4OVERRUN:
 612                 printf("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
 613                        ahd_inb(ahd, MODE_PTR));
 614                 break;
 615         case DUMP_CARD_STATE:
 616         {
 617                 ahd_dump_card_state(ahd);
 618                 break;
 619         }
 620         case PDATA_REINIT:
 621         {
 622                 printf("%s: PDATA_REINIT - DFCNTRL = 0x%x "
 623                        "SG_CACHE_SHADOW = 0x%x\n",
 624                        ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
 625                        ahd_inb(ahd, SG_CACHE_SHADOW));
 626                 ahd_reinitialize_dataptrs(ahd);
 627                 break;
 628         }
 629         case HOST_MSG_LOOP:
 630         {
 631                 struct ahd_devinfo devinfo;
 632
 633                 /*
 634                  * The sequencer has encountered a message phase
 635                  * that requires host assistance for completion.
 636                  * While handling the message phase(s), we will be
 637                  * notified by the sequencer after each byte is
 638                  * transfered so we can track bus phase changes.
 639                  *
 640                  * If this is the first time we've seen a HOST_MSG_LOOP
 641                  * interrupt, initialize the state of the host message
 642                  * loop.
 643                  */
 644                 ahd_fetch_devinfo(ahd, &devinfo);
 645                 if (ahd->msg_type == MSG_TYPE_NONE) {
 646                         struct scb *scb;
 647                         u_int scb_index;
 648                         u_int bus_phase;
 649
 650                         bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
 651                         if (bus_phase != P_MESGIN
 652                          && bus_phase != P_MESGOUT) {
 653                                 printf("ahd_intr: HOST_MSG_LOOP bad "
 654                                        "phase 0x%x\n",
 655                                       bus_phase);
 656                                 /*
 657                                  * Probably transitioned to bus free before
 658                                  * we got here.  Just punt the message.
 659                                  */
 660                                 ahd_dump_card_state(ahd);
 661                                 ahd_clear_intstat(ahd);
 662                                 ahd_restart(ahd);
 663                                 return;
 664                         }
 665
 666                         scb_index = ahd_get_scbptr(ahd);
 667                         scb = ahd_lookup_scb(ahd, scb_index);
 668                         if (devinfo.role == ROLE_INITIATOR) {
 669                                 if (bus_phase == P_MESGOUT)
 670                                         ahd_setup_initiator_msgout(ahd,
 671                                                                    &devinfo,
 672                                                                    scb);
 673                                 else {
 674                                         ahd->msg_type =
 675                                             MSG_TYPE_INITIATOR_MSGIN;
 676                                         ahd->msgin_index = 0;
 677                                 }
 678                         }
 679 #if AHD_TARGET_MODE
 680                         else {
 681                                 if (bus_phase == P_MESGOUT) {
 682                                         ahd->msg_type =
 683                                             MSG_TYPE_TARGET_MSGOUT;
 684                                         ahd->msgin_index = 0;
 685                                 }
 686                                 else
 687                                         ahd_setup_target_msgin(ahd,
 688                                                                &devinfo,
 689                                                                scb);
 690                         }
 691 #endif
 692                 }
 693
 694                 ahd_handle_message_phase(ahd);
 695                 break;
 696         }
 697         case NO_MATCH:
 698         {
 699                 /* Ensure we don't leave the selection hardware on */
 700                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 701                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
 702
 703                 printf("%s:%c:%d: no active SCB for reconnecting "
 704                        "target - issuing BUS DEVICE RESET\n",
 705                        ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
 706                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
 707                        "REG0 == 0x%x ACCUM = 0x%x\n",
 708                        ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
 709                        ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
 710                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
 711                        "SINDEX == 0x%x\n",
 712                        ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
 713                        ahd_find_busy_tcl(ahd,
 714                                          BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
 715                                                    ahd_inb(ahd, SAVED_LUN))),
 716                        ahd_inw(ahd, SINDEX));
 717                 printf("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
 718                        "SCB_CONTROL == 0x%x\n",
 719                        ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
 720                        ahd_inb_scbram(ahd, SCB_LUN),
 721                        ahd_inb_scbram(ahd, SCB_CONTROL));
 722                 printf("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
 723                        ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
 724                 printf("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
 725                 printf("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
 726                 ahd_dump_card_state(ahd);
 727                 ahd->msgout_buf[0] = MSG_BUS_DEV_RESET;
 728                 ahd->msgout_len = 1;
 729                 ahd->msgout_index = 0;
 730                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 731                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
 732                 ahd_assert_atn(ahd);
 733                 break;
 734         }
 735         case PROTO_VIOLATION:
 736         {
 737                 ahd_handle_proto_violation(ahd);
 738                 break;
 739         }
 740         case IGN_WIDE_RES:
 741         {
 742                 struct ahd_devinfo devinfo;
 743
 744                 ahd_fetch_devinfo(ahd, &devinfo);
 745                 ahd_handle_ign_wide_residue(ahd, &devinfo);
 746                 break;
 747         }
 748         case BAD_PHASE:
 749         {
 750                 u_int lastphase;
 751
 752                 lastphase = ahd_inb(ahd, LASTPHASE);
 753                 printf("%s:%c:%d: unknown scsi bus phase %x, "
 754                        "lastphase = 0x%x.  Attempting to continue\n",
 755                        ahd_name(ahd), 'A',
 756                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
 757                        lastphase, ahd_inb(ahd, SCSISIGI));
 758                 break;
 759         }
 760         case MISSED_BUSFREE:
 761         {
 762                 u_int lastphase;
 763
 764                 lastphase = ahd_inb(ahd, LASTPHASE);
 765                 printf("%s:%c:%d: Missed busfree. "
 766                        "Lastphase = 0x%x, Curphase = 0x%x\n",
 767                        ahd_name(ahd), 'A',
 768                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
 769                        lastphase, ahd_inb(ahd, SCSISIGI));
 770                 ahd_restart(ahd);
 771                 return;
 772         }
 773         case DATA_OVERRUN:
 774         {
 775                 /*
 776                  * When the sequencer detects an overrun, it
 777                  * places the controller in "BITBUCKET" mode
 778                  * and allows the target to complete its transfer.
 779                  * Unfortunately, none of the counters get updated
 780                  * when the controller is in this mode, so we have
 781                  * no way of knowing how large the overrun was.
 782                  */
 783                 struct  scb *scb;
 784                 u_int   scbindex = ahd_get_scbptr(ahd);
 785                 u_int   lastphase = ahd_inb(ahd, LASTPHASE);
 786
 787                 scb = ahd_lookup_scb(ahd, scbindex);
 788                 ahd_print_path(ahd, scb);
 789                 printf("data overrun detected %s."
 790                        "  Tag == 0x%x.\n",
 791                        ahd_lookup_phase_entry(lastphase)->phasemsg,
 792                        SCB_GET_TAG(scb));
 793                 ahd_print_path(ahd, scb);
 794                 printf("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
 795                        ahd_inb(ahd, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
 796                        ahd_get_transfer_length(scb), scb->sg_count);
 797                 ahd_dump_sglist(scb);
 798
 799                 /*
 800                  * Set this and it will take effect when the
 801                  * target does a command complete.
 802                  */
 803                 ahd_freeze_devq(ahd, scb);
 804                 ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
 805                 ahd_freeze_scb(scb);
 806                 break;
 807         }
 808         case MKMSG_FAILED:
 809         {
 810                 struct ahd_devinfo devinfo;
 811                 struct scb *scb;
 812                 u_int scbid;
 813
 814                 ahd_fetch_devinfo(ahd, &devinfo);
 815                 printf("%s:%c:%d:%d: Attempt to issue message failed\n",
 816                        ahd_name(ahd), devinfo.channel, devinfo.target,
 817                        devinfo.lun);
 818                 scbid = ahd_get_scbptr(ahd);
 819                 scb = ahd_lookup_scb(ahd, scbid);
 820                 if (scb != NULL
 821                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
 822                         /*
 823                          * Ensure that we didn't put a second instance of this
 824                          * SCB into the QINFIFO.
 825                          */
 826                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
 827                                            SCB_GET_CHANNEL(ahd, scb),
 828                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
 829                                            ROLE_INITIATOR, /*status*/0,
 830                                            SEARCH_REMOVE);
 831                 ahd_outb(ahd, SCB_CONTROL,
 832                          ahd_inb(ahd, SCB_CONTROL) & ~MK_MESSAGE);
 833                 break;
 834         }
 835         default:
 836                 printf("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
 837                        seqintcode);
 838                 break;
 839         }
 840         /*
 841          *  The sequencer is paused immediately on
 842          *  a SEQINT, so we should restart it when
 843          *  we're done.
 844          */
 845         ahd_unpause(ahd);
 846 }
 847
 848 void
 849 ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
 850 {
 851         struct scb      *scb;
 852         u_int            status0;
 853         u_int            status3;
 854         u_int            status;
 855         u_int            lqistat1;
 856         u_int            lqostat0;
 857         u_int            scbid;
 858
 859         ahd_update_modes(ahd);
 860         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 861
 862         status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
 863         status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
 864         status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
 865         lqistat1 = ahd_inb(ahd, LQISTAT1);
 866         lqostat0 = ahd_inb(ahd, LQOSTAT0);
 867         if ((status0 & (SELDI|SELDO)) != 0) {
 868                 u_int simode0;
 869
 870                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 871                 simode0 = ahd_inb(ahd, SIMODE0);
 872                 status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
 873                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 874         }
 875         scbid = ahd_get_scbptr(ahd);
 876         scb = ahd_lookup_scb(ahd, scbid);
 877         if (scb != NULL
 878          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
 879                 scb = NULL;
 880
 881         /* Make sure the sequencer is in a safe location. */
 882         ahd_clear_critical_section(ahd);
 883
 884         if ((status0 & IOERR) != 0) {
 885                 u_int now_lvd;
 886
 887                 now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
 888                 printf("%s: Transceiver State Has Changed to %s mode\n",
 889                        ahd_name(ahd), now_lvd ? "LVD" : "SE");
 890                 ahd_outb(ahd, CLRSINT0, CLRIOERR);
 891                 /*
 892                  * A change in I/O mode is equivalent to a bus reset.
 893                  */
 894                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
 895                 ahd_pause(ahd);
 896                 ahd_setup_iocell_workaround(ahd);
 897                 ahd_unpause(ahd);
 898         } else if ((status0 & OVERRUN) != 0) {
 899                 printf("%s: SCSI offset overrun detected.  Resetting bus.\n",
 900                        ahd_name(ahd));
 901                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 902         } else if ((status & SCSIRSTI) != 0) {
 903                 printf("%s: Someone reset channel A\n", ahd_name(ahd));
 904                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
 905         } else if ((status & SCSIPERR) != 0) {
 906                 ahd_handle_transmission_error(ahd);
 907         } else if (lqostat0 != 0) {
 908                 printf("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
 909                 ahd_outb(ahd, CLRLQOINT0, lqostat0);
 910                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
 911                         ahd_outb(ahd, CLRLQOINT1, 0);
 912                 }
 913         } else if ((status & SELTO) != 0) {
 914                 u_int scbid;
 915
 916                 /* Stop the selection */
 917                 ahd_outb(ahd, SCSISEQ0, 0);
 918
 919                 /* No more pending messages */
 920                 ahd_clear_msg_state(ahd);
 921
 922                 /* Clear interrupt state */
 923                 ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
 924
 925                 /*
 926                  * Although the driver does not care about the
 927                  * 'Selection in Progress' status bit, the busy
 928                  * LED does.  SELINGO is only cleared by a sucessfull
 929                  * selection, so we must manually clear it to insure
 930                  * the LED turns off just incase no future successful
 931                  * selections occur (e.g. no devices on the bus).
 932                  */
 933                 ahd_outb(ahd, CLRSINT0, CLRSELINGO);
 934
 935                 scbid = ahd_inw(ahd, WAITING_TID_HEAD);
 936 #ifdef AHD_DEBUG
 937                 if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
 938                         ahd_print_path(ahd, scb);
 939                         printf("Saw Selection Timeout for SCB 0x%x\n", scbid);
 940                 }
 941 #endif
 942                 scb = ahd_lookup_scb(ahd, scbid);
 943                 if (scb == NULL) {
 944                         printf("%s: ahd_intr - referenced scb not "
 945                                "valid during SELTO scb(0x%x)\n",
 946                                ahd_name(ahd), scbid);
 947                         ahd_dump_card_state(ahd);
 948                         panic("For diagnostics");
 949                 } else {
 950                         ahd_set_transaction_status(scb, CAM_SEL_TIMEOUT);
 951                         ahd_freeze_devq(ahd, scb);
 952                 }
 953                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
 954                 ahd_iocell_first_selection(ahd);
 955                 ahd_restart(ahd);
 956         } else if ((status0 & (SELDI|SELDO)) != 0) {
 957                 ahd_iocell_first_selection(ahd);
 958                 ahd_unpause(ahd);
 959         } else if (status3 != 0) {
 960                 printf("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
 961                        ahd_name(ahd), status3);
 962                 ahd_outb(ahd, CLRSINT3, status3);
 963         } else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
 964                 ahd_handle_lqiphase_error(ahd, lqistat1);
 965         } else if ((status & BUSFREE) != 0) {
 966                 u_int busfreetime;
 967                 u_int lqostat1;
 968                 int   restart;
 969                 int   clear_fifo;
 970                 int   packetized;
 971                 u_int mode;
 972
 973                 /*
 974                  * Clear our selection hardware as soon as possible.
 975                  * We may have an entry in the waiting Q for this target,
 976                  * that is affected by this busfree and we don't want to
 977                  * go about selecting the target while we handle the event.
 978                  */
 979                 ahd_outb(ahd, SCSISEQ0, 0);
 980
 981                 /*
 982                  * Determine what we were up to at the time of
 983                  * the busfree.
 984                  */
 985                 mode = AHD_MODE_SCSI;
 986                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 987                 busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
 988                 lqostat1 = ahd_inb(ahd, LQOSTAT1);
 989                 switch (busfreetime) {
 990                 case BUSFREE_DFF0:
 991                 case BUSFREE_DFF1:
 992                 {
 993                         u_int   scbid;
 994                         struct  scb *scb;
 995
 996                         mode = busfreetime == BUSFREE_DFF0
 997                              ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
 998                         ahd_set_modes(ahd, mode, mode);
 999                         scbid = ahd_get_scbptr(ahd);
1000                         scb = ahd_lookup_scb(ahd, scbid);
1001                         if (scb == NULL) {
1002                                 printf("%s: Invalid SCB in DFF%d "
1003                                        "during unexpected busfree\n",
1004                                        ahd_name(ahd), mode);
1005                                 packetized = 0;
1006                         } else
1007                                 packetized = (scb->flags & SCB_PACKETIZED) != 0;
1008                         clear_fifo = 1;
1009                         break;
1010                 }
1011                 case BUSFREE_LQO:
1012                         clear_fifo = 0;
1013                         packetized = 1;
1014                         break;
1015                 default:
1016                         clear_fifo = 0;
1017                         packetized =  (lqostat1 & LQOBUSFREE) != 0;
1018                         if (!packetized
1019                          && ahd_inb(ahd, LASTPHASE) == P_BUSFREE)
1020                                 packetized = 1;
1021                         break;
1022                 }
1023
1024 #ifdef AHD_DEBUG
1025                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
1026                         printf("Saw Busfree.  Busfreetime = 0x%x.\n",
1027                                busfreetime);
1028 #endif
1029                 /*
1030                  * Busfrees that occur in non-packetized phases are
1031                  * handled by the nonpkt_busfree handler.
1032                  */
1033                 if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
1034                         restart = ahd_handle_pkt_busfree(ahd, busfreetime);
1035                 } else {
1036                         restart = ahd_handle_nonpkt_busfree(ahd);
1037                 }
1038                 /*
1039                  * Clear the busfree interrupt status.  The setting of
1040                  * the interrupt is a pulse, so we do not need to muck
1041                  * with the ENBUSFREE logic.  This also ensures that if
1042                  * the bus has moved on to another connection, busfree
1043                  * protection is still in force.
1044                  */
1045                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1046
1047                 if (clear_fifo)
1048                         ahd_clear_fifo(ahd, mode);
1049
1050                 ahd_clear_msg_state(ahd);
1051                 ahd_clear_intstat(ahd);
1052                 if (restart) {
1053                         ahd_restart(ahd);
1054                 } else {
1055                         ahd_unpause(ahd);
1056                 }
1057         } else {
1058                 printf("%s: Missing case in ahd_handle_scsiint. status = %x\n",
1059                        ahd_name(ahd), status);
1060                 ahd_dump_card_state(ahd);
1061                 ahd_clear_intstat(ahd);
1062                 ahd_unpause(ahd);
1063         }
1064 }
1065
1066 static void
1067 ahd_handle_transmission_error(struct ahd_softc *ahd)
1068 {
1069         u_int            lqistat1;
1070         u_int            lqistat2;
1071         u_int            msg_out;
1072         u_int            curphase;
1073         u_int            lastphase;
1074         u_int            perrdiag;
1075
1076         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1077         lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
1078         lqistat2 = ahd_inb(ahd, LQISTAT2);
1079         if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
1080          && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
1081                 u_int lqistate;
1082
1083                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1084                 lqistate = ahd_inb(ahd, LQISTATE);
1085                 if ((lqistate >= 0x1E && lqistate <= 0x24)
1086                  || (lqistate == 0x29)) {
1087                         printf("%s: NLQCRC found via LQISTATE\n",
1088                                ahd_name(ahd));
1089                         lqistat1 |= LQICRCI_NLQ;
1090                 }
1091                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1092         }
1093
1094         ahd_outb(ahd, CLRLQIINT1, lqistat1);
1095         lastphase = ahd_inb(ahd, LASTPHASE);
1096         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1097         perrdiag = ahd_inb(ahd, PERRDIAG);
1098         msg_out = MSG_INITIATOR_DET_ERR;
1099         ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
1100         printf("%s: Transmission error detected\n", ahd_name(ahd));
1101         printf("%s: lqistat1 == 0x%x, LASTPHASE == 0x0%x, "
1102                "curphase = 0x%x, perrdiag == 0x%x\n",
1103                ahd_name(ahd), lqistat1, lastphase, curphase, perrdiag);
1104         ahd_dump_card_state(ahd);
1105         if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
1106                 printf("%s: Gross protocol error during incoming "
1107                        "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
1108                        ahd_name(ahd), lqistat1);
1109                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1110                 return;
1111         } else if ((lqistat1 & LQICRCI_LQ) != 0) {
1112                 /*
1113                  * A CRC error has been detected on an incoming LQ.
1114                  * The bus is currently hung on the last ACK.
1115                  * Hit LQIRETRY to release the last ack, and
1116                  * wait for the sequencer to determine that ATNO
1117                  * is asserted while in message out to take us
1118                  * to our host message loop.  No NONPACKREQ or
1119                  * LQIPHASE type errors will occur in this
1120                  * scenario.  After this first LQIRETRY, the LQI
1121                  * manager will be in ISELO where it will
1122                  * happily sit until another packet phase begins.
1123                  * Unexpected bus free detection is enabled
1124                  * through any phases that occur after we release
1125                  * this last ack until the LQI manager sees a
1126                  * packet phase.  This implies we may have to
1127                  * ignore a perfectly valid "unexected busfree"
1128                  * after our "initiator detected error" message is
1129                  * sent.  A busfree is the expected response after
1130                  * we tell the target that it's L_Q was corrupted.
1131                  * (SPI4R09 10.7.3.3.3)
1132                  */
1133                 ahd_outb(ahd, LQCTL2, LQIRETRY);
1134                 printf("LQIRetry for LQICRCI_LQ to release ACK\n");
1135         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
1136                 u_int scbid;
1137                 struct scb *scb;
1138
1139                 /*
1140                  * We detected a CRC error in a NON-LQ packet.
1141                  * The hardware has varying behavior in this situation
1142                  * depending on whether this packet was part of a
1143                  * stream or not.
1144                  *
1145                  * PKT by PKT mode:
1146                  * The hardware has already acked the complete packet.
1147                  * If the target honors our outstanding ATN condition,
1148                  * we should be (or soon will be) in MSGOUT phase.
1149                  * This will trigger the LQIPHASE_LQ status bit as the
1150                  * hardware was expecting another LQ.  Unexpected
1151                  * busfree detection is enabled.  Once LQIPHASE_LQ is
1152                  * true (first entry into host message loop is much
1153                  * the same), we must clear LQIPHASE_LQ and hit
1154                  * LQIRETRY so the hardware is ready to handle
1155                  * a future LQ.  NONPACKREQ will not be asserted again
1156                  * once we hit LQIRETRY until another packet is
1157                  * processed.  The target may either go busfree
1158                  * or start another packet in response to our message.
1159                  *
1160                  * Read Streaming P0 asserted:
1161                  * If we raise ATN and the target completes the entire
1162                  * stream (P0 asserted during the last packet), the
1163                  * hardware will ack all data and return to the ISTART
1164                  * state.  When the target reponds to our ATN condition,
1165                  * LQIPHASE_LQ will be asserted.  We should respond to
1166                  * this with an LQIRETRY to prepare for any future
1167                  * packets.  NONPACKREQ will not be asserted again
1168                  * once we hit LQIRETRY until another packet is
1169                  * processed.  The target may either go busfree or
1170                  * start another packet in response to our message.
1171                  * Busfree detection is enabled.
1172                  *
1173                  * Read Streaming P0 not asserted:
1174                  * If we raise ATN and the target transitions to
1175                  * MSGOUT in or after a packet where P0 is not
1176                  * asserted, the hardware will assert LQIPHASE_NLQ.
1177                  * We should respond to the LQIPHASE_NLQ with an
1178                  * LQICONTINUE.  Should the target stay in a non-pkt
1179                  * phase after we send our message, the hardware
1180                  * will assert LQIPHASE_LQ.  Recovery is then just as
1181                  * listed above for the read streaming with P0 asserted.
1182                  * Busfree detection is enabled.
1183                  */
1184                 printf("LQICRC_NLQ\n");
1185                 ahd_set_active_fifo(ahd);
1186                 scbid = ahd_get_scbptr(ahd);
1187                 scb = ahd_lookup_scb(ahd, scbid);
1188                 if (scb == NULL) {
1189                         printf("%s: No SCB valid for LQICRC_NLQ.  "
1190                                "Resetting bus\n", ahd_name(ahd));
1191                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1192                         return;
1193                 }
1194                 scb->flags |= SCB_TRANSMISSION_ERROR;
1195         } else if ((lqistat1 & LQIBADLQI) != 0) {
1196                 printf("Need to handle BADLQI!\n");
1197                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1198                 return;
1199         } else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
1200                 if ((curphase & ~P_DATAIN_DT) != 0) {
1201                         /* Ack the byte.  So we can continue. */
1202                         printf("Acking %s to clear perror\n",
1203                                ahd_lookup_phase_entry(curphase)->phasemsg);
1204                         ahd_inb(ahd, SCSIDAT);
1205                 }
1206
1207                 if (curphase == P_MESGIN)
1208                         msg_out = MSG_PARITY_ERROR;
1209         }
1210
1211         /*
1212          * We've set the hardware to assert ATN if we
1213          * get a parity error on "in" phases, so all we
1214          * need to do is stuff the message buffer with
1215          * the appropriate message.  "In" phases have set
1216          * mesg_out to something other than MSG_NOP.
1217          */
1218         ahd->send_msg_perror = msg_out;
1219         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1220         ahd_outb(ahd, CLRINT, CLRSCSIINT);
1221         ahd_unpause(ahd);
1222 }
1223
1224 static void
1225 ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
1226 {
1227         /*
1228          * Clear the sources of the interrupts.
1229          */
1230         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1231         ahd_outb(ahd, CLRLQIINT1, lqistat1);
1232
1233         /*
1234          * If the "illegal" phase changes were in response
1235          * to our ATN to flag a CRC error, AND we ended up
1236          * on packet boundaries, clear the error, restart the
1237          * LQI manager as appropriate, and go on our merry
1238          * way toward sending the message.  Otherwise, reset
1239          * the bus to clear the error.
1240          */
1241         ahd_set_active_fifo(ahd);
1242         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
1243          && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
1244                 if ((lqistat1 & LQIPHASE_LQ) != 0) {
1245                         printf("LQIRETRY for LQIPHASE_LQ\n");
1246                         ahd_outb(ahd, LQCTL2, LQIRETRY);
1247                 } else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
1248                         printf("LQICONTINUE for LQIPHASE_NLQ\n");
1249                         ahd_outb(ahd, LQCTL2, LQIRETRY);
1250                 } else
1251                         panic("ahd_handle_lqiphase_error: No phase errors\n");
1252                 ahd_dump_card_state(ahd);
1253                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
1254                 ahd_unpause(ahd);
1255         } else {
1256                 printf("Reseting Channel for LQI Phase error\n");
1257                 ahd_dump_card_state(ahd);
1258                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1259         }
1260 }
1261
1262 /*
1263  * Packetized unexpected or expected busfree.
1264  * Entered in MODE_SCSI.
1265  */
1266 static int
1267 ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
1268 {
1269         u_int lqostat1;
1270
1271         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
1272                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
1273         lqostat1 = ahd_inb(ahd, LQOSTAT1);
1274         if ((lqostat1 & LQOBUSFREE) != 0) {
1275                 struct scb *scb;
1276                 u_int scbid;
1277                 u_int waiting_h;
1278                 u_int waiting_t;
1279                 u_int next;
1280
1281                 if ((busfreetime & BUSFREE_LQO) == 0)
1282                         printf("%s: Warning, BUSFREE time is 0x%x.  "
1283                                "Expected BUSFREE_LQO.\n",
1284                                ahd_name(ahd), busfreetime);
1285
1286                 scbid = ahd_get_scbptr(ahd);
1287                 scb = ahd_lookup_scb(ahd, scbid);
1288                 if (scb == NULL)
1289                        panic("SCB not valid during LQOBUSFREE");
1290                 ahd_print_path(ahd, scb);
1291                 printf("Probable outgoing LQ CRC error.  Retrying command\n");
1292
1293                 /*
1294                  * Return the LQO manager to its idle loop.  It will
1295                  * not do this automatically if the busfree occurs
1296                  * after the first REQ of either the LQ or command
1297                  * packet or between the LQ and command packet.
1298                  */
1299                 ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
1300
1301                 /*
1302                  * Clear the status.
1303                  */
1304                 ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
1305                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
1306                         ahd_outb(ahd, CLRLQOINT1, 0);
1307                 }
1308                 /*
1309                  * The LQO manager detected an unexpected busfree
1310                  * either:
1311                  *
1312                  * 1) During an outgoing LQ.
1313                  * 2) After an outgoing LQ but before the first
1314                  *    REQ of the command packet.
1315                  * 3) During an outgoing command packet.
1316                  *
1317                  * In all cases, CURRSCB is pointing to the
1318                  * SCB that encountered the failure.  Clean
1319                  * up the queue, clear SELDO and LQOBUSFREE,
1320                  * and allow the sequencer to restart the select
1321                  * out at its lesure.
1322                  */
1323                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
1324                 ahd_outb(ahd, CLRSINT0, CLRSELDO);
1325                 waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
1326                 if (waiting_h != scbid) {
1327
1328                         ahd_outw(ahd, WAITING_TID_HEAD, scbid);
1329                         waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
1330                         next = SCB_LIST_NULL;
1331                         if (waiting_t == waiting_h) {
1332                                 ahd_outw(ahd, WAITING_TID_TAIL, scbid);
1333                         } else {
1334                                 ahd_set_scbptr(ahd, waiting_h);
1335                                 next = ahd_inw(ahd, SCB_NEXT2);
1336                         }
1337                         ahd_set_scbptr(ahd, scbid);
1338                         ahd_outw(ahd, SCB_NEXT2, next);
1339                 }
1340
1341                 /* Return unpausing the sequencer. */
1342                 return (0);
1343         }
1344         if (ahd->src_mode != AHD_MODE_SCSI) {
1345                 u_int   scbid;
1346                 struct  scb *scb;
1347
1348                 scbid = ahd_get_scbptr(ahd);
1349                 scb = ahd_lookup_scb(ahd, scbid);
1350                 ahd_print_path(ahd, scb);
1351                 printf("Unexpected PKT busfree condition\n");
1352                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
1353                                SCB_GET_LUN(scb), SCB_GET_TAG(scb),
1354                                ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
1355
1356                 /* Return restarting the sequencer. */
1357                 return (1);
1358         }
1359         printf("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
1360         ahd_dump_card_state(ahd);
1361         /* Restart the sequencer. */
1362         return (1);
1363 }
1364
1365 /*
1366  * Non-packetized unexpected or expected busfree.
1367  */
1368 static int
1369 ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
1370 {
1371         struct  ahd_devinfo devinfo;
1372         struct  scb *scb;
1373         u_int   lastphase;
1374         u_int   saved_scsiid;
1375         u_int   saved_lun;
1376         u_int   target;
1377         u_int   initiator_role_id;
1378         u_int   scbid;
1379         int     printerror;
1380
1381         /*
1382          * Look at what phase we were last in.  If its message out,
1383          * chances are pretty good that the busfree was in response
1384          * to one of our abort requests.
1385          */
1386         lastphase = ahd_inb(ahd, LASTPHASE);
1387         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
1388         saved_lun = ahd_inb(ahd, SAVED_LUN);
1389         target = SCSIID_TARGET(ahd, saved_scsiid);
1390         initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1391         ahd_compile_devinfo(&devinfo, initiator_role_id,
1392                             target, saved_lun, 'A', ROLE_INITIATOR);
1393         printerror = 1;
1394
1395         scbid = ahd_get_scbptr(ahd);
1396         scb = ahd_lookup_scb(ahd, scbid);
1397         if (scb != NULL
1398          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1399                 scb = NULL;
1400
1401         if (lastphase == P_MESGOUT) {
1402                 u_int tag;
1403
1404                 tag = SCB_LIST_NULL;
1405                 if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT_TAG, TRUE)
1406                  || ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT, TRUE)) {
1407                         int found;
1408                         int sent_msg;
1409
1410                         if (scb == NULL) {
1411                                 ahd_print_devinfo(ahd, &devinfo);
1412                                 printf("Abort for unidentified "
1413                                        "connection completed.\n");
1414                                 /* restart the sequencer. */
1415                                 return (1);
1416                         }
1417                         sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
1418                         ahd_print_path(ahd, scb);
1419                         printf("SCB %d - Abort%s Completed.\n",
1420                                SCB_GET_TAG(scb),
1421                                sent_msg == MSG_ABORT_TAG ? "" : " Tag");
1422
1423                         if (sent_msg == MSG_ABORT_TAG)
1424                                 tag = SCB_GET_TAG(scb);
1425
1426                         if ((scb->flags & SCB_CMDPHASE_ABORT) != 0) {
1427                                 /*
1428                                  * This abort is in response to an
1429                                  * unexpected switch to command phase
1430                                  * for a packetized connection.  Since
1431                                  * the identify message was never sent,
1432                                  * "saved lun" is 0.  We really want to
1433                                  * abort only the SCB that encountered
1434                                  * this error, which could have a different
1435                                  * lun.  The SCB will be retried so the OS
1436                                  * will see the UA after renegotiating to
1437                                  * packetized.
1438                                  */
1439                                 tag = SCB_GET_TAG(scb);
1440                                 saved_lun = scb->hscb->lun;
1441                         }
1442                         found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
1443                                                tag, ROLE_INITIATOR,
1444                                                CAM_REQ_ABORTED);
1445                         printf("found == 0x%x\n", found);
1446                         printerror = 0;
1447                 } else if (ahd_sent_msg(ahd, AHDMSG_1B,
1448                                         MSG_BUS_DEV_RESET, TRUE)) {
1449 #ifdef __FreeBSD__
1450                         /*
1451                          * Don't mark the user's request for this BDR
1452                          * as completing with CAM_BDR_SENT.  CAM3
1453                          * specifies CAM_REQ_CMP.
1454                          */
1455                         if (scb != NULL
1456                          && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
1457                          && ahd_match_scb(ahd, scb, target, 'A',
1458                                           CAM_LUN_WILDCARD, SCB_LIST_NULL,
1459                                           ROLE_INITIATOR))
1460                                 ahd_set_transaction_status(scb, CAM_REQ_CMP);
1461 #endif
1462                         ahd_handle_devreset(ahd, &devinfo, CAM_BDR_SENT,
1463                                             "Bus Device Reset",
1464                                             /*verbose_level*/0);
1465                         printerror = 0;
1466                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, FALSE)) {
1467                         struct ahd_initiator_tinfo *tinfo;
1468                         struct ahd_tmode_tstate *tstate;
1469
1470                         /*
1471                          * PPR Rejected.  Try non-ppr negotiation
1472                          * and retry command.
1473                          */
1474                         tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
1475                                                     devinfo.our_scsiid,
1476                                                     devinfo.target, &tstate);
1477                         tinfo->curr.transport_version = 2;
1478                         tinfo->goal.transport_version = 2;
1479                         tinfo->goal.ppr_options = 0;
1480                         ahd_qinfifo_requeue_tail(ahd, scb);
1481                         printerror = 0;
1482                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, FALSE)
1483                         || ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, FALSE)) {
1484                         /*
1485                          * Negotiation Rejected.  Go-async and
1486                          * retry command.
1487                          */
1488                         ahd_set_width(ahd, &devinfo,
1489                                       MSG_EXT_WDTR_BUS_8_BIT,
1490                                       AHD_TRANS_CUR|AHD_TRANS_GOAL,
1491                                       /*paused*/TRUE);
1492                         ahd_set_syncrate(ahd, &devinfo,
1493                                         /*period*/0, /*offset*/0,
1494                                         /*ppr_options*/0,
1495                                         AHD_TRANS_CUR|AHD_TRANS_GOAL,
1496                                         /*paused*/TRUE);
1497                         ahd_qinfifo_requeue_tail(ahd, scb);
1498                         printerror = 0;
1499                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
1500                         && ahd_sent_msg(ahd, AHDMSG_1B,
1501                                          MSG_INITIATOR_DET_ERR, TRUE)) {
1502
1503 #ifdef AHD_DEBUG
1504                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
1505                                 printf("Expected IDE Busfree\n");
1506 #endif
1507                         printerror = 0;
1508                 }
1509         }
1510
1511         /*
1512          * The busfree required flag is honored at the end of
1513          * the message phases.  We check it last in case we
1514          * had to send some other message that caused a busfree.
1515          */
1516         if (printerror != 0
1517          && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
1518          && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
1519
1520                 ahd_freeze_devq(ahd, scb);
1521                 ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
1522                 ahd_freeze_scb(scb);
1523                 if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
1524                         ahd_print_path(ahd, scb);
1525                         printf("Now %spacketized.\n",
1526                                (scb->flags & SCB_PACKETIZED) == 0
1527                              ? "" : "non-");
1528                         ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
1529                                        SCB_GET_CHANNEL(ahd, scb),
1530                                        SCB_GET_LUN(scb), SCB_LIST_NULL,
1531                                        ROLE_INITIATOR, CAM_REQ_ABORTED);
1532                 } else {
1533 #ifdef AHD_DEBUG
1534                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
1535                                 printf("PPR Negotiation Busfree.\n");
1536 #endif
1537                         ahd_done(ahd, scb);
1538                 }
1539                 printerror = 0;
1540         }
1541         if (printerror != 0) {
1542                 int aborted;
1543
1544                 aborted = 0;
1545                 if (scb != NULL) {
1546                         u_int tag;
1547
1548                         if ((scb->hscb->control & TAG_ENB) != 0)
1549                                 tag = SCB_GET_TAG(scb);
1550                         else
1551                                 tag = SCB_LIST_NULL;
1552                         ahd_print_path(ahd, scb);
1553                         aborted = ahd_abort_scbs(ahd, target, 'A',
1554                                        SCB_GET_LUN(scb), tag,
1555                                        ROLE_INITIATOR,
1556                                        CAM_UNEXP_BUSFREE);
1557                 } else {
1558                         /*
1559                          * We had not fully identified this connection,
1560                          * so we cannot abort anything.
1561                          */
1562                         printf("%s: ", ahd_name(ahd));
1563                 }
1564                 if (lastphase != P_BUSFREE)
1565                         ahd_force_renegotiation(ahd, &devinfo);
1566                 printf("Unexpected busfree %s, %d SCBs aborted, "
1567                        "PRGMCNT == 0x%x\n",
1568                        ahd_lookup_phase_entry(lastphase)->phasemsg,
1569                        aborted,
1570                        ahd_inb(ahd, PRGMCNT)
1571                         | (ahd_inb(ahd, PRGMCNT+1) << 8));
1572                 ahd_dump_card_state(ahd);
1573         }
1574         /* Always restart the sequencer. */
1575         return (1);
1576 }
1577
1578 static void
1579 ahd_handle_proto_violation(struct ahd_softc *ahd)
1580 {
1581         struct  ahd_devinfo devinfo;
1582         struct  scb *scb;
1583         u_int   scbid;
1584         u_int   seq_flags;
1585         u_int   curphase;
1586         u_int   lastphase;
1587         int     found;
1588
1589         ahd_fetch_devinfo(ahd, &devinfo);
1590         scbid = ahd_get_scbptr(ahd);
1591         scb = ahd_lookup_scb(ahd, scbid);
1592         seq_flags = ahd_inb(ahd, SEQ_FLAGS);
1593         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1594         lastphase = ahd_inb(ahd, LASTPHASE);
1595         if ((seq_flags & NOT_IDENTIFIED) != 0) {
1596
1597                 /*
1598                  * The reconnecting target either did not send an
1599                  * identify message, or did, but we didn't find an SCB
1600                  * to match.
1601                  */
1602                 ahd_print_devinfo(ahd, &devinfo);
1603                 printf("Target did not send an IDENTIFY message. "
1604                        "LASTPHASE = 0x%x.\n", lastphase);
1605                 scb = NULL;
1606         } else if (scb == NULL) {
1607                 /*
1608                  * We don't seem to have an SCB active for this
1609                  * transaction.  Print an error and reset the bus.
1610                  */
1611                 ahd_print_devinfo(ahd, &devinfo);
1612                 printf("No SCB found during protocol violation\n");
1613                 goto proto_violation_reset;
1614         } else {
1615                 ahd_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
1616                 if ((seq_flags & NO_CDB_SENT) != 0) {
1617                         ahd_print_path(ahd, scb);
1618                         printf("No or incomplete CDB sent to device.\n");
1619                 } else if ((ahd_inb(ahd, SCB_CONTROL) & STATUS_RCVD) == 0) {
1620                         /*
1621                          * The target never bothered to provide status to
1622                          * us prior to completing the command.  Since we don't
1623                          * know the disposition of this command, we must attempt
1624                          * to abort it.  Assert ATN and prepare to send an abort
1625                          * message.
1626                          */
1627                         ahd_print_path(ahd, scb);
1628                         printf("Completed command without status.\n");
1629                 } else {
1630                         ahd_print_path(ahd, scb);
1631                         printf("Unknown protocol violation.\n");
1632                         ahd_dump_card_state(ahd);
1633                 }
1634         }
1635         if ((lastphase & ~P_DATAIN_DT) == 0) {
1636 proto_violation_reset:
1637                 /*
1638                  * Target either went directly to data
1639                  * phase or didn't respond to our ATN.
1640                  * The only safe thing to do is to blow
1641                  * it away with a bus reset.
1642                  */
1643                 found = ahd_reset_channel(ahd, 'A', TRUE);
1644                 printf("%s: Issued Channel %c Bus Reset. "
1645                        "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
1646         } else {
1647                 /*
1648                  * Leave the selection hardware off in case
1649                  * this abort attempt will affect yet to
1650                  * be sent commands.
1651                  */
1652                 ahd_outb(ahd, SCSISEQ0,
1653                          ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
1654                 ahd_assert_atn(ahd);
1655                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
1656                 if (scb == NULL) {
1657                         ahd_print_devinfo(ahd, &devinfo);
1658                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
1659                         ahd->msgout_len = 1;
1660                         ahd->msgout_index = 0;
1661                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1662                 } else {
1663                         ahd_print_path(ahd, scb);
1664                         scb->flags |= SCB_ABORT;
1665                 }
1666                 printf("Protocol violation %s.  Attempting to abort.\n",
1667                        ahd_lookup_phase_entry(curphase)->phasemsg);
1668         }
1669 }
1670
1671 /*
1672  * Force renegotiation to occur the next time we initiate
1673  * a command to the current device.
1674  */
1675 static void
1676 ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
1677 {
1678         struct  ahd_initiator_tinfo *targ_info;
1679         struct  ahd_tmode_tstate *tstate;
1680
1681         printf("Forcing renegotiation (%d:%c:%d)\n",
1682                devinfo->our_scsiid, devinfo->channel,
1683                devinfo->target);
1684         targ_info = ahd_fetch_transinfo(ahd,
1685                                         devinfo->channel,
1686                                         devinfo->our_scsiid,
1687                                         devinfo->target,
1688                                         &tstate);
1689         ahd_update_neg_request(ahd, devinfo, tstate,
1690                                targ_info, /*force*/TRUE);
1691 }
1692
1693 #define AHD_MAX_STEPS 2000
1694 void
1695 ahd_clear_critical_section(struct ahd_softc *ahd)
1696 {
1697         ahd_mode_state  saved_modes;
1698         int             stepping;
1699         int             steps;
1700
1701         if (ahd->num_critical_sections == 0)
1702                 return;
1703
1704         stepping = FALSE;
1705         steps = 0;
1706         saved_modes = ahd_save_modes(ahd);
1707         for (;;) {
1708                 struct  cs *cs;
1709                 u_int   seqaddr;
1710                 u_int   i;
1711
1712                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1713                 seqaddr = ahd_inb(ahd, CURADDR)
1714                         | (ahd_inb(ahd, CURADDR+1) << 8);
1715
1716                 cs = ahd->critical_sections;
1717                 for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
1718
1719                         if (cs->begin < seqaddr && cs->end >= seqaddr)
1720                                 break;
1721                 }
1722
1723                 if (i == ahd->num_critical_sections)
1724                         break;
1725
1726                 if (steps > AHD_MAX_STEPS) {
1727                         printf("%s: Infinite loop in critical section\n",
1728                                ahd_name(ahd));
1729                         ahd_dump_card_state(ahd);
1730                         panic("critical section loop");
1731                 }
1732
1733                 steps++;
1734                 if (stepping == FALSE) {
1735
1736                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1737                         ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
1738                         stepping = TRUE;
1739                 }
1740                 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
1741                 ahd_outb(ahd, HCNTRL, ahd->unpause);
1742                 do {
1743                         ahd_delay(200);
1744                 } while (!ahd_is_paused(ahd));
1745                 ahd_update_modes(ahd);
1746         }
1747         if (stepping) {
1748                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1749                 ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
1750         }
1751         ahd_restore_modes(ahd, saved_modes);
1752 }
1753
1754 /*
1755  * Clear any pending interrupt status.
1756  */
1757 void
1758 ahd_clear_intstat(struct ahd_softc *ahd)
1759 {
1760         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
1761                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
1762         /* Clear any interrupt conditions this may have caused */
1763         ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
1764                                  |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
1765         ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
1766                                  |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
1767                                  |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
1768         ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
1769                                  |CLRLQOATNPKT|CLRLQOTCRC);
1770         ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
1771                                  |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
1772         ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
1773         ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
1774                                 |CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
1775         ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO|CLRIOERR);
1776         ahd_outb(ahd, CLRINT, CLRSCSIINT);
1777 }
1778
1779 /**************************** Debugging Routines ******************************/
1780 #ifdef AHD_DEBUG
1781 uint32_t ahd_debug = AHD_DEBUG_OPTS;
1782 #endif
1783 void
1784 ahd_print_scb(struct scb *scb)
1785 {
1786         struct hardware_scb *hscb;
1787         int i;
1788
1789         hscb = scb->hscb;
1790         printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
1791                (void *)scb,
1792                hscb->control,
1793                hscb->scsiid,
1794                hscb->lun,
1795                hscb->cdb_len);
1796         printf("Shared Data: ");
1797         for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
1798                 printf("%#02x", hscb->shared_data.idata.cdb[i]);
1799         printf("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
1800                (uint32_t)((ahd_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
1801                (uint32_t)(ahd_le64toh(hscb->dataptr) & 0xFFFFFFFF),
1802                ahd_le32toh(hscb->datacnt),
1803                ahd_le32toh(hscb->sgptr),
1804                SCB_GET_TAG(scb));
1805         ahd_dump_sglist(scb);
1806 }
1807
1808 void
1809 ahd_dump_sglist(struct scb *scb)
1810 {
1811         int i;
1812
1813         if (scb->sg_count > 0) {
1814                 if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
1815                         struct ahd_dma64_seg *sg_list;
1816
1817                         sg_list = (struct ahd_dma64_seg*)scb->sg_list;
1818                         for (i = 0; i < scb->sg_count; i++) {
1819                                 uint64_t addr;
1820
1821                                 addr = ahd_le64toh(sg_list[i].addr);
1822                                 printf("sg[%d] - Addr 0x%x%x : Length %d\n",
1823                                        i,
1824                                        (uint32_t)((addr >> 32) & 0xFFFFFFFF),
1825                                        (uint32_t)(addr & 0xFFFFFFFF),
1826                                        ahd_le32toh(sg_list[i].len));
1827                         }
1828                 } else {
1829                         struct ahd_dma_seg *sg_list;
1830
1831                         sg_list = (struct ahd_dma_seg*)scb->sg_list;
1832                         for (i = 0; i < scb->sg_count; i++) {
1833                                 printf("sg[%d] - Addr 0x%x%x : Length %d\n",
1834                                        i,
1835                                        (ahd_le32toh(sg_list[i].len) >> 24
1836                                         & SG_HIGH_ADDR_BITS),
1837                                        ahd_le32toh(sg_list[i].addr),
1838                                        ahd_le32toh(sg_list[i].len)
1839                                        & AHD_SG_LEN_MASK);
1840                         }
1841                 }
1842         }
1843 }
1844
1845 /************************* Transfer Negotiation *******************************/
1846 /*
1847  * Allocate per target mode instance (ID we respond to as a target)
1848  * transfer negotiation data structures.
1849  */
1850 static struct ahd_tmode_tstate *
1851 ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
1852 {
1853         struct ahd_tmode_tstate *master_tstate;
1854         struct ahd_tmode_tstate *tstate;
1855         int i;
1856
1857         master_tstate = ahd->enabled_targets[ahd->our_id];
1858         if (ahd->enabled_targets[scsi_id] != NULL
1859          && ahd->enabled_targets[scsi_id] != master_tstate)
1860                 panic("%s: ahd_alloc_tstate - Target already allocated",
1861                       ahd_name(ahd));
1862         tstate = malloc(sizeof(*tstate), M_DEVBUF, M_NOWAIT);
1863         if (tstate == NULL)
1864                 return (NULL);
1865
1866         /*
1867          * If we have allocated a master tstate, copy user settings from
1868          * the master tstate (taken from SRAM or the EEPROM) for this
1869          * channel, but reset our current and goal settings to async/narrow
1870          * until an initiator talks to us.
1871          */
1872         if (master_tstate != NULL) {
1873                 memcpy(tstate, master_tstate, sizeof(*tstate));
1874                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
1875                 for (i = 0; i < 16; i++) {
1876                         memset(&tstate->transinfo[i].curr, 0,
1877                               sizeof(tstate->transinfo[i].curr));
1878                         memset(&tstate->transinfo[i].goal, 0,
1879                               sizeof(tstate->transinfo[i].goal));
1880                 }
1881         } else
1882                 memset(tstate, 0, sizeof(*tstate));
1883         ahd->enabled_targets[scsi_id] = tstate;
1884         return (tstate);
1885 }
1886
1887 #ifdef AHD_TARGET_MODE
1888 /*
1889  * Free per target mode instance (ID we respond to as a target)
1890  * transfer negotiation data structures.
1891  */
1892 static void
1893 ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
1894 {
1895         struct ahd_tmode_tstate *tstate;
1896
1897         /*
1898          * Don't clean up our "master" tstate.
1899          * It has our default user settings.
1900          */
1901         if (scsi_id == ahd->our_id
1902          && force == FALSE)
1903                 return;
1904
1905         tstate = ahd->enabled_targets[scsi_id];
1906         if (tstate != NULL)
1907                 free(tstate, M_DEVBUF);
1908         ahd->enabled_targets[scsi_id] = NULL;
1909 }
1910 #endif
1911
1912 /*
1913  * Called when we have an active connection to a target on the bus,
1914  * this function finds the nearest period to the input period limited
1915  * by the capabilities of the bus connectivity of and sync settings for
1916  * the target.
1917  */
1918 void
1919 ahd_devlimited_syncrate(struct ahd_softc *ahd,
1920                         struct ahd_initiator_tinfo *tinfo,
1921                         u_int *period, u_int *ppr_options, role_t role)
1922 {
1923         struct  ahd_transinfo *transinfo;
1924         u_int   maxsync;
1925
1926         if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
1927          && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
1928                 maxsync = AHD_SYNCRATE_PACED;
1929         } else {
1930                 maxsync = AHD_SYNCRATE_ULTRA;
1931                 /* Can't do DT related options on an SE bus */
1932                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
1933         }
1934         /*
1935          * Never allow a value higher than our current goal
1936          * period otherwise we may allow a target initiated
1937          * negotiation to go above the limit as set by the
1938          * user.  In the case of an initiator initiated
1939          * sync negotiation, we limit based on the user
1940          * setting.  This allows the system to still accept
1941          * incoming negotiations even if target initiated
1942          * negotiation is not performed.
1943          */
1944         if (role == ROLE_TARGET)
1945                 transinfo = &tinfo->user;
1946         else
1947                 transinfo = &tinfo->goal;
1948         *ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
1949         if (transinfo->period == 0) {
1950                 *period = 0;
1951                 *ppr_options = 0;
1952         } else {
1953                 *period = MAX(*period, transinfo->period);
1954                 ahd_find_syncrate(ahd, period, ppr_options, maxsync);
1955         }
1956 }
1957
1958 /*
1959  * Look up the valid period to SCSIRATE conversion in our table.
1960  * Return the period and offset that should be sent to the target
1961  * if this was the beginning of an SDTR.
1962  */
1963 void
1964 ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
1965                   u_int *ppr_options, u_int maxsync)
1966 {
1967         /* Skip all PACED only entries if IU is not available */
1968         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
1969          && maxsync < AHD_SYNCRATE_DT)
1970                 maxsync = AHD_SYNCRATE_DT;
1971
1972         /* Skip all DT only entries if DT is not available */
1973         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
1974          && maxsync < AHD_SYNCRATE_ULTRA2)
1975                 maxsync = AHD_SYNCRATE_ULTRA2;
1976
1977         if (*period < maxsync)
1978                 *period = maxsync;
1979
1980         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
1981          && *period > AHD_SYNCRATE_MIN_DT)
1982                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1983
1984         if (*period > AHD_SYNCRATE_MIN)
1985                 *period = 0;
1986
1987         /* Honor PPR option conformance rules. */
1988         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
1989                 *ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
1990
1991         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
1992                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
1993 }
1994
1995 /*
1996  * Truncate the given synchronous offset to a value the
1997  * current adapter type and syncrate are capable of.
1998  */
1999 void
2000 ahd_validate_offset(struct ahd_softc *ahd,
2001                     struct ahd_initiator_tinfo *tinfo,
2002                     u_int period, u_int *offset, int wide,
2003                     role_t role)
2004 {
2005         u_int maxoffset;
2006
2007         /* Limit offset to what we can do */
2008         if (period == 0)
2009                 maxoffset = 0;
2010         else if (period <= AHD_SYNCRATE_PACED)
2011                 maxoffset = MAX_OFFSET_PACED;
2012         else
2013                 maxoffset = MAX_OFFSET;
2014         *offset = MIN(*offset, maxoffset);
2015         if (tinfo != NULL) {
2016                 if (role == ROLE_TARGET)
2017                         *offset = MIN(*offset, tinfo->user.offset);
2018                 else
2019                         *offset = MIN(*offset, tinfo->goal.offset);
2020         }
2021 }
2022
2023 /*
2024  * Truncate the given transfer width parameter to a value the
2025  * current adapter type is capable of.
2026  */
2027 void
2028 ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
2029                    u_int *bus_width, role_t role)
2030 {
2031         switch (*bus_width) {
2032         default:
2033                 if (ahd->features & AHD_WIDE) {
2034                         /* Respond Wide */
2035                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
2036                         break;
2037                 }
2038                 /* FALLTHROUGH */
2039         case MSG_EXT_WDTR_BUS_8_BIT:
2040                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
2041                 break;
2042         }
2043         if (tinfo != NULL) {
2044                 if (role == ROLE_TARGET)
2045                         *bus_width = MIN(tinfo->user.width, *bus_width);
2046                 else
2047                         *bus_width = MIN(tinfo->goal.width, *bus_width);
2048         }
2049 }
2050
2051 /*
2052  * Update the bitmask of targets for which the controller should
2053  * negotiate with at the next convenient oportunity.  This currently
2054  * means the next time we send the initial identify messages for
2055  * a new transaction.
2056  */
2057 int
2058 ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2059                        struct ahd_tmode_tstate *tstate,
2060                        struct ahd_initiator_tinfo *tinfo, int force)
2061 {
2062         u_int auto_negotiate_orig;
2063
2064         auto_negotiate_orig = tstate->auto_negotiate;
2065         if (tinfo->curr.period != tinfo->goal.period
2066          || tinfo->curr.width != tinfo->goal.width
2067          || tinfo->curr.offset != tinfo->goal.offset
2068          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
2069          || (force
2070           && (tinfo->goal.period != 0
2071            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
2072            || tinfo->goal.ppr_options != 0)))
2073                 tstate->auto_negotiate |= devinfo->target_mask;
2074         else
2075                 tstate->auto_negotiate &= ~devinfo->target_mask;
2076
2077         return (auto_negotiate_orig != tstate->auto_negotiate);
2078 }
2079
2080 /*
2081  * Update the user/goal/curr tables of synchronous negotiation
2082  * parameters as well as, in the case of a current or active update,
2083  * any data structures on the host controller.  In the case of an
2084  * active update, the specified target is currently talking to us on
2085  * the bus, so the transfer parameter update must take effect
2086  * immediately.
2087  */
2088 void
2089 ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2090                  u_int period, u_int offset, u_int ppr_options,
2091                  u_int type, int paused)
2092 {
2093         struct  ahd_initiator_tinfo *tinfo;
2094         struct  ahd_tmode_tstate *tstate;
2095         u_int   old_period;
2096         u_int   old_offset;
2097         u_int   old_ppr;
2098         int     active;
2099         int     update_needed;
2100
2101         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
2102         update_needed = 0;
2103
2104         if (period == 0 || offset == 0) {
2105                 period = 0;
2106                 offset = 0;
2107         }
2108
2109         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
2110                                     devinfo->target, &tstate);
2111
2112         if ((type & AHD_TRANS_USER) != 0) {
2113                 tinfo->user.period = period;
2114                 tinfo->user.offset = offset;
2115                 tinfo->user.ppr_options = ppr_options;
2116         }
2117
2118         if ((type & AHD_TRANS_GOAL) != 0) {
2119                 tinfo->goal.period = period;
2120                 tinfo->goal.offset = offset;
2121                 tinfo->goal.ppr_options = ppr_options;
2122         }
2123
2124         old_period = tinfo->curr.period;
2125         old_offset = tinfo->curr.offset;
2126         old_ppr    = tinfo->curr.ppr_options;
2127
2128         if ((type & AHD_TRANS_CUR) != 0
2129          && (old_period != period
2130           || old_offset != offset
2131           || old_ppr != ppr_options)) {
2132
2133                 update_needed++;
2134
2135                 tinfo->curr.period = period;
2136                 tinfo->curr.offset = offset;
2137                 tinfo->curr.ppr_options = ppr_options;
2138
2139                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
2140                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
2141                 if (bootverbose) {
2142                         if (offset != 0) {
2143                                 printf("%s: target %d synchronous with "
2144                                        "period = 0x%x, offset = 0x%x%s\n",
2145                                        ahd_name(ahd), devinfo->target,
2146                                        period, offset,
2147                                        (ppr_options & MSG_EXT_PPR_DT_REQ)
2148                                        ? " (DT)" : "");
2149                         } else {
2150                                 printf("%s: target %d using "
2151                                        "asynchronous transfers\n",
2152                                        ahd_name(ahd), devinfo->target);
2153                         }
2154                 }
2155         }
2156         /*
2157          * Always refresh the neg-table to handle the case of the
2158          * sequencer setting the ENATNO bit for a MK_MESSAGE request.
2159          * We will always renegotiate in that case if this is a
2160          * packetized request.  Also manage the busfree expected flag
2161          * from this common routine so that we catch changes due to
2162          * WDTR or SDTR messages.
2163          */
2164         if ((type & AHD_TRANS_CUR) != 0) {
2165                 if (!paused)
2166                         ahd_pause(ahd);
2167                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
2168                 if (!paused)
2169                         ahd_unpause(ahd);
2170                 if (ahd->msg_type != MSG_TYPE_NONE) {
2171                         if ((old_ppr & MSG_EXT_PPR_IU_REQ)
2172                          != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
2173 #ifdef AHD_DEBUG
2174                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2175                                         printf("Expecting IU Change busfree\n");
2176 #endif
2177                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
2178                                                |  MSG_FLAG_IU_REQ_CHANGED;
2179                         }
2180                         if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
2181 #ifdef AHD_DEBUG
2182                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2183                                         printf("PPR with IU_REQ outstanding\n");
2184 #endif
2185                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
2186                         }
2187                 }
2188         }
2189
2190         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
2191                                                 tinfo, /*force*/FALSE);
2192
2193         if (update_needed && active)
2194                 ahd_update_pending_scbs(ahd);
2195 }
2196
2197 /*
2198  * Update the user/goal/curr tables of wide negotiation
2199  * parameters as well as, in the case of a current or active update,
2200  * any data structures on the host controller.  In the case of an
2201  * active update, the specified target is currently talking to us on
2202  * the bus, so the transfer parameter update must take effect
2203  * immediately.
2204  */
2205 void
2206 ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2207               u_int width, u_int type, int paused)
2208 {
2209         struct  ahd_initiator_tinfo *tinfo;
2210         struct  ahd_tmode_tstate *tstate;
2211         u_int   oldwidth;
2212         int     active;
2213         int     update_needed;
2214
2215         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
2216         update_needed = 0;
2217         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
2218                                     devinfo->target, &tstate);
2219
2220         if ((type & AHD_TRANS_USER) != 0)
2221                 tinfo->user.width = width;
2222
2223         if ((type & AHD_TRANS_GOAL) != 0)
2224                 tinfo->goal.width = width;
2225
2226         oldwidth = tinfo->curr.width;
2227         if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
2228
2229                 update_needed++;
2230
2231                 tinfo->curr.width = width;
2232                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
2233                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
2234                 if (bootverbose) {
2235                         printf("%s: target %d using %dbit transfers\n",
2236                                ahd_name(ahd), devinfo->target,
2237                                8 * (0x01 << width));
2238                 }
2239         }
2240
2241         if ((type & AHD_TRANS_CUR) != 0) {
2242                 if (!paused)
2243                         ahd_pause(ahd);
2244                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
2245                 if (!paused)
2246                         ahd_unpause(ahd);
2247         }
2248
2249         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
2250                                                 tinfo, /*force*/FALSE);
2251         if (update_needed && active)
2252                 ahd_update_pending_scbs(ahd);
2253
2254 }
2255
2256 /*
2257  * Update the current state of tagged queuing for a given target.
2258  */
2259 void
2260 ahd_set_tags(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2261              ahd_queue_alg alg)
2262 {
2263         ahd_platform_set_tags(ahd, devinfo, alg);
2264         ahd_send_async(ahd, devinfo->channel, devinfo->target,
2265                        devinfo->lun, AC_TRANSFER_NEG, &alg);
2266 }
2267
2268 static void
2269 ahd_update_neg_table(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2270                      struct ahd_transinfo *tinfo)
2271 {
2272         ahd_mode_state  saved_modes;
2273         u_int           period;
2274         u_int           ppr_opts;
2275         u_int           con_opts;
2276         u_int           offset;
2277         u_int           precomp;
2278
2279         saved_modes = ahd_save_modes(ahd);
2280         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2281
2282         ahd_outb(ahd, NEGOADDR, devinfo->target);
2283         period = tinfo->period;
2284         offset = tinfo->offset;
2285         precomp = 0;
2286         if (period == 0)
2287                 period = AHD_SYNCRATE_ASYNC;
2288         if (period == AHD_SYNCRATE_160) {
2289                 period = AHD_SYNCRATE_REVA_160;
2290                 precomp = 0;
2291                 if ((ahd->flags & AHD_CPQ_BOARD) == 0)
2292                         precomp |= AHD_PRECOMP_FASTSLEW;
2293                 if ((tinfo->ppr_options & MSG_EXT_PPR_PCOMP_EN) != 0)
2294                         precomp |= AHD_PRECOMP_CUTBACK_29;
2295         }
2296         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP);
2297         ahd_outb(ahd, ANNEXDAT, precomp);
2298
2299         ahd_outb(ahd, NEGPERIOD, period);
2300         ppr_opts = tinfo->ppr_options
2301                  & (MSG_EXT_PPR_QAS_REQ|MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_IU_REQ);
2302         /*
2303          * When the SPI4 spec was finalized, PACE transfers
2304          * was not made a configurable option in the PPR message.
2305          * Instead it is assumed to be enabled for any
2306          * syncrate faster than 80MHz.  Nevertheless, Harpoon
2307          * allows this to be configurable.
2308          *
2309          * Harpoon also assumes at most 2 data bytes per negotiated
2310          * REQ/ACK offset.  Paced transfers take 4, so we must
2311          * adjust our offset.
2312          */
2313         if (period <= AHD_SYNCRATE_PACED) {
2314                 ppr_opts |= PPROPT_PACE;
2315                 offset *= 2;
2316         }
2317         ahd_outb(ahd, NEGPPROPTS, ppr_opts);
2318         ahd_outb(ahd, NEGOFFSET, offset);
2319
2320         con_opts = 0;
2321         if (tinfo->width == MSG_EXT_WDTR_BUS_16_BIT)
2322                 con_opts |= WIDEXFER;
2323
2324         /*
2325          * During packetized transfers, the target will
2326          * give us the oportunity to send command packets
2327          * without us asserting attention.
2328          */
2329         if ((tinfo->ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
2330                 con_opts |= ENAUTOATNO;
2331         ahd_outb(ahd, NEGCONOPTS, con_opts);
2332         ahd_restore_modes(ahd, saved_modes);
2333 }
2334
2335 /*
2336  * When the transfer settings for a connection change, setup for
2337  * negotiation in pending SCBs to effect the change as quickly as
2338  * possible.  We also cancel any negotiations that are scheduled
2339  * for inflight SCBs that have not been started yet.
2340  */
2341 static void
2342 ahd_update_pending_scbs(struct ahd_softc *ahd)
2343 {
2344         struct          scb *pending_scb;
2345         int             pending_scb_count;
2346         int             i;
2347         int             paused;
2348         u_int           saved_scbptr;
2349         ahd_mode_state  saved_modes;
2350
2351         /*
2352          * Traverse the pending SCB list and ensure that all of the
2353          * SCBs there have the proper settings.  We can only safely
2354          * clear the negotiation required flag (setting requires the
2355          * execution queue to be modified) and this is only possible
2356          * if we are not already attempting to select out for this
2357          * SCB.  For this reason, all callers only call this routine
2358          * if we are changing the negotiation settings for the currently
2359          * active transaction on the bus.
2360          */
2361         pending_scb_count = 0;
2362         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
2363                 struct ahd_devinfo devinfo;
2364                 struct hardware_scb *pending_hscb;
2365                 struct ahd_initiator_tinfo *tinfo;
2366                 struct ahd_tmode_tstate *tstate;
2367
2368                 ahd_scb_devinfo(ahd, &devinfo, pending_scb);
2369                 tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
2370                                             devinfo.our_scsiid,
2371                                             devinfo.target, &tstate);
2372                 pending_hscb = pending_scb->hscb;
2373                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2374                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2375                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2376                         pending_hscb->control &= ~MK_MESSAGE;
2377                 }
2378                 ahd_sync_scb(ahd, pending_scb,
2379                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2380                 pending_scb_count++;
2381         }
2382
2383         if (pending_scb_count == 0)
2384                 return;
2385
2386         if (ahd_is_paused(ahd)) {
2387                 paused = 1;
2388         } else {
2389                 paused = 0;
2390                 ahd_pause(ahd);
2391         }
2392
2393         /*
2394          * Force the sequencer to reinitialize the selection for
2395          * the command at the head of the execution queue if it
2396          * has already been setup.  The negotiation changes may
2397          * effect whether we select-out with ATN.
2398          */
2399         ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2400         saved_modes = ahd_save_modes(ahd);
2401         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2402         saved_scbptr = ahd_get_scbptr(ahd);
2403         /* Ensure that the hscbs down on the card match the new information */
2404         for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
2405                 struct  hardware_scb *pending_hscb;
2406                 u_int   control;
2407                 u_int   scb_tag;
2408
2409                 ahd_set_scbptr(ahd, i);
2410                 scb_tag = i;
2411                 pending_scb = ahd_lookup_scb(ahd, scb_tag);
2412                 if (pending_scb == NULL)
2413                         continue;
2414
2415                 pending_hscb = pending_scb->hscb;
2416                 control = ahd_inb_scbram(ahd, SCB_CONTROL);
2417                 control &= ~MK_MESSAGE;
2418                 control |= pending_hscb->control & MK_MESSAGE;
2419                 ahd_outb(ahd, SCB_CONTROL, control);
2420         }
2421         ahd_set_scbptr(ahd,saved_scbptr);
2422         ahd_restore_modes(ahd, saved_modes);
2423
2424         if (paused == 0)
2425                 ahd_unpause(ahd);
2426 }
2427
2428 /**************************** Pathing Information *****************************/
2429 static void
2430 ahd_fetch_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
2431 {
2432         ahd_mode_state  saved_modes;
2433         u_int           saved_scsiid;
2434         role_t          role;
2435         int             our_id;
2436
2437         saved_modes = ahd_save_modes(ahd);
2438         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2439
2440         if (ahd_inb(ahd, SSTAT0) & TARGET)
2441                 role = ROLE_TARGET;
2442         else
2443                 role = ROLE_INITIATOR;
2444
2445         if (role == ROLE_TARGET
2446          && (ahd_inb(ahd, SEQ_FLAGS) & CMDPHASE_PENDING) != 0) {
2447                 /* We were selected, so pull our id from TARGIDIN */
2448                 our_id = ahd_inb(ahd, TARGIDIN) & OID;
2449         } else if (role == ROLE_TARGET)
2450                 our_id = ahd_inb(ahd, TOWNID);
2451         else
2452                 our_id = ahd_inb(ahd, IOWNID);
2453
2454         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
2455         ahd_compile_devinfo(devinfo,
2456                             our_id,
2457                             SCSIID_TARGET(ahd, saved_scsiid),
2458                             ahd_inb(ahd, SAVED_LUN),
2459                             SCSIID_CHANNEL(ahd, saved_scsiid),
2460                             role);
2461         ahd_restore_modes(ahd, saved_modes);
2462 }
2463
2464 static void
2465 ahd_print_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
2466 {
2467         printf("%s:%c:%d:%d:", ahd_name(ahd), 'A',
2468                devinfo->target, devinfo->lun);
2469 }
2470
2471 struct ahd_phase_table_entry*
2472 ahd_lookup_phase_entry(int phase)
2473 {
2474         struct ahd_phase_table_entry *entry;
2475         struct ahd_phase_table_entry *last_entry;
2476
2477         /*
2478          * num_phases doesn't include the default entry which
2479          * will be returned if the phase doesn't match.
2480          */
2481         last_entry = &ahd_phase_table[num_phases];
2482         for (entry = ahd_phase_table; entry < last_entry; entry++) {
2483                 if (phase == entry->phase)
2484                         break;
2485         }
2486         return (entry);
2487 }
2488
2489 void
2490 ahd_compile_devinfo(struct ahd_devinfo *devinfo, u_int our_id, u_int target,
2491                     u_int lun, char channel, role_t role)
2492 {
2493         devinfo->our_scsiid = our_id;
2494         devinfo->target = target;
2495         devinfo->lun = lun;
2496         devinfo->target_offset = target;
2497         devinfo->channel = channel;
2498         devinfo->role = role;
2499         if (channel == 'B')
2500                 devinfo->target_offset += 8;
2501         devinfo->target_mask = (0x01 << devinfo->target_offset);
2502 }
2503
2504 static void
2505 ahd_scb_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2506                 struct scb *scb)
2507 {
2508         role_t  role;
2509         int     our_id;
2510
2511         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2512         role = ROLE_INITIATOR;
2513         if ((scb->hscb->control & TARGET_SCB) != 0)
2514                 role = ROLE_TARGET;
2515         ahd_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahd, scb),
2516                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahd, scb), role);
2517 }
2518
2519
2520 /************************ Message Phase Processing ****************************/
2521 /*
2522  * When an initiator transaction with the MK_MESSAGE flag either reconnects
2523  * or enters the initial message out phase, we are interrupted.  Fill our
2524  * outgoing message buffer with the appropriate message and beging handing
2525  * the message phase(s) manually.
2526  */
2527 static void
2528 ahd_setup_initiator_msgout(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2529                            struct scb *scb)
2530 {
2531         /*
2532          * To facilitate adding multiple messages together,
2533          * each routine should increment the index and len
2534          * variables instead of setting them explicitly.
2535          */
2536         ahd->msgout_index = 0;
2537         ahd->msgout_len = 0;
2538
2539         if (ahd_currently_packetized(ahd))
2540                 ahd->msg_flags |= MSG_FLAG_PACKETIZED;
2541
2542         if (ahd->send_msg_perror
2543          && ahd_inb(ahd, MSG_OUT) == HOST_MSG) {
2544                 ahd->msgout_buf[ahd->msgout_index++] = ahd->send_msg_perror;
2545                 ahd->msgout_len++;
2546                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2547                 printf("Setting up for Parity Error delivery\n");
2548                 return;
2549         } else if (scb == NULL) {
2550                 printf("%s: WARNING. No pending message for "
2551                        "I_T msgin.  Issuing NO-OP\n", ahd_name(ahd));
2552                 ahd->msgout_buf[ahd->msgout_index++] = MSG_NOOP;
2553                 ahd->msgout_len++;
2554                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2555                 return;
2556         }
2557
2558         if ((scb->flags & SCB_DEVICE_RESET) == 0
2559          && (scb->flags & SCB_PACKETIZED) == 0
2560          && ahd_inb(ahd, MSG_OUT) == MSG_IDENTIFYFLAG) {
2561                 u_int identify_msg;
2562
2563                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2564                 if ((scb->hscb->control & DISCENB) != 0)
2565                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
2566                 ahd->msgout_buf[ahd->msgout_index++] = identify_msg;
2567                 ahd->msgout_len++;
2568
2569                 if ((scb->hscb->control & TAG_ENB) != 0) {
2570                         ahd->msgout_buf[ahd->msgout_index++] =
2571                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2572                         ahd->msgout_buf[ahd->msgout_index++] = SCB_GET_TAG(scb);
2573                         ahd->msgout_len += 2;
2574                 }
2575         }
2576
2577         if (scb->flags & SCB_DEVICE_RESET) {
2578                 ahd->msgout_buf[ahd->msgout_index++] = MSG_BUS_DEV_RESET;
2579                 ahd->msgout_len++;
2580                 ahd_print_path(ahd, scb);
2581                 printf("Bus Device Reset Message Sent\n");
2582                 /*
2583                  * Clear our selection hardware in advance of
2584                  * the busfree.  We may have an entry in the waiting
2585                  * Q for this target, and we don't want to go about
2586                  * selecting while we handle the busfree and blow it
2587                  * away.
2588                  */
2589                 ahd_outb(ahd, SCSISEQ0, 0);
2590         } else if ((scb->flags & SCB_ABORT) != 0) {
2591
2592                 if ((scb->hscb->control & TAG_ENB) != 0) {
2593                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT_TAG;
2594                 } else {
2595                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT;
2596                 }
2597                 ahd->msgout_len++;
2598                 ahd_print_path(ahd, scb);
2599                 printf("Abort%s Message Sent\n",
2600                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2601                 /*
2602                  * Clear our selection hardware in advance of
2603                  * the busfree.  We may have an entry in the waiting
2604                  * Q for this target, and we don't want to go about
2605                  * selecting while we handle the busfree and blow it
2606                  * away.
2607                  */
2608                 ahd_outb(ahd, SCSISEQ0, 0);
2609         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2610                 ahd_build_transfer_msg(ahd, devinfo);
2611                 /*
2612                  * Clear our selection hardware in advance of potential
2613                  * PPR IU status change busfree.  We may have an entry in
2614                  * the waiting Q for this target, and we don't want to go
2615                  * about selecting while we handle the busfree and blow
2616                  * it away.
2617                  */
2618                 ahd_outb(ahd, SCSISEQ0, 0);
2619         } else {
2620                 printf("ahd_intr: AWAITING_MSG for an SCB that "
2621                        "does not have a waiting message\n");
2622                 printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2623                        devinfo->target_mask);
2624                 panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2625                       "SCB flags = %x", SCB_GET_TAG(scb), scb->hscb->control,
2626                       ahd_inb(ahd, MSG_OUT), scb->flags);
2627         }
2628
2629         /*
2630          * Clear the MK_MESSAGE flag from the SCB so we aren't
2631          * asked to send this message again.
2632          */
2633         ahd_outb(ahd, SCB_CONTROL,
2634                  ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
2635         scb->hscb->control &= ~MK_MESSAGE;
2636         ahd->msgout_index = 0;
2637         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2638 }
2639
2640 /*
2641  * Build an appropriate transfer negotiation message for the
2642  * currently active target.
2643  */
2644 static void
2645 ahd_build_transfer_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
2646 {
2647         /*
2648          * We need to initiate transfer negotiations.
2649          * If our current and goal settings are identical,
2650          * we want to renegotiate due to a check condition.
2651          */
2652         struct  ahd_initiator_tinfo *tinfo;
2653         struct  ahd_tmode_tstate *tstate;
2654         int     dowide;
2655         int     dosync;
2656         int     doppr;
2657         int     use_ppr;
2658         u_int   period;
2659         u_int   ppr_options;
2660         u_int   offset;
2661
2662         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
2663                                     devinfo->target, &tstate);
2664         /*
2665          * Filter our period based on the current connection.
2666          * If we can't perform DT transfers on this segment (not in LVD
2667          * mode for instance), then our decision to issue a PPR message
2668          * may change.
2669          */
2670         period = tinfo->goal.period;
2671         ppr_options = tinfo->goal.ppr_options;
2672         /* Target initiated PPR is not allowed in the SCSI spec */
2673         if (devinfo->role == ROLE_TARGET)
2674                 ppr_options = 0;
2675         ahd_devlimited_syncrate(ahd, tinfo, &period,
2676                                 &ppr_options, devinfo->role);
2677         dowide = tinfo->curr.width != tinfo->goal.width;
2678         dosync = tinfo->curr.period != period;
2679         doppr = tinfo->curr.ppr_options != ppr_options;
2680
2681         if (!dowide && !dosync && !doppr) {
2682                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
2683                 dosync = tinfo->goal.period != 0;
2684                 doppr = tinfo->goal.ppr_options != 0;
2685         }
2686
2687         if (!dowide && !dosync && !doppr) {
2688                 panic("ahd_intr: AWAITING_MSG for negotiation, "
2689                       "but no negotiation needed\n");
2690         }
2691
2692         use_ppr = (tinfo->curr.transport_version >= 3) || doppr;
2693         /* Target initiated PPR is not allowed in the SCSI spec */
2694         if (devinfo->role == ROLE_TARGET)
2695                 use_ppr = 0;
2696
2697         /*
2698          * Both the PPR message and SDTR message require the
2699          * goal syncrate to be limited to what the target device
2700          * is capable of handling (based on whether an LVD->SE
2701          * expander is on the bus), so combine these two cases.
2702          * Regardless, guarantee that if we are using WDTR and SDTR
2703          * messages that WDTR comes first.
2704          */
2705         if (use_ppr || (dosync && !dowide)) {
2706
2707                 offset = tinfo->goal.offset;
2708                 ahd_validate_offset(ahd, tinfo, period, &offset,
2709                                     use_ppr ? tinfo->goal.width
2710                                             : tinfo->curr.width,
2711                                     devinfo->role);
2712                 if (use_ppr) {
2713                         ahd_construct_ppr(ahd, devinfo, period, offset,
2714                                           tinfo->goal.width, ppr_options);
2715                 } else {
2716                         ahd_construct_sdtr(ahd, devinfo, period, offset);
2717                 }
2718         } else {
2719                 ahd_construct_wdtr(ahd, devinfo, tinfo->goal.width);
2720         }
2721 }
2722
2723 /*
2724  * Build a synchronous negotiation message in our message
2725  * buffer based on the input parameters.
2726  */
2727 static void
2728 ahd_construct_sdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2729                    u_int period, u_int offset)
2730 {
2731         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
2732         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_SDTR_LEN;
2733         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_SDTR;
2734         ahd->msgout_buf[ahd->msgout_index++] = period;
2735         ahd->msgout_buf[ahd->msgout_index++] = offset;
2736         ahd->msgout_len += 5;
2737         if (bootverbose) {
2738                 printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
2739                        ahd_name(ahd), devinfo->channel, devinfo->target,
2740                        devinfo->lun, period, offset);
2741         }
2742 }
2743
2744 /*
2745  * Build a wide negotiateion message in our message
2746  * buffer based on the input parameters.
2747  */
2748 static void
2749 ahd_construct_wdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2750                    u_int bus_width)
2751 {
2752         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
2753         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_WDTR_LEN;
2754         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_WDTR;
2755         ahd->msgout_buf[ahd->msgout_index++] = bus_width;
2756         ahd->msgout_len += 4;
2757         if (bootverbose) {
2758                 printf("(%s:%c:%d:%d): Sending WDTR %x\n",
2759                        ahd_name(ahd), devinfo->channel, devinfo->target,
2760                        devinfo->lun, bus_width);
2761         }
2762 }
2763
2764 /*
2765  * Build a parallel protocol request message in our message
2766  * buffer based on the input parameters.
2767  */
2768 static void
2769 ahd_construct_ppr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2770                   u_int period, u_int offset, u_int bus_width,
2771                   u_int ppr_options)
2772 {
2773         /*
2774          * Always request precompensation from
2775          * the other target if we are running
2776          * at paced syncrates.
2777          */
2778         if (period <= AHD_SYNCRATE_PACED)
2779                 ppr_options |= MSG_EXT_PPR_PCOMP_EN;
2780         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
2781         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_PPR_LEN;
2782         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_PPR;
2783         ahd->msgout_buf[ahd->msgout_index++] = period;
2784         ahd->msgout_buf[ahd->msgout_index++] = 0;
2785         ahd->msgout_buf[ahd->msgout_index++] = offset;
2786         ahd->msgout_buf[ahd->msgout_index++] = bus_width;
2787         ahd->msgout_buf[ahd->msgout_index++] = ppr_options;
2788         ahd->msgout_len += 8;
2789         if (bootverbose) {
2790                 printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
2791                        "offset %x, ppr_options %x\n", ahd_name(ahd),
2792                        devinfo->channel, devinfo->target, devinfo->lun,
2793                        bus_width, period, offset, ppr_options);
2794         }
2795 }
2796
2797 /*
2798  * Clear any active message state.
2799  */
2800 static void
2801 ahd_clear_msg_state(struct ahd_softc *ahd)
2802 {
2803         ahd_mode_state saved_modes;
2804
2805         saved_modes = ahd_save_modes(ahd);
2806         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2807         ahd->send_msg_perror = 0;
2808         ahd->msg_flags = MSG_FLAG_NONE;
2809         ahd->msgout_len = 0;
2810         ahd->msgin_index = 0;
2811         ahd->msg_type = MSG_TYPE_NONE;
2812         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
2813                 /*
2814                  * The target didn't care to respond to our
2815                  * message request, so clear ATN.
2816                  */
2817                 ahd_outb(ahd, CLRSINT1, CLRATNO);
2818         }
2819         ahd_outb(ahd, MSG_OUT, MSG_NOOP);
2820         ahd_outb(ahd, SEQ_FLAGS2,
2821                  ahd_inb(ahd, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
2822         ahd_restore_modes(ahd, saved_modes);
2823 }
2824
2825 /*
2826  * Manual message loop handler.
2827  */
2828 static void
2829 ahd_handle_message_phase(struct ahd_softc *ahd)
2830 {
2831         struct  ahd_devinfo devinfo;
2832         u_int   bus_phase;
2833         int     end_session;
2834
2835         ahd_fetch_devinfo(ahd, &devinfo);
2836         end_session = FALSE;
2837         bus_phase = ahd_inb(ahd, LASTPHASE);
2838
2839         if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0) {
2840                 printf("LQIRETRY for LQIPHASE_OUTPKT\n");
2841                 ahd_outb(ahd, LQCTL2, LQIRETRY);
2842         }
2843 reswitch:
2844         switch (ahd->msg_type) {
2845         case MSG_TYPE_INITIATOR_MSGOUT:
2846         {
2847                 int lastbyte;
2848                 int phasemis;
2849                 int msgdone;
2850
2851                 if (ahd->msgout_len == 0 && ahd->send_msg_perror == 0)
2852                         panic("HOST_MSG_LOOP interrupt with no active message");
2853
2854 #ifdef AHD_DEBUG
2855                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
2856                         ahd_print_devinfo(ahd, &devinfo);
2857                         printf("INITIATOR_MSG_OUT");
2858                 }
2859 #endif
2860                 phasemis = bus_phase != P_MESGOUT;
2861                 if (phasemis) {
2862 #ifdef AHD_DEBUG
2863                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
2864                                 printf(" PHASEMIS %s\n",
2865                                        ahd_lookup_phase_entry(bus_phase)
2866                                                              ->phasemsg);
2867                         }
2868 #endif
2869                         if (bus_phase == P_MESGIN) {
2870                                 /*
2871                                  * Change gears and see if
2872                                  * this messages is of interest to
2873                                  * us or should be passed back to
2874                                  * the sequencer.
2875                                  */
2876                                 ahd_outb(ahd, CLRSINT1, CLRATNO);
2877                                 ahd->send_msg_perror = 0;
2878                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGIN;
2879                                 ahd->msgin_index = 0;
2880                                 goto reswitch;
2881                         }
2882                         end_session = TRUE;
2883                         break;
2884                 }
2885
2886                 if (ahd->send_msg_perror) {
2887                         ahd_outb(ahd, CLRSINT1, CLRATNO);
2888                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
2889 #ifdef AHD_DEBUG
2890                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2891                                 printf(" byte 0x%x\n", ahd->send_msg_perror);
2892 #endif
2893                         /*
2894                          * If we are notifying the target of a CRC error
2895                          * during packetized operations, the target is
2896                          * within its rights to acknowledge our message
2897                          * with a busfree.
2898                          */
2899                         if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0
2900                          && ahd->send_msg_perror == MSG_INITIATOR_DET_ERR)
2901                                 ahd->msg_flags |= MSG_FLAG_EXPECT_IDE_BUSFREE;
2902
2903                         ahd_outb(ahd, RETURN_2, ahd->send_msg_perror);
2904                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
2905                         break;
2906                 }
2907
2908                 msgdone = ahd->msgout_index == ahd->msgout_len;
2909                 if (msgdone) {
2910                         /*
2911                          * The target has requested a retry.
2912                          * Re-assert ATN, reset our message index to
2913                          * 0, and try again.
2914                          */
2915                         ahd->msgout_index = 0;
2916                         ahd_assert_atn(ahd);
2917                 }
2918
2919                 lastbyte = ahd->msgout_index == (ahd->msgout_len - 1);
2920                 if (lastbyte) {
2921                         /* Last byte is signified by dropping ATN */
2922                         ahd_outb(ahd, CLRSINT1, CLRATNO);
2923                 }
2924
2925                 /*
2926                  * Clear our interrupt status and present
2927                  * the next byte on the bus.
2928                  */
2929                 ahd_outb(ahd, CLRSINT1, CLRREQINIT);
2930 #ifdef AHD_DEBUG
2931                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2932                         printf(" byte 0x%x\n",
2933                                ahd->msgout_buf[ahd->msgout_index]);
2934 #endif
2935                 ahd_outb(ahd, RETURN_2, ahd->msgout_buf[ahd->msgout_index++]);
2936                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
2937                 break;
2938         }
2939         case MSG_TYPE_INITIATOR_MSGIN:
2940         {
2941                 int phasemis;
2942                 int message_done;
2943
2944 #ifdef AHD_DEBUG
2945                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
2946                         ahd_print_devinfo(ahd, &devinfo);
2947                         printf("INITIATOR_MSG_IN");
2948                 }
2949 #endif
2950                 phasemis = bus_phase != P_MESGIN;
2951                 if (phasemis) {
2952 #ifdef AHD_DEBUG
2953                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
2954                                 printf(" PHASEMIS %s\n",
2955                                        ahd_lookup_phase_entry(bus_phase)
2956                                                              ->phasemsg);
2957                         }
2958 #endif
2959                         ahd->msgin_index = 0;
2960                         if (bus_phase == P_MESGOUT
2961                          && (ahd->send_msg_perror != 0
2962                           || (ahd->msgout_len != 0
2963                            && ahd->msgout_index == 0))) {
2964                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2965                                 goto reswitch;
2966                         }
2967                         end_session = TRUE;
2968                         break;
2969                 }
2970
2971                 /* Pull the byte in without acking it */
2972                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIBUS);
2973 #ifdef AHD_DEBUG
2974                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2975                         printf(" byte 0x%x\n",
2976                                ahd->msgin_buf[ahd->msgin_index]);
2977 #endif
2978
2979                 message_done = ahd_parse_msg(ahd, &devinfo);
2980
2981                 if (message_done) {
2982                         /*
2983                          * Clear our incoming message buffer in case there
2984                          * is another message following this one.
2985                          */
2986                         ahd->msgin_index = 0;
2987
2988                         /*
2989                          * If this message illicited a response,
2990                          * assert ATN so the target takes us to the
2991                          * message out phase.
2992                          */
2993                         if (ahd->msgout_len != 0)
2994                                 ahd_assert_atn(ahd);
2995                 } else
2996                         ahd->msgin_index++;
2997
2998                 if (message_done == MSGLOOP_TERMINATED) {
2999                         end_session = TRUE;
3000                 } else {
3001                         /* Ack the byte */
3002                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
3003                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_READ);
3004                 }
3005                 break;
3006         }
3007         case MSG_TYPE_TARGET_MSGIN:
3008         {
3009                 int msgdone;
3010                 int msgout_request;
3011
3012                 /*
3013                  * By default, the message loop will continue.
3014                  */
3015                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
3016
3017                 if (ahd->msgout_len == 0)
3018                         panic("Target MSGIN with no active message");
3019
3020                 /*
3021                  * If we interrupted a mesgout session, the initiator
3022                  * will not know this until our first REQ.  So, we
3023                  * only honor mesgout requests after we've sent our
3024                  * first byte.
3025                  */
3026                 if ((ahd_inb(ahd, SCSISIGI) & ATNI) != 0
3027                  && ahd->msgout_index > 0)
3028                         msgout_request = TRUE;
3029                 else
3030                         msgout_request = FALSE;
3031
3032                 if (msgout_request) {
3033
3034                         /*
3035                          * Change gears and see if
3036                          * this messages is of interest to
3037                          * us or should be passed back to
3038                          * the sequencer.
3039                          */
3040                         ahd->msg_type = MSG_TYPE_TARGET_MSGOUT;
3041                         ahd_outb(ahd, SCSISIGO, P_MESGOUT | BSYO);
3042                         ahd->msgin_index = 0;
3043                         /* Dummy read to REQ for first byte */
3044                         ahd_inb(ahd, SCSIDAT);
3045                         ahd_outb(ahd, SXFRCTL0,
3046                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
3047                         break;
3048                 }
3049
3050                 msgdone = ahd->msgout_index == ahd->msgout_len;
3051                 if (msgdone) {
3052                         ahd_outb(ahd, SXFRCTL0,
3053                                  ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
3054                         end_session = TRUE;
3055                         break;
3056                 }
3057
3058                 /*
3059                  * Present the next byte on the bus.
3060                  */
3061                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) | SPIOEN);
3062                 ahd_outb(ahd, SCSIDAT, ahd->msgout_buf[ahd->msgout_index++]);
3063                 break;
3064         }
3065         case MSG_TYPE_TARGET_MSGOUT:
3066         {
3067                 int lastbyte;
3068                 int msgdone;
3069
3070                 /*
3071                  * By default, the message loop will continue.
3072                  */
3073                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
3074
3075                 /*
3076                  * The initiator signals that this is
3077                  * the last byte by dropping ATN.
3078                  */
3079                 lastbyte = (ahd_inb(ahd, SCSISIGI) & ATNI) == 0;
3080
3081                 /*
3082                  * Read the latched byte, but turn off SPIOEN first
3083                  * so that we don't inadvertently cause a REQ for the
3084                  * next byte.
3085                  */
3086                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
3087                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIDAT);
3088                 msgdone = ahd_parse_msg(ahd, &devinfo);
3089                 if (msgdone == MSGLOOP_TERMINATED) {
3090                         /*
3091                          * The message is *really* done in that it caused
3092                          * us to go to bus free.  The sequencer has already
3093                          * been reset at this point, so pull the ejection
3094                          * handle.
3095                          */
3096                         return;
3097                 }
3098
3099                 ahd->msgin_index++;
3100
3101                 /*
3102                  * XXX Read spec about initiator dropping ATN too soon
3103                  *     and use msgdone to detect it.
3104                  */
3105                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
3106                         ahd->msgin_index = 0;
3107
3108                         /*
3109                          * If this message illicited a response, transition
3110                          * to the Message in phase and send it.
3111                          */
3112                         if (ahd->msgout_len != 0) {
3113                                 ahd_outb(ahd, SCSISIGO, P_MESGIN | BSYO);
3114                                 ahd_outb(ahd, SXFRCTL0,
3115                                          ahd_inb(ahd, SXFRCTL0) | SPIOEN);
3116                                 ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
3117                                 ahd->msgin_index = 0;
3118                                 break;
3119                         }
3120                 }
3121
3122                 if (lastbyte)
3123                         end_session = TRUE;
3124                 else {
3125                         /* Ask for the next byte. */
3126                         ahd_outb(ahd, SXFRCTL0,
3127                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
3128                 }
3129
3130                 break;
3131         }
3132         default:
3133                 panic("Unknown REQINIT message type");
3134         }
3135
3136         if (end_session) {
3137                 if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0) {
3138                         printf("%s: Returning to Idle Loop\n",
3139                                ahd_name(ahd));
3140                         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
3141                         ahd_clear_msg_state(ahd);
3142                         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
3143                 } else {
3144                         ahd_clear_msg_state(ahd);
3145                         ahd_outb(ahd, RETURN_1, EXIT_MSG_LOOP);
3146                 }
3147         }
3148 }
3149
3150 /*
3151  * See if we sent a particular extended message to the target.
3152  * If "full" is true, return true only if the target saw the full
3153  * message.  If "full" is false, return true if the target saw at
3154  * least the first byte of the message.
3155  */
3156 static int
3157 ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type, u_int msgval, int full)
3158 {
3159         int found;
3160         u_int index;
3161
3162         found = FALSE;
3163         index = 0;
3164
3165         while (index < ahd->msgout_len) {
3166                 if (ahd->msgout_buf[index] == MSG_EXTENDED) {
3167                         u_int end_index;
3168
3169                         end_index = index + 1 + ahd->msgout_buf[index + 1];
3170                         if (ahd->msgout_buf[index+2] == msgval
3171                          && type == AHDMSG_EXT) {
3172
3173                                 if (full) {
3174                                         if (ahd->msgout_index > end_index)
3175                                                 found = TRUE;
3176                                 } else if (ahd->msgout_index > index)
3177                                         found = TRUE;
3178                         }
3179                         index = end_index;
3180                 } else if (ahd->msgout_buf[index] >= MSG_SIMPLE_TASK
3181                         && ahd->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
3182
3183                         /* Skip tag type and tag id or residue param*/
3184                         index += 2;
3185                 } else {
3186                         /* Single byte message */
3187                         if (type == AHDMSG_1B
3188                          && ahd->msgout_buf[index] == msgval
3189                          && ahd->msgout_index > index)
3190                                 found = TRUE;
3191                         index++;
3192                 }
3193
3194                 if (found)
3195                         break;
3196         }
3197         return (found);
3198 }
3199
3200 /*
3201  * Wait for a complete incoming message, parse it, and respond accordingly.
3202  */
3203 static int
3204 ahd_parse_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3205 {
3206         struct  ahd_initiator_tinfo *tinfo;
3207         struct  ahd_tmode_tstate *tstate;
3208         int     reject;
3209         int     done;
3210         int     response;
3211
3212         done = MSGLOOP_IN_PROG;
3213         response = FALSE;
3214         reject = FALSE;
3215         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3216                                     devinfo->target, &tstate);
3217
3218         /*
3219          * Parse as much of the message as is availible,
3220          * rejecting it if we don't support it.  When
3221          * the entire message is availible and has been
3222          * handled, return MSGLOOP_MSGCOMPLETE, indicating
3223          * that we have parsed an entire message.
3224          *
3225          * In the case of extended messages, we accept the length
3226          * byte outright and perform more checking once we know the
3227          * extended message type.
3228          */
3229         switch (ahd->msgin_buf[0]) {
3230         case MSG_DISCONNECT:
3231         case MSG_SAVEDATAPOINTER:
3232         case MSG_CMDCOMPLETE:
3233         case MSG_RESTOREPOINTERS:
3234         case MSG_IGN_WIDE_RESIDUE:
3235                 /*
3236                  * End our message loop as these are messages
3237                  * the sequencer handles on its own.
3238                  */
3239                 done = MSGLOOP_TERMINATED;
3240                 break;
3241         case MSG_MESSAGE_REJECT:
3242                 response = ahd_handle_msg_reject(ahd, devinfo);
3243                 /* FALLTHROUGH */
3244         case MSG_NOOP:
3245                 done = MSGLOOP_MSGCOMPLETE;
3246                 break;
3247         case MSG_EXTENDED:
3248         {
3249                 /* Wait for enough of the message to begin validation */
3250                 if (ahd->msgin_index < 2)
3251                         break;
3252                 switch (ahd->msgin_buf[2]) {
3253                 case MSG_EXT_SDTR:
3254                 {
3255                         u_int    period;
3256                         u_int    ppr_options;
3257                         u_int    offset;
3258                         u_int    saved_offset;
3259
3260                         if (ahd->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3261                                 reject = TRUE;
3262                                 break;
3263                         }
3264
3265                         /*
3266                          * Wait until we have both args before validating
3267                          * and acting on this message.
3268                          *
3269                          * Add one to MSG_EXT_SDTR_LEN to account for
3270                          * the extended message preamble.
3271                          */
3272                         if (ahd->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3273                                 break;
3274
3275                         period = ahd->msgin_buf[3];
3276                         ppr_options = 0;
3277                         saved_offset = offset = ahd->msgin_buf[4];
3278                         ahd_devlimited_syncrate(ahd, tinfo, &period,
3279                                                 &ppr_options, devinfo->role);
3280                         ahd_validate_offset(ahd, tinfo, period, &offset,
3281                                             tinfo->curr.width, devinfo->role);
3282                         if (bootverbose) {
3283                                 printf("(%s:%c:%d:%d): Received "
3284                                        "SDTR period %x, offset %x\n\t"
3285                                        "Filtered to period %x, offset %x\n",
3286                                        ahd_name(ahd), devinfo->channel,
3287                                        devinfo->target, devinfo->lun,
3288                                        ahd->msgin_buf[3], saved_offset,
3289                                        period, offset);
3290                         }
3291                         ahd_set_syncrate(ahd, devinfo, period,
3292                                          offset, ppr_options,
3293                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
3294                                          /*paused*/TRUE);
3295
3296                         /*
3297                          * See if we initiated Sync Negotiation
3298                          * and didn't have to fall down to async
3299                          * transfers.
3300                          */
3301                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, TRUE)) {
3302                                 /* We started it */
3303                                 if (saved_offset != offset) {
3304                                         /* Went too low - force async */
3305                                         reject = TRUE;
3306                                 }
3307                         } else {
3308                                 /*
3309                                  * Send our own SDTR in reply
3310                                  */
3311                                 if (bootverbose
3312                                  && devinfo->role == ROLE_INITIATOR) {
3313                                         printf("(%s:%c:%d:%d): Target "
3314                                                "Initiated SDTR\n",
3315                                                ahd_name(ahd), devinfo->channel,
3316                                                devinfo->target, devinfo->lun);
3317                                 }
3318                                 ahd->msgout_index = 0;
3319                                 ahd->msgout_len = 0;
3320                                 ahd_construct_sdtr(ahd, devinfo,
3321                                                    period, offset);
3322                                 ahd->msgout_index = 0;
3323                                 response = TRUE;
3324                         }
3325                         done = MSGLOOP_MSGCOMPLETE;
3326                         break;
3327                 }
3328                 case MSG_EXT_WDTR:
3329                 {
3330                         u_int bus_width;
3331                         u_int saved_width;
3332                         u_int sending_reply;
3333
3334                         sending_reply = FALSE;
3335                         if (ahd->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3336                                 reject = TRUE;
3337                                 break;
3338                         }
3339
3340                         /*
3341                          * Wait until we have our arg before validating
3342                          * and acting on this message.
3343                          *
3344                          * Add one to MSG_EXT_WDTR_LEN to account for
3345                          * the extended message preamble.
3346                          */
3347                         if (ahd->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3348                                 break;
3349
3350                         bus_width = ahd->msgin_buf[3];
3351                         saved_width = bus_width;
3352                         ahd_validate_width(ahd, tinfo, &bus_width,
3353                                            devinfo->role);
3354                         if (bootverbose) {
3355                                 printf("(%s:%c:%d:%d): Received WDTR "
3356                                        "%x filtered to %x\n",
3357                                        ahd_name(ahd), devinfo->channel,
3358                                        devinfo->target, devinfo->lun,
3359                                        saved_width, bus_width);
3360                         }
3361
3362                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, TRUE)) {
3363                                 /*
3364                                  * Don't send a WDTR back to the
3365                                  * target, since we asked first.
3366                                  * If the width went higher than our
3367                                  * request, reject it.
3368                                  */
3369                                 if (saved_width > bus_width) {
3370                                         reject = TRUE;
3371                                         printf("(%s:%c:%d:%d): requested %dBit "
3372                                                "transfers.  Rejecting...\n",
3373                                                ahd_name(ahd), devinfo->channel,
3374                                                devinfo->target, devinfo->lun,
3375                                                8 * (0x01 << bus_width));
3376                                         bus_width = 0;
3377                                 }
3378                         } else {
3379                                 /*
3380                                  * Send our own WDTR in reply
3381                                  */
3382                                 if (bootverbose
3383                                  && devinfo->role == ROLE_INITIATOR) {
3384                                         printf("(%s:%c:%d:%d): Target "
3385                                                "Initiated WDTR\n",
3386                                                ahd_name(ahd), devinfo->channel,
3387                                                devinfo->target, devinfo->lun);
3388                                 }
3389                                 ahd->msgout_index = 0;
3390                                 ahd->msgout_len = 0;
3391                                 ahd_construct_wdtr(ahd, devinfo, bus_width);
3392                                 ahd->msgout_index = 0;
3393                                 response = TRUE;
3394                                 sending_reply = TRUE;
3395                         }
3396                         ahd_set_width(ahd, devinfo, bus_width,
3397                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
3398                                       /*paused*/TRUE);
3399                         /* After a wide message, we are async */
3400                         ahd_set_syncrate(ahd, devinfo, /*period*/0,
3401                                          /*offset*/0, /*ppr_options*/0,
3402                                          AHD_TRANS_ACTIVE, /*paused*/TRUE);
3403                         if (sending_reply == FALSE && reject == FALSE) {
3404
3405                                 if (tinfo->goal.period) {
3406                                         ahd->msgout_index = 0;
3407                                         ahd->msgout_len = 0;
3408                                         ahd_build_transfer_msg(ahd, devinfo);
3409                                         ahd->msgout_index = 0;
3410                                         response = TRUE;
3411                                 }
3412                         }
3413                         done = MSGLOOP_MSGCOMPLETE;
3414                         break;
3415                 }
3416                 case MSG_EXT_PPR:
3417                 {
3418                         u_int   period;
3419                         u_int   offset;
3420                         u_int   bus_width;
3421                         u_int   ppr_options;
3422                         u_int   saved_width;
3423                         u_int   saved_offset;
3424                         u_int   saved_ppr_options;
3425
3426                         if (ahd->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3427                                 reject = TRUE;
3428                                 break;
3429                         }
3430
3431                         /*
3432                          * Wait until we have all args before validating
3433                          * and acting on this message.
3434                          *
3435                          * Add one to MSG_EXT_PPR_LEN to account for
3436                          * the extended message preamble.
3437                          */
3438                         if (ahd->msgin_index < (MSG_EXT_PPR_LEN + 1))
3439                                 break;
3440
3441                         period = ahd->msgin_buf[3];
3442                         offset = ahd->msgin_buf[5];
3443                         bus_width = ahd->msgin_buf[6];
3444                         saved_width = bus_width;
3445                         ppr_options = ahd->msgin_buf[7];
3446                         /*
3447                          * According to the spec, a DT only
3448                          * period factor with no DT option
3449                          * set implies async.
3450                          */
3451                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3452                          && period <= 9)
3453                                 offset = 0;
3454                         saved_ppr_options = ppr_options;
3455                         saved_offset = offset;
3456
3457                         /*
3458                          * Transfer options are only available if we
3459                          * are negotiating wide.
3460                          */
3461                         if (bus_width == 0)
3462                                 ppr_options &= MSG_EXT_PPR_QAS_REQ;
3463
3464                         ahd_validate_width(ahd, tinfo, &bus_width,
3465                                            devinfo->role);
3466                         ahd_devlimited_syncrate(ahd, tinfo, &period,
3467                                                 &ppr_options, devinfo->role);
3468                         ahd_validate_offset(ahd, tinfo, period, &offset,
3469                                             bus_width, devinfo->role);
3470
3471                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, TRUE)) {
3472                                 /*
3473                                  * If we are unable to do any of the
3474                                  * requested options (we went too low),
3475                                  * then we'll have to reject the message.
3476                                  */
3477                                 if (saved_width > bus_width
3478                                  || saved_offset != offset
3479                                  || saved_ppr_options != ppr_options) {
3480                                         reject = TRUE;
3481                                         period = 0;
3482                                         offset = 0;
3483                                         bus_width = 0;
3484                                         ppr_options = 0;
3485                                 }
3486                         } else {
3487                                 if (devinfo->role != ROLE_TARGET)
3488                                         printf("(%s:%c:%d:%d): Target "
3489                                                "Initiated PPR\n",
3490                                                ahd_name(ahd), devinfo->channel,
3491                                                devinfo->target, devinfo->lun);
3492                                 else
3493                                         printf("(%s:%c:%d:%d): Initiator "
3494                                                "Initiated PPR\n",
3495                                                ahd_name(ahd), devinfo->channel,
3496                                                devinfo->target, devinfo->lun);
3497                                 ahd->msgout_index = 0;
3498                                 ahd->msgout_len = 0;
3499                                 ahd_construct_ppr(ahd, devinfo, period, offset,
3500                                                   bus_width, ppr_options);
3501                                 ahd->msgout_index = 0;
3502                                 response = TRUE;
3503                         }
3504                         if (bootverbose) {
3505                                 printf("(%s:%c:%d:%d): Received PPR width %x, "
3506                                        "period %x, offset %x,options %x\n"
3507                                        "\tFiltered to width %x, period %x, "
3508                                        "offset %x, options %x\n",
3509                                        ahd_name(ahd), devinfo->channel,
3510                                        devinfo->target, devinfo->lun,
3511                                        saved_width, ahd->msgin_buf[3],
3512                                        saved_offset, saved_ppr_options,
3513                                        bus_width, period, offset, ppr_options);
3514                         }
3515                         ahd_set_width(ahd, devinfo, bus_width,
3516                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
3517                                       /*paused*/TRUE);
3518                         ahd_set_syncrate(ahd, devinfo, period,
3519                                          offset, ppr_options,
3520                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
3521                                          /*paused*/TRUE);
3522
3523                         done = MSGLOOP_MSGCOMPLETE;
3524                         break;
3525                 }
3526                 default:
3527                         /* Unknown extended message.  Reject it. */
3528                         reject = TRUE;
3529                         break;
3530                 }
3531                 break;
3532         }
3533 #ifdef AHD_TARGET_MODE
3534         case MSG_BUS_DEV_RESET:
3535                 ahd_handle_devreset(ahd, devinfo,
3536                                     CAM_BDR_SENT,
3537                                     "Bus Device Reset Received",
3538                                     /*verbose_level*/0);
3539                 ahd_restart(ahd);
3540                 done = MSGLOOP_TERMINATED;
3541                 break;
3542         case MSG_ABORT_TAG:
3543         case MSG_ABORT:
3544         case MSG_CLEAR_QUEUE:
3545         {
3546                 int tag;
3547
3548                 /* Target mode messages */
3549                 if (devinfo->role != ROLE_TARGET) {
3550                         reject = TRUE;
3551                         break;
3552                 }
3553                 tag = SCB_LIST_NULL;
3554                 if (ahd->msgin_buf[0] == MSG_ABORT_TAG)
3555                         tag = ahd_inb(ahd, INITIATOR_TAG);
3556                 ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
3557                                devinfo->lun, tag, ROLE_TARGET,
3558                                CAM_REQ_ABORTED);
3559
3560                 tstate = ahd->enabled_targets[devinfo->our_scsiid];
3561                 if (tstate != NULL) {
3562                         struct ahd_tmode_lstate* lstate;
3563
3564                         lstate = tstate->enabled_luns[devinfo->lun];
3565                         if (lstate != NULL) {
3566                                 ahd_queue_lstate_event(ahd, lstate,
3567                                                        devinfo->our_scsiid,
3568                                                        ahd->msgin_buf[0],
3569                                                        /*arg*/tag);
3570                                 ahd_send_lstate_events(ahd, lstate);
3571                         }
3572                 }
3573                 ahd_restart(ahd);
3574                 done = MSGLOOP_TERMINATED;
3575                 break;
3576         }
3577 #endif
3578         case MSG_QAS_REQUEST:
3579                 printf("%s: QAS request.  SCSISIGI == 0x%x\n",
3580                        ahd_name(ahd), ahd_inb(ahd, SCSISIGI));
3581                 /* FALLTHROUGH */
3582         case MSG_TERM_IO_PROC:
3583         default:
3584                 reject = TRUE;
3585                 break;
3586         }
3587
3588         if (reject) {
3589                 /*
3590                  * Setup to reject the message.
3591                  */
3592                 ahd->msgout_index = 0;
3593                 ahd->msgout_len = 1;
3594                 ahd->msgout_buf[0] = MSG_MESSAGE_REJECT;
3595                 done = MSGLOOP_MSGCOMPLETE;
3596                 response = TRUE;
3597         }
3598
3599         if (done != MSGLOOP_IN_PROG && !response)
3600                 /* Clear the outgoing message buffer */
3601                 ahd->msgout_len = 0;
3602
3603         return (done);
3604 }
3605
3606 /*
3607  * Process a message reject message.
3608  */
3609 static int
3610 ahd_handle_msg_reject(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3611 {
3612         /*
3613          * What we care about here is if we had an
3614          * outstanding SDTR or WDTR message for this
3615          * target.  If we did, this is a signal that
3616          * the target is refusing negotiation.
3617          */
3618         struct scb *scb;
3619         struct ahd_initiator_tinfo *tinfo;
3620         struct ahd_tmode_tstate *tstate;
3621         u_int scb_index;
3622         u_int last_msg;
3623         int   response = 0;
3624
3625         scb_index = ahd_get_scbptr(ahd);
3626         scb = ahd_lookup_scb(ahd, scb_index);
3627         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel,
3628                                     devinfo->our_scsiid,
3629                                     devinfo->target, &tstate);
3630         /* Might be necessary */
3631         last_msg = ahd_inb(ahd, LAST_MSG);
3632
3633         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
3634                 /*
3635                  * Target does not support the PPR message.
3636                  * Attempt to negotiate SPI-2 style.
3637                  */
3638                 if (bootverbose) {
3639                         printf("(%s:%c:%d:%d): PPR Rejected. "
3640                                "Trying WDTR/SDTR\n",
3641                                ahd_name(ahd), devinfo->channel,
3642                                devinfo->target, devinfo->lun);
3643                 }
3644                 tinfo->goal.ppr_options = 0;
3645                 tinfo->curr.transport_version = 2;
3646                 tinfo->goal.transport_version = 2;
3647                 ahd->msgout_index = 0;
3648                 ahd->msgout_len = 0;
3649                 ahd_build_transfer_msg(ahd, devinfo);
3650                 ahd->msgout_index = 0;
3651                 response = 1;
3652         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
3653
3654                 /* note 8bit xfers */
3655                 printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
3656                        "8bit transfers\n", ahd_name(ahd),
3657                        devinfo->channel, devinfo->target, devinfo->lun);
3658                 ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
3659                               AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
3660                               /*paused*/TRUE);
3661                 /*
3662                  * No need to clear the sync rate.  If the target
3663                  * did not accept the command, our syncrate is
3664                  * unaffected.  If the target started the negotiation,
3665                  * but rejected our response, we already cleared the
3666                  * sync rate before sending our WDTR.
3667                  */
3668                 if (tinfo->goal.period) {
3669
3670                         /* Start the sync negotiation */
3671                         ahd->msgout_index = 0;
3672                         ahd->msgout_len = 0;
3673                         ahd_build_transfer_msg(ahd, devinfo);
3674                         ahd->msgout_index = 0;
3675                         response = 1;
3676                 }
3677         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
3678                 /* note asynch xfers and clear flag */
3679                 ahd_set_syncrate(ahd, devinfo, /*period*/0,
3680                                  /*offset*/0, /*ppr_options*/0,
3681                                  AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
3682                                  /*paused*/TRUE);
3683                 printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
3684                        "Using asynchronous transfers\n",
3685                        ahd_name(ahd), devinfo->channel,
3686                        devinfo->target, devinfo->lun);
3687         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
3688                 int tag_type;
3689                 int mask;
3690
3691                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
3692
3693                 if (tag_type == MSG_SIMPLE_TASK) {
3694                         printf("(%s:%c:%d:%d): refuses tagged commands.  "
3695                                "Performing non-tagged I/O\n", ahd_name(ahd),
3696                                devinfo->channel, devinfo->target, devinfo->lun);
3697                         ahd_set_tags(ahd, devinfo, AHD_QUEUE_NONE);
3698                         mask = ~0x23;
3699                 } else {
3700                         printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
3701                                "Performing simple queue tagged I/O only\n",
3702                                ahd_name(ahd), devinfo->channel, devinfo->target,
3703                                devinfo->lun, tag_type == MSG_ORDERED_TASK
3704                                ? "ordered" : "head of queue");
3705                         ahd_set_tags(ahd, devinfo, AHD_QUEUE_BASIC);
3706                         mask = ~0x03;
3707                 }
3708
3709                 /*
3710                  * Resend the identify for this CCB as the target
3711                  * may believe that the selection is invalid otherwise.
3712                  */
3713                 ahd_outb(ahd, SCB_CONTROL,
3714                          ahd_inb_scbram(ahd, SCB_CONTROL) & mask);
3715                 scb->hscb->control &= mask;
3716                 ahd_set_transaction_tag(scb, /*enabled*/FALSE,
3717                                         /*type*/MSG_SIMPLE_TASK);
3718                 ahd_outb(ahd, MSG_OUT, MSG_IDENTIFYFLAG);
3719                 ahd_assert_atn(ahd);
3720                 ahd_busy_tcl(ahd, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
3721                              SCB_GET_TAG(scb));
3722
3723                 /*
3724                  * Requeue all tagged commands for this target
3725                  * currently in our posession so they can be
3726                  * converted to untagged commands.
3727                  */
3728                 ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
3729                                    SCB_GET_CHANNEL(ahd, scb),
3730                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
3731                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
3732                                    SEARCH_COMPLETE);
3733         } else {
3734                 /*
3735                  * Otherwise, we ignore it.
3736                  */
3737                 printf("%s:%c:%d: Message reject for %x -- ignored\n",
3738                        ahd_name(ahd), devinfo->channel, devinfo->target,
3739                        last_msg);
3740         }
3741         return (response);
3742 }
3743
3744 /*
3745  * Process an ingnore wide residue message.
3746  */
3747 static void
3748 ahd_handle_ign_wide_residue(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3749 {
3750         u_int scb_index;
3751         struct scb *scb;
3752
3753         scb_index = ahd_get_scbptr(ahd);
3754         scb = ahd_lookup_scb(ahd, scb_index);
3755         /*
3756          * XXX Actually check data direction in the sequencer?
3757          * Perhaps add datadir to some spare bits in the hscb?
3758          */
3759         if ((ahd_inb(ahd, SEQ_FLAGS) & DPHASE) == 0
3760          || ahd_get_transfer_dir(scb) != CAM_DIR_IN) {
3761                 /*
3762                  * Ignore the message if we haven't
3763                  * seen an appropriate data phase yet.
3764                  */
3765         } else {
3766                 /*
3767                  * If the residual occurred on the last
3768                  * transfer and the transfer request was
3769                  * expected to end on an odd count, do
3770                  * nothing.  Otherwise, subtract a byte
3771                  * and update the residual count accordingly.
3772                  */
3773                 uint32_t sgptr;
3774
3775                 sgptr = ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
3776                 if ((sgptr & SG_LIST_NULL) != 0
3777                  && ahd_inb(ahd, DATA_COUNT_ODD) == 1) {
3778                         /*
3779                          * If the residual occurred on the last
3780                          * transfer and the transfer request was
3781                          * expected to end on an odd count, do
3782                          * nothing.
3783                          */
3784                 } else {
3785                         uint32_t data_cnt;
3786                         uint64_t data_addr;
3787                         uint32_t sglen;
3788
3789                         /* Pull in the rest of the sgptr */
3790                         sgptr |=
3791                             (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR + 3) << 24)
3792                           | (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR + 2) << 16)
3793                           | (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR + 1) << 8);
3794                         sgptr &= SG_PTR_MASK;
3795                         data_cnt =
3796                             (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24)
3797                           | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+2) << 16)
3798                           | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+1) << 8)
3799                           | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT));
3800
3801                         data_addr = (((uint64_t)ahd_inb(ahd, SHADDR + 7)) << 56)
3802                                   | (((uint64_t)ahd_inb(ahd, SHADDR + 6)) << 48)
3803                                   | (((uint64_t)ahd_inb(ahd, SHADDR + 5)) << 40)
3804                                   | (((uint64_t)ahd_inb(ahd, SHADDR + 4)) << 32)
3805                                   | (ahd_inb(ahd, SHADDR + 3) << 24)
3806                                   | (ahd_inb(ahd, SHADDR + 2) << 16)
3807                                   | (ahd_inb(ahd, SHADDR + 1) << 8)
3808                                   | (ahd_inb(ahd, SHADDR));
3809
3810                         data_cnt += 1;
3811                         data_addr -= 1;
3812
3813                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
3814                                 struct ahd_dma64_seg *sg;
3815
3816                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
3817
3818                                 /*
3819                                  * The residual sg ptr points to the next S/G
3820                                  * to load so we must go back one.
3821                                  */
3822                                 sg--;
3823                                 sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
3824                                 if (sg != scb->sg_list
3825                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
3826
3827                                         sg--;
3828                                         sglen = ahd_le32toh(sg->len);
3829                                         /*
3830                                          * Preserve High Address and SG_LIST
3831                                          * bits while setting the count to 1.
3832                                          */
3833                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
3834                                         data_addr = ahd_le64toh(sg->addr)
3835                                                   + (sglen & AHD_SG_LEN_MASK)
3836                                                   - 1;
3837
3838                                         /*
3839                                          * Increment sg so it points to the
3840                                          * "next" sg.
3841                                          */
3842                                         sg++;
3843                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
3844                                                                    sg);
3845                                 }
3846                         } else {
3847                                 struct ahd_dma_seg *sg;
3848
3849                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
3850
3851                                 /*
3852                                  * The residual sg ptr points to the next S/G
3853                                  * to load so we must go back one.
3854                                  */
3855                                 sg--;
3856                                 sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
3857                                 if (sg != scb->sg_list
3858                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
3859
3860                                         sg--;
3861                                         sglen = ahd_le32toh(sg->len);
3862                                         /*
3863                                          * Preserve High Address and SG_LIST
3864                                          * bits while setting the count to 1.
3865                                          */
3866                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
3867                                         data_addr = ahd_le32toh(sg->addr)
3868                                                   + (sglen & AHD_SG_LEN_MASK)
3869                                                   - 1;
3870
3871                                         /*
3872                                          * Increment sg so it points to the
3873                                          * "next" sg.
3874                                          */
3875                                         sg++;
3876                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
3877                                                                   sg);
3878                                 }
3879                         }
3880                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR + 3, sgptr >> 24);
3881                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR + 2, sgptr >> 16);
3882                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR + 1, sgptr >> 8);
3883                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR, sgptr);
3884
3885                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, data_cnt >> 24);
3886                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 2, data_cnt >> 16);
3887                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 1, data_cnt >> 8);
3888                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT, data_cnt);
3889
3890                         /*
3891                          * The FIFO's pointers will be updated if/when the
3892                          * sequencer re-enters a data phase.
3893                          */
3894                 }
3895         }
3896 }
3897
3898
3899 /*
3900  * Reinitialize the data pointers for the active transfer
3901  * based on its current residual.
3902  */
3903 static void
3904 ahd_reinitialize_dataptrs(struct ahd_softc *ahd)
3905 {
3906         struct           scb *scb;
3907         ahd_mode_state   saved_modes;
3908         u_int            scb_index;
3909         u_int            wait;
3910         uint32_t         sgptr;
3911         uint32_t         resid;
3912         uint64_t         dataptr;
3913
3914         AHD_ASSERT_MODES(ahd, AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK,
3915                          AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK);
3916
3917         scb_index = ahd_get_scbptr(ahd);
3918         scb = ahd_lookup_scb(ahd, scb_index);
3919
3920         /*
3921          * Release and reacquire the FIFO so we
3922          * have a clean slate.
3923          */
3924         ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
3925         wait = 1000;
3926         do {
3927                 ahd_delay(100);
3928         } while (--wait && !(ahd_inb(ahd, MDFFSTAT) & FIFOFREE));
3929         if (wait == 0) {
3930                 ahd_print_path(ahd, scb);
3931                 printf("ahd_reinitialize_dataptrs: Forcing FIFO free.\n");
3932                 ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
3933         }
3934         saved_modes = ahd_save_modes(ahd);
3935         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3936         ahd_outb(ahd, DFFSTAT,
3937                  ahd_inb(ahd, DFFSTAT) | (saved_modes == 0x11 ? CURRFIFO : 0));
3938
3939         /*
3940          * Determine initial values for data_addr and data_cnt
3941          * for resuming the data phase.
3942          */
3943         sgptr = (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR + 3) << 24)
3944               | (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR + 2) << 16)
3945               | (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR + 1) << 8)
3946               | ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
3947         sgptr &= SG_PTR_MASK;
3948
3949         resid = (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 2) << 16)
3950               | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 1) << 8)
3951               | ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT);
3952
3953         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
3954                 struct ahd_dma64_seg *sg;
3955
3956                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
3957
3958                 /* The residual sg_ptr always points to the next sg */
3959                 sg--;
3960
3961                 dataptr = ahd_le64toh(sg->addr)
3962                         + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
3963                         - resid;
3964                 ahd_outb(ahd, HADDR + 7, dataptr >> 56);
3965                 ahd_outb(ahd, HADDR + 6, dataptr >> 48);
3966                 ahd_outb(ahd, HADDR + 5, dataptr >> 40);
3967                 ahd_outb(ahd, HADDR + 4, dataptr >> 32);
3968         } else {
3969                 struct   ahd_dma_seg *sg;
3970
3971                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
3972
3973                 /* The residual sg_ptr always points to the next sg */
3974                 sg--;
3975
3976                 dataptr = ahd_le32toh(sg->addr)
3977                         + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
3978                         - resid;
3979                 ahd_outb(ahd, HADDR + 4,
3980                          (ahd_le32toh(sg->len) & ~AHD_SG_LEN_MASK) >> 24);
3981         }
3982         ahd_outb(ahd, HADDR + 3, dataptr >> 24);
3983         ahd_outb(ahd, HADDR + 2, dataptr >> 16);
3984         ahd_outb(ahd, HADDR + 1, dataptr >> 8);
3985         ahd_outb(ahd, HADDR, dataptr);
3986         ahd_outb(ahd, HCNT + 2, resid >> 16);
3987         ahd_outb(ahd, HCNT + 1, resid >> 8);
3988         ahd_outb(ahd, HCNT, resid);
3989 }
3990
3991 /*
3992  * Handle the effects of issuing a bus device reset message.
3993  */
3994 static void
3995 ahd_handle_devreset(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3996                     cam_status status, char *message, int verbose_level)
3997 {
3998 #ifdef AHD_TARGET_MODE
3999         struct ahd_tmode_tstate* tstate;
4000         u_int lun;
4001 #endif
4002         int found;
4003
4004         found = ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
4005                                CAM_LUN_WILDCARD, SCB_LIST_NULL, devinfo->role,
4006                                status);
4007
4008 #ifdef AHD_TARGET_MODE
4009         /*
4010          * Send an immediate notify ccb to all target mord peripheral
4011          * drivers affected by this action.
4012          */
4013         tstate = ahd->enabled_targets[devinfo->our_scsiid];
4014         if (tstate != NULL) {
4015                 for (lun = 0; lun < AHD_NUM_LUNS; lun++) {
4016                         struct ahd_tmode_lstate* lstate;
4017
4018                         lstate = tstate->enabled_luns[lun];
4019                         if (lstate == NULL)
4020                                 continue;
4021
4022                         ahd_queue_lstate_event(ahd, lstate, devinfo->our_scsiid,
4023                                                MSG_BUS_DEV_RESET, /*arg*/0);
4024                         ahd_send_lstate_events(ahd, lstate);
4025                 }
4026         }
4027 #endif
4028
4029         /*
4030          * Go back to async/narrow transfers and renegotiate.
4031          */
4032         ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4033                       AHD_TRANS_CUR, /*paused*/TRUE);
4034         ahd_set_syncrate(ahd, devinfo, /*period*/0, /*offset*/0,
4035                          /*ppr_options*/0, AHD_TRANS_CUR, /*paused*/TRUE);
4036
4037         ahd_send_async(ahd, devinfo->channel, devinfo->target,
4038                        CAM_LUN_WILDCARD, AC_SENT_BDR, NULL);
4039
4040         if (message != NULL
4041          && (verbose_level <= bootverbose))
4042                 printf("%s: %s on %c:%d. %d SCBs aborted\n", ahd_name(ahd),
4043                        message, devinfo->channel, devinfo->target, found);
4044 }
4045
4046 #ifdef AHD_TARGET_MODE
4047 static void
4048 ahd_setup_target_msgin(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4049                        struct scb *scb)
4050 {
4051
4052         /*
4053          * To facilitate adding multiple messages together,
4054          * each routine should increment the index and len
4055          * variables instead of setting them explicitly.
4056          */
4057         ahd->msgout_index = 0;
4058         ahd->msgout_len = 0;
4059
4060         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
4061                 ahd_build_transfer_msg(ahd, devinfo);
4062         else
4063                 panic("ahd_intr: AWAITING target message with no message");
4064
4065         ahd->msgout_index = 0;
4066         ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
4067 }
4068 #endif
4069 /**************************** Initialization **********************************/
4070 static bus_size_t
4071 ahd_sglist_size(struct ahd_softc *ahd)
4072 {
4073         bus_size_t list_size;
4074
4075         list_size = sizeof(struct ahd_dma_seg) * AHD_NSEG;
4076         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
4077                 list_size = sizeof(struct ahd_dma64_seg) * AHD_NSEG;
4078         return (list_size);
4079 }
4080
4081 /*
4082  * Calculate the optimum S/G List allocation size.  S/G elements used
4083  * for a given transaction must be physically contiguous.  Assume the
4084  * OS will allocate full pages to us, so it doesn't make sense to request
4085  * less than a page.
4086  */
4087 static bus_size_t
4088 ahd_sglist_allocsize(struct ahd_softc *ahd)
4089 {
4090         bus_size_t sg_list_increment;
4091         bus_size_t sg_list_size;
4092         bus_size_t max_list_size;
4093         bus_size_t best_list_size;
4094
4095         /* Start out with the minimum required for AHD_NSEG. */
4096         sg_list_increment = ahd_sglist_size(ahd);
4097         sg_list_size = sg_list_increment;
4098
4099         /* Get us as close as possible to a page in size. */
4100         while ((sg_list_size + sg_list_increment) <= PAGE_SIZE)
4101                 sg_list_size += sg_list_increment;
4102
4103         /*
4104          * Try to reduce the amount of wastage by allocating
4105          * multiple pages.
4106          */
4107         best_list_size = sg_list_size;
4108         max_list_size = roundup(sg_list_increment, PAGE_SIZE);
4109         if (max_list_size < 4 * PAGE_SIZE)
4110                 max_list_size = 4 * PAGE_SIZE;
4111         if (max_list_size > (AHD_SCB_MAX_ALLOC * sg_list_increment))
4112                 max_list_size = (AHD_SCB_MAX_ALLOC * sg_list_increment);
4113         while ((sg_list_size + sg_list_increment) <= max_list_size
4114            &&  (sg_list_size % PAGE_SIZE) != 0) {
4115                 bus_size_t new_mod;
4116                 bus_size_t best_mod;
4117
4118                 sg_list_size += sg_list_increment;
4119                 new_mod = sg_list_size % PAGE_SIZE;
4120                 best_mod = best_list_size % PAGE_SIZE;
4121                 if (new_mod > best_mod || new_mod == 0) {
4122                         best_list_size = sg_list_size;
4123                 }
4124         }
4125         return (best_list_size);
4126 }
4127
4128 /*
4129  * Allocate a controller structure for a new device
4130  * and perform initial initializion.
4131  */
4132 struct ahd_softc *
4133 ahd_alloc(void *platform_arg, char *name)
4134 {
4135         struct  ahd_softc *ahd;
4136
4137 #ifndef __FreeBSD__
4138         ahd = malloc(sizeof(*ahd), M_DEVBUF, M_NOWAIT);
4139         if (!ahd) {
4140                 printf("aic7xxx: cannot malloc softc!\n");
4141                 free(name, M_DEVBUF);
4142                 return NULL;
4143         }
4144 #else
4145         ahd = device_get_softc((device_t)platform_arg);
4146 #endif
4147         memset(ahd, 0, sizeof(*ahd));
4148         ahd->seep_config = malloc(sizeof(*ahd->seep_config),
4149                                   M_DEVBUF, M_NOWAIT);
4150         if (ahd->seep_config == NULL) {
4151 #ifndef __FreeBSD__
4152                 free(ahd, M_DEVBUF);
4153 #endif
4154                 free(name, M_DEVBUF);
4155                 return (NULL);
4156         }
4157         LIST_INIT(&ahd->pending_scbs);
4158         /* We don't know our unit number until the OSM sets it */
4159         ahd->name = name;
4160         ahd->unit = -1;
4161         ahd->description = NULL;
4162         ahd->bus_description = NULL;
4163         ahd->channel = 'A';
4164         ahd->chip = AHD_NONE;
4165         ahd->features = AHD_FENONE;
4166         ahd->bugs = AHD_BUGNONE;
4167         ahd->flags = AHD_SPCHK_ENB_A|AHD_RESET_BUS_A|AHD_TERM_ENB_A
4168                    | AHD_EXTENDED_TRANS_A|AHD_STPWLEVEL_A;
4169         ahd_timer_init(&ahd->reset_timer);
4170
4171         if (ahd_platform_alloc(ahd, platform_arg) != 0) {
4172                 ahd_free(ahd);
4173                 ahd = NULL;
4174         }
4175 #ifdef AHD_DEBUG
4176         if ((ahd_debug & AHD_SHOW_MEMORY) != 0) {
4177                 printf("%s: scb size = 0x%x, hscb size - 0x%x\n",
4178                        ahd_name(ahd), sizeof(struct scb),
4179                        sizeof(struct hardware_scb));
4180         }
4181 #endif
4182         return (ahd);
4183 }
4184
4185 int
4186 ahd_softc_init(struct ahd_softc *ahd)
4187 {
4188
4189         ahd->unpause = 0;
4190         ahd->pause = PAUSE;
4191         return (0);
4192 }
4193
4194 void
4195 ahd_softc_insert(struct ahd_softc *ahd)
4196 {
4197         struct ahd_softc *list_ahd;
4198
4199 #if AHD_PCI_CONFIG > 0
4200         /*
4201          * Second Function PCI devices need to inherit some
4202          * settings from function 0.
4203          */
4204         if ((ahd->features & AHD_MULTI_FUNC) != 0) {
4205                 TAILQ_FOREACH(list_ahd, &ahd_tailq, links) {
4206                         ahd_dev_softc_t list_pci;
4207                         ahd_dev_softc_t pci;
4208
4209                         list_pci = list_ahd->dev_softc;
4210                         pci = ahd->dev_softc;
4211                         if (ahd_get_pci_slot(list_pci) == ahd_get_pci_slot(pci)
4212                          && ahd_get_pci_bus(list_pci) == ahd_get_pci_bus(pci)) {
4213                                 struct ahd_softc *master;
4214                                 struct ahd_softc *slave;
4215
4216                                 if (ahd_get_pci_function(list_pci) == 0) {
4217                                         master = list_ahd;
4218                                         slave = ahd;
4219                                 } else {
4220                                         master = ahd;
4221                                         slave = list_ahd;
4222                                 }
4223                                 slave->flags &= ~AHD_BIOS_ENABLED;
4224                                 slave->flags |=
4225                                     master->flags & AHD_BIOS_ENABLED;
4226                                 slave->flags &= ~AHD_PRIMARY_CHANNEL;
4227                                 slave->flags |=
4228                                     master->flags & AHD_PRIMARY_CHANNEL;
4229                                 break;
4230                         }
4231                 }
4232         }
4233 #endif
4234
4235         /*
4236          * Insertion sort into our list of softcs.
4237          */
4238         list_ahd = TAILQ_FIRST(&ahd_tailq);
4239         while (list_ahd != NULL
4240             && ahd_softc_comp(list_ahd, ahd) <= 0)
4241                 list_ahd = TAILQ_NEXT(list_ahd, links);
4242         if (list_ahd != NULL)
4243                 TAILQ_INSERT_BEFORE(list_ahd, ahd, links);
4244         else
4245                 TAILQ_INSERT_TAIL(&ahd_tailq, ahd, links);
4246         ahd->init_level++;
4247 }
4248
4249 /*
4250  * Verify that the passed in softc pointer is for a
4251  * controller that is still configured.
4252  */
4253 struct ahd_softc *
4254 ahd_find_softc(struct ahd_softc *ahd)
4255 {
4256         struct ahd_softc *list_ahd;
4257
4258         TAILQ_FOREACH(list_ahd, &ahd_tailq, links) {
4259                 if (list_ahd == ahd)
4260                         return (ahd);
4261         }
4262         return (NULL);
4263 }
4264
4265 void
4266 ahd_set_unit(struct ahd_softc *ahd, int unit)
4267 {
4268         ahd->unit = unit;
4269 }
4270
4271 void
4272 ahd_set_name(struct ahd_softc *ahd, char *name)
4273 {
4274         if (ahd->name != NULL)
4275                 free(ahd->name, M_DEVBUF);
4276         ahd->name = name;
4277 }
4278
4279 void
4280 ahd_free(struct ahd_softc *ahd)
4281 {
4282         int i;
4283
4284         ahd_fini_scbdata(ahd);
4285         switch (ahd->init_level) {
4286         default:
4287         case 5:
4288                 ahd_shutdown(ahd);
4289                 TAILQ_REMOVE(&ahd_tailq, ahd, links);
4290                 /* FALLTHROUGH */
4291         case 4:
4292                 ahd_dmamap_unload(ahd, ahd->shared_data_dmat,
4293                                   ahd->shared_data_dmamap);
4294                 /* FALLTHROUGH */
4295         case 3:
4296                 ahd_dmamem_free(ahd, ahd->shared_data_dmat, ahd->qoutfifo,
4297                                 ahd->shared_data_dmamap);
4298                 ahd_dmamap_destroy(ahd, ahd->shared_data_dmat,
4299                                    ahd->shared_data_dmamap);
4300                 /* FALLTHROUGH */
4301         case 2:
4302                 ahd_dma_tag_destroy(ahd, ahd->shared_data_dmat);
4303         case 1:
4304 #ifndef __linux__
4305                 ahd_dma_tag_destroy(ahd, ahd->buffer_dmat);
4306 #endif
4307                 break;
4308         case 0:
4309                 break;
4310         }
4311
4312 #ifndef __linux__
4313         ahd_dma_tag_destroy(ahd, ahd->parent_dmat);
4314 #endif
4315         ahd_platform_free(ahd);
4316         for (i = 0; i < AHD_NUM_TARGETS; i++) {
4317                 struct ahd_tmode_tstate *tstate;
4318
4319                 tstate = ahd->enabled_targets[i];
4320                 if (tstate != NULL) {
4321 #if AHD_TARGET_MODE
4322                         int j;
4323
4324                         for (j = 0; j < AHD_NUM_LUNS; j++) {
4325                                 struct ahd_tmode_lstate *lstate;
4326
4327                                 lstate = tstate->enabled_luns[j];
4328                                 if (lstate != NULL) {
4329                                         xpt_free_path(lstate->path);
4330                                         free(lstate, M_DEVBUF);
4331                                 }
4332                         }
4333 #endif
4334                         free(tstate, M_DEVBUF);
4335                 }
4336         }
4337 #if AHD_TARGET_MODE
4338         if (ahd->black_hole != NULL) {
4339                 xpt_free_path(ahd->black_hole->path);
4340                 free(ahd->black_hole, M_DEVBUF);
4341         }
4342 #endif
4343         if (ahd->name != NULL)
4344                 free(ahd->name, M_DEVBUF);
4345         if (ahd->seep_config != NULL)
4346                 free(ahd->seep_config, M_DEVBUF);
4347 #ifndef __FreeBSD__
4348         free(ahd, M_DEVBUF);
4349 #endif
4350         return;
4351 }
4352
4353 void
4354 ahd_shutdown(void *arg)
4355 {
4356         struct  ahd_softc *ahd;
4357
4358         ahd = (struct ahd_softc *)arg;
4359
4360         /* This will reset most registers to 0, but not all */
4361         ahd_reset(ahd);
4362 }
4363
4364 /*
4365  * Reset the controller and record some information about it
4366  * that is only available just after a reset.
4367  */
4368 int
4369 ahd_reset(struct ahd_softc *ahd)
4370 {
4371         u_int    sxfrctl1;
4372         int      wait;
4373         uint32_t cmd;
4374
4375         /*
4376          * Preserve the value of the SXFRCTL1 register for all channels.
4377          * It contains settings that affect termination and we don't want
4378          * to disturb the integrity of the bus.
4379          */
4380         ahd_pause(ahd);
4381         sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
4382
4383         cmd = ahd_pci_read_config(ahd->dev_softc, PCIR_COMMAND, /*bytes*/2);
4384         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
4385                 uint32_t mod_cmd;
4386
4387                 /*
4388                  * A4 Razor #632
4389                  * During the assertion of CHIPRST, the chip
4390                  * does not disable its parity logic prior to
4391                  * the start of the reset.  This may cause a
4392                  * parity error to be detected and thus a
4393                  * spurious SERR or PERR assertion.  Disble
4394                  * PERR and SERR responses during the CHIPRST.
4395                  */
4396                 mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
4397                 ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
4398                                      mod_cmd, /*bytes*/2);
4399         }
4400         ahd_outb(ahd, HCNTRL, CHIPRST | ahd->pause);
4401
4402         /*
4403          * Ensure that the reset has finished.  We delay 1000us
4404          * prior to reading the register to make sure the chip
4405          * has sufficiently completed its reset to handle register
4406          * accesses.
4407          */
4408         wait = 1000;
4409         do {
4410                 ahd_delay(1000);
4411         } while (--wait && !(ahd_inb(ahd, HCNTRL) & CHIPRSTACK));
4412
4413         if (wait == 0) {
4414                 printf("%s: WARNING - Failed chip reset!  "
4415                        "Trying to initialize anyway.\n", ahd_name(ahd));
4416         }
4417         ahd_outb(ahd, HCNTRL, ahd->pause);
4418
4419         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
4420                 /*
4421                  * Clear any latched PCI error status and restore
4422                  * previous SERR and PERR response enables.
4423                  */
4424                 ahd_pci_write_config(ahd->dev_softc, PCIR_STATUS + 1,
4425                                      0xFF, /*bytes*/1);
4426                 ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
4427                                      cmd, /*bytes*/2);
4428         }
4429         /* After a reset, we know the state of the mode register. */
4430         ahd_known_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4431
4432         /* Determine chip configuration */
4433         ahd->features &= ~AHD_WIDE;
4434         if ((ahd_inb(ahd, SBLKCTL) & SELWIDE) != 0)
4435                 ahd->features |= AHD_WIDE;
4436
4437         /*
4438          * Restore SXFRCTL1.
4439          *
4440          * We must always initialize STPWEN to 1 before we
4441          * restore the saved values.  STPWEN is initialized
4442          * to a tri-state condition which can only be cleared
4443          * by turning it on.
4444          */
4445         ahd_outb(ahd, SXFRCTL1, sxfrctl1|STPWEN);
4446         ahd_outb(ahd, SXFRCTL1, sxfrctl1);
4447
4448         /*
4449          * If a recovery action has forced a chip reset,
4450          * re-initialize the chip to our likeing.
4451          */
4452         if (ahd->init_level > 0)
4453                 ahd_chip_init(ahd);
4454
4455         return (0);
4456 }
4457
4458 /*
4459  * Determine the number of SCBs available on the controller
4460  */
4461 int
4462 ahd_probe_scbs(struct ahd_softc *ahd) {
4463         int i;
4464
4465         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
4466                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
4467         for (i = 0; i < AHD_SCB_MAX; i++) {
4468                 int j;
4469
4470                 ahd_set_scbptr(ahd, i);
4471                 ahd_outw(ahd, SCB_BASE, i);
4472                 for (j = 2; j < 64; j++)
4473                         ahd_outb(ahd, SCB_BASE+j, 0);
4474                 /* Start out life as unallocated (needing an abort) */
4475                 ahd_outb(ahd, SCB_CONTROL, MK_MESSAGE);
4476                 if (ahd_inw_scbram(ahd, SCB_BASE) != i)
4477                         break;
4478                 ahd_set_scbptr(ahd, 0);
4479                 if (ahd_inw_scbram(ahd, SCB_BASE) != 0)
4480                         break;
4481         }
4482         return (i);
4483 }
4484
4485 static void
4486 ahd_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
4487 {
4488         bus_addr_t *baddr;
4489
4490         baddr = (bus_addr_t *)arg;
4491         *baddr = segs->ds_addr;
4492 }
4493
4494 static void
4495 ahd_initialize_hscbs(struct ahd_softc *ahd)
4496 {
4497         int i;
4498
4499         for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
4500                 ahd_set_scbptr(ahd, i);
4501
4502                 /* Clear the control byte. */
4503                 ahd_outb(ahd, SCB_CONTROL, 0);
4504
4505                 /* Set the next pointer */
4506                 ahd_outw(ahd, SCB_NEXT, SCB_LIST_NULL);
4507         }
4508 }
4509
4510 static int
4511 ahd_init_scbdata(struct ahd_softc *ahd)
4512 {
4513         struct  scb_data *scb_data;
4514         int     i;
4515
4516         scb_data = &ahd->scb_data;
4517         TAILQ_INIT(&scb_data->free_scbs);
4518         for (i = 0; i < AHD_NUM_TARGETS * AHD_NUM_LUNS_NONPKT; i++)
4519                 LIST_INIT(&scb_data->free_scb_lists[i]);
4520         LIST_INIT(&scb_data->any_dev_free_scb_list);
4521         SLIST_INIT(&scb_data->hscb_maps);
4522         SLIST_INIT(&scb_data->sg_maps);
4523         SLIST_INIT(&scb_data->sense_maps);
4524
4525         /* Determine the number of hardware SCBs and initialize them */
4526         scb_data->maxhscbs = ahd_probe_scbs(ahd);
4527         if (scb_data->maxhscbs == 0) {
4528                 printf("%s: No SCB space found\n", ahd_name(ahd));
4529                 return (ENXIO);
4530         }
4531
4532         ahd_initialize_hscbs(ahd);
4533
4534         /*
4535          * Create our DMA tags.  These tags define the kinds of device
4536          * accessible memory allocations and memory mappings we will
4537          * need to perform during normal operation.
4538          *
4539          * Unless we need to further restrict the allocation, we rely
4540          * on the restrictions of the parent dmat, hence the common
4541          * use of MAXADDR and MAXSIZE.
4542          */
4543
4544         /* DMA tag for our hardware scb structures */
4545         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
4546                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4547                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4548                                /*highaddr*/BUS_SPACE_MAXADDR,
4549                                /*filter*/NULL, /*filterarg*/NULL,
4550                                PAGE_SIZE, /*nsegments*/1,
4551                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4552                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
4553                 goto error_exit;
4554         }
4555
4556         scb_data->init_level++;
4557
4558         /* DMA tag for our S/G structures. */
4559         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
4560                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4561                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4562                                /*highaddr*/BUS_SPACE_MAXADDR,
4563                                /*filter*/NULL, /*filterarg*/NULL,
4564                                ahd_sglist_allocsize(ahd), /*nsegments*/1,
4565                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4566                                /*flags*/0, &scb_data->sg_dmat) != 0) {
4567                 goto error_exit;
4568         }
4569 #ifdef AHD_DEBUG
4570         if ((ahd_debug & AHD_SHOW_MEMORY) != 0)
4571                 printf("%s: ahd_sglist_allocsize = 0x%x\n", ahd_name(ahd),
4572                        ahd_sglist_allocsize(ahd));
4573 #endif
4574
4575         scb_data->init_level++;
4576
4577         /* DMA tag for our sense buffers.  We allocate in page sized chunks */
4578         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
4579                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4580                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4581                                /*highaddr*/BUS_SPACE_MAXADDR,
4582                                /*filter*/NULL, /*filterarg*/NULL,
4583                                PAGE_SIZE, /*nsegments*/1,
4584                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4585                                /*flags*/0, &scb_data->sense_dmat) != 0) {
4586                 goto error_exit;
4587         }
4588
4589         scb_data->init_level++;
4590
4591         /* Perform initial CCB allocation */
4592         ahd_alloc_scbs(ahd);
4593
4594         if (scb_data->numscbs == 0) {
4595                 printf("%s: ahd_init_scbdata - "
4596                        "Unable to allocate initial scbs\n",
4597                        ahd_name(ahd));
4598                 goto error_exit;
4599         }
4600
4601         /*
4602          * Note that we were successfull
4603          */
4604         return (0);
4605
4606 error_exit:
4607
4608         return (ENOMEM);
4609 }
4610
4611 static struct scb *
4612 ahd_find_scb_by_tag(struct ahd_softc *ahd, u_int tag)
4613 {
4614         struct scb *scb;
4615
4616         /*
4617          * Look on the pending list.
4618          */
4619         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
4620                 if (SCB_GET_TAG(scb) == tag)
4621                         return (scb);
4622         }
4623
4624         /*
4625          * Then on all of the collision free lists.
4626          */
4627         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
4628                 struct scb *list_scb;
4629
4630                 list_scb = scb;
4631                 do {
4632                         if (SCB_GET_TAG(list_scb) == tag)
4633                                 return (list_scb);
4634                         list_scb = LIST_NEXT(list_scb, collision_links);
4635                 } while (list_scb);
4636         }
4637
4638         /*
4639          * And finally on the generic free list.
4640          */
4641         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
4642                 if (SCB_GET_TAG(scb) == tag)
4643                         return (scb);
4644         }
4645
4646         return (NULL);
4647 }
4648
4649 static void
4650 ahd_fini_scbdata(struct ahd_softc *ahd)
4651 {
4652         struct scb_data *scb_data;
4653
4654         scb_data = &ahd->scb_data;
4655         if (scb_data == NULL)
4656                 return;
4657
4658         switch (scb_data->init_level) {
4659         default:
4660         case 7:
4661         {
4662                 struct map_node *sns_map;
4663
4664                 while ((sns_map = SLIST_FIRST(&scb_data->sense_maps)) != NULL) {
4665                         SLIST_REMOVE_HEAD(&scb_data->sense_maps, links);
4666                         ahd_dmamap_unload(ahd, scb_data->sense_dmat,
4667                                           sns_map->dmamap);
4668                         ahd_dmamem_free(ahd, scb_data->sense_dmat,
4669                                         sns_map->vaddr, sns_map->dmamap);
4670                         free(sns_map, M_DEVBUF);
4671                 }
4672                 ahd_dma_tag_destroy(ahd, scb_data->sense_dmat);
4673                 /* FALLTHROUGH */
4674         }
4675         case 6:
4676         {
4677                 struct map_node *sg_map;
4678
4679                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps)) != NULL) {
4680                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
4681                         ahd_dmamap_unload(ahd, scb_data->sg_dmat,
4682                                           sg_map->dmamap);
4683                         ahd_dmamem_free(ahd, scb_data->sg_dmat,
4684                                         sg_map->vaddr, sg_map->dmamap);
4685                         free(sg_map, M_DEVBUF);
4686                 }
4687                 ahd_dma_tag_destroy(ahd, scb_data->sg_dmat);
4688                 /* FALLTHROUGH */
4689         }
4690         case 5:
4691         {
4692                 struct map_node *hscb_map;
4693
4694                 while ((hscb_map = SLIST_FIRST(&scb_data->hscb_maps)) != NULL) {
4695                         SLIST_REMOVE_HEAD(&scb_data->hscb_maps, links);
4696                         ahd_dmamap_unload(ahd, scb_data->hscb_dmat,
4697                                           hscb_map->dmamap);
4698                         ahd_dmamem_free(ahd, scb_data->hscb_dmat,
4699                                         hscb_map->vaddr, hscb_map->dmamap);
4700                         free(hscb_map, M_DEVBUF);
4701                 }
4702                 ahd_dma_tag_destroy(ahd, scb_data->hscb_dmat);
4703                 /* FALLTHROUGH */
4704         }
4705         case 4:
4706         case 3:
4707         case 2:
4708         case 1:
4709         case 0:
4710                 break;
4711         }
4712 }
4713
4714 /*
4715  * DSP filter Bypass must be enabled until the first selection
4716  * after a change in bus mode (Razor #491 and #493).
4717  */
4718 static void
4719 ahd_setup_iocell_workaround(struct ahd_softc *ahd)
4720 {
4721         ahd_mode_state saved_modes;
4722
4723         saved_modes = ahd_save_modes(ahd);
4724         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
4725         ahd_outb(ahd, DSPDATACTL, ahd_inb(ahd, DSPDATACTL)
4726                | BYPASSENAB | RCVROFFSTDIS | XMITOFFSTDIS);
4727         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) | (ENSELDO|ENSELDI));
4728         ahd_restore_modes(ahd, saved_modes);
4729 }
4730
4731 static void
4732 ahd_iocell_first_selection(struct ahd_softc *ahd)
4733 {
4734         ahd_mode_state  saved_modes;
4735         u_int           sblkctl;
4736
4737         saved_modes = ahd_save_modes(ahd);
4738         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4739         sblkctl = ahd_inb(ahd, SBLKCTL);
4740         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
4741         if ((sblkctl & ENAB40) != 0) {
4742                 ahd_outb(ahd, DSPDATACTL,
4743                          ahd_inb(ahd, DSPDATACTL) & ~BYPASSENAB);
4744         }
4745         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) & ~(ENSELDO|ENSELDI));
4746         ahd_outb(ahd, CLRINT, CLRSCSIINT);
4747         ahd_restore_modes(ahd, saved_modes);
4748 }
4749
4750 /*************************** SCB Management ***********************************/
4751 static void
4752 ahd_add_col_list(struct ahd_softc *ahd, struct scb *scb, u_int col_idx)
4753 {
4754         struct  scb_list *free_list;
4755         struct  scb_tailq *free_tailq;
4756         struct  scb *first_scb;
4757
4758         scb->flags |= SCB_ON_COL_LIST;
4759         AHD_SET_SCB_COL_IDX(scb, col_idx);
4760         free_list = &ahd->scb_data.free_scb_lists[col_idx];
4761         free_tailq = &ahd->scb_data.free_scbs;
4762         first_scb = LIST_FIRST(free_list);
4763         if (first_scb != NULL) {
4764                 LIST_INSERT_AFTER(first_scb, scb, collision_links);
4765         } else {
4766                 LIST_INSERT_HEAD(free_list, scb, collision_links);
4767                 TAILQ_INSERT_TAIL(free_tailq, scb, links.tqe);
4768         }
4769 }
4770
4771 static void
4772 ahd_rem_col_list(struct ahd_softc *ahd, struct scb *scb)
4773 {
4774         struct  scb_list *free_list;
4775         struct  scb_tailq *free_tailq;
4776         struct  scb *first_scb;
4777         u_int   col_idx;
4778
4779         scb->flags &= ~SCB_ON_COL_LIST;
4780         col_idx = AHD_GET_SCB_COL_IDX(ahd, scb);
4781         free_list = &ahd->scb_data.free_scb_lists[col_idx];
4782         free_tailq = &ahd->scb_data.free_scbs;
4783         first_scb = LIST_FIRST(free_list);
4784         if (first_scb == scb) {
4785                 struct scb *next_scb;
4786
4787                 /*
4788                  * Maintain order in the collision free
4789                  * lists for fairness if this device has
4790                  * other colliding tags active.
4791                  */
4792                 next_scb = LIST_NEXT(scb, collision_links);
4793                 if (next_scb != NULL) {
4794                         TAILQ_INSERT_AFTER(free_tailq, scb,
4795                                            next_scb, links.tqe);
4796                 }
4797                 TAILQ_REMOVE(free_tailq, scb, links.tqe);
4798         }
4799         LIST_REMOVE(scb, collision_links);
4800 }
4801
4802 /*
4803  * Get a free scb. If there are none, see if we can allocate a new SCB.
4804  */
4805 struct scb *
4806 ahd_get_scb(struct ahd_softc *ahd, u_int col_idx)
4807 {
4808         struct scb *scb;
4809         int tries;
4810
4811         tries = 0;
4812 look_again:
4813         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
4814                 if (AHD_GET_SCB_COL_IDX(ahd, scb) != col_idx) {
4815                         ahd_rem_col_list(ahd, scb);
4816                         goto found;
4817                 }
4818         }
4819         if ((scb = LIST_FIRST(&ahd->scb_data.any_dev_free_scb_list)) == NULL) {
4820
4821                 if (tries++ != 0)
4822                         return (NULL);
4823                 ahd_alloc_scbs(ahd);
4824                 goto look_again;
4825         }
4826         LIST_REMOVE(scb, links.le);
4827         if (col_idx != AHD_NEVER_COL_IDX
4828          && (scb->col_scb != NULL)
4829          && (scb->col_scb->flags & SCB_ACTIVE) == 0) {
4830                 LIST_REMOVE(scb->col_scb, links.le);
4831                 ahd_add_col_list(ahd, scb->col_scb, col_idx);
4832         }
4833 found:
4834         scb->flags |= SCB_ACTIVE;
4835         return (scb);
4836 }
4837
4838 /*
4839  * Return an SCB resource to the free list.
4840  */
4841 void
4842 ahd_free_scb(struct ahd_softc *ahd, struct scb *scb)
4843 {
4844
4845         /* Clean up for the next user */
4846         scb->flags = SCB_FLAG_NONE;
4847         scb->hscb->control = 0;
4848         ahd->scb_data.scbindex[scb->hscb->tag] = NULL;
4849
4850         if (scb->col_scb == NULL) {
4851
4852                 /*
4853                  * No collision possible.  Just free normally.
4854                  */
4855                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
4856                                  scb, links.le);
4857         } else if ((scb->col_scb->flags & SCB_ON_COL_LIST) != 0) {
4858
4859                 /*
4860                  * The SCB we might have collided with is on
4861                  * a free collision list.  Put both SCBs on
4862                  * the generic list.
4863                  */
4864                 ahd_rem_col_list(ahd, scb->col_scb);
4865                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
4866                                  scb, links.le);
4867                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
4868                                  scb->col_scb, links.le);
4869         } else if ((scb->col_scb->flags
4870                   & (SCB_PACKETIZED|SCB_ACTIVE)) == SCB_ACTIVE
4871                 && (scb->col_scb->hscb->control & TAG_ENB) != 0) {
4872
4873                 /*
4874                  * The SCB we might collide with on the next allocation
4875                  * is still active in a non-packetized, tagged, context.
4876                  * Put us on the SCB collision list.
4877                  */
4878                 ahd_add_col_list(ahd, scb,
4879                                  AHD_GET_SCB_COL_IDX(ahd, scb->col_scb));
4880         } else {
4881                 /*
4882                  * The SCB we might collide with on the next allocation
4883                  * is either active in a packetized context, or free.
4884                  * Since we can't collide, put this SCB on the generic
4885                  * free list.
4886                  */
4887                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
4888                                  scb, links.le);
4889         }
4890
4891         ahd_platform_scb_free(ahd, scb);
4892 }
4893
4894 void
4895 ahd_alloc_scbs(struct ahd_softc *ahd)
4896 {
4897         struct scb_data *scb_data;
4898         struct scb      *next_scb;
4899         struct hardware_scb *hscb;
4900         struct map_node *hscb_map;
4901         struct map_node *sg_map;
4902         struct map_node *sense_map;
4903         uint8_t         *segs;
4904         uint8_t         *sense_data;
4905         bus_addr_t       hscb_busaddr;
4906         bus_addr_t       sg_busaddr;
4907         bus_addr_t       sense_busaddr;
4908         int              newcount;
4909         int              i;
4910
4911         scb_data = &ahd->scb_data;
4912         if (scb_data->numscbs >= AHD_SCB_MAX_ALLOC)
4913                 /* Can't allocate any more */
4914                 return;
4915
4916         if (scb_data->scbs_left != 0) {
4917                 int offset;
4918
4919                 offset = (PAGE_SIZE / sizeof(*hscb)) - scb_data->scbs_left;
4920                 hscb_map = SLIST_FIRST(&scb_data->hscb_maps);
4921                 hscb = &((struct hardware_scb *)hscb_map->vaddr)[offset];
4922                 hscb_busaddr = hscb_map->physaddr + (offset * sizeof(*hscb));
4923         } else {
4924                 hscb_map = malloc(sizeof(*hscb_map), M_DEVBUF, M_NOWAIT);
4925
4926                 if (hscb_map == NULL)
4927                         return;
4928
4929                 /* Allocate the next batch of hardware SCBs */
4930                 if (ahd_dmamem_alloc(ahd, scb_data->hscb_dmat,
4931                                      (void **)&hscb_map->vaddr,
4932                                      BUS_DMA_NOWAIT, &hscb_map->dmamap) != 0) {
4933                         free(hscb_map, M_DEVBUF);
4934                         return;
4935                 }
4936
4937                 SLIST_INSERT_HEAD(&scb_data->hscb_maps, hscb_map, links);
4938
4939                 ahd_dmamap_load(ahd, scb_data->hscb_dmat, hscb_map->dmamap,
4940                                 hscb_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
4941                                 &hscb_map->physaddr, /*flags*/0);
4942
4943                 hscb = (struct hardware_scb *)hscb_map->vaddr;
4944                 hscb_busaddr = hscb_map->physaddr;
4945                 scb_data->scbs_left = PAGE_SIZE / sizeof(*hscb);
4946         }
4947
4948         if (scb_data->sgs_left != 0) {
4949                 int offset;
4950
4951                 offset = ahd_sglist_allocsize(ahd)
4952                        - (scb_data->sgs_left * ahd_sglist_size(ahd));
4953                 sg_map = SLIST_FIRST(&scb_data->sg_maps);
4954                 segs = sg_map->vaddr + offset;
4955                 sg_busaddr = sg_map->physaddr + offset;
4956         } else {
4957                 sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
4958
4959                 if (sg_map == NULL)
4960                         return;
4961
4962                 /* Allocate the next batch of S/G lists */
4963                 if (ahd_dmamem_alloc(ahd, scb_data->sg_dmat,
4964                                      (void **)&sg_map->vaddr,
4965                                      BUS_DMA_NOWAIT, &sg_map->dmamap) != 0) {
4966                         free(sg_map, M_DEVBUF);
4967                         return;
4968                 }
4969
4970                 SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
4971
4972                 ahd_dmamap_load(ahd, scb_data->sg_dmat, sg_map->dmamap,
4973                                 sg_map->vaddr, ahd_sglist_allocsize(ahd),
4974                                 ahd_dmamap_cb, &sg_map->physaddr, /*flags*/0);
4975
4976                 segs = sg_map->vaddr;
4977                 sg_busaddr = sg_map->physaddr;
4978                 scb_data->sgs_left =
4979                     ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd);
4980 #ifdef AHD_DEBUG
4981                 if (ahd_debug & AHD_SHOW_MEMORY)
4982                         printf("Mapped SG data\n");
4983 #endif
4984         }
4985
4986         if (scb_data->sense_left != 0) {
4987                 int offset;
4988
4989                 offset = PAGE_SIZE - (AHD_SENSE_BUFSIZE * scb_data->sense_left);
4990                 sense_map = SLIST_FIRST(&scb_data->sense_maps);
4991                 sense_data = sense_map->vaddr + offset;
4992                 sense_busaddr = sense_map->physaddr + offset;
4993         } else {
4994                 sense_map = malloc(sizeof(*sense_map), M_DEVBUF, M_NOWAIT);
4995
4996                 if (sense_map == NULL)
4997                         return;
4998
4999                 /* Allocate the next batch of sense buffers */
5000                 if (ahd_dmamem_alloc(ahd, scb_data->sense_dmat,
5001                                      (void **)&sense_map->vaddr,
5002                                      BUS_DMA_NOWAIT, &sense_map->dmamap) != 0) {
5003                         free(sense_map, M_DEVBUF);
5004                         return;
5005                 }
5006
5007                 SLIST_INSERT_HEAD(&scb_data->sense_maps, sense_map, links);
5008
5009                 ahd_dmamap_load(ahd, scb_data->sense_dmat, sense_map->dmamap,
5010                                 sense_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
5011                                 &sense_map->physaddr, /*flags*/0);
5012
5013                 sense_data = sense_map->vaddr;
5014                 sense_busaddr = sense_map->physaddr;
5015                 scb_data->sense_left = PAGE_SIZE / AHD_SENSE_BUFSIZE;
5016 #ifdef AHD_DEBUG
5017                 if (ahd_debug & AHD_SHOW_MEMORY)
5018                         printf("Mapped sense data\n");
5019 #endif
5020         }
5021
5022         newcount = MIN(scb_data->sense_left, scb_data->scbs_left);
5023         newcount = MIN(newcount, scb_data->sgs_left);
5024         newcount = MIN(newcount, (AHD_SCB_MAX_ALLOC - scb_data->numscbs));
5025         scb_data->sense_left -= newcount;
5026         scb_data->scbs_left -= newcount;
5027         scb_data->sgs_left -= newcount;
5028         for (i = 0; i < newcount; i++) {
5029                 u_int col_tag;
5030
5031                 struct scb_platform_data *pdata;
5032 #ifndef __linux__
5033                 int error;
5034 #endif
5035                 next_scb = (struct scb *)malloc(sizeof(*next_scb),
5036                                                 M_DEVBUF, M_NOWAIT);
5037                 if (next_scb == NULL)
5038                         break;
5039
5040                 pdata = (struct scb_platform_data *)malloc(sizeof(*pdata),
5041                                                            M_DEVBUF, M_NOWAIT);
5042                 if (pdata == NULL) {
5043                         free(next_scb, M_DEVBUF);
5044                         break;
5045                 }
5046                 next_scb->platform_data = pdata;
5047                 next_scb->hscb_map = hscb_map;
5048                 next_scb->sg_map = sg_map;
5049                 next_scb->sense_map = sense_map;
5050                 next_scb->sg_list = segs;
5051                 next_scb->sense_data = sense_data;
5052                 next_scb->sense_busaddr = sense_busaddr;
5053                 next_scb->hscb = hscb;
5054                 hscb->hscb_busaddr = ahd_htole32(hscb_busaddr);
5055
5056                 /*
5057                  * The sequencer always starts with the second entry.
5058                  * The first entry is embedded in the scb.
5059                  */
5060                 next_scb->sg_list_busaddr = sg_busaddr;
5061                 if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
5062                         next_scb->sg_list_busaddr
5063                             += sizeof(struct ahd_dma64_seg);
5064                 else
5065                         next_scb->sg_list_busaddr += sizeof(struct ahd_dma_seg);
5066                 next_scb->ahd_softc = ahd;
5067                 next_scb->flags = SCB_FLAG_NONE;
5068 #ifndef __linux__
5069                 error = ahd_dmamap_create(ahd, ahd->buffer_dmat, /*flags*/0,
5070                                           &next_scb->dmamap);
5071                 if (error != 0) {
5072                         free(next_scb, M_DEVBUF);
5073                         free(pdata, M_DEVBUF);
5074                         break;
5075                 }
5076 #endif
5077                 next_scb->hscb->tag = ahd_htole16(scb_data->numscbs);
5078                 col_tag = scb_data->numscbs ^ 0x100;
5079                 next_scb->col_scb = ahd_find_scb_by_tag(ahd, col_tag);
5080                 if (next_scb->col_scb != NULL)
5081                         next_scb->col_scb->col_scb = next_scb;
5082                 ahd_free_scb(ahd, next_scb);
5083                 hscb++;
5084                 hscb_busaddr += sizeof(*hscb);
5085                 segs += ahd_sglist_size(ahd);
5086                 sg_busaddr += ahd_sglist_size(ahd);
5087                 sense_data += AHD_SENSE_BUFSIZE;
5088                 sense_busaddr += AHD_SENSE_BUFSIZE;
5089                 scb_data->numscbs++;
5090         }
5091 }
5092
5093 void
5094 ahd_controller_info(struct ahd_softc *ahd, char *buf)
5095 {
5096         const char *speed;
5097         const char *type;
5098         int len;
5099
5100         len = sprintf(buf, "%s: ", ahd_chip_names[ahd->chip & AHD_CHIPID_MASK]);
5101         buf += len;
5102
5103         speed = "Ultra320 ";
5104         if ((ahd->features & AHD_WIDE) != 0) {
5105                 type = "Wide ";
5106         } else {
5107                 type = "Single ";
5108         }
5109         len = sprintf(buf, "%s%sChannel %c, SCSI Id=%d, ",
5110                       speed, type, ahd->channel, ahd->our_id);
5111         buf += len;
5112
5113         sprintf(buf, "%s, %d SCBs", ahd->bus_description,
5114                 ahd->scb_data.maxhscbs);
5115 }
5116
5117 static const char *channel_strings[] = {
5118         "Primary Low",
5119         "Primary High",
5120         "Secondary Low",
5121         "Secondary High"
5122 };
5123
5124 static const char *termstat_strings[] = {
5125         "Terminated Correctly",
5126         "Over Terminated",
5127         "Under Terminated",
5128         "Not Configured"
5129 };
5130
5131 /*
5132  * Start the board, ready for normal operation
5133  */
5134 int
5135 ahd_init(struct ahd_softc *ahd)
5136 {
5137         uint8_t         *base_vaddr;
5138         uint8_t         *next_vaddr;
5139         bus_addr_t       next_baddr;
5140         size_t           driver_data_size;
5141         int              i;
5142         int              error;
5143         u_int            warn_user;
5144         uint8_t          current_sensing;
5145         uint8_t          fstat;
5146
5147         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
5148
5149         /*
5150          * Verify that the compiler hasn't over-agressively
5151          * padded important structures.
5152          */
5153         if (sizeof(struct hardware_scb) != 64)
5154                 panic("Hardware SCB size is incorrect");
5155
5156 #ifdef AHD_DEBUG
5157         if ((ahd_debug & AHD_DEBUG_SEQUENCER) != 0)
5158                 ahd->flags |= AHD_SEQUENCER_DEBUG;
5159 #endif
5160
5161         /*
5162          * Default to allowing initiator operations.
5163          */
5164         ahd->flags |= AHD_INITIATORROLE;
5165
5166         /*
5167          * Only allow target mode features if this unit has them enabled.
5168          */
5169         if ((AHD_TMODE_ENABLE & (0x1 << ahd->unit)) == 0)
5170                 ahd->features &= ~AHD_TARGETMODE;
5171
5172 #ifndef __linux__
5173         /* DMA tag for mapping buffers into device visible space. */
5174         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
5175                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5176                                /*lowaddr*/BUS_SPACE_MAXADDR,
5177                                /*highaddr*/BUS_SPACE_MAXADDR,
5178                                /*filter*/NULL, /*filterarg*/NULL,
5179                                /*maxsize*/MAXBSIZE, /*nsegments*/AHD_NSEG,
5180                                /*maxsegsz*/AHD_MAXTRANSFER_SIZE,
5181                                /*flags*/BUS_DMA_ALLOCNOW,
5182                                &ahd->buffer_dmat) != 0) {
5183                 return (ENOMEM);
5184         }
5185 #endif
5186
5187         ahd->init_level++;
5188
5189         /*
5190          * DMA tag for our command fifos and other data in system memory
5191          * the card's sequencer must be able to access.  For initiator
5192          * roles, we need to allocate space for the qoutfifo.  When providing
5193          * for the target mode role, we must additionally provide space for
5194          * the incoming target command fifo.
5195          */
5196         driver_data_size = AHD_SCB_MAX * sizeof(uint16_t)
5197                          + sizeof(struct hardware_scb);
5198         if ((ahd->features & AHD_TARGETMODE) != 0)
5199                 driver_data_size += AHD_TMODE_CMDS * sizeof(struct target_cmd);
5200         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0)
5201                 driver_data_size += PKT_OVERRUN_BUFSIZE;
5202         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
5203                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5204                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5205                                /*highaddr*/BUS_SPACE_MAXADDR,
5206                                /*filter*/NULL, /*filterarg*/NULL,
5207                                driver_data_size,
5208                                /*nsegments*/1,
5209                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5210                                /*flags*/0, &ahd->shared_data_dmat) != 0) {
5211                 return (ENOMEM);
5212         }
5213
5214         ahd->init_level++;
5215
5216         /* Allocation of driver data */
5217         if (ahd_dmamem_alloc(ahd, ahd->shared_data_dmat,
5218                              (void **)&base_vaddr,
5219                              BUS_DMA_NOWAIT, &ahd->shared_data_dmamap) != 0) {
5220                 return (ENOMEM);
5221         }
5222
5223         ahd->init_level++;
5224
5225         /* And permanently map it in */
5226         ahd_dmamap_load(ahd, ahd->shared_data_dmat, ahd->shared_data_dmamap,
5227                         base_vaddr, driver_data_size, ahd_dmamap_cb,
5228                         &ahd->shared_data_busaddr, /*flags*/0);
5229         ahd->qoutfifo = (uint16_t *)base_vaddr;
5230         next_vaddr = (uint8_t *)&ahd->qoutfifo[AHD_QOUT_SIZE];
5231         next_baddr = ahd->shared_data_busaddr + AHD_QOUT_SIZE*sizeof(uint16_t);
5232         if ((ahd->features & AHD_TARGETMODE) != 0) {
5233                 ahd->targetcmds = (struct target_cmd *)next_vaddr;
5234                 next_vaddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
5235                 next_baddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
5236         }
5237
5238         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0) {
5239                 ahd->overrun_buf = next_vaddr;
5240                 next_vaddr += PKT_OVERRUN_BUFSIZE;
5241                 next_baddr += PKT_OVERRUN_BUFSIZE;
5242         }
5243
5244         /*
5245          * We need one SCB to serve as the "next SCB".  Since the
5246          * tag identifier in this SCB will never be used, there is
5247          * no point in using a valid HSCB tag from an SCB pulled from
5248          * the standard free pool.  So, we allocate this "sentinel"
5249          * specially from the DMA safe memory chunk used for the QOUTFIFO.
5250          */
5251         ahd->next_queued_hscb = (struct hardware_scb *)next_vaddr;
5252         ahd->next_queued_hscb->hscb_busaddr = next_baddr;
5253
5254         ahd->init_level++;
5255
5256         /* Allocate SCB data now that buffer_dmat is initialized */
5257         if (ahd_init_scbdata(ahd) != 0)
5258                 return (ENOMEM);
5259
5260         if ((ahd->flags & AHD_INITIATORROLE) == 0)
5261                 ahd->flags &= ~AHD_RESET_BUS_A;
5262
5263         ahd_chip_init(ahd);
5264
5265         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
5266
5267         if ((ahd->flags & AHD_CURRENT_SENSING) == 0)
5268                 goto init_done;
5269
5270         /*
5271          * Verify termination based on current draw and
5272          * warn user if the bus is over/under terminated.
5273          */
5274         error = ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL,
5275                                    CURSENSE_ENB);
5276         if (error != 0) {
5277                 printf("%s: current sensing timeout 1\n", ahd_name(ahd));
5278                 goto init_done;
5279         }
5280         for (i = 20, fstat = FLX_FSTAT_BUSY;
5281              (fstat & FLX_FSTAT_BUSY) != 0 && i; i--) {
5282                 error = ahd_read_flexport(ahd, FLXADDR_FLEXSTAT, &fstat);
5283                 if (error != 0) {
5284                         printf("%s: current sensing timeout 2\n",
5285                                ahd_name(ahd));
5286                         goto init_done;
5287                 }
5288         }
5289         if (i == 0) {
5290                 printf("%s: Timedout during current-sensing test\n",
5291                        ahd_name(ahd));
5292                 goto init_done;
5293         }
5294
5295         /* Latch Current Sensing status. */
5296         error = ahd_read_flexport(ahd, FLXADDR_CURRENT_STAT, &current_sensing);
5297         if (error != 0) {
5298                 printf("%s: current sensing timeout 3\n", ahd_name(ahd));
5299                 goto init_done;
5300         }
5301
5302         /* Diable current sensing. */
5303         ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, 0);
5304
5305 #ifdef AHD_DEBUG
5306         if ((ahd_debug & AHD_SHOW_TERMCTL) != 0) {
5307                 printf("%s: current_sensing == 0x%x\n",
5308                        ahd_name(ahd), current_sensing);
5309         }
5310 #endif
5311         warn_user = 0;
5312         for (i = 0; i < 4; i++, current_sensing >>= FLX_CSTAT_SHIFT) {
5313                 u_int term_stat;
5314
5315                 term_stat = (current_sensing & FLX_CSTAT_MASK);
5316                 switch (term_stat) {
5317                 case FLX_CSTAT_OVER:
5318                 case FLX_CSTAT_UNDER:
5319                         warn_user++;
5320                 case FLX_CSTAT_INVALID:
5321                 case FLX_CSTAT_OKAY:
5322                         if (warn_user == 0 && bootverbose == 0)
5323                                 break;
5324                         printf("%s: %s Channel %s\n", ahd_name(ahd),
5325                                channel_strings[i], termstat_strings[term_stat]);
5326                         break;
5327                 }
5328         }
5329         if (warn_user) {
5330                 printf("%s: WARNING. Termination is not configured correctly.\n"
5331                        "%s: WARNING. SCSI bus operations may FAIL.\n",
5332                        ahd_name(ahd), ahd_name(ahd));
5333         }
5334 init_done:
5335         ahd_restart(ahd);
5336         return (0);
5337 }
5338
5339 /*
5340  * (Re)initialize chip state after a chip reset.
5341  */
5342 static void
5343 ahd_chip_init(struct ahd_softc *ahd)
5344 {
5345         uint32_t busaddr;
5346         u_int    sxfrctl1;
5347         u_int    scsiseq_template;
5348         u_int    wait;
5349         u_int    i;
5350         u_int    target;
5351
5352         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5353         /*
5354          * Take the LED out of diagnostic mode
5355          */
5356         ahd_outb(ahd, SBLKCTL, ahd_inb(ahd, SBLKCTL) & ~(DIAGLEDEN|DIAGLEDON));
5357
5358         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1. */
5359         ahd_outb(ahd, IOWNID, ahd->our_id);
5360         ahd_outb(ahd, TOWNID, ahd->our_id);
5361         sxfrctl1 = (ahd->flags & AHD_TERM_ENB_A) != 0 ? STPWEN : 0;
5362         sxfrctl1 |= (ahd->flags & AHD_SPCHK_ENB_A) != 0 ? ENSPCHK : 0;
5363         if ((ahd->bugs & AHD_LONG_SETIMO_BUG)
5364          && (ahd->seltime != STIMESEL_MIN)) {
5365                 /*
5366                  * The selection timer duration is twice as long
5367                  * as it should be.  Halve it by adding "1" to
5368                  * the user specified setting.
5369                  */
5370                 sxfrctl1 |= ahd->seltime + STIMESEL_BUG_ADJ;
5371         } else {
5372                 sxfrctl1 |= ahd->seltime;
5373         }
5374
5375         ahd_outb(ahd, SXFRCTL0, DFON);
5376         ahd_outb(ahd, SXFRCTL1, sxfrctl1|ahd->seltime|ENSTIMER|ACTNEGEN);
5377         ahd_outb(ahd, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5378
5379         /*
5380          * Now that termination is set, wait for up
5381          * to 500ms for our transceivers to settle.  If
5382          * the adapter does not have a cable attached,
5383          * the tranceivers may never settle, so don't
5384          * complain if we fail here.
5385          */
5386         for (wait = 10000;
5387              (ahd_inb(ahd, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
5388              wait--)
5389                 ahd_delay(100);
5390
5391         /* Clear any false bus resets due to the transceivers settling */
5392         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
5393         ahd_outb(ahd, CLRINT, CLRSCSIINT);
5394
5395         /* Initialize mode specific S/G state. */
5396         for (i = 0; i < 2; i++) {
5397                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
5398                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
5399                 ahd_outw(ahd, LONGJMP_SCB, SCB_LIST_NULL);
5400                 ahd_outb(ahd, SG_STATE, 0);
5401                 ahd_outb(ahd, CLRSEQINTSRC, 0xFF);
5402                 ahd_outb(ahd, SEQIMODE,
5403                          ENSAVEPTRS|ENCFG4DATA|ENCFG4ISTAT
5404                         |ENCFG4TSTAT|ENCFG4ICMD|ENCFG4TCMD);
5405         }
5406
5407         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
5408         ahd_outb(ahd, DSCOMMAND0, ahd_inb(ahd, DSCOMMAND0)|MPARCKEN|CACHETHEN);
5409         ahd_outb(ahd, DFF_THRSH, RD_DFTHRSH_75|WR_DFTHRSH_75);
5410         ahd_outb(ahd, SIMODE0, ENIOERR|ENOVERRUN);
5411         ahd_outb(ahd, SIMODE3, ENNTRAMPERR|ENOSRAMPERR);
5412         ahd_outb(ahd, OPTIONMODE, AUTOACKEN|BUSFREEREV|AUTO_MSGOUT_DE);
5413         if ((ahd->chip & AHD_BUS_MASK) == AHD_PCIX)
5414                 /*
5415                  * Do not issue a target abort when a split completion
5416                  * error occurs.  Let our PCIX interrupt handler deal
5417                  * with it instead. H2A4 Razor #625
5418                  */
5419                 ahd_outb(ahd, PCIXCTL, ahd_inb(ahd, PCIXCTL) | SPLTSTADIS);
5420
5421         /*
5422          * Tweak IOCELL settings.
5423          */
5424         if ((ahd->flags & AHD_CPQ_BOARD) != 0) {
5425                 for (i = 0; i < NUMDSPS; i++) {
5426                         ahd_outb(ahd, DSPSELECT, i);
5427                         ahd_outb(ahd, WRTBIASCTL, WRTBIASCTL_CPQ_DEFAULT);
5428                 }
5429         }
5430         ahd_setup_iocell_workaround(ahd);
5431
5432         /*
5433          * Enable LQI Manager interrupts.
5434          */
5435         ahd_outb(ahd, LQIMODE1, ENLQIPHASE_LQ|ENLQIPHASE_NLQ|ENLIQABORT
5436                               | ENLQICRCI_LQ|ENLQICRCI_NLQ|ENLQIBADLQI
5437                               | ENLQIOVERI_LQ|ENLQIOVERI_NLQ);
5438         ahd_outb(ahd, LQOMODE0, ENLQOATNLQ|ENLQOATNPKT|ENLQOTCRC);
5439         /*
5440          * An interrupt from LQOBUSFREE is made redundant by the
5441          * BUSFREE interrupt.  We choose to have the sequencer catch
5442          * LQOPHCHGINPKT errors manually for the command phase at the
5443          * start of a packetized selection case.
5444         ahd_outb(ahd, LQOMODE1, ENLQOBUSFREE|ENLQOPHACHGINPKT);
5445          */
5446         ahd_outb(ahd, LQOMODE1, 0);
5447
5448         /*
5449          * Setup sequencer interrupt handler.
5450          */
5451         ahd_outw(ahd, INTVEC1_ADDR, ahd_resolve_seqaddr(ahd, LABEL_seq_isr));
5452
5453         /*
5454          * Setup SCB Offset registers.
5455          */
5456         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
5457                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb,
5458                          pkt_long_lun));
5459         } else {
5460                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb, lun));
5461         }
5462         ahd_outb(ahd, CMDLENPTR, offsetof(struct hardware_scb, cdb_len));
5463         ahd_outb(ahd, ATTRPTR,
5464                  offsetof(struct hardware_scb, task_attribute_nonpkt_tag));
5465         ahd_outb(ahd, FLAGPTR, offsetof(struct hardware_scb, task_management));
5466         ahd_outb(ahd, CMDPTR, offsetof(struct hardware_scb,
5467                                        shared_data.idata.cdb));
5468         ahd_outb(ahd, QNEXTPTR,
5469                  offsetof(struct hardware_scb, next_hscb_busaddr));
5470         ahd_outb(ahd, ABRTBITPTR, MK_MESSAGE_BIT_OFFSET);
5471         ahd_outb(ahd, ABRTBYTEPTR, offsetof(struct hardware_scb, control));
5472         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
5473                 ahd_outb(ahd, LUNLEN,
5474                          sizeof(ahd->next_queued_hscb->pkt_long_lun) - 1);
5475         } else {
5476                 ahd_outb(ahd, LUNLEN, sizeof(ahd->next_queued_hscb->lun) - 1);
5477         }
5478         ahd_outb(ahd, CDBLIMIT, SCB_CDB_LEN_PTR - 1);
5479         ahd_outb(ahd, MAXCMD, 0xFF);
5480         ahd_outb(ahd, SCBAUTOPTR,
5481                  AUSCBPTR_EN | offsetof(struct hardware_scb, tag));
5482
5483         /* We haven't been enabled for target mode yet. */
5484         ahd_outb(ahd, MULTARGID, 0);
5485         ahd_outb(ahd, MULTARGID + 1, 0);
5486
5487         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5488         /*
5489          * Clear the spare bytes in the neg table to avoid
5490          * spurious parity errors.
5491          */
5492         for (target = 0; target < AHD_NUM_TARGETS; target++) {
5493
5494                 ahd_outb(ahd, NEGOADDR, target);
5495                 ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP);
5496                 for (i = 0; i < AHD_NUM_ANNEXCOLS; i++)
5497                         ahd_outb(ahd, ANNEXDAT, 0);
5498         }
5499         ahd_outb(ahd, CLRSINT3, NTRAMPERR|OSRAMPERR);
5500         ahd_outb(ahd, CLRINT, CLRSCSIINT);
5501
5502         /*
5503          * Always enable abort on incoming L_Qs if this feature is
5504          * supported.  We use this to catch invalid SCB references.
5505          */
5506         if ((ahd->bugs & AHD_ABORT_LQI_BUG) == 0)
5507                 ahd_outb(ahd, LQCTL1, ABORTPENDING);
5508         else
5509                 ahd_outb(ahd, LQCTL1, 0);
5510
5511         /* All of our queues are empty */
5512         ahd->qoutfifonext = 0;
5513         ahd->qoutfifonext_valid_tag = QOUTFIFO_ENTRY_VALID_LE;
5514         ahd_outb(ahd, QOUTFIFO_ENTRY_VALID_TAG, QOUTFIFO_ENTRY_VALID >> 8);
5515         for (i = 0; i < AHD_QOUT_SIZE; i++)
5516                 ahd->qoutfifo[i] = 0;
5517         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_PREREAD);
5518
5519         ahd->qinfifonext = 0;
5520         for (i = 0; i < AHD_QIN_SIZE; i++)
5521                 ahd->qinfifo[i] = SCB_LIST_NULL;
5522
5523         if ((ahd->features & AHD_TARGETMODE) != 0) {
5524                 /* All target command blocks start out invalid. */
5525                 for (i = 0; i < AHD_TMODE_CMDS; i++)
5526                         ahd->targetcmds[i].cmd_valid = 0;
5527                 ahd_sync_tqinfifo(ahd, BUS_DMASYNC_PREREAD);
5528                 ahd->tqinfifonext = 1;
5529                 ahd_outb(ahd, KERNEL_TQINPOS, ahd->tqinfifonext - 1);
5530                 ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
5531         }
5532
5533         /* Initialize Scratch Ram. */
5534         ahd_outb(ahd, SEQ_FLAGS, 0);
5535         ahd_outb(ahd, SEQ_FLAGS2, 0);
5536
5537         /* We don't have any waiting selections */
5538         ahd_outw(ahd, WAITING_TID_HEAD, SCB_LIST_NULL);
5539         ahd_outw(ahd, WAITING_TID_TAIL, SCB_LIST_NULL);
5540         for (i = 0; i < AHD_NUM_TARGETS; i++)
5541                 ahd_outw(ahd, WAITING_SCB_TAILS + (2 * i), SCB_LIST_NULL);
5542
5543         /*
5544          * Nobody is waiting to be DMAed into the QOUTFIFO.
5545          */
5546         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
5547         ahd_outw(ahd, COMPLETE_SCB_DMAINPROG_HEAD, SCB_LIST_NULL);
5548         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
5549
5550         /*
5551          * The Freeze Count is 0.
5552          */
5553         ahd_outw(ahd, QFREEZE_COUNT, 0);
5554
5555         /*
5556          * Tell the sequencer where it can find our arrays in memory.
5557          */
5558         busaddr = ahd->shared_data_busaddr;
5559         ahd_outb(ahd, SHARED_DATA_ADDR, busaddr & 0xFF);
5560         ahd_outb(ahd, SHARED_DATA_ADDR + 1, (busaddr >> 8) & 0xFF);
5561         ahd_outb(ahd, SHARED_DATA_ADDR + 2, (busaddr >> 16) & 0xFF);
5562         ahd_outb(ahd, SHARED_DATA_ADDR + 3, (busaddr >> 24) & 0xFF);
5563         ahd_outb(ahd, QOUTFIFO_NEXT_ADDR, busaddr & 0xFF);
5564         ahd_outb(ahd, QOUTFIFO_NEXT_ADDR + 1, (busaddr >> 8) & 0xFF);
5565         ahd_outb(ahd, QOUTFIFO_NEXT_ADDR + 2, (busaddr >> 16) & 0xFF);
5566         ahd_outb(ahd, QOUTFIFO_NEXT_ADDR + 3, (busaddr >> 24) & 0xFF);
5567
5568         /*
5569          * Setup the allowed SCSI Sequences based on operational mode.
5570          * If we are a target, we'll enable select in operations once
5571          * we've had a lun enabled.
5572          */
5573         scsiseq_template = ENAUTOATNP;
5574         if ((ahd->flags & AHD_INITIATORROLE) != 0)
5575                 scsiseq_template |= ENRSELI;
5576         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq_template);
5577
5578         /* There are no busy SCBs yet. */
5579         for (target = 0; target < AHD_NUM_TARGETS; target++) {
5580                 int lun;
5581
5582                 for (lun = 0; lun < AHD_NUM_LUNS_NONPKT; lun++)
5583                         ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(target, 'A', lun));
5584         }
5585
5586         /*
5587          * Always enable abort on incoming L_Qs if this feature is
5588          * supported.  We use this to catch invalid SCB references.
5589          */
5590         if ((ahd->bugs & AHD_ABORT_LQI_BUG) == 0)
5591                 ahd_outb(ahd, LQCTL1, ABORTPENDING);
5592         else
5593                 ahd_outb(ahd, LQCTL1, 0);
5594
5595         /*
5596          * Initialize the group code to command length table.
5597          * Vendor Unique codes are set to 0 so we only capture
5598          * the first byte of the cdb.  These can be overridden
5599          * when target mode is enabled.
5600          */
5601         ahd_outb(ahd, CMDSIZE_TABLE, 5);
5602         ahd_outb(ahd, CMDSIZE_TABLE + 1, 9);
5603         ahd_outb(ahd, CMDSIZE_TABLE + 2, 9);
5604         ahd_outb(ahd, CMDSIZE_TABLE + 3, 0);
5605         ahd_outb(ahd, CMDSIZE_TABLE + 4, 15);
5606         ahd_outb(ahd, CMDSIZE_TABLE + 5, 11);
5607         ahd_outb(ahd, CMDSIZE_TABLE + 6, 0);
5608         ahd_outb(ahd, CMDSIZE_TABLE + 7, 0);
5609
5610         /* Tell the sequencer of our initial queue positions */
5611         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
5612         ahd_outb(ahd, QOFF_CTLSTA, SCB_QSIZE_512);
5613         ahd->qinfifonext = 0;
5614         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
5615         ahd_set_hescb_qoff(ahd, 0);
5616         ahd_set_snscb_qoff(ahd, 0);
5617         ahd_set_sescb_qoff(ahd, 0);
5618         ahd_set_sdscb_qoff(ahd, 0);
5619
5620         /*
5621          * Tell the sequencer which SCB will be the next one it receives.
5622          */
5623         busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
5624         ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 0, busaddr & 0xFF);
5625         ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 1, (busaddr >> 8) & 0xFF);
5626         ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 2, (busaddr >> 16) & 0xFF);
5627         ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 3, (busaddr >> 24) & 0xFF);
5628         ahd_loadseq(ahd);
5629         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5630 }
5631
5632 /*
5633  * Setup default device and controller settings.
5634  * This should only be called if our probe has
5635  * determined that no configuration data is available.
5636  */
5637 int
5638 ahd_default_config(struct ahd_softc *ahd)
5639 {
5640         int     targ;
5641
5642         ahd->our_id = 7;
5643
5644         /*
5645          * Allocate a tstate to house information for our
5646          * initiator presence on the bus as well as the user
5647          * data for any target mode initiator.
5648          */
5649         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
5650                 printf("%s: unable to allocate ahd_tmode_tstate.  "
5651                        "Failing attach\n", ahd_name(ahd));
5652                 return (ENOMEM);
5653         }
5654
5655         for (targ = 0; targ < AHD_NUM_TARGETS; targ++) {
5656                 struct   ahd_devinfo devinfo;
5657                 struct   ahd_initiator_tinfo *tinfo;
5658                 struct   ahd_tmode_tstate *tstate;
5659                 uint16_t target_mask;
5660
5661                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
5662                                             targ, &tstate);
5663                 /*
5664                  * We support SPC2 and SPI4.
5665                  */
5666                 tinfo->user.protocol_version = 4;
5667                 tinfo->user.transport_version = 4;
5668
5669                 target_mask = 0x01 << targ;
5670                 ahd->user_discenable |= target_mask;
5671                 tstate->discenable |= target_mask;
5672                 ahd->user_tagenable |= target_mask;
5673 #ifdef AHD_FORCE_160
5674                 tinfo->user.period = AHD_SYNCRATE_DT;
5675 #else
5676                 tinfo->user.period = AHD_SYNCRATE_160;
5677 #endif
5678                 tinfo->user.offset= ~0;
5679                 tinfo->user.ppr_options = MSG_EXT_PPR_RD_STRM
5680                                         | MSG_EXT_PPR_WR_FLOW
5681                                         | MSG_EXT_PPR_IU_REQ
5682                                         | MSG_EXT_PPR_QAS_REQ
5683                                         | MSG_EXT_PPR_DT_REQ;
5684
5685                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5686
5687                 /*
5688                  * Start out Async/Narrow/Untagged and with
5689                  * conservative protocol support.
5690                  */
5691                 tinfo->goal.protocol_version = 2;
5692                 tinfo->goal.transport_version = 2;
5693                 tinfo->curr.protocol_version = 2;
5694                 tinfo->curr.transport_version = 2;
5695                 ahd_compile_devinfo(&devinfo, ahd->our_id,
5696                                     targ, CAM_LUN_WILDCARD,
5697                                     'A', ROLE_INITIATOR);
5698                 tstate->tagenable &= ~target_mask;
5699                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5700                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
5701                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
5702                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
5703                                  /*paused*/TRUE);
5704                 /*
5705                  * The neg table must be initialized even if the
5706                  * new settings above are the same as those from
5707                  * when our xfer info data structures were allocated
5708                  * and initialized.
5709                  */
5710                 ahd_update_neg_table(ahd, &devinfo, &tinfo->curr);
5711         }
5712         return (0);
5713 }
5714
5715 /*
5716  * Parse device configuration information.
5717  */
5718 int
5719 ahd_parse_cfgdata(struct ahd_softc *ahd, struct seeprom_config *sc)
5720 {
5721         int targ;
5722         int max_targ;
5723
5724         max_targ = sc->max_targets & CFMAXTARG;
5725         ahd->our_id = sc->brtime_id & CFSCSIID;
5726
5727         /*
5728          * Allocate a tstate to house information for our
5729          * initiator presence on the bus as well as the user
5730          * data for any target mode initiator.
5731          */
5732         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
5733                 printf("%s: unable to allocate ahd_tmode_tstate.  "
5734                        "Failing attach\n", ahd_name(ahd));
5735                 return (ENOMEM);
5736         }
5737
5738         for (targ = 0; targ < max_targ; targ++) {
5739                 struct   ahd_devinfo devinfo;
5740                 struct   ahd_initiator_tinfo *tinfo;
5741                 struct   ahd_transinfo *user_tinfo;
5742                 struct   ahd_tmode_tstate *tstate;
5743                 uint16_t target_mask;
5744
5745                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
5746                                             targ, &tstate);
5747                 user_tinfo = &tinfo->user;
5748
5749                 /*
5750                  * We support SPC2 and SPI4.
5751                  */
5752                 tinfo->user.protocol_version = 4;
5753                 tinfo->user.transport_version = 4;
5754
5755                 target_mask = 0x01 << targ;
5756                 ahd->user_discenable &= ~target_mask;
5757                 tstate->discenable &= ~target_mask;
5758                 ahd->user_tagenable &= ~target_mask;
5759                 if (sc->device_flags[targ] & CFDISC) {
5760                         tstate->discenable |= target_mask;
5761                         ahd->user_discenable |= target_mask;
5762                         ahd->user_tagenable |= target_mask;
5763                 } else {
5764                         /*
5765                          * Cannot be packetized without disconnection.
5766                          */
5767                         sc->device_flags[targ] &= ~CFPACKETIZED;
5768                 }
5769
5770                 user_tinfo->ppr_options = 0;
5771                 user_tinfo->period = (sc->device_flags[targ] & CFXFER);
5772                 if (user_tinfo->period < CFXFER_ASYNC) {
5773                         if (user_tinfo->period <= AHD_PERIOD_10MHz)
5774                                 user_tinfo->ppr_options |= MSG_EXT_PPR_DT_REQ;
5775                         user_tinfo->offset = MAX_OFFSET;
5776                 } else  {
5777                         user_tinfo->offset = 0;
5778                         user_tinfo->period = AHD_PERIOD_ASYNC;
5779                 }
5780 #ifdef AHD_FORCE_160
5781                 if (user_tinfo->period <= AHD_SYNCRATE_160)
5782                         user_tinfo->period = AHD_SYNCRATE_DT;
5783 #endif
5784
5785                 if ((sc->device_flags[targ] & CFPACKETIZED) != 0)
5786                         user_tinfo->ppr_options |= MSG_EXT_PPR_RD_STRM
5787                                                 |  MSG_EXT_PPR_WR_FLOW
5788                                                 |  MSG_EXT_PPR_IU_REQ;
5789
5790                 if ((sc->device_flags[targ] & CFQAS) != 0)
5791                         user_tinfo->ppr_options |= MSG_EXT_PPR_QAS_REQ;
5792
5793                 if ((sc->device_flags[targ] & CFWIDEB) != 0)
5794                         user_tinfo->width = MSG_EXT_WDTR_BUS_16_BIT;
5795                 else
5796                         user_tinfo->width = MSG_EXT_WDTR_BUS_8_BIT;
5797 #ifdef AHD_DEBUG
5798                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
5799                         printf("(%d): %x:%x:%x:%x\n", targ, user_tinfo->width,
5800                                user_tinfo->period, user_tinfo->offset,
5801                                user_tinfo->ppr_options);
5802 #endif
5803                 /*
5804                  * Start out Async/Narrow/Untagged and with
5805                  * conservative protocol support.
5806                  */
5807                 tstate->tagenable &= ~target_mask;
5808                 tinfo->goal.protocol_version = 2;
5809                 tinfo->goal.transport_version = 2;
5810                 tinfo->curr.protocol_version = 2;
5811                 tinfo->curr.transport_version = 2;
5812                 ahd_compile_devinfo(&devinfo, ahd->our_id,
5813                                     targ, CAM_LUN_WILDCARD,
5814                                     'A', ROLE_INITIATOR);
5815                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5816                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
5817                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
5818                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
5819                                  /*paused*/TRUE);
5820                 /*
5821                  * The neg table must be initialized even if the
5822                  * new settings above are the same as those from
5823                  * when our xfer info data structures were allocated
5824                  * and initialized.
5825                  */
5826                 ahd_update_neg_table(ahd, &devinfo, &tinfo->curr);
5827         }
5828
5829         ahd->flags &= ~AHD_SPCHK_ENB_A;
5830         if (sc->bios_control & CFSPARITY)
5831                 ahd->flags |= AHD_SPCHK_ENB_A;
5832
5833         ahd->flags &= ~AHD_RESET_BUS_A;
5834         if (sc->bios_control & CFRESETB)
5835                 ahd->flags |= AHD_RESET_BUS_A;
5836
5837         ahd->flags &= ~AHD_EXTENDED_TRANS_A;
5838         if (sc->bios_control & CFEXTEND)
5839                 ahd->flags |= AHD_EXTENDED_TRANS_A;
5840
5841         ahd->flags &= ~AHD_BIOS_ENABLED;
5842         if ((sc->bios_control & CFBIOSSTATE) == CFBS_ENABLED)
5843                 ahd->flags |= AHD_BIOS_ENABLED;
5844
5845         ahd->flags &= ~AHD_STPWLEVEL_A;
5846         if ((sc->adapter_control & CFSTPWLEVEL) != 0)
5847                 ahd->flags |= AHD_STPWLEVEL_A;
5848
5849         return (0);
5850 }
5851
5852 void
5853 ahd_intr_enable(struct ahd_softc *ahd, int enable)
5854 {
5855         u_int hcntrl;
5856
5857         hcntrl = ahd_inb(ahd, HCNTRL);
5858         hcntrl &= ~INTEN;
5859         ahd->pause &= ~INTEN;
5860         ahd->unpause &= ~INTEN;
5861         if (enable) {
5862                 hcntrl |= INTEN;
5863                 ahd->pause |= INTEN;
5864                 ahd->unpause |= INTEN;
5865         }
5866         ahd_outb(ahd, HCNTRL, hcntrl);
5867 }
5868
5869 /*
5870  * Ensure that the card is paused in a location
5871  * outside of all critical sections and that all
5872  * pending work is completed prior to returning.
5873  * This routine should only be called from outside
5874  * an interrupt context.
5875  */
5876 void
5877 ahd_pause_and_flushwork(struct ahd_softc *ahd)
5878 {
5879         int intstat;
5880         int maxloops;
5881
5882         maxloops = 1000;
5883         ahd->flags |= AHD_ALL_INTERRUPTS;
5884         intstat = 0;
5885         do {
5886                 ahd_intr(ahd);
5887                 ahd_pause(ahd);
5888                 ahd_clear_critical_section(ahd);
5889                 if (intstat == 0xFF && (ahd->features & AHD_REMOVABLE) != 0)
5890                         break;
5891                 maxloops--;
5892         } while (((intstat = ahd_inb(ahd, INTSTAT)) & INT_PEND) && --maxloops);
5893         if (maxloops == 0) {
5894                 printf("Infinite interrupt loop, INTSTAT = %x",
5895                       ahd_inb(ahd, INTSTAT));
5896         }
5897         ahd_platform_flushwork(ahd);
5898         ahd->flags &= ~AHD_ALL_INTERRUPTS;
5899 }
5900
5901 int
5902 ahd_suspend(struct ahd_softc *ahd)
5903 {
5904 #if 0
5905         uint8_t *ptr;
5906         int      i;
5907
5908         ahd_pause_and_flushwork(ahd);
5909
5910         if (LIST_FIRST(&ahd->pending_scbs) != NULL)
5911                 return (EBUSY);
5912
5913 #if AHD_TARGET_MODE
5914         /*
5915          * XXX What about ATIOs that have not yet been serviced?
5916          * Perhaps we should just refuse to be suspended if we
5917          * are acting in a target role.
5918          */
5919         if (ahd->pending_device != NULL)
5920                 return (EBUSY);
5921 #endif
5922
5923         /* Save volatile registers */
5924         ahd->suspend_state.channel[0].scsiseq = ahd_inb(ahd, SCSISEQ0);
5925         ahd->suspend_state.channel[0].sxfrctl0 = ahd_inb(ahd, SXFRCTL0);
5926         ahd->suspend_state.channel[0].sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
5927         ahd->suspend_state.channel[0].simode0 = ahd_inb(ahd, SIMODE0);
5928         ahd->suspend_state.channel[0].simode1 = ahd_inb(ahd, SIMODE1);
5929         ahd->suspend_state.channel[0].seltimer = ahd_inb(ahd, SELTIMER);
5930         ahd->suspend_state.channel[0].seqctl = ahd_inb(ahd, SEQCTL0);
5931         ahd->suspend_state.dscommand0 = ahd_inb(ahd, DSCOMMAND0);
5932         ahd->suspend_state.dspcistatus = ahd_inb(ahd, DSPCISTATUS);
5933
5934         if ((ahd->features & AHD_DT) != 0) {
5935                 u_int sfunct;
5936
5937                 sfunct = ahd_inb(ahd, SFUNCT) & ~ALT_MODE;
5938                 ahd_outb(ahd, SFUNCT, sfunct | ALT_MODE);
5939                 ahd->suspend_state.optionmode = ahd_inb(ahd, OPTIONMODE);
5940                 ahd_outb(ahd, SFUNCT, sfunct);
5941                 ahd->suspend_state.crccontrol1 = ahd_inb(ahd, CRCCONTROL1);
5942         }
5943
5944         if ((ahd->features & AHD_MULTI_FUNC) != 0)
5945                 ahd->suspend_state.scbbaddr = ahd_inb(ahd, SCBBADDR);
5946
5947         if ((ahd->features & AHD_ULTRA2) != 0)
5948                 ahd->suspend_state.dff_thrsh = ahd_inb(ahd, DFF_THRSH);
5949
5950         ptr = ahd->suspend_state.scratch_ram;
5951         for (i = 0; i < 64; i++)
5952                 *ptr++ = ahd_inb(ahd, SRAM_BASE + i);
5953
5954         if ((ahd->features & AHD_MORE_SRAM) != 0) {
5955                 for (i = 0; i < 16; i++)
5956                         *ptr++ = ahd_inb(ahd, TARG_OFFSET + i);
5957         }
5958
5959         ptr = ahd->suspend_state.btt;
5960         for (i = 0;i < AHD_NUM_TARGETS; i++) {
5961                 int j;
5962
5963                 for (j = 0;j < AHD_NUM_LUNS_NONPKT; j++) {
5964                         u_int tcl;
5965
5966                         tcl = BUILD_TCL_RAW(i, 'A', j);
5967                         *ptr = ahd_find_busy_tcl(ahd, tcl);
5968                 }
5969         }
5970         ahd_shutdown(ahd);
5971 #endif
5972         return (0);
5973 }
5974
5975 int
5976 ahd_resume(struct ahd_softc *ahd)
5977 {
5978 #if 0
5979         uint8_t *ptr;
5980         int      i;
5981
5982         ahd_reset(ahd);
5983
5984         ahd_build_free_scb_list(ahd);
5985
5986         /* Restore volatile registers */
5987         ahd_outb(ahd, SCSISEQ0, ahd->suspend_state.channel[0].scsiseq);
5988         ahd_outb(ahd, SXFRCTL0, ahd->suspend_state.channel[0].sxfrctl0);
5989         ahd_outb(ahd, SXFRCTL1, ahd->suspend_state.channel[0].sxfrctl1);
5990         ahd_outb(ahd, SIMODE0, ahd->suspend_state.channel[0].simode0);
5991         ahd_outb(ahd, SIMODE1, ahd->suspend_state.channel[0].simode1);
5992         ahd_outb(ahd, SELTIMER, ahd->suspend_state.channel[0].seltimer);
5993         ahd_outb(ahd, SEQCTL0, ahd->suspend_state.channel[0].seqctl);
5994         if ((ahd->features & AHD_ULTRA2) != 0)
5995                 ahd_outb(ahd, SCSIID_ULTRA2, ahd->our_id);
5996         else
5997                 ahd_outb(ahd, SCSIID, ahd->our_id);
5998
5999         ahd_outb(ahd, DSCOMMAND0, ahd->suspend_state.dscommand0);
6000         ahd_outb(ahd, DSPCISTATUS, ahd->suspend_state.dspcistatus);
6001
6002         if ((ahd->features & AHD_DT) != 0) {
6003                 u_int sfunct;
6004
6005                 sfunct = ahd_inb(ahd, SFUNCT) & ~ALT_MODE;
6006                 ahd_outb(ahd, SFUNCT, sfunct | ALT_MODE);
6007                 ahd_outb(ahd, OPTIONMODE, ahd->suspend_state.optionmode);
6008                 ahd_outb(ahd, SFUNCT, sfunct);
6009                 ahd_outb(ahd, CRCCONTROL1, ahd->suspend_state.crccontrol1);
6010         }
6011
6012         if ((ahd->features & AHD_MULTI_FUNC) != 0)
6013                 ahd_outb(ahd, SCBBADDR, ahd->suspend_state.scbbaddr);
6014
6015         if ((ahd->features & AHD_ULTRA2) != 0)
6016                 ahd_outb(ahd, DFF_THRSH, ahd->suspend_state.dff_thrsh);
6017
6018         ptr = ahd->suspend_state.scratch_ram;
6019         for (i = 0; i < 64; i++)
6020                 ahd_outb(ahd, SRAM_BASE + i, *ptr++);
6021
6022         if ((ahd->features & AHD_MORE_SRAM) != 0) {
6023                 for (i = 0; i < 16; i++)
6024                         ahd_outb(ahd, TARG_OFFSET + i, *ptr++);
6025         }
6026
6027         ptr = ahd->suspend_state.btt;
6028         for (i = 0;i < AHD_NUM_TARGETS; i++) {
6029                 int j;
6030
6031                 for (j = 0;j < AHD_NUM_LUNS; j++) {
6032                         u_int tcl;
6033
6034                         tcl = BUILD_TCL(i << 4, j);
6035                         ahd_busy_tcl(ahd, tcl, *ptr);
6036                 }
6037         }
6038 #endif
6039         return (0);
6040 }
6041
6042 /************************** Busy Target Table *********************************/
6043 /*
6044  * Set SCBPTR to the SCB that contains the busy
6045  * table entry for TCL.  Return the offset into
6046  * the SCB that contains the entry for TCL.
6047  * saved_scbid is dereferenced and set to the
6048  * scbid that should be restored once manipualtion
6049  * of the TCL entry is complete.
6050  */
6051 static __inline u_int
6052 ahd_index_busy_tcl(struct ahd_softc *ahd, u_int *saved_scbid, u_int tcl)
6053 {
6054
6055         *saved_scbid = ahd_get_scbptr(ahd);
6056
6057         /*
6058          * Index to the SCB that contains the busy entry.
6059          */
6060         ahd_set_scbptr(ahd, TCL_LUN(tcl)
6061                      | ((TCL_TARGET_OFFSET(tcl) & 0xC) << 4));
6062
6063         /*
6064          * And now calculate the SCB offset to the entry.
6065          * Each entry is 2 bytes wide, hence the
6066          * multiplication by 2.
6067          */
6068         return (((TCL_TARGET_OFFSET(tcl) & 0x3) << 1) + SCB_DISCONNECTED_LISTS);
6069 }
6070
6071 /*
6072  * Return the untagged transaction id for a given target/channel lun.
6073  * Optionally, clear the entry.
6074  */
6075 u_int
6076 ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl)
6077 {
6078         u_int scbid;
6079         u_int scb_offset;
6080         u_int saved_scbptr;
6081
6082         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
6083         scbid = ahd_inw_scbram(ahd, scb_offset);
6084         ahd_set_scbptr(ahd, saved_scbptr);
6085         return (scbid);
6086 }
6087
6088 void
6089 ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl, u_int scbid)
6090 {
6091         u_int scb_offset;
6092         u_int saved_scbptr;
6093
6094         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
6095         ahd_outw(ahd, scb_offset, scbid);
6096         ahd_set_scbptr(ahd, saved_scbptr);
6097 }
6098
6099 /************************** SCB and SCB queue management **********************/
6100 int
6101 ahd_match_scb(struct ahd_softc *ahd, struct scb *scb, int target,
6102               char channel, int lun, u_int tag, role_t role)
6103 {
6104         int targ = SCB_GET_TARGET(ahd, scb);
6105         char chan = SCB_GET_CHANNEL(ahd, scb);
6106         int slun = SCB_GET_LUN(scb);
6107         int match;
6108
6109         match = ((chan == channel) || (channel == ALL_CHANNELS));
6110         if (match != 0)
6111                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
6112         if (match != 0)
6113                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
6114         if (match != 0) {
6115 #if AHD_TARGET_MODE
6116                 int group;
6117
6118                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
6119                 if (role == ROLE_INITIATOR) {
6120                         match = (group != XPT_FC_GROUP_TMODE)
6121                               && ((tag == SCB_GET_TAG(scb))
6122                                || (tag == SCB_LIST_NULL));
6123                 } else if (role == ROLE_TARGET) {
6124                         match = (group == XPT_FC_GROUP_TMODE)
6125                               && ((tag == scb->io_ctx->csio.tag_id)
6126                                || (tag == SCB_LIST_NULL));
6127                 }
6128 #else /* !AHD_TARGET_MODE */
6129                 match = ((tag == SCB_GET_TAG(scb)) || (tag == SCB_LIST_NULL));
6130 #endif /* AHD_TARGET_MODE */
6131         }
6132
6133         return match;
6134 }
6135
6136 void
6137 ahd_freeze_devq(struct ahd_softc *ahd, struct scb *scb)
6138 {
6139         int     target;
6140         char    channel;
6141         int     lun;
6142
6143         target = SCB_GET_TARGET(ahd, scb);
6144         lun = SCB_GET_LUN(scb);
6145         channel = SCB_GET_CHANNEL(ahd, scb);
6146
6147         ahd_search_qinfifo(ahd, target, channel, lun,
6148                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
6149                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
6150
6151         ahd_platform_freeze_devq(ahd, scb);
6152 }
6153
6154 void
6155 ahd_qinfifo_requeue_tail(struct ahd_softc *ahd, struct scb *scb)
6156 {
6157         struct scb *prev_scb;
6158
6159         prev_scb = NULL;
6160         if (ahd_qinfifo_count(ahd) != 0) {
6161                 u_int prev_tag;
6162                 u_int prev_pos;
6163
6164                 prev_pos = AHD_QIN_WRAP(ahd->qinfifonext - 1);
6165                 prev_tag = ahd->qinfifo[prev_pos];
6166                 prev_scb = ahd_lookup_scb(ahd, prev_tag);
6167         }
6168         ahd_qinfifo_requeue(ahd, prev_scb, scb);
6169         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
6170 }
6171
6172 static void
6173 ahd_qinfifo_requeue(struct ahd_softc *ahd, struct scb *prev_scb,
6174                     struct scb *scb)
6175 {
6176         if (prev_scb == NULL) {
6177                 uint32_t busaddr;
6178
6179                 busaddr = ahd_le32toh(scb->hscb->hscb_busaddr);
6180                 ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 0, busaddr & 0xFF);
6181                 ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 1, (busaddr >> 8) & 0xFF);
6182                 ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 2, (busaddr >> 16) & 0xFF);
6183                 ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 3, (busaddr >> 24) & 0xFF);
6184         } else {
6185                 prev_scb->hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
6186                 ahd_sync_scb(ahd, prev_scb,
6187                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
6188         }
6189         ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
6190         ahd->qinfifonext++;
6191         scb->hscb->next_hscb_busaddr = ahd->next_queued_hscb->hscb_busaddr;
6192         ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
6193 }
6194
6195 static int
6196 ahd_qinfifo_count(struct ahd_softc *ahd)
6197 {
6198         u_int qinpos;
6199         u_int wrap_qinpos;
6200         u_int wrap_qinfifonext;
6201
6202         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
6203         qinpos = ahd_get_snscb_qoff(ahd);
6204         wrap_qinpos = AHD_QIN_WRAP(qinpos);
6205         wrap_qinfifonext = AHD_QIN_WRAP(ahd->qinfifonext);
6206         if (wrap_qinfifonext > wrap_qinpos)
6207                 return (wrap_qinfifonext - wrap_qinpos);
6208         else
6209                 return (wrap_qinfifonext
6210                       + NUM_ELEMENTS(ahd->qinfifo) - wrap_qinpos);
6211 }
6212
6213 int
6214 ahd_search_qinfifo(struct ahd_softc *ahd, int target, char channel,
6215                    int lun, u_int tag, role_t role, uint32_t status,
6216                    ahd_search_action action)
6217 {
6218         struct scb      *scb;
6219         struct scb      *prev_scb;
6220         ahd_mode_state   saved_modes;
6221         u_int            qinstart;
6222         u_int            qinpos;
6223         u_int            qintail;
6224         u_int            tid_next;
6225         u_int            tid_prev;
6226         u_int            scbid;
6227         u_int            savedscbptr;
6228         uint32_t         busaddr;
6229         int              found;
6230         int              targets;
6231
6232         /* Must be in CCHAN mode */
6233         saved_modes = ahd_save_modes(ahd);
6234         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
6235
6236         /*
6237          * Halt any pending SCB DMA.  The sequencer will reinitiate
6238          * this dma if the qinfifo is not empty once we unpause.
6239          */
6240         if ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN|CCSCBDIR))
6241          == (CCARREN|CCSCBEN|CCSCBDIR)) {
6242                 ahd_outb(ahd, CCSCBCTL,
6243                          ahd_inb(ahd, CCSCBCTL) & ~(CCARREN|CCSCBEN));
6244                 while ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN)) != 0)
6245                         ;
6246         }
6247         /* Determine sequencer's position in the qinfifo. */
6248         qintail = AHD_QIN_WRAP(ahd->qinfifonext);
6249         qinstart = ahd_get_snscb_qoff(ahd);
6250         qinpos = AHD_QIN_WRAP(qinstart);
6251         found = 0;
6252         prev_scb = NULL;
6253
6254         if (action == SEARCH_PRINT) {
6255                 printf("qinstart = %d qinfifonext = %d\nQINFIFO:",
6256                        qinstart, ahd->qinfifonext);
6257         }
6258
6259         /*
6260          * Start with an empty queue.  Entries that are not chosen
6261          * for removal will be re-added to the queue as we go.
6262          */
6263         ahd->qinfifonext = qinstart;
6264         busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
6265         ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 0, busaddr & 0xFF);
6266         ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 1, (busaddr >> 8) & 0xFF);
6267         ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 2, (busaddr >> 16) & 0xFF);
6268         ahd_outb(ahd, NEXT_QUEUED_SCB_ADDR + 3, (busaddr >> 24) & 0xFF);
6269
6270         while (qinpos != qintail) {
6271                 scb = ahd_lookup_scb(ahd, ahd->qinfifo[qinpos]);
6272                 if (scb == NULL) {
6273                         printf("qinpos = %d, SCB index = %d\n",
6274                                 qinpos, ahd->qinfifo[qinpos]);
6275                         panic("Loop 1\n");
6276                 }
6277
6278                 if (ahd_match_scb(ahd, scb, target, channel, lun, tag, role)) {
6279                         /*
6280                          * We found an scb that needs to be acted on.
6281                          */
6282                         found++;
6283                         switch (action) {
6284                         case SEARCH_COMPLETE:
6285                         {
6286                                 cam_status ostat;
6287                                 cam_status cstat;
6288
6289                                 ostat = ahd_get_transaction_status(scb);
6290                                 if (ostat == CAM_REQ_INPROG)
6291                                         ahd_set_transaction_status(scb,
6292                                                                    status);
6293                                 cstat = ahd_get_transaction_status(scb);
6294                                 if (cstat != CAM_REQ_CMP)
6295                                         ahd_freeze_scb(scb);
6296                                 if ((scb->flags & SCB_ACTIVE) == 0)
6297                                         printf("Inactive SCB in qinfifo\n");
6298                                 ahd_done(ahd, scb);
6299
6300                                 /* FALLTHROUGH */
6301                         }
6302                         case SEARCH_REMOVE:
6303                                 break;
6304                         case SEARCH_PRINT:
6305                                 printf(" 0x%x", ahd->qinfifo[qinpos]);
6306                                 /* FALLTHROUGH */
6307                         case SEARCH_COUNT:
6308                                 ahd_qinfifo_requeue(ahd, prev_scb, scb);
6309                                 prev_scb = scb;
6310                                 break;
6311                         }
6312                 } else {
6313                         ahd_qinfifo_requeue(ahd, prev_scb, scb);
6314                         prev_scb = scb;
6315                 }
6316                 qinpos = AHD_QIN_WRAP(qinpos+1);
6317         }
6318
6319         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
6320
6321         if (action == SEARCH_PRINT)
6322                 printf("\nWAITING_TID_QUEUES:\n");
6323
6324         /*
6325          * Search waiting for selection lists.  We traverse the
6326          * list of "their ids" waiting for selection and, if
6327          * appropriate, traverse the SCBs of each "their id"
6328          * looking for matches.
6329          */
6330         savedscbptr = ahd_get_scbptr(ahd);
6331         tid_next = ahd_inw(ahd, WAITING_TID_HEAD);
6332         tid_prev = SCB_LIST_NULL;
6333         targets = 0;
6334         for (scbid = tid_next; !SCBID_IS_NULL(scbid); scbid = tid_next) {
6335                 u_int tid_head;
6336
6337                 /*
6338                  * We limit based on the number of SCBs since
6339                  * MK_MESSAGE SCBs are not in the per-tid lists.
6340                  */
6341                 targets++;
6342                 if (targets > AHD_SCB_MAX) {
6343                         panic("TID LIST LOOP");
6344                 }
6345                 if (scbid >= ahd->scb_data.numscbs) {
6346                         printf("%s: Waiting TID List inconsistency. "
6347                                "SCB index == 0x%x, yet numscbs == 0x%x.",
6348                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
6349                         ahd_dump_card_state(ahd);
6350                         panic("for safety");
6351                 }
6352                 scb = ahd_lookup_scb(ahd, scbid);
6353                 if (scb == NULL) {
6354                         printf("%s: SCB = 0x%x Not Active!\n",
6355                                ahd_name(ahd), scbid);
6356                         panic("Waiting TID List traversal\n");
6357                 }
6358                 ahd_set_scbptr(ahd, scbid);
6359                 tid_next = ahd_inw_scbram(ahd, SCB_NEXT2);
6360                 if (ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
6361                                   SCB_LIST_NULL, ROLE_UNKNOWN) == 0) {
6362                         tid_prev = scbid;
6363                         continue;
6364                 }
6365
6366                 /*
6367                  * We found a list of scbs that needs to be searched.
6368                  */
6369                 if (action == SEARCH_PRINT)
6370                         printf("       %d ( ", SCB_GET_TARGET(ahd, scb));
6371                 tid_head = scbid;
6372                 found += ahd_search_scb_list(ahd, target, channel,
6373                                              lun, tag, role, status,
6374                                              action, &tid_head,
6375                                              SCB_GET_TARGET(ahd, scb));
6376                 if (tid_head != scbid)
6377                         ahd_stitch_tid_list(ahd, tid_prev, tid_head, tid_next);
6378                 if (!SCBID_IS_NULL(tid_head))
6379                         tid_prev = tid_head;
6380                 if (action == SEARCH_PRINT)
6381                         printf(")\n");
6382         }
6383         ahd_set_scbptr(ahd, savedscbptr);
6384
6385         ahd_restore_modes(ahd, saved_modes);
6386         return (found);
6387 }
6388
6389 static int
6390 ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel,
6391                     int lun, u_int tag, role_t role, uint32_t status,
6392                     ahd_search_action action, u_int *list_head, u_int tid)
6393 {
6394         struct  scb *scb;
6395         u_int   scbid;
6396         u_int   next;
6397         u_int   prev;
6398         int     found;
6399
6400         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
6401         found = 0;
6402         prev = SCB_LIST_NULL;
6403         next = *list_head;
6404         for (scbid = next; !SCBID_IS_NULL(scbid); scbid = next) {
6405                 if (scbid >= ahd->scb_data.numscbs) {
6406                         printf("%s:SCB List inconsistency. "
6407                                "SCB == 0x%x, yet numscbs == 0x%x.",
6408                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
6409                         ahd_dump_card_state(ahd);
6410                         panic("for safety");
6411                 }
6412                 scb = ahd_lookup_scb(ahd, scbid);
6413                 if (scb == NULL) {
6414                         printf("%s: SCB = %d Not Active!\n",
6415                                ahd_name(ahd), scbid);
6416                         panic("Waiting List traversal\n");
6417                 }
6418                 ahd_set_scbptr(ahd, scbid);
6419                 next = ahd_inw_scbram(ahd, SCB_NEXT);
6420                 if (ahd_match_scb(ahd, scb, target, channel,
6421                                   lun, SCB_LIST_NULL, role) == 0) {
6422                         prev = scbid;
6423                         continue;
6424                 }
6425                 found++;
6426                 switch (action) {
6427                 case SEARCH_COMPLETE:
6428                 {
6429                         cam_status ostat;
6430                         cam_status cstat;
6431
6432                         ostat = ahd_get_transaction_status(scb);
6433                         if (ostat == CAM_REQ_INPROG)
6434                                 ahd_set_transaction_status(scb, status);
6435                         cstat = ahd_get_transaction_status(scb);
6436                         if (cstat != CAM_REQ_CMP)
6437                                 ahd_freeze_scb(scb);
6438                         if ((scb->flags & SCB_ACTIVE) == 0)
6439                                 printf("Inactive SCB in Waiting List\n");
6440                         ahd_done(ahd, scb);
6441                         /* FALLTHROUGH */
6442                 }
6443                 case SEARCH_REMOVE:
6444                         ahd_rem_wscb(ahd, scbid, prev, next, tid);
6445                         if (prev == SCB_LIST_NULL)
6446                                 *list_head = next;
6447                         break;
6448                 case SEARCH_PRINT:
6449                         printf("0x%x ", scbid);
6450                 case SEARCH_COUNT:
6451                         prev = scbid;
6452                         break;
6453                 }
6454                 if (found > AHD_SCB_MAX)
6455                         panic("SCB LIST LOOP");
6456         }
6457         return (found);
6458 }
6459
6460 static void
6461 ahd_stitch_tid_list(struct ahd_softc *ahd, u_int tid_prev,
6462                     u_int tid_cur, u_int tid_next)
6463 {
6464         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
6465
6466         if (SCBID_IS_NULL(tid_cur)) {
6467
6468                 /* Bypass current TID list */
6469                 if (SCBID_IS_NULL(tid_prev)) {
6470                         ahd_outw(ahd, WAITING_TID_HEAD, tid_next);
6471                 } else {
6472                         ahd_set_scbptr(ahd, tid_prev);
6473                         ahd_outw(ahd, SCB_NEXT2, tid_next);
6474                 }
6475                 if (SCBID_IS_NULL(tid_next))
6476                         ahd_outw(ahd, WAITING_TID_TAIL, tid_prev);
6477         } else {
6478
6479                 /* Stitch through tid_cur */
6480                 if (SCBID_IS_NULL(tid_prev)) {
6481                         ahd_outw(ahd, WAITING_TID_HEAD, tid_cur);
6482                 } else {
6483                         ahd_set_scbptr(ahd, tid_prev);
6484                         ahd_outw(ahd, SCB_NEXT2, tid_cur);
6485                 }
6486                 ahd_set_scbptr(ahd, tid_cur);
6487                 ahd_outw(ahd, SCB_NEXT2, tid_next);
6488
6489                 if (SCBID_IS_NULL(tid_next))
6490                         ahd_outw(ahd, WAITING_TID_TAIL, tid_cur);
6491         }
6492 }
6493
6494 /*
6495  * Manipulate the waiting for selection list and return the
6496  * scb that follows the one that we remove.
6497  */
6498 static u_int
6499 ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
6500              u_int prev, u_int next, u_int tid)
6501 {
6502         u_int tail_offset;
6503
6504         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
6505         if (!SCBID_IS_NULL(prev)) {
6506                 ahd_set_scbptr(ahd, prev);
6507                 ahd_outw(ahd, SCB_NEXT, next);
6508         }
6509
6510         /*
6511          * SCBs that had MK_MESSAGE set in them will not
6512          * be queued to the per-target lists, so don't
6513          * blindly clear the tail pointer.
6514          */
6515         tail_offset = WAITING_SCB_TAILS + (2 * tid);
6516         if (SCBID_IS_NULL(next)
6517          && ahd_inw(ahd, tail_offset) == scbid)
6518                 ahd_outw(ahd, tail_offset, prev);
6519         ahd_add_scb_to_free_list(ahd, scbid);
6520         return (next);
6521 }
6522
6523 /*
6524  * Add the SCB as selected by SCBPTR onto the on chip list of
6525  * free hardware SCBs.  This list is empty/unused if we are not
6526  * performing SCB paging.
6527  */
6528 static void
6529 ahd_add_scb_to_free_list(struct ahd_softc *ahd, u_int scbid)
6530 {
6531 /* XXX Need some other mechanism to designate "free". */
6532         /*
6533          * Invalidate the tag so that our abort
6534          * routines don't think it's active.
6535         ahd_outb(ahd, SCB_TAG, SCB_LIST_NULL);
6536          */
6537 }
6538
6539 /******************************** Error Handling ******************************/
6540 /*
6541  * Abort all SCBs that match the given description (target/channel/lun/tag),
6542  * setting their status to the passed in status if the status has not already
6543  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
6544  * is paused before it is called.
6545  */
6546 int
6547 ahd_abort_scbs(struct ahd_softc *ahd, int target, char channel,
6548                int lun, u_int tag, role_t role, uint32_t status)
6549 {
6550         struct  scb *scbp;
6551         struct  scb *scbp_next;
6552         u_int   active_scb;
6553         u_int   i, j;
6554         u_int   maxtarget;
6555         u_int   minlun;
6556         u_int   maxlun;
6557
6558         int     found;
6559
6560         /* restore this when we're done */
6561         active_scb = ahd_get_scbptr(ahd);
6562
6563         found = ahd_search_qinfifo(ahd, target, channel, lun, SCB_LIST_NULL,
6564                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
6565
6566         /*
6567          * Clean out the busy target table for any untagged commands.
6568          */
6569         i = 0;
6570         maxtarget = 16;
6571         if (target != CAM_TARGET_WILDCARD) {
6572                 i = target;
6573                 if (channel == 'B')
6574                         i += 8;
6575                 maxtarget = i + 1;
6576         }
6577
6578         if (lun == CAM_LUN_WILDCARD) {
6579                 minlun = 0;
6580                 maxlun = AHD_NUM_LUNS_NONPKT;
6581         } else if (lun >= AHD_NUM_LUNS_NONPKT) {
6582                 minlun = maxlun = 0;
6583         } else {
6584                 minlun = lun;
6585                 maxlun = lun + 1;
6586         }
6587
6588         if (role != ROLE_TARGET) {
6589                 for (;i < maxtarget; i++) {
6590                         for (j = minlun;j < maxlun; j++) {
6591                                 u_int scbid;
6592                                 u_int tcl;
6593
6594                                 tcl = BUILD_TCL_RAW(i, 'A', j);
6595                                 scbid = ahd_find_busy_tcl(ahd, tcl);
6596                                 scbp = ahd_lookup_scb(ahd, scbid);
6597                                 if (scbp == NULL
6598                                  || ahd_match_scb(ahd, scbp, target, channel,
6599                                                   lun, tag, role) == 0)
6600                                         continue;
6601                                 ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(i, 'A', j));
6602                         }
6603                 }
6604         }
6605
6606         /*
6607          * Go through the pending CCB list and look for
6608          * commands for this target that are still active.
6609          * These are other tagged commands that were
6610          * disconnected when the reset occurred.
6611          */
6612         scbp_next = LIST_FIRST(&ahd->pending_scbs);
6613         while (scbp_next != NULL) {
6614                 scbp = scbp_next;
6615                 scbp_next = LIST_NEXT(scbp, pending_links);
6616                 if (ahd_match_scb(ahd, scbp, target, channel, lun, tag, role)) {
6617                         cam_status ostat;
6618
6619                         ostat = ahd_get_transaction_status(scbp);
6620                         if (ostat == CAM_REQ_INPROG)
6621                                 ahd_set_transaction_status(scbp, status);
6622                         if (ahd_get_transaction_status(scbp) != CAM_REQ_CMP)
6623                                 ahd_freeze_scb(scbp);
6624                         if ((scbp->flags & SCB_ACTIVE) == 0)
6625                                 printf("Inactive SCB on pending list\n");
6626                         ahd_done(ahd, scbp);
6627                         found++;
6628                 }
6629         }
6630         ahd_set_scbptr(ahd, active_scb);
6631         ahd_platform_abort_scbs(ahd, target, channel, lun, tag, role, status);
6632         return found;
6633 }
6634
6635 static void
6636 ahd_reset_current_bus(struct ahd_softc *ahd)
6637 {
6638         uint8_t scsiseq;
6639
6640         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
6641         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) & ~ENSCSIRST);
6642         scsiseq = ahd_inb(ahd, SCSISEQ0) & ~(ENSELO|ENARBO|SCSIRSTO);
6643         ahd_outb(ahd, SCSISEQ0, scsiseq | SCSIRSTO);
6644         ahd_delay(AHD_BUSRESET_DELAY);
6645         /* Turn off the bus reset */
6646         ahd_outb(ahd, SCSISEQ0, scsiseq);
6647         if ((ahd->bugs & AHD_SCSIRST_BUG) != 0) {
6648                 /*
6649                  * 2A Razor #474
6650                  * Certain chip state is not cleared for
6651                  * SCSI bus resets that we initiate, so
6652                  * we must reset the chip.
6653                  */
6654                 ahd_delay(AHD_BUSRESET_DELAY);
6655                 ahd_reset(ahd);
6656                 ahd_intr_enable(ahd, /*enable*/TRUE);
6657                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
6658         }
6659
6660         ahd_clear_intstat(ahd);
6661 }
6662
6663 int
6664 ahd_reset_channel(struct ahd_softc *ahd, char channel, int initiate_reset)
6665 {
6666         struct  ahd_devinfo devinfo;
6667         u_int   initiator;
6668         u_int   target;
6669         u_int   max_scsiid;
6670         int     found;
6671         u_int   fifo;
6672         u_int   next_fifo;
6673
6674         ahd->pending_device = NULL;
6675
6676         ahd_compile_devinfo(&devinfo,
6677                             CAM_TARGET_WILDCARD,
6678                             CAM_TARGET_WILDCARD,
6679                             CAM_LUN_WILDCARD,
6680                             channel, ROLE_UNKNOWN);
6681         ahd_pause(ahd);
6682
6683         /* Make sure the sequencer is in a safe location. */
6684         ahd_clear_critical_section(ahd);
6685
6686         /*
6687          * Run our command complete fifos to ensure that we perform
6688          * completion processing on any commands that 'completed'
6689          * before the reset occurred.
6690          */
6691         ahd_run_qoutfifo(ahd);
6692 #if AHD_TARGET_MODE
6693         if ((ahd->flags & AHD_TARGETROLE) != 0) {
6694                 ahd_run_tqinfifo(ahd, /*paused*/TRUE);
6695         }
6696 #endif
6697         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6698
6699         /*
6700          * Disable selections so no automatic hardware
6701          * functions will modify chip state.
6702          */
6703         ahd_outb(ahd, SCSISEQ0, 0);
6704         ahd_outb(ahd, SCSISEQ1, 0);
6705
6706         /*
6707          * Safely shut down our DMA engines.  Always start with
6708          * the FIFO that is not currently active (if any are
6709          * actively connected).
6710          */
6711         next_fifo = fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
6712         do {
6713                 next_fifo = next_fifo ^ CURRFIFO;
6714                 ahd_set_modes(ahd, next_fifo, next_fifo);
6715                 ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) & ~SCSIEN);
6716                 while ((ahd_inb(ahd, DFCNTRL) & SCSIENACK) != 0)
6717                         ahd_delay(10);
6718                 ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) & ~HDMAEN);
6719                 while ((ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0)
6720                         ahd_delay(10);
6721                 /*
6722                  * Set CURRFIFO to the now inactive channel.
6723                  */
6724                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6725                 ahd_outb(ahd, DFFSTAT, next_fifo);
6726         } while (next_fifo != fifo);
6727
6728         /*
6729          * Reset the bus if we are initiating this reset
6730          */
6731         ahd_clear_msg_state(ahd);
6732         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) & ~(ENBUSFREE|ENSCSIRST));
6733         if (initiate_reset)
6734                 ahd_reset_current_bus(ahd);
6735         ahd_clear_intstat(ahd);
6736
6737         /*
6738          * Clean up all the state information for the
6739          * pending transactions on this bus.
6740          */
6741         found = ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, channel,
6742                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
6743                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
6744
6745         /*
6746          * Cleanup anything left in the FIFOs.
6747          */
6748         ahd_clear_fifo(ahd, 0);
6749         ahd_clear_fifo(ahd, 1);
6750
6751         /*
6752          * Revert to async/narrow transfers until we renegotiate.
6753          */
6754         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
6755         for (target = 0; target <= max_scsiid; target++) {
6756
6757                 if (ahd->enabled_targets[target] == NULL)
6758                         continue;
6759                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
6760                         struct ahd_devinfo devinfo;
6761
6762                         ahd_compile_devinfo(&devinfo, target, initiator,
6763                                             CAM_LUN_WILDCARD,
6764                                             'A', ROLE_UNKNOWN);
6765                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6766                                       AHD_TRANS_CUR, /*paused*/TRUE);
6767                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
6768                                          /*offset*/0, /*ppr_options*/0,
6769                                          AHD_TRANS_CUR, /*paused*/TRUE);
6770                 }
6771         }
6772
6773 #ifdef AHD_TARGET_MODE
6774         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
6775
6776         /*
6777          * Send an immediate notify ccb to all target more peripheral
6778          * drivers affected by this action.
6779          */
6780         for (target = 0; target <= max_scsiid; target++) {
6781                 struct ahd_tmode_tstate* tstate;
6782                 u_int lun;
6783
6784                 tstate = ahd->enabled_targets[target];
6785                 if (tstate == NULL)
6786                         continue;
6787                 for (lun = 0; lun < AHD_NUM_LUNS; lun++) {
6788                         struct ahd_tmode_lstate* lstate;
6789
6790                         lstate = tstate->enabled_luns[lun];
6791                         if (lstate == NULL)
6792                                 continue;
6793
6794                         ahd_queue_lstate_event(ahd, lstate, CAM_TARGET_WILDCARD,
6795                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
6796                         ahd_send_lstate_events(ahd, lstate);
6797                 }
6798         }
6799 #endif
6800         /* Notify the XPT that a bus reset occurred */
6801         ahd_send_async(ahd, devinfo.channel, CAM_TARGET_WILDCARD,
6802                        CAM_LUN_WILDCARD, AC_BUS_RESET, NULL);
6803
6804         /*
6805          * Freeze the SIMQ until our poller can determine that
6806          * the bus reset has really gone away.  We set the initial
6807          * timer to 0 to have the check performed as soon as possible
6808          * from the timer context.
6809          */
6810         if ((ahd->flags & AHD_RESET_POLL_ACTIVE) == 0) {
6811                 ahd->flags |= AHD_RESET_POLL_ACTIVE;
6812                 ahd_freeze_simq(ahd);
6813                 ahd_timer_reset(&ahd->reset_timer, 0, ahd_reset_poll, ahd);
6814         }
6815         ahd_restart(ahd);
6816         return (found);
6817 }
6818
6819
6820 #define AHD_RESET_POLL_US 1000
6821 static void
6822 ahd_reset_poll(void *arg)
6823 {
6824         struct  ahd_softc *ahd;
6825         u_int   scsiseq1;
6826         u_long  l;
6827         u_long  s;
6828
6829         ahd_list_lock(&l);
6830         ahd = ahd_find_softc((struct ahd_softc *)arg);
6831         if (ahd == NULL) {
6832                 printf("ahd_reset_poll: Instance %p no longer exists\n", arg);
6833                 ahd_list_unlock(&l);
6834                 return;
6835         }
6836         ahd_lock(ahd, &s);
6837         ahd_pause(ahd);
6838         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
6839         if ((ahd_inb(ahd, SSTAT1) & SCSIRSTI) != 0) {
6840                 ahd_timer_reset(&ahd->reset_timer, AHD_RESET_POLL_US,
6841                                 ahd_reset_poll, ahd);
6842                 ahd_unpause(ahd);
6843                 ahd_unlock(ahd, &s);
6844                 ahd_list_unlock(&l);
6845                 return;
6846         }
6847
6848         /* Reset is now low.  Complete chip reinitialization. */
6849         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) | ENSCSIRST);
6850         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
6851         ahd_outb(ahd, SCSISEQ1, scsiseq1 & (ENSELI|ENRSELI|ENAUTOATNP));
6852         ahd_unpause(ahd);
6853         ahd->flags &= ~AHD_RESET_POLL_ACTIVE;
6854         ahd_unlock(ahd, &s);
6855         ahd_release_simq(ahd);
6856         ahd_list_unlock(&l);
6857 }
6858
6859
6860 /****************************** Status Processing *****************************/
6861 void
6862 ahd_handle_scb_status(struct ahd_softc *ahd, struct scb *scb)
6863 {
6864         if (scb->hscb->shared_data.istatus.scsi_status != 0) {
6865                 ahd_handle_scsi_status(ahd, scb);
6866         } else {
6867                 ahd_calc_residual(ahd, scb);
6868                 ahd_done(ahd, scb);
6869         }
6870 }
6871
6872 void
6873 ahd_handle_scsi_status(struct ahd_softc *ahd, struct scb *scb)
6874 {
6875         struct hardware_scb *hscb;
6876         u_int  qfreeze_cnt;
6877         ahd_mode_state saved_modes;
6878
6879         /*
6880          * The sequencer freezes its select-out queue
6881          * anytime a SCSI status error occurs.  We must
6882          * handle the error and decrement the QFREEZE count
6883          * to allow the sequencer to continue.
6884          */
6885         hscb = scb->hscb;
6886
6887         /* Freeze the queue until the client sees the error. */
6888         ahd_pause(ahd);
6889         saved_modes = ahd_save_modes(ahd);
6890         ahd_clear_critical_section(ahd);
6891         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6892         ahd_freeze_devq(ahd, scb);
6893         ahd_freeze_scb(scb);
6894         qfreeze_cnt = ahd_inw(ahd, QFREEZE_COUNT);
6895         if (qfreeze_cnt == 0) {
6896                 printf("%s: Bad status with 0 qfreeze count!\n", ahd_name(ahd));
6897         } else {
6898                 qfreeze_cnt--;
6899                 ahd_outw(ahd, QFREEZE_COUNT, qfreeze_cnt);
6900         }
6901         if (qfreeze_cnt == 0)
6902                 ahd_outb(ahd, SEQ_FLAGS2,
6903                          ahd_inb(ahd, SEQ_FLAGS2) & ~SELECTOUT_QFROZEN);
6904         ahd_unpause(ahd);
6905         /* Don't want to clobber the original sense code */
6906         if ((scb->flags & SCB_SENSE) != 0) {
6907                 /*
6908                  * Clear the SCB_SENSE Flag and perform
6909                  * a normal command completion.
6910                  */
6911                 scb->flags &= ~SCB_SENSE;
6912                 ahd_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
6913                 ahd_done(ahd, scb);
6914                 return;
6915         }
6916         ahd_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
6917         ahd_set_scsi_status(scb, hscb->shared_data.istatus.scsi_status);
6918         switch (hscb->shared_data.istatus.scsi_status) {
6919         case STATUS_PKT_SENSE:
6920         {
6921                 struct scsi_status_iu_header *siu;
6922
6923                 ahd_sync_sense(ahd, scb, BUS_DMASYNC_POSTREAD);
6924                 siu = (struct scsi_status_iu_header *)scb->sense_data;
6925                 ahd_set_scsi_status(scb, siu->status);
6926 #ifdef AHD_DEBUG
6927                 if ((ahd_debug & AHD_SHOW_SENSE) != 0)
6928                         ahd_print_path(ahd, scb);
6929                         printf("SCB 0x%x Received PKT Status of 0x%x\n",
6930                                SCB_GET_TAG(scb), siu->status);
6931                         printf("\tflags = 0x%x, sense len = 0x%x, "
6932                                "pktfail = 0x%x\n",
6933                                siu->flags, scsi_4btoul(siu->sense_length),
6934                                scsi_4btoul(siu->pkt_failures_length));
6935 #endif
6936                 if ((siu->flags & SIU_RSPVALID) != 0) {
6937                         ahd_print_path(ahd, scb);
6938                         if (scsi_4btoul(siu->pkt_failures_length) < 4) {
6939                                 printf("Unable to parse pkt_failures\n");
6940                         } else {
6941
6942                                 switch (SIU_PKTFAIL_CODE(siu)) {
6943                                 case SIU_PFC_NONE:
6944                                         printf("No packet failure found\n");
6945                                         break;
6946                                 case SIU_PFC_CIU_FIELDS_INVALID:
6947                                         printf("Invalid Command IU Field\n");
6948                                         break;
6949                                 case SIU_PFC_TMF_NOT_SUPPORTED:
6950                                         printf("TMF not supportd\n");
6951                                         break;
6952                                 case SIU_PFC_TMF_FAILED:
6953                                         printf("TMF failed\n");
6954                                         break;
6955                                 case SIU_PFC_INVALID_TYPE_CODE:
6956                                         printf("Invalid L_Q Type code\n");
6957                                         break;
6958                                 case SIU_PFC_ILLEGAL_REQUEST:
6959                                         printf("Illegal request\n");
6960                                 default:
6961                                         break;
6962                                 }
6963                         }
6964                         if (siu->status == SCSI_STATUS_OK)
6965                                 ahd_set_transaction_status(scb,
6966                                                            CAM_REQ_CMP_ERR);
6967                 }
6968                 if ((siu->flags & SIU_SNSVALID) != 0) {
6969                         scb->flags |= SCB_PKT_SENSE;
6970 #ifdef AHD_DEBUG
6971                         if ((ahd_debug & AHD_SHOW_SENSE) != 0)
6972                                 printf("Sense data available\n");
6973 #endif
6974                 }
6975                 ahd_done(ahd, scb);
6976                 break;
6977         }
6978         case SCSI_STATUS_CMD_TERMINATED:
6979         case SCSI_STATUS_CHECK_COND:
6980         {
6981                 struct ahd_devinfo devinfo;
6982                 struct ahd_dma_seg *sg;
6983                 struct scsi_sense *sc;
6984                 struct ahd_initiator_tinfo *targ_info;
6985                 struct ahd_tmode_tstate *tstate;
6986                 struct ahd_transinfo *tinfo;
6987 #ifdef AHD_DEBUG
6988                 if (ahd_debug & AHD_SHOW_SENSE) {
6989                         ahd_print_path(ahd, scb);
6990                         printf("SCB %d: requests Check Status\n",
6991                                SCB_GET_TAG(scb));
6992                 }
6993 #endif
6994
6995                 if (ahd_perform_autosense(scb) == 0)
6996                         break;
6997
6998                 ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
6999                                     SCB_GET_TARGET(ahd, scb),
7000                                     SCB_GET_LUN(scb),
7001                                     SCB_GET_CHANNEL(ahd, scb),
7002                                     ROLE_INITIATOR);
7003                 targ_info = ahd_fetch_transinfo(ahd,
7004                                                 devinfo.channel,
7005                                                 devinfo.our_scsiid,
7006                                                 devinfo.target,
7007                                                 &tstate);
7008                 tinfo = &targ_info->curr;
7009                 sg = scb->sg_list;
7010                 sc = (struct scsi_sense *)hscb->shared_data.idata.cdb;
7011                 /*
7012                  * Save off the residual if there is one.
7013                  */
7014                 ahd_update_residual(ahd, scb);
7015 #ifdef AHD_DEBUG
7016                 if (ahd_debug & AHD_SHOW_SENSE) {
7017                         ahd_print_path(ahd, scb);
7018                         printf("Sending Sense\n");
7019                 }
7020 #endif
7021                 scb->sg_count = 0;
7022                 sg = ahd_sg_setup(ahd, scb, sg, ahd_get_sense_bufaddr(ahd, scb),
7023                                   ahd_get_sense_bufsize(ahd, scb),
7024                                   /*last*/TRUE);
7025                 sc->opcode = REQUEST_SENSE;
7026                 sc->byte2 = 0;
7027                 if (tinfo->protocol_version <= SCSI_REV_2
7028                  && SCB_GET_LUN(scb) < 8)
7029                         sc->byte2 = SCB_GET_LUN(scb) << 5;
7030                 sc->unused[0] = 0;
7031                 sc->unused[1] = 0;
7032                 sc->length = ahd_get_sense_bufsize(ahd, scb);
7033                 sc->control = 0;
7034
7035                 /*
7036                  * We can't allow the target to disconnect.
7037                  * This will be an untagged transaction and
7038                  * having the target disconnect will make this
7039                  * transaction indestinguishable from outstanding
7040                  * tagged transactions.
7041                  */
7042                 hscb->control = 0;
7043
7044                 /*
7045                  * This request sense could be because the
7046                  * the device lost power or in some other
7047                  * way has lost our transfer negotiations.
7048                  * Renegotiate if appropriate.  Unit attention
7049                  * errors will be reported before any data
7050                  * phases occur.
7051                  */
7052                 if (ahd_get_residual(scb) == ahd_get_transfer_length(scb)) {
7053                         ahd_update_neg_request(ahd, &devinfo,
7054                                                tstate, targ_info,
7055                                                /*force*/TRUE);
7056                 }
7057                 if (tstate->auto_negotiate & devinfo.target_mask) {
7058                         hscb->control |= MK_MESSAGE;
7059                         scb->flags &=
7060                             ~(SCB_NEGOTIATE|SCB_ABORT|SCB_DEVICE_RESET);
7061                         scb->flags |= SCB_AUTO_NEGOTIATE;
7062                 }
7063                 hscb->cdb_len = sizeof(*sc);
7064                 ahd_setup_data_scb(ahd, scb);
7065                 scb->flags |= SCB_SENSE;
7066                 ahd_queue_scb(ahd, scb);
7067 #ifdef __FreeBSD__
7068                 /*
7069                  * Ensure we have enough time to actually
7070                  * retrieve the sense.
7071                  */
7072                 untimeout(ahd_timeout, (caddr_t)scb,
7073                           scb->io_ctx->ccb_h.timeout_ch);
7074                 scb->io_ctx->ccb_h.timeout_ch =
7075                     timeout(ahd_timeout, (caddr_t)scb, 5 * hz);
7076 #endif
7077                 break;
7078         }
7079         case SCSI_STATUS_OK:
7080                 printf("%s: Interrupted for staus of 0???\n",
7081                        ahd_name(ahd));
7082                 /* FALLTHROUGH */
7083         default:
7084                 ahd_done(ahd, scb);
7085                 break;
7086         }
7087 }
7088
7089 /*
7090  * Calculate the residual for a just completed SCB.
7091  */
7092 void
7093 ahd_calc_residual(struct ahd_softc *ahd, struct scb *scb)
7094 {
7095         struct hardware_scb *hscb;
7096         struct initiator_status *spkt;
7097         uint32_t sgptr;
7098         uint32_t resid_sgptr;
7099         uint32_t resid;
7100
7101         /*
7102          * 5 cases.
7103          * 1) No residual.
7104          *    SG_STATUS_VALID clear in sgptr.
7105          * 2) Transferless command
7106          * 3) Never performed any transfers.
7107          *    sgptr has SG_FULL_RESID set.
7108          * 4) No residual but target did not
7109          *    save data pointers after the
7110          *    last transfer, so sgptr was
7111          *    never updated.
7112          * 5) We have a partial residual.
7113          *    Use residual_sgptr to determine
7114          *    where we are.
7115          */
7116
7117         hscb = scb->hscb;
7118         sgptr = ahd_le32toh(hscb->sgptr);
7119         if ((sgptr & SG_STATUS_VALID) == 0)
7120                 /* Case 1 */
7121                 return;
7122         sgptr &= ~SG_STATUS_VALID;
7123
7124         if ((sgptr & SG_LIST_NULL) != 0)
7125                 /* Case 2 */
7126                 return;
7127
7128         /*
7129          * Residual fields are the same in both
7130          * target and initiator status packets,
7131          * so we can always use the initiator fields
7132          * regardless of the role for this SCB.
7133          */
7134         spkt = &hscb->shared_data.istatus;
7135         resid_sgptr = ahd_le32toh(spkt->residual_sgptr);
7136         if ((sgptr & SG_FULL_RESID) != 0) {
7137                 /* Case 3 */
7138                 resid = ahd_get_transfer_length(scb);
7139         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
7140                 /* Case 4 */
7141                 return;
7142         } else if ((resid_sgptr & SG_OVERRUN_RESID) != 0) {
7143                 ahd_print_path(ahd, scb);
7144                 printf("data overrun detected Tag == 0x%x.\n",
7145                        SCB_GET_TAG(scb));
7146                 ahd_freeze_devq(ahd, scb);
7147                 ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
7148                 ahd_freeze_scb(scb);
7149                 return;
7150         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
7151                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
7152                 /* NOTREACHED */
7153         } else {
7154                 struct ahd_dma_seg *sg;
7155
7156                 /*
7157                  * Remainder of the SG where the transfer
7158                  * stopped.
7159                  */
7160                 resid = ahd_le32toh(spkt->residual_datacnt) & AHD_SG_LEN_MASK;
7161                 sg = ahd_sg_bus_to_virt(ahd, scb, resid_sgptr & SG_PTR_MASK);
7162
7163                 /* The residual sg_ptr always points to the next sg */
7164                 sg--;
7165
7166                 /*
7167                  * Add up the contents of all residual
7168                  * SG segments that are after the SG where
7169                  * the transfer stopped.
7170                  */
7171                 while ((ahd_le32toh(sg->len) & AHD_DMA_LAST_SEG) == 0) {
7172                         sg++;
7173                         resid += ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
7174                 }
7175         }
7176         if ((scb->flags & SCB_SENSE) == 0)
7177                 ahd_set_residual(scb, resid);
7178         else
7179                 ahd_set_sense_residual(scb, resid);
7180
7181 #ifdef AHD_DEBUG
7182         if ((ahd_debug & AHD_SHOW_MISC) != 0) {
7183                 ahd_print_path(ahd, scb);
7184                 printf("Handled Residual of %d bytes\n", resid);
7185         }
7186 #endif
7187 }
7188
7189 /******************************* Target Mode **********************************/
7190 #ifdef AHD_TARGET_MODE
7191 /*
7192  * Add a target mode event to this lun's queue
7193  */
7194 static void
7195 ahd_queue_lstate_event(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate,
7196                        u_int initiator_id, u_int event_type, u_int event_arg)
7197 {
7198         struct ahd_tmode_event *event;
7199         int pending;
7200
7201         xpt_freeze_devq(lstate->path, /*count*/1);
7202         if (lstate->event_w_idx >= lstate->event_r_idx)
7203                 pending = lstate->event_w_idx - lstate->event_r_idx;
7204         else
7205                 pending = AHD_TMODE_EVENT_BUFFER_SIZE + 1
7206                         - (lstate->event_r_idx - lstate->event_w_idx);
7207
7208         if (event_type == EVENT_TYPE_BUS_RESET
7209          || event_type == MSG_BUS_DEV_RESET) {
7210                 /*
7211                  * Any earlier events are irrelevant, so reset our buffer.
7212                  * This has the effect of allowing us to deal with reset
7213                  * floods (an external device holding down the reset line)
7214                  * without losing the event that is really interesting.
7215                  */
7216                 lstate->event_r_idx = 0;
7217                 lstate->event_w_idx = 0;
7218                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
7219         }
7220
7221         if (pending == AHD_TMODE_EVENT_BUFFER_SIZE) {
7222                 xpt_print_path(lstate->path);
7223                 printf("immediate event %x:%x lost\n",
7224                        lstate->event_buffer[lstate->event_r_idx].event_type,
7225                        lstate->event_buffer[lstate->event_r_idx].event_arg);
7226                 lstate->event_r_idx++;
7227                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
7228                         lstate->event_r_idx = 0;
7229                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
7230         }
7231
7232         event = &lstate->event_buffer[lstate->event_w_idx];
7233         event->initiator_id = initiator_id;
7234         event->event_type = event_type;
7235         event->event_arg = event_arg;
7236         lstate->event_w_idx++;
7237         if (lstate->event_w_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
7238                 lstate->event_w_idx = 0;
7239 }
7240
7241 /*
7242  * Send any target mode events queued up waiting
7243  * for immediate notify resources.
7244  */
7245 void
7246 ahd_send_lstate_events(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate)
7247 {
7248         struct ccb_hdr *ccbh;
7249         struct ccb_immed_notify *inot;
7250
7251         while (lstate->event_r_idx != lstate->event_w_idx
7252             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
7253                 struct ahd_tmode_event *event;
7254
7255                 event = &lstate->event_buffer[lstate->event_r_idx];
7256                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
7257                 inot = (struct ccb_immed_notify *)ccbh;
7258                 switch (event->event_type) {
7259                 case EVENT_TYPE_BUS_RESET:
7260                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
7261                         break;
7262                 default:
7263                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
7264                         inot->message_args[0] = event->event_type;
7265                         inot->message_args[1] = event->event_arg;
7266                         break;
7267                 }
7268                 inot->initiator_id = event->initiator_id;
7269                 inot->sense_len = 0;
7270                 xpt_done((union ccb *)inot);
7271                 lstate->event_r_idx++;
7272                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
7273                         lstate->event_r_idx = 0;
7274         }
7275 }
7276 #endif
7277
7278 /******************** Sequencer Program Patching/Download *********************/
7279
7280 #ifdef AHD_DUMP_SEQ
7281 void
7282 ahd_dumpseq(struct ahd_softc* ahd)
7283 {
7284         int i;
7285         int max_prog;
7286
7287         max_prog = 2048;
7288
7289         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
7290         ahd_outb(ahd, PRGMCNT, 0);
7291         ahd_outb(ahd, PRGMCNT+1, 0);
7292         for (i = 0; i < max_prog; i++) {
7293                 uint8_t ins_bytes[4];
7294
7295                 ahd_insb(ahd, SEQRAM, ins_bytes, 4);
7296                 printf("0x%08x\n", ins_bytes[0] << 24
7297                                  | ins_bytes[1] << 16
7298                                  | ins_bytes[2] << 8
7299                                  | ins_bytes[3]);
7300         }
7301 }
7302 #endif
7303
7304 static void
7305 ahd_loadseq(struct ahd_softc *ahd)
7306 {
7307         struct  cs cs_table[num_critical_sections];
7308         u_int   begin_set[num_critical_sections];
7309         u_int   end_set[num_critical_sections];
7310         struct  patch *cur_patch;
7311         u_int   cs_count;
7312         u_int   cur_cs;
7313         u_int   i;
7314         int     downloaded;
7315         u_int   skip_addr;
7316         u_int   sg_prefetch_cnt;
7317         u_int   sg_prefetch_cnt_limit;
7318         u_int   sg_prefetch_align;
7319         u_int   sg_size;
7320         uint8_t download_consts[DOWNLOAD_CONST_COUNT];
7321
7322         if (bootverbose)
7323                 printf("%s: Downloading Sequencer Program...",
7324                        ahd_name(ahd));
7325
7326 #if DOWNLOAD_CONST_COUNT != 7
7327 #error "Download Const Mismatch"
7328 #endif
7329         /*
7330          * Start out with 0 critical sections
7331          * that apply to this firmware load.
7332          */
7333         cs_count = 0;
7334         cur_cs = 0;
7335         memset(begin_set, 0, sizeof(begin_set));
7336         memset(end_set, 0, sizeof(end_set));
7337
7338         /*
7339          * Setup downloadable constant table.
7340          *
7341          * The computation for the S/G prefetch variables is
7342          * a bit complicated.  We would like to always fetch
7343          * in terms of cachelined sized increments.  However,
7344          * if the cacheline is not an even multiple of the
7345          * SG element size or is larger than our SG RAM, using
7346          * just the cache size might leave us with only a portion
7347          * of an SG element at the tail of a prefetch.  If the
7348          * cacheline is larger than our S/G prefetch buffer less
7349          * the size of an SG element, we may round down to a cacheline
7350          * that doesn't contain any or all of the S/G of interest
7351          * within the bounds of our S/G ram.  Provide variables to
7352          * the sequencer that will allow it to handle these edge
7353          * cases.
7354          */
7355         /* Start by aligning to the nearest cacheline. */
7356         sg_prefetch_align = ahd->pci_cachesize;
7357         if (sg_prefetch_align == 0)
7358                 sg_prefetch_cnt = 8;
7359         /* Round down to the nearest power of 2. */
7360         while (powerof2(sg_prefetch_align) == 0)
7361                 sg_prefetch_align--;
7362         /*
7363          * If the cacheline boundary is greater than half our prefetch RAM
7364          * we risk not being able to fetch even a single complete S/G
7365          * segment if we align to that boundary.
7366          */
7367         if (sg_prefetch_align > CCSGADDR_MAX/2)
7368                 sg_prefetch_align = CCSGADDR_MAX/2;
7369         /* Start by fetching a single cacheline. */
7370         sg_prefetch_cnt = sg_prefetch_align;
7371         /*
7372          * Increment the prefetch count by cachelines until
7373          * at least one S/G element will fit.
7374          */
7375         sg_size = sizeof(struct ahd_dma_seg);
7376         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
7377                 sg_size = sizeof(struct ahd_dma64_seg);
7378         while (sg_prefetch_cnt < sg_size)
7379                 sg_prefetch_cnt += sg_prefetch_align;
7380         /*
7381          * If the cacheline is not an even multiple of
7382          * the S/G size, we may only get a partial S/G when
7383          * we align. Add a cacheline if this is the case.
7384          */
7385         if ((sg_prefetch_align % sg_size) != 0
7386          && (sg_prefetch_cnt < CCSGADDR_MAX))
7387                 sg_prefetch_cnt += sg_prefetch_align;
7388         /*
7389          * Lastly, compute a value that the sequencer can use
7390          * to determine if the remainder of the CCSGRAM buffer
7391          * has a full S/G element in it.
7392          */
7393         sg_prefetch_cnt_limit = -(sg_prefetch_cnt - sg_size + 1);
7394         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
7395         download_consts[SG_PREFETCH_CNT_LIMIT] = sg_prefetch_cnt_limit;
7396         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_align - 1);
7397         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_align - 1);
7398         download_consts[SG_SIZEOF] = sg_size;
7399         download_consts[PKT_OVERRUN_BUFOFFSET] =
7400                 (ahd->overrun_buf - (uint8_t *)ahd->qoutfifo) / 256;
7401         download_consts[SCB_TRANSFER_SIZE] = SCB_TRANSFER_SIZE_1BYTE_LUN;
7402         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0)
7403                 download_consts[SCB_TRANSFER_SIZE] = SCB_TRANSFER_SIZE_FULL_LUN;
7404         cur_patch = patches;
7405         downloaded = 0;
7406         skip_addr = 0;
7407         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
7408         ahd_outb(ahd, PRGMCNT, 0);
7409         ahd_outb(ahd, PRGMCNT+1, 0);
7410
7411         for (i = 0; i < sizeof(seqprog)/4; i++) {
7412                 if (ahd_check_patch(ahd, &cur_patch, i, &skip_addr) == 0) {
7413                         /*
7414                          * Don't download this instruction as it
7415                          * is in a patch that was removed.
7416                          */
7417                         continue;
7418                 }
7419                 /*
7420                  * Move through the CS table until we find a CS
7421                  * that might apply to this instruction.
7422                  */
7423                 for (; cur_cs < num_critical_sections; cur_cs++) {
7424                         if (critical_sections[cur_cs].end <= i) {
7425                                 if (begin_set[cs_count] == TRUE
7426                                  && end_set[cs_count] == FALSE) {
7427                                         cs_table[cs_count].end = downloaded;
7428                                         end_set[cs_count] = TRUE;
7429                                         cs_count++;
7430                                 }
7431                                 continue;
7432                         }
7433                         if (critical_sections[cur_cs].begin <= i
7434                          && begin_set[cs_count] == FALSE) {
7435                                 cs_table[cs_count].begin = downloaded;
7436                                 begin_set[cs_count] = TRUE;
7437                         }
7438                         break;
7439                 }
7440                 ahd_download_instr(ahd, i, download_consts);
7441                 downloaded++;
7442         }
7443
7444         ahd->num_critical_sections = cs_count;
7445         if (cs_count != 0) {
7446
7447                 cs_count *= sizeof(struct cs);
7448                 ahd->critical_sections = malloc(cs_count, M_DEVBUF, M_NOWAIT);
7449                 if (ahd->critical_sections == NULL)
7450                         panic("ahd_loadseq: Could not malloc");
7451                 memcpy(ahd->critical_sections, cs_table, cs_count);
7452         }
7453         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE);
7454
7455         if (bootverbose)
7456                 printf(" %d instructions downloaded\n", downloaded);
7457 }
7458
7459 static int
7460 ahd_check_patch(struct ahd_softc *ahd, struct patch **start_patch,
7461                 u_int start_instr, u_int *skip_addr)
7462 {
7463         struct  patch *cur_patch;
7464         struct  patch *last_patch;
7465         u_int   num_patches;
7466
7467         num_patches = sizeof(patches)/sizeof(struct patch);
7468         last_patch = &patches[num_patches];
7469         cur_patch = *start_patch;
7470
7471         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
7472
7473                 if (cur_patch->patch_func(ahd) == 0) {
7474
7475                         /* Start rejecting code */
7476                         *skip_addr = start_instr + cur_patch->skip_instr;
7477                         cur_patch += cur_patch->skip_patch;
7478                 } else {
7479                         /* Accepted this patch.  Advance to the next
7480                          * one and wait for our intruction pointer to
7481                          * hit this point.
7482                          */
7483                         cur_patch++;
7484                 }
7485         }
7486
7487         *start_patch = cur_patch;
7488         if (start_instr < *skip_addr)
7489                 /* Still skipping */
7490                 return (0);
7491
7492         return (1);
7493 }
7494
7495 static u_int
7496 ahd_resolve_seqaddr(struct ahd_softc *ahd, u_int address)
7497 {
7498         struct patch *cur_patch;
7499         int address_offset;
7500         u_int skip_addr;
7501         u_int i;
7502
7503         address_offset = 0;
7504         cur_patch = patches;
7505         skip_addr = 0;
7506
7507         for (i = 0; i < address;) {
7508
7509                 ahd_check_patch(ahd, &cur_patch, i, &skip_addr);
7510
7511                 if (skip_addr > i) {
7512                         int end_addr;
7513
7514                         end_addr = MIN(address, skip_addr);
7515                         address_offset += end_addr - i;
7516                         i = skip_addr;
7517                 } else {
7518                         i++;
7519                 }
7520         }
7521         return (address - address_offset);
7522 }
7523
7524 static void
7525 ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts)
7526 {
7527         union   ins_formats instr;
7528         struct  ins_format1 *fmt1_ins;
7529         struct  ins_format3 *fmt3_ins;
7530         u_int   opcode;
7531
7532         /*
7533          * The firmware is always compiled into a little endian format.
7534          */
7535         instr.integer = ahd_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
7536
7537         fmt1_ins = &instr.format1;
7538         fmt3_ins = NULL;
7539
7540         /* Pull the opcode */
7541         opcode = instr.format1.opcode;
7542         switch (opcode) {
7543         case AIC_OP_JMP:
7544         case AIC_OP_JC:
7545         case AIC_OP_JNC:
7546         case AIC_OP_CALL:
7547         case AIC_OP_JNE:
7548         case AIC_OP_JNZ:
7549         case AIC_OP_JE:
7550         case AIC_OP_JZ:
7551         {
7552                 fmt3_ins = &instr.format3;
7553                 fmt3_ins->address = ahd_resolve_seqaddr(ahd, fmt3_ins->address);
7554                 /* FALLTHROUGH */
7555         }
7556         case AIC_OP_OR:
7557         case AIC_OP_AND:
7558         case AIC_OP_XOR:
7559         case AIC_OP_ADD:
7560         case AIC_OP_ADC:
7561         case AIC_OP_BMOV:
7562                 if (fmt1_ins->parity != 0) {
7563                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
7564                 }
7565                 fmt1_ins->parity = 0;
7566                 /* FALLTHROUGH */
7567         case AIC_OP_ROL:
7568         {
7569                 int i, count;
7570
7571                 /* Calculate odd parity for the instruction */
7572                 for (i = 0, count = 0; i < 31; i++) {
7573                         uint32_t mask;
7574
7575                         mask = 0x01 << i;
7576                         if ((instr.integer & mask) != 0)
7577                                 count++;
7578                 }
7579                 if ((count & 0x01) == 0)
7580                         instr.format1.parity = 1;
7581
7582                 /* The sequencer is a little endian cpu */
7583                 instr.integer = ahd_htole32(instr.integer);
7584                 ahd_outsb(ahd, SEQRAM, instr.bytes, 4);
7585                 break;
7586         }
7587         default:
7588                 panic("Unknown opcode encountered in seq program");
7589                 break;
7590         }
7591 }
7592
7593 void
7594 ahd_dump_all_cards_state()
7595 {
7596         struct ahd_softc *list_ahd;
7597
7598         TAILQ_FOREACH(list_ahd, &ahd_tailq, links) {
7599                 ahd_dump_card_state(list_ahd);
7600         }
7601 }
7602
7603 int
7604 ahd_print_register(ahd_reg_parse_entry_t *table, u_int num_entries,
7605                    const char *name, u_int address, u_int value,
7606                    u_int *cur_column, u_int wrap_point)
7607 {
7608         int     printed;
7609         u_int   printed_mask;
7610
7611         if (*cur_column >= wrap_point) {
7612                 printf("\n");
7613                 *cur_column = 0;
7614         }
7615         printed = printf("%s[0x%x]", name, value);
7616         if (table == NULL) {
7617                 printed += printf(" ");
7618                 *cur_column += printed;
7619                 return (printed);
7620         }
7621         printed_mask = 0;
7622         while (printed_mask != 0xFF) {
7623                 int entry;
7624
7625                 for (entry = 0; entry < num_entries; entry++) {
7626                         if (((value & table[entry].mask)
7627                           != table[entry].value)
7628                          || ((printed_mask & table[entry].mask)
7629                           == table[entry].mask))
7630                                 continue;
7631
7632                         printed += printf("%s%s",
7633                                           printed_mask == 0 ? ":(" : "|",
7634                                           table[entry].name);
7635                         printed_mask |= table[entry].mask;
7636
7637                         break;
7638                 }
7639                 if (entry >= num_entries)
7640                         break;
7641         }
7642         if (printed_mask != 0)
7643                 printed += printf(") ");
7644         else
7645                 printed += printf(" ");
7646         *cur_column += printed;
7647         return (printed);
7648 }
7649
7650 void
7651 ahd_dump_card_state(struct ahd_softc *ahd)
7652 {
7653         struct scb      *scb;
7654         ahd_mode_state   saved_modes;
7655         u_int            dffstat;
7656         int              paused;
7657         u_int            scb_index;
7658         u_int            i;
7659         u_int            cur_col;
7660
7661         if (ahd_is_paused(ahd)) {
7662                 paused = 1;
7663         } else {
7664                 paused = 0;
7665                 ahd_pause(ahd);
7666         }
7667         saved_modes = ahd_save_modes(ahd);
7668         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7669         printf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
7670                "%s: Dumping Card State at program address 0x%x Mode 0x%x\n",
7671                ahd_name(ahd),
7672                ahd_inb(ahd, CURADDR) | (ahd_inb(ahd, CURADDR+1) << 8),
7673                ahd_build_mode_state(ahd, ahd->saved_src_mode,
7674                                     ahd->saved_dst_mode));
7675         if (paused)
7676                 printf("Card was paused\n");
7677         /*
7678          * Mode independent registers.
7679          */
7680         cur_col = 0;
7681         ahd_scsiseq0_print(ahd_inb(ahd, SCSISEQ0), &cur_col, 50);
7682         ahd_scsiseq1_print(ahd_inb(ahd, SCSISEQ1), &cur_col, 50);
7683         ahd_seqintctl_print(ahd_inb(ahd, SEQINTCTL), &cur_col, 50);
7684         ahd_scsisigi_print(ahd_inb(ahd, SCSISIGI), &cur_col, 50);
7685         ahd_scsiphase_print(ahd_inb(ahd, SCSIPHASE), &cur_col, 50);
7686         ahd_lastphase_print(ahd_inb(ahd, LASTPHASE), &cur_col, 50);
7687         ahd_seq_flags_print(ahd_inb(ahd, SEQ_FLAGS), &cur_col, 50);
7688         ahd_seq_flags2_print(ahd_inb(ahd, SEQ_FLAGS2), &cur_col, 50);
7689         ahd_sstat0_print(ahd_inb(ahd, SSTAT0), &cur_col, 50);
7690         ahd_sstat1_print(ahd_inb(ahd, SSTAT1), &cur_col, 50);
7691         ahd_sstat2_print(ahd_inb(ahd, SSTAT2), &cur_col, 50);
7692         ahd_sstat3_print(ahd_inb(ahd, SSTAT3), &cur_col, 50);
7693         ahd_perrdiag_print(ahd_inb(ahd, PERRDIAG), &cur_col, 50);
7694         printf("\nSCB Count = %d LASTSCB 0x%x CURRSCB 0x%x NEXTSCB 0x%x\n",
7695                ahd->scb_data.numscbs, ahd_inw(ahd, LASTSCB),
7696                ahd_inw(ahd, CURRSCB), ahd_inw(ahd, NEXTSCB));
7697         cur_col = 0;
7698         /* QINFIFO */
7699         ahd_search_qinfifo(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
7700                            CAM_LUN_WILDCARD, SCB_LIST_NULL,
7701                            ROLE_UNKNOWN, /*status*/0, SEARCH_PRINT);
7702         printf("Pending list:");
7703         i = 0;
7704         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
7705                 if (i++ > AHD_SCB_MAX)
7706                         break;
7707                 if (scb != LIST_FIRST(&ahd->pending_scbs))
7708                         printf(", ");
7709                 cur_col = printf("\n%3d", SCB_GET_TAG(scb));
7710                 ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
7711                 ahd_scb_control_print(ahd_inb(ahd, SCB_CONTROL), &cur_col, 50);
7712                 ahd_scb_scsiid_print(ahd_inb(ahd, SCB_SCSIID), &cur_col, 50);
7713         }
7714         printf("\n");
7715
7716         printf("Kernel Free SCB list: ");
7717         i = 0;
7718         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
7719                 struct scb *list_scb;
7720
7721                 list_scb = scb;
7722                 do {
7723                         printf("%d ", SCB_GET_TAG(list_scb));
7724                         list_scb = LIST_NEXT(list_scb, collision_links);
7725                 } while (list_scb && i++ < AHD_SCB_MAX);
7726         }
7727
7728         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
7729                 if (i++ > AHD_SCB_MAX)
7730                         break;
7731                 printf("%d ", SCB_GET_TAG(scb));
7732         }
7733         printf("\n");
7734
7735         printf("Sequencer Complete DMA-inprog list: ");
7736         scb_index = ahd_inw(ahd, COMPLETE_SCB_DMAINPROG_HEAD);
7737         i = 0;
7738         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
7739                 ahd_set_scbptr(ahd, scb_index);
7740                 printf("%d ", scb_index);
7741                 scb_index = ahd_inw(ahd, SCB_NEXT_COMPLETE);
7742         }
7743         printf("\n");
7744
7745         printf("Sequencer Complete list: ");
7746         scb_index = ahd_inw(ahd, COMPLETE_SCB_HEAD);
7747         i = 0;
7748         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
7749                 ahd_set_scbptr(ahd, scb_index);
7750                 printf("%d ", scb_index);
7751                 scb_index = ahd_inw(ahd, SCB_NEXT_COMPLETE);
7752         }
7753         printf("\n");
7754
7755
7756         printf("Sequencer DMA-Up and Complete list: ");
7757         scb_index = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
7758         i = 0;
7759         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
7760                 ahd_set_scbptr(ahd, scb_index);
7761                 printf("%d ", scb_index);
7762                 scb_index = ahd_inw(ahd, SCB_NEXT_COMPLETE);
7763         }
7764         printf("\n");
7765         cur_col = 0;
7766         ahd_simode1_print(ahd_inb(ahd, SIMODE1), &cur_col, 50);
7767         ahd_lqistat0_print(ahd_inb(ahd, LQISTAT0), &cur_col, 50);
7768         ahd_lqistat1_print(ahd_inb(ahd, LQISTAT1), &cur_col, 50);
7769         ahd_lqistat2_print(ahd_inb(ahd, LQISTAT2), &cur_col, 50);
7770         dffstat = ahd_inb(ahd, DFFSTAT);
7771         for (i = 0; i < 2; i++) {
7772 #ifdef AHD_DEBUG
7773                 struct scb *fifo_scb;
7774 #endif
7775                 u_int       fifo_scbptr;
7776
7777                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
7778                 fifo_scbptr = ahd_inb(ahd, SCBPTR);
7779                 printf("\n%s: FIFO%d %s, LONGJMP == 0x%x, "
7780                        "SCB 0x%x, LJSCB 0x%x\n",
7781                        ahd_name(ahd), i,
7782                        (dffstat & (FIFO0FREE << i)) ? "Free" : "Active",
7783                        ahd_inw(ahd, LONGJMP_ADDR), fifo_scbptr,
7784                        ahd_inw(ahd, LONGJMP_SCB));
7785                 cur_col = 0;
7786                 ahd_seqimode_print(ahd_inb(ahd, SEQIMODE), &cur_col, 50);
7787                 ahd_seqintsrc_print(ahd_inb(ahd, SEQINTSRC), &cur_col, 50);
7788                 ahd_dfcntrl_print(ahd_inb(ahd, DFCNTRL), &cur_col, 50);
7789                 ahd_dfstatus_print(ahd_inb(ahd, DFSTATUS), &cur_col, 50);
7790                 ahd_sg_cache_shadow_print(ahd_inb(ahd, SG_CACHE_SHADOW),
7791                                           &cur_col, 50);
7792                 ahd_sg_state_print(ahd_inb(ahd, SG_STATE), &cur_col, 50);
7793                 ahd_dffsxfrctl_print(ahd_inb(ahd, DFFSXFRCTL), &cur_col, 50);
7794                 ahd_soffcnt_print(ahd_inb(ahd, SOFFCNT), &cur_col, 50);
7795                 ahd_mdffstat_print(ahd_inb(ahd, MDFFSTAT), &cur_col, 50);
7796                 if (cur_col > 50) {
7797                         printf("\n");
7798                         cur_col = 0;
7799                 }
7800                 cur_col += printf("SHADDR = 0x%x%x, SHCNT = 0x%x",
7801                                   ahd_inl(ahd, SHADDR+4),
7802                                   ahd_inl(ahd, SHADDR),
7803                                   (ahd_inb(ahd, SHCNT)
7804                                 | (ahd_inb(ahd, SHCNT + 1) << 8)
7805                                 | (ahd_inb(ahd, SHCNT + 2) << 16)));
7806                 if (cur_col > 50) {
7807                         printf("\n");
7808                         cur_col = 0;
7809                 }
7810                 cur_col += printf("HADDR = 0x%x%x, HCNT = 0x%x",
7811                                   ahd_inl(ahd, HADDR+4),
7812                                   ahd_inl(ahd, HADDR),
7813                                   (ahd_inb(ahd, HCNT)
7814                                 | (ahd_inb(ahd, HCNT + 1) << 8)
7815                                 | (ahd_inb(ahd, HCNT + 2) << 16)));
7816                 ahd_ccsgctl_print(ahd_inb(ahd, CCSGCTL), &cur_col, 50);
7817 #ifdef AHD_DEBUG
7818                 if ((ahd_debug & AHD_SHOW_SG) != 0) {
7819                         fifo_scb = ahd_lookup_scb(ahd, fifo_scbptr);
7820                         if (fifo_scb != NULL)
7821                                 ahd_dump_sglist(fifo_scb);
7822                 }
7823 #endif
7824         }
7825         printf("LQIN: ");
7826         for (i = 0; i < 20; i++)
7827                 printf("0x%x ", ahd_inb(ahd, LQIN + i));
7828         printf("\n");
7829         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
7830         printf("%s: LQISTATE = 0x%x, LQOSTATE = 0x%x, OPTIONMODE = 0x%x\n",
7831                ahd_name(ahd), ahd_inb(ahd, LQISTATE), ahd_inb(ahd, LQOSTATE),
7832                ahd_inb(ahd, OPTIONMODE));
7833         printf("%s: OS_SPACE_CNT = 0x%x MAXCMDCNT = 0x%x\n",
7834                ahd_name(ahd), ahd_inb(ahd, OS_SPACE_CNT),
7835                ahd_inb(ahd, MAXCMDCNT));
7836         ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
7837         printf("%s: REG0 == 0x%x, SINDEX = 0x%x, DINDEX = 0x%x\n",
7838                ahd_name(ahd), ahd_inw(ahd, REG0), ahd_inw(ahd, SINDEX),
7839                ahd_inw(ahd, DINDEX));
7840         printf("%s: SCBPTR == 0x%x, SCB_NEXT == 0x%x, SCB_NEXT2 == 0x%x\n",
7841                ahd_name(ahd), ahd_get_scbptr(ahd), ahd_inw(ahd, SCB_NEXT),
7842                ahd_inw(ahd, SCB_NEXT2));
7843         printf("CDB %x %x %x %x %x %x\n",
7844                ahd_inb(ahd, SCB_CDB_STORE),
7845                ahd_inb(ahd, SCB_CDB_STORE+1),
7846                ahd_inb(ahd, SCB_CDB_STORE+2),
7847                ahd_inb(ahd, SCB_CDB_STORE+3),
7848                ahd_inb(ahd, SCB_CDB_STORE+4),
7849                ahd_inb(ahd, SCB_CDB_STORE+5));
7850         printf("STACK:");
7851         for(i = 0; i < SEQ_STACK_SIZE; i++)
7852                printf(" 0x%x", ahd_inb(ahd, STACK)|(ahd_inb(ahd, STACK) << 8));
7853         printf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
7854         ahd_platform_dump_card_state(ahd);
7855         ahd_restore_modes(ahd, saved_modes);
7856         if (paused == 0)
7857                 ahd_unpause(ahd);
7858 }
7859
7860 void
7861 ahd_dump_scbs(struct ahd_softc *ahd)
7862 {
7863         ahd_mode_state saved_modes;
7864         int            i;
7865
7866         saved_modes = ahd_save_modes(ahd);
7867         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7868         for (i = 0; i < AHD_SCB_MAX; i++) {
7869                 ahd_set_scbptr(ahd, i);
7870                 printf("%3d", i);
7871                 printf("(CTRL 0x%x ID 0x%x N 0x%x N2 0x%x SG 0x%x, RSG 0x%x)\n",
7872                        ahd_inb(ahd, SCB_CONTROL),
7873                        ahd_inb(ahd, SCB_SCSIID), ahd_inw(ahd, SCB_NEXT),
7874                        ahd_inw(ahd, SCB_NEXT2), ahd_inl(ahd, SCB_SGPTR),
7875                        ahd_inl(ahd, SCB_RESIDUAL_SGPTR));
7876         }
7877         printf("\n");
7878         ahd_restore_modes(ahd, saved_modes);
7879 }
7880
7881 /**************************** Flexport Logic **********************************/
7882 /*
7883  * Read count 16bit words from 16bit word address start_addr from the
7884  * SEEPROM attached to the controller, into buf, using the controller's
7885  * SEEPROM reading state machine.
7886  */
7887 int
7888 ahd_read_seeprom(struct ahd_softc *ahd, uint16_t *buf,
7889                  u_int start_addr, u_int count)
7890 {
7891         u_int cur_addr;
7892         u_int end_addr;
7893         int   error;
7894
7895         /*
7896          * If we never make it through the loop even once,
7897          * we were passed invalid arguments.
7898          */
7899         error = EINVAL;
7900         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7901         end_addr = start_addr + count;
7902         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
7903                 ahd_outb(ahd, SEEADR, cur_addr);
7904                 ahd_outb(ahd, SEECTL, SEEOP_READ | SEESTART);
7905
7906                 error = ahd_wait_seeprom(ahd);
7907                 if (error)
7908                         break;
7909                 *buf++ = ahd_inw(ahd, SEEDAT);
7910         }
7911         return (error);
7912 }
7913
7914 /*
7915  * Write count 16bit words from buf, into SEEPROM attache to the
7916  * controller starting at 16bit word address start_addr, using the
7917  * controller's SEEPROM writing state machine.
7918  */
7919 int
7920 ahd_write_seeprom(struct ahd_softc *ahd, uint16_t *buf,
7921                   u_int start_addr, u_int count)
7922 {
7923         u_int cur_addr;
7924         u_int end_addr;
7925         int   error;
7926         int   retval;
7927
7928         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7929         error = ENOENT;
7930
7931         /* Place the chip into write-enable mode */
7932         ahd_outb(ahd, SEEADR, SEEOP_EWEN_ADDR);
7933         ahd_outb(ahd, SEECTL, SEEOP_EWEN | SEESTART);
7934         error = ahd_wait_seeprom(ahd);
7935         if (error)
7936                 return (error);
7937
7938         /*
7939          * Write the data.  If we don't get throught the loop at
7940          * least once, the arguments were invalid.
7941          */
7942         retval = EINVAL;
7943         end_addr = start_addr + count;
7944         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
7945                 ahd_outw(ahd, SEEDAT, *buf++);
7946                 ahd_outb(ahd, SEEADR, cur_addr);
7947                 ahd_outb(ahd, SEECTL, SEEOP_WRITE | SEESTART);
7948
7949                 retval = ahd_wait_seeprom(ahd);
7950                 if (retval)
7951                         break;
7952         }
7953
7954         /*
7955          * Disable writes.
7956          */
7957         ahd_outb(ahd, SEEADR, SEEOP_EWDS_ADDR);
7958         ahd_outb(ahd, SEECTL, SEEOP_EWDS | SEESTART);
7959         error = ahd_wait_seeprom(ahd);
7960         if (error)
7961                 return (error);
7962         return (retval);
7963 }
7964
7965 /*
7966  * Wait ~100us for the serial eeprom to satisfy our request.
7967  */
7968 int
7969 ahd_wait_seeprom(struct ahd_softc *ahd)
7970 {
7971         int cnt;
7972
7973         cnt = 20;
7974         while ((ahd_inb(ahd, SEESTAT) & (SEEARBACK|SEEBUSY)) != 0 && --cnt)
7975                 ahd_delay(5);
7976
7977         if (cnt == 0)
7978                 return (ETIMEDOUT);
7979         return (0);
7980 }
7981
7982 int
7983 ahd_verify_cksum(struct seeprom_config *sc)
7984 {
7985         int i;
7986         int maxaddr;
7987         uint32_t checksum;
7988         uint16_t *scarray;
7989
7990         maxaddr = (sizeof(*sc)/2) - 1;
7991         checksum = 0;
7992         scarray = (uint16_t *)sc;
7993
7994         for (i = 0; i < maxaddr; i++)
7995                 checksum = checksum + scarray[i];
7996         if (checksum == 0
7997          || (checksum & 0xFFFF) != sc->checksum) {
7998                 return (0);
7999         } else {
8000                 return (1);
8001         }
8002 }
8003
8004 int
8005 ahd_acquire_seeprom(struct ahd_softc *ahd)
8006 {
8007         /*
8008          * We should be able to determine the SEEPROM type
8009          * from the flexport logic, but unfortunately not
8010          * all implementations have this logic and there is
8011          * no programatic method for determining if the logic
8012          * is present.
8013          */
8014         return (1);
8015 #if 0
8016         uint8_t seetype;
8017         int     error;
8018
8019         error = ahd_read_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, &seetype);
8020         if (error != 0
8021          || ((seetype & FLX_ROMSTAT_SEECFG) == FLX_ROMSTAT_SEE_NONE))
8022                 return (0);
8023         return (1);
8024 #endif
8025 }
8026
8027 void
8028 ahd_release_seeprom(struct ahd_softc *ahd)
8029 {
8030         /* Currently a no-op */
8031 }
8032
8033 int
8034 ahd_write_flexport(struct ahd_softc *ahd, u_int addr, u_int value)
8035 {
8036         int error;
8037
8038         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8039         if (addr > 7)
8040                 panic("ahd_write_flexport: address out of range");
8041         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
8042         error = ahd_wait_flexport(ahd);
8043         if (error != 0)
8044                 return (error);
8045         ahd_outb(ahd, BRDDAT, value);
8046         ahd_flush_device_writes(ahd);
8047         ahd_outb(ahd, BRDCTL, BRDSTB|BRDEN|(addr << 3));
8048         ahd_flush_device_writes(ahd);
8049         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
8050         ahd_flush_device_writes(ahd);
8051         ahd_outb(ahd, BRDCTL, 0);
8052         ahd_flush_device_writes(ahd);
8053         return (0);
8054 }
8055
8056 int
8057 ahd_read_flexport(struct ahd_softc *ahd, u_int addr, uint8_t *value)
8058 {
8059         int     error;
8060
8061         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8062         if (addr > 7)
8063                 panic("ahd_read_flexport: address out of range");
8064         ahd_outb(ahd, BRDCTL, BRDRW|BRDEN|(addr << 3));
8065         error = ahd_wait_flexport(ahd);
8066         if (error != 0)
8067                 return (error);
8068         *value = ahd_inb(ahd, BRDDAT);
8069         ahd_outb(ahd, BRDCTL, 0);
8070         ahd_flush_device_writes(ahd);
8071         return (0);
8072 }
8073
8074 /*
8075  * Wait at most 2 seconds for flexport arbitration to succeed.
8076  */
8077 int
8078 ahd_wait_flexport(struct ahd_softc *ahd)
8079 {
8080         int cnt;
8081
8082         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8083         cnt = 1000000 * 2 / 5;
8084         while ((ahd_inb(ahd, BRDCTL) & FLXARBACK) == 0 && --cnt)
8085                 ahd_delay(5);
8086
8087         if (cnt == 0)
8088                 return (ETIMEDOUT);
8089         return (0);
8090 }
8091
8092 /************************* Target Mode ****************************************/
8093 #ifdef AHD_TARGET_MODE
8094 cam_status
8095 ahd_find_tmode_devs(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb,
8096                     struct ahd_tmode_tstate **tstate,
8097                     struct ahd_tmode_lstate **lstate,
8098                     int notfound_failure)
8099 {
8100
8101         if ((ahd->features & AHD_TARGETMODE) == 0)
8102                 return (CAM_REQ_INVALID);
8103
8104         /*
8105          * Handle the 'black hole' device that sucks up
8106          * requests to unattached luns on enabled targets.
8107          */
8108         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
8109          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
8110                 *tstate = NULL;
8111                 *lstate = ahd->black_hole;
8112         } else {
8113                 u_int max_id;
8114
8115                 max_id = (ahd->features & AHD_WIDE) ? 15 : 7;
8116                 if (ccb->ccb_h.target_id > max_id)
8117                         return (CAM_TID_INVALID);
8118
8119                 if (ccb->ccb_h.target_lun >= AHD_NUM_LUNS)
8120                         return (CAM_LUN_INVALID);
8121
8122                 *tstate = ahd->enabled_targets[ccb->ccb_h.target_id];
8123                 *lstate = NULL;
8124                 if (*tstate != NULL)
8125                         *lstate =
8126                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
8127         }
8128
8129         if (notfound_failure != 0 && *lstate == NULL)
8130                 return (CAM_PATH_INVALID);
8131
8132         return (CAM_REQ_CMP);
8133 }
8134
8135 void
8136 ahd_handle_en_lun(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb)
8137 {
8138 #if NOT_YET
8139         struct     ahd_tmode_tstate *tstate;
8140         struct     ahd_tmode_lstate *lstate;
8141         struct     ccb_en_lun *cel;
8142         cam_status status;
8143         u_int      target;
8144         u_int      lun;
8145         u_int      target_mask;
8146         u_long     s;
8147         char       channel;
8148
8149         status = ahd_find_tmode_devs(ahd, sim, ccb, &tstate, &lstate,
8150                                      /*notfound_failure*/FALSE);
8151
8152         if (status != CAM_REQ_CMP) {
8153                 ccb->ccb_h.status = status;
8154                 return;
8155         }
8156
8157         if ((ahd->features & AHD_MULTIROLE) != 0) {
8158                 u_int      our_id;
8159
8160                 our_id = ahd->our_id;
8161                 if (ccb->ccb_h.target_id != our_id) {
8162                         if ((ahd->features & AHD_MULTI_TID) != 0
8163                          && (ahd->flags & AHD_INITIATORROLE) != 0) {
8164                                 /*
8165                                  * Only allow additional targets if
8166                                  * the initiator role is disabled.
8167                                  * The hardware cannot handle a re-select-in
8168                                  * on the initiator id during a re-select-out
8169                                  * on a different target id.
8170                                  */
8171                                 status = CAM_TID_INVALID;
8172                         } else if ((ahd->flags & AHD_INITIATORROLE) != 0
8173                                 || ahd->enabled_luns > 0) {
8174                                 /*
8175                                  * Only allow our target id to change
8176                                  * if the initiator role is not configured
8177                                  * and there are no enabled luns which
8178                                  * are attached to the currently registered
8179                                  * scsi id.
8180                                  */
8181                                 status = CAM_TID_INVALID;
8182                         }
8183                 }
8184         }
8185
8186         if (status != CAM_REQ_CMP) {
8187                 ccb->ccb_h.status = status;
8188                 return;
8189         }
8190
8191         /*
8192          * We now have an id that is valid.
8193          * If we aren't in target mode, switch modes.
8194          */
8195         if ((ahd->flags & AHD_TARGETROLE) == 0
8196          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
8197                 u_long  s;
8198
8199                 printf("Configuring Target Mode\n");
8200                 ahd_lock(ahd, &s);
8201                 if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
8202                         ccb->ccb_h.status = CAM_BUSY;
8203                         ahd_unlock(ahd, &s);
8204                         return;
8205                 }
8206                 ahd->flags |= AHD_TARGETROLE;
8207                 if ((ahd->features & AHD_MULTIROLE) == 0)
8208                         ahd->flags &= ~AHD_INITIATORROLE;
8209                 ahd_pause(ahd);
8210                 ahd_loadseq(ahd);
8211                 ahd_unlock(ahd, &s);
8212         }
8213         cel = &ccb->cel;
8214         target = ccb->ccb_h.target_id;
8215         lun = ccb->ccb_h.target_lun;
8216         channel = SIM_CHANNEL(ahd, sim);
8217         target_mask = 0x01 << target;
8218         if (channel == 'B')
8219                 target_mask <<= 8;
8220
8221         if (cel->enable != 0) {
8222                 u_int scsiseq1;
8223
8224                 /* Are we already enabled?? */
8225                 if (lstate != NULL) {
8226                         xpt_print_path(ccb->ccb_h.path);
8227                         printf("Lun already enabled\n");
8228                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
8229                         return;
8230                 }
8231
8232                 if (cel->grp6_len != 0
8233                  || cel->grp7_len != 0) {
8234                         /*
8235                          * Don't (yet?) support vendor
8236                          * specific commands.
8237                          */
8238                         ccb->ccb_h.status = CAM_REQ_INVALID;
8239                         printf("Non-zero Group Codes\n");
8240                         return;
8241                 }
8242
8243                 /*
8244                  * Seems to be okay.
8245                  * Setup our data structures.
8246                  */
8247                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
8248                         tstate = ahd_alloc_tstate(ahd, target, channel);
8249                         if (tstate == NULL) {
8250                                 xpt_print_path(ccb->ccb_h.path);
8251                                 printf("Couldn't allocate tstate\n");
8252                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
8253                                 return;
8254                         }
8255                 }
8256                 lstate = malloc(sizeof(*lstate), M_DEVBUF, M_NOWAIT);
8257                 if (lstate == NULL) {
8258                         xpt_print_path(ccb->ccb_h.path);
8259                         printf("Couldn't allocate lstate\n");
8260                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
8261                         return;
8262                 }
8263                 memset(lstate, 0, sizeof(*lstate));
8264                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
8265                                          xpt_path_path_id(ccb->ccb_h.path),
8266                                          xpt_path_target_id(ccb->ccb_h.path),
8267                                          xpt_path_lun_id(ccb->ccb_h.path));
8268                 if (status != CAM_REQ_CMP) {
8269                         free(lstate, M_DEVBUF);
8270                         xpt_print_path(ccb->ccb_h.path);
8271                         printf("Couldn't allocate path\n");
8272                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
8273                         return;
8274                 }
8275                 SLIST_INIT(&lstate->accept_tios);
8276                 SLIST_INIT(&lstate->immed_notifies);
8277                 ahd_lock(ahd, &s);
8278                 ahd_pause(ahd);
8279                 if (target != CAM_TARGET_WILDCARD) {
8280                         tstate->enabled_luns[lun] = lstate;
8281                         ahd->enabled_luns++;
8282
8283                         if ((ahd->features & AHD_MULTI_TID) != 0) {
8284                                 u_int targid_mask;
8285
8286                                 targid_mask = ahd_inb(ahd, TARGID)
8287                                             | (ahd_inb(ahd, TARGID + 1) << 8);
8288
8289                                 targid_mask |= target_mask;
8290                                 ahd_outb(ahd, TARGID, targid_mask);
8291                                 ahd_outb(ahd, TARGID+1, (targid_mask >> 8));
8292
8293                                 ahd_update_scsiid(ahd, targid_mask);
8294                         } else {
8295                                 u_int our_id;
8296                                 char  channel;
8297
8298                                 channel = SIM_CHANNEL(ahd, sim);
8299                                 our_id = SIM_SCSI_ID(ahd, sim);
8300
8301                                 /*
8302                                  * This can only happen if selections
8303                                  * are not enabled
8304                                  */
8305                                 if (target != our_id) {
8306                                         u_int sblkctl;
8307                                         char  cur_channel;
8308                                         int   swap;
8309
8310                                         sblkctl = ahd_inb(ahd, SBLKCTL);
8311                                         cur_channel = (sblkctl & SELBUSB)
8312                                                     ? 'B' : 'A';
8313                                         if ((ahd->features & AHD_TWIN) == 0)
8314                                                 cur_channel = 'A';
8315                                         swap = cur_channel != channel;
8316                                         ahd->our_id = target;
8317
8318                                         if (swap)
8319                                                 ahd_outb(ahd, SBLKCTL,
8320                                                          sblkctl ^ SELBUSB);
8321
8322                                         ahd_outb(ahd, SCSIID, target);
8323
8324                                         if (swap)
8325                                                 ahd_outb(ahd, SBLKCTL, sblkctl);
8326                                 }
8327                         }
8328                 } else
8329                         ahd->black_hole = lstate;
8330                 /* Allow select-in operations */
8331                 if (ahd->black_hole != NULL && ahd->enabled_luns > 0) {
8332                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
8333                         scsiseq1 |= ENSELI;
8334                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
8335                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
8336                         scsiseq1 |= ENSELI;
8337                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
8338                 }
8339                 ahd_unpause(ahd);
8340                 ahd_unlock(ahd, &s);
8341                 ccb->ccb_h.status = CAM_REQ_CMP;
8342                 xpt_print_path(ccb->ccb_h.path);
8343                 printf("Lun now enabled for target mode\n");
8344         } else {
8345                 struct scb *scb;
8346                 int i, empty;
8347
8348                 if (lstate == NULL) {
8349                         ccb->ccb_h.status = CAM_LUN_INVALID;
8350                         return;
8351                 }
8352
8353                 ahd_lock(ahd, &s);
8354
8355                 ccb->ccb_h.status = CAM_REQ_CMP;
8356                 LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
8357                         struct ccb_hdr *ccbh;
8358
8359                         ccbh = &scb->io_ctx->ccb_h;
8360                         if (ccbh->func_code == XPT_CONT_TARGET_IO
8361                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
8362                                 printf("CTIO pending\n");
8363                                 ccb->ccb_h.status = CAM_REQ_INVALID;
8364                                 ahd_unlock(ahd, &s);
8365                                 return;
8366                         }
8367                 }
8368
8369                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
8370                         printf("ATIOs pending\n");
8371                         ccb->ccb_h.status = CAM_REQ_INVALID;
8372                 }
8373
8374                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
8375                         printf("INOTs pending\n");
8376                         ccb->ccb_h.status = CAM_REQ_INVALID;
8377                 }
8378
8379                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
8380                         ahd_unlock(ahd, &s);
8381                         return;
8382                 }
8383
8384                 xpt_print_path(ccb->ccb_h.path);
8385                 printf("Target mode disabled\n");
8386                 xpt_free_path(lstate->path);
8387                 free(lstate, M_DEVBUF);
8388
8389                 ahd_pause(ahd);
8390                 /* Can we clean up the target too? */
8391                 if (target != CAM_TARGET_WILDCARD) {
8392                         tstate->enabled_luns[lun] = NULL;
8393                         ahd->enabled_luns--;
8394                         for (empty = 1, i = 0; i < 8; i++)
8395                                 if (tstate->enabled_luns[i] != NULL) {
8396                                         empty = 0;
8397                                         break;
8398                                 }
8399
8400                         if (empty) {
8401                                 ahd_free_tstate(ahd, target, channel,
8402                                                 /*force*/FALSE);
8403                                 if (ahd->features & AHD_MULTI_TID) {
8404                                         u_int targid_mask;
8405
8406                                         targid_mask = ahd_inb(ahd, TARGID)
8407                                                     | (ahd_inb(ahd, TARGID + 1)
8408                                                        << 8);
8409
8410                                         targid_mask &= ~target_mask;
8411                                         ahd_outb(ahd, TARGID, targid_mask);
8412                                         ahd_outb(ahd, TARGID+1,
8413                                                  (targid_mask >> 8));
8414                                         ahd_update_scsiid(ahd, targid_mask);
8415                                 }
8416                         }
8417                 } else {
8418
8419                         ahd->black_hole = NULL;
8420
8421                         /*
8422                          * We can't allow selections without
8423                          * our black hole device.
8424                          */
8425                         empty = TRUE;
8426                 }
8427                 if (ahd->enabled_luns == 0) {
8428                         /* Disallow select-in */
8429                         u_int scsiseq1;
8430
8431                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
8432                         scsiseq1 &= ~ENSELI;
8433                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
8434                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
8435                         scsiseq1 &= ~ENSELI;
8436                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
8437
8438                         if ((ahd->features & AHD_MULTIROLE) == 0) {
8439                                 printf("Configuring Initiator Mode\n");
8440                                 ahd->flags &= ~AHD_TARGETROLE;
8441                                 ahd->flags |= AHD_INITIATORROLE;
8442                                 ahd_pause(ahd);
8443                                 ahd_loadseq(ahd);
8444                         }
8445                 }
8446                 ahd_unpause(ahd);
8447                 ahd_unlock(ahd, &s);
8448         }
8449 #endif
8450 }
8451
8452 static void
8453 ahd_update_scsiid(struct ahd_softc *ahd, u_int targid_mask)
8454 {
8455 #if NOT_YET
8456         u_int scsiid_mask;
8457         u_int scsiid;
8458
8459         if ((ahd->features & AHD_MULTI_TID) == 0)
8460                 panic("ahd_update_scsiid called on non-multitid unit\n");
8461
8462         /*
8463          * Since we will rely on the TARGID mask
8464          * for selection enables, ensure that OID
8465          * in SCSIID is not set to some other ID
8466          * that we don't want to allow selections on.
8467          */
8468         if ((ahd->features & AHD_ULTRA2) != 0)
8469                 scsiid = ahd_inb(ahd, SCSIID_ULTRA2);
8470         else
8471                 scsiid = ahd_inb(ahd, SCSIID);
8472         scsiid_mask = 0x1 << (scsiid & OID);
8473         if ((targid_mask & scsiid_mask) == 0) {
8474                 u_int our_id;
8475
8476                 /* ffs counts from 1 */
8477                 our_id = ffs(targid_mask);
8478                 if (our_id == 0)
8479                         our_id = ahd->our_id;
8480                 else
8481                         our_id--;
8482                 scsiid &= TID;
8483                 scsiid |= our_id;
8484         }
8485         if ((ahd->features & AHD_ULTRA2) != 0)
8486                 ahd_outb(ahd, SCSIID_ULTRA2, scsiid);
8487         else
8488                 ahd_outb(ahd, SCSIID, scsiid);
8489 #endif
8490 }
8491
8492 void
8493 ahd_run_tqinfifo(struct ahd_softc *ahd, int paused)
8494 {
8495         struct target_cmd *cmd;
8496
8497         ahd_sync_tqinfifo(ahd, BUS_DMASYNC_POSTREAD);
8498         while ((cmd = &ahd->targetcmds[ahd->tqinfifonext])->cmd_valid != 0) {
8499
8500                 /*
8501                  * Only advance through the queue if we
8502                  * have the resources to process the command.
8503                  */
8504                 if (ahd_handle_target_cmd(ahd, cmd) != 0)
8505                         break;
8506
8507                 cmd->cmd_valid = 0;
8508                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
8509                                 ahd->shared_data_dmamap,
8510                                 ahd_targetcmd_offset(ahd, ahd->tqinfifonext),
8511                                 sizeof(struct target_cmd),
8512                                 BUS_DMASYNC_PREREAD);
8513                 ahd->tqinfifonext++;
8514
8515                 /*
8516                  * Lazily update our position in the target mode incoming
8517                  * command queue as seen by the sequencer.
8518                  */
8519                 if ((ahd->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
8520                         u_int hs_mailbox;
8521
8522                         hs_mailbox = ahd_inb(ahd, HS_MAILBOX);
8523                         hs_mailbox &= ~HOST_TQINPOS;
8524                         hs_mailbox |= ahd->tqinfifonext & HOST_TQINPOS;
8525                         ahd_outb(ahd, HS_MAILBOX, hs_mailbox);
8526                 }
8527         }
8528 }
8529
8530 static int
8531 ahd_handle_target_cmd(struct ahd_softc *ahd, struct target_cmd *cmd)
8532 {
8533         struct    ahd_tmode_tstate *tstate;
8534         struct    ahd_tmode_lstate *lstate;
8535         struct    ccb_accept_tio *atio;
8536         uint8_t *byte;
8537         int       initiator;
8538         int       target;
8539         int       lun;
8540
8541         initiator = SCSIID_TARGET(ahd, cmd->scsiid);
8542         target = SCSIID_OUR_ID(cmd->scsiid);
8543         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
8544
8545         byte = cmd->bytes;
8546         tstate = ahd->enabled_targets[target];
8547         lstate = NULL;
8548         if (tstate != NULL)
8549                 lstate = tstate->enabled_luns[lun];
8550
8551         /*
8552          * Commands for disabled luns go to the black hole driver.
8553          */
8554         if (lstate == NULL)
8555                 lstate = ahd->black_hole;
8556
8557         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
8558         if (atio == NULL) {
8559                 ahd->flags |= AHD_TQINFIFO_BLOCKED;
8560                 /*
8561                  * Wait for more ATIOs from the peripheral driver for this lun.
8562                  */
8563                 return (1);
8564         } else
8565                 ahd->flags &= ~AHD_TQINFIFO_BLOCKED;
8566 #ifdef AHD_DEBUG
8567         if ((ahd_debug & AHD_SHOW_TQIN) != 0)
8568                 printf("Incoming command from %d for %d:%d%s\n",
8569                        initiator, target, lun,
8570                        lstate == ahd->black_hole ? "(Black Holed)" : "");
8571 #endif
8572         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
8573
8574         if (lstate == ahd->black_hole) {
8575                 /* Fill in the wildcards */
8576                 atio->ccb_h.target_id = target;
8577                 atio->ccb_h.target_lun = lun;
8578         }
8579
8580         /*
8581          * Package it up and send it off to
8582          * whomever has this lun enabled.
8583          */
8584         atio->sense_len = 0;
8585         atio->init_id = initiator;
8586         if (byte[0] != 0xFF) {
8587                 /* Tag was included */
8588                 atio->tag_action = *byte++;
8589                 atio->tag_id = *byte++;
8590                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
8591         } else {
8592                 atio->ccb_h.flags = 0;
8593         }
8594         byte++;
8595
8596         /* Okay.  Now determine the cdb size based on the command code */
8597         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
8598         case 0:
8599                 atio->cdb_len = 6;
8600                 break;
8601         case 1:
8602         case 2:
8603                 atio->cdb_len = 10;
8604                 break;
8605         case 4:
8606                 atio->cdb_len = 16;
8607                 break;
8608         case 5:
8609                 atio->cdb_len = 12;
8610                 break;
8611         case 3:
8612         default:
8613                 /* Only copy the opcode. */
8614                 atio->cdb_len = 1;
8615                 printf("Reserved or VU command code type encountered\n");
8616                 break;
8617         }
8618
8619         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
8620
8621         atio->ccb_h.status |= CAM_CDB_RECVD;
8622
8623         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
8624                 /*
8625                  * We weren't allowed to disconnect.
8626                  * We're hanging on the bus until a
8627                  * continue target I/O comes in response
8628                  * to this accept tio.
8629                  */
8630 #ifdef AHD_DEBUG
8631                 if ((ahd_debug & AHD_SHOW_TQIN) != 0)
8632                         printf("Received Immediate Command %d:%d:%d - %p\n",
8633                                initiator, target, lun, ahd->pending_device);
8634 #endif
8635                 ahd->pending_device = lstate;
8636                 ahd_freeze_ccb((union ccb *)atio);
8637                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
8638         }
8639         xpt_done((union ccb*)atio);
8640         return (0);
8641 }
8642
8643 #endif