sys/dev/aic7xxx/aic79xx.c

   1 /*-
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002, 2004 Justin T. Gibbs.
   5  * Copyright (c) 2000-2003 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#246 $
  41  */
  42
  43 #ifdef __linux__
  44 #include "aic79xx_osm.h"
  45 #include "aic79xx_inline.h"
  46 #include "aicasm/aicasm_insformat.h"
  47 #else
  48 #include <sys/cdefs.h>
  49 __FBSDID("$FreeBSD$");
  50 #include <dev/aic7xxx/aic79xx_osm.h>
  51 #include <dev/aic7xxx/aic79xx_inline.h>
  52 #include <dev/aic7xxx/aicasm/aicasm_insformat.h>
  53 #endif
  54
  55 /******************************** Globals *************************************/
  56 struct ahd_softc_tailq ahd_tailq = TAILQ_HEAD_INITIALIZER(ahd_tailq);
  57 uint32_t ahd_attach_to_HostRAID_controllers = 1;
  58
  59 /***************************** Lookup Tables **********************************/
  60 char *ahd_chip_names[] =
  61 {
  62         "NONE",
  63         "aic7901",
  64         "aic7902",
  65         "aic7901A"
  66 };
  67 static const u_int num_chip_names = NUM_ELEMENTS(ahd_chip_names);
  68
  69 /*
  70  * Hardware error codes.
  71  */
  72 struct ahd_hard_error_entry {
  73         uint8_t errno;
  74         char *errmesg;
  75 };
  76
  77 static struct ahd_hard_error_entry ahd_hard_errors[] = {
  78         { DSCTMOUT,     "Discard Timer has timed out" },
  79         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  80         { SQPARERR,     "Sequencer Parity Error" },
  81         { DPARERR,      "Data-path Parity Error" },
  82         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  83         { CIOPARERR,    "CIOBUS Parity Error" },
  84 };
  85 static const u_int num_errors = NUM_ELEMENTS(ahd_hard_errors);
  86
  87 static struct ahd_phase_table_entry ahd_phase_table[] =
  88 {
  89         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  90         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  91         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  92         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  93         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
  94         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
  95         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
  96         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  97         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
  98         { 0,            MSG_NOOP,               "in unknown phase"      }
  99 };
 100
 101 /*
 102  * In most cases we only wish to itterate over real phases, so
 103  * exclude the last element from the count.
 104  */
 105 static const u_int num_phases = NUM_ELEMENTS(ahd_phase_table) - 1;
 106
 107 /* Our Sequencer Program */
 108 #include "aic79xx_seq.h"
 109
 110 /**************************** Function Declarations ***************************/
 111 static void             ahd_handle_transmission_error(struct ahd_softc *ahd);
 112 static void             ahd_handle_lqiphase_error(struct ahd_softc *ahd,
 113                                                   u_int lqistat1);
 114 static int              ahd_handle_pkt_busfree(struct ahd_softc *ahd,
 115                                                u_int busfreetime);
 116 static int              ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
 117 static void             ahd_handle_proto_violation(struct ahd_softc *ahd);
 118 static void             ahd_force_renegotiation(struct ahd_softc *ahd,
 119                                                 struct ahd_devinfo *devinfo);
 120
 121 static struct ahd_tmode_tstate*
 122                         ahd_alloc_tstate(struct ahd_softc *ahd,
 123                                          u_int scsi_id, char channel);
 124 #ifdef AHD_TARGET_MODE
 125 static void             ahd_free_tstate(struct ahd_softc *ahd,
 126                                         u_int scsi_id, char channel, int force);
 127 #endif
 128 static void             ahd_devlimited_syncrate(struct ahd_softc *ahd,
 129                                                 struct ahd_initiator_tinfo *,
 130                                                 u_int *period,
 131                                                 u_int *ppr_options,
 132                                                 role_t role);
 133 static void             ahd_update_neg_table(struct ahd_softc *ahd,
 134                                              struct ahd_devinfo *devinfo,
 135                                              struct ahd_transinfo *tinfo);
 136 static void             ahd_update_pending_scbs(struct ahd_softc *ahd);
 137 static void             ahd_fetch_devinfo(struct ahd_softc *ahd,
 138                                           struct ahd_devinfo *devinfo);
 139 static void             ahd_scb_devinfo(struct ahd_softc *ahd,
 140                                         struct ahd_devinfo *devinfo,
 141                                         struct scb *scb);
 142 static void             ahd_setup_initiator_msgout(struct ahd_softc *ahd,
 143                                                    struct ahd_devinfo *devinfo,
 144                                                    struct scb *scb);
 145 static void             ahd_build_transfer_msg(struct ahd_softc *ahd,
 146                                                struct ahd_devinfo *devinfo);
 147 static void             ahd_construct_sdtr(struct ahd_softc *ahd,
 148                                            struct ahd_devinfo *devinfo,
 149                                            u_int period, u_int offset);
 150 static void             ahd_construct_wdtr(struct ahd_softc *ahd,
 151                                            struct ahd_devinfo *devinfo,
 152                                            u_int bus_width);
 153 static void             ahd_construct_ppr(struct ahd_softc *ahd,
 154                                           struct ahd_devinfo *devinfo,
 155                                           u_int period, u_int offset,
 156                                           u_int bus_width, u_int ppr_options);
 157 static void             ahd_clear_msg_state(struct ahd_softc *ahd);
 158 static void             ahd_handle_message_phase(struct ahd_softc *ahd);
 159 typedef enum {
 160         AHDMSG_1B,
 161         AHDMSG_2B,
 162         AHDMSG_EXT
 163 } ahd_msgtype;
 164 static int              ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
 165                                      u_int msgval, int full);
 166 static int              ahd_parse_msg(struct ahd_softc *ahd,
 167                                       struct ahd_devinfo *devinfo);
 168 static int              ahd_handle_msg_reject(struct ahd_softc *ahd,
 169                                               struct ahd_devinfo *devinfo);
 170 static void             ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
 171                                                 struct ahd_devinfo *devinfo);
 172 static void             ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
 173 static void             ahd_handle_devreset(struct ahd_softc *ahd,
 174                                             struct ahd_devinfo *devinfo,
 175                                             u_int lun, cam_status status,
 176                                             char *message, int verbose_level);
 177 #ifdef AHD_TARGET_MODE
 178 static void             ahd_setup_target_msgin(struct ahd_softc *ahd,
 179                                                struct ahd_devinfo *devinfo,
 180                                                struct scb *scb);
 181 #endif
 182
 183 static u_int            ahd_sglist_size(struct ahd_softc *ahd);
 184 static u_int            ahd_sglist_allocsize(struct ahd_softc *ahd);
 185 static bus_dmamap_callback_t
 186                         ahd_dmamap_cb;
 187 static void             ahd_initialize_hscbs(struct ahd_softc *ahd);
 188 static int              ahd_init_scbdata(struct ahd_softc *ahd);
 189 static void             ahd_fini_scbdata(struct ahd_softc *ahd);
 190 static void             ahd_setup_iocell_workaround(struct ahd_softc *ahd);
 191 static void             ahd_iocell_first_selection(struct ahd_softc *ahd);
 192 static void             ahd_add_col_list(struct ahd_softc *ahd,
 193                                          struct scb *scb, u_int col_idx);
 194 static void             ahd_rem_col_list(struct ahd_softc *ahd,
 195                                          struct scb *scb);
 196 static void             ahd_chip_init(struct ahd_softc *ahd);
 197 static void             ahd_qinfifo_requeue(struct ahd_softc *ahd,
 198                                             struct scb *prev_scb,
 199                                             struct scb *scb);
 200 static int              ahd_qinfifo_count(struct ahd_softc *ahd);
 201 static int              ahd_search_scb_list(struct ahd_softc *ahd, int target,
 202                                             char channel, int lun, u_int tag,
 203                                             role_t role, uint32_t status,
 204                                             ahd_search_action action,
 205                                             u_int *list_head, u_int *list_tail,
 206                                             u_int tid);
 207 static void             ahd_stitch_tid_list(struct ahd_softc *ahd,
 208                                             u_int tid_prev, u_int tid_cur,
 209                                             u_int tid_next);
 210 static void             ahd_add_scb_to_free_list(struct ahd_softc *ahd,
 211                                                  u_int scbid);
 212 static u_int            ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
 213                                      u_int prev, u_int next, u_int tid);
 214 static void             ahd_reset_current_bus(struct ahd_softc *ahd);
 215 static ahd_callback_t   ahd_reset_poll;
 216 static ahd_callback_t   ahd_stat_timer;
 217 #ifdef AHD_DUMP_SEQ
 218 static void             ahd_dumpseq(struct ahd_softc *ahd);
 219 #endif
 220 static void             ahd_loadseq(struct ahd_softc *ahd);
 221 static int              ahd_check_patch(struct ahd_softc *ahd,
 222                                         struct patch **start_patch,
 223                                         u_int start_instr, u_int *skip_addr);
 224 static u_int            ahd_resolve_seqaddr(struct ahd_softc *ahd,
 225                                             u_int address);
 226 static void             ahd_download_instr(struct ahd_softc *ahd,
 227                                            u_int instrptr, uint8_t *dconsts);
 228 static int              ahd_probe_stack_size(struct ahd_softc *ahd);
 229 static int              ahd_other_scb_timeout(struct ahd_softc *ahd,
 230                                               struct scb *scb,
 231                                               struct scb *other_scb);
 232 static int              ahd_scb_active_in_fifo(struct ahd_softc *ahd,
 233                                                struct scb *scb);
 234 static void             ahd_run_data_fifo(struct ahd_softc *ahd,
 235                                           struct scb *scb);
 236
 237 #ifdef AHD_TARGET_MODE
 238 static void             ahd_queue_lstate_event(struct ahd_softc *ahd,
 239                                                struct ahd_tmode_lstate *lstate,
 240                                                u_int initiator_id,
 241                                                u_int event_type,
 242                                                u_int event_arg);
 243 static void             ahd_update_scsiid(struct ahd_softc *ahd,
 244                                           u_int targid_mask);
 245 static int              ahd_handle_target_cmd(struct ahd_softc *ahd,
 246                                               struct target_cmd *cmd);
 247 #endif
 248
 249 /******************************** Private Inlines *****************************/
 250 static __inline void    ahd_assert_atn(struct ahd_softc *ahd);
 251 static __inline int     ahd_currently_packetized(struct ahd_softc *ahd);
 252 static __inline int     ahd_set_active_fifo(struct ahd_softc *ahd);
 253
 254 static __inline void
 255 ahd_assert_atn(struct ahd_softc *ahd)
 256 {
 257         ahd_outb(ahd, SCSISIGO, ATNO);
 258 }
 259
 260 /*
 261  * Determine if the current connection has a packetized
 262  * agreement.  This does not necessarily mean that we
 263  * are currently in a packetized transfer.  We could
 264  * just as easily be sending or receiving a message.
 265  */
 266 static __inline int
 267 ahd_currently_packetized(struct ahd_softc *ahd)
 268 {
 269         ahd_mode_state   saved_modes;
 270         int              packetized;
 271
 272         saved_modes = ahd_save_modes(ahd);
 273         if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
 274                 /*
 275                  * The packetized bit refers to the last
 276                  * connection, not the current one.  Check
 277                  * for non-zero LQISTATE instead.
 278                  */
 279                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 280                 packetized = ahd_inb(ahd, LQISTATE) != 0;
 281         } else {
 282                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 283                 packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
 284         }
 285         ahd_restore_modes(ahd, saved_modes);
 286         return (packetized);
 287 }
 288
 289 static __inline int
 290 ahd_set_active_fifo(struct ahd_softc *ahd)
 291 {
 292         u_int active_fifo;
 293
 294         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 295         active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
 296         switch (active_fifo) {
 297         case 0:
 298         case 1:
 299                 ahd_set_modes(ahd, active_fifo, active_fifo);
 300                 return (1);
 301         default:
 302                 return (0);
 303         }
 304 }
 305
 306 /************************* Sequencer Execution Control ************************/
 307 /*
 308  * Restart the sequencer program from address zero
 309  */
 310 void
 311 ahd_restart(struct ahd_softc *ahd)
 312 {
 313
 314         ahd_pause(ahd);
 315
 316         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 317
 318         /* No more pending messages */
 319         ahd_clear_msg_state(ahd);
 320         ahd_outb(ahd, SCSISIGO, 0);             /* De-assert BSY */
 321         ahd_outb(ahd, MSG_OUT, MSG_NOOP);       /* No message to send */
 322         ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
 323         ahd_outb(ahd, SEQINTCTL, 0);
 324         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
 325         ahd_outb(ahd, SEQ_FLAGS, 0);
 326         ahd_outb(ahd, SAVED_SCSIID, 0xFF);
 327         ahd_outb(ahd, SAVED_LUN, 0xFF);
 328
 329         /*
 330          * Ensure that the sequencer's idea of TQINPOS
 331          * matches our own.  The sequencer increments TQINPOS
 332          * only after it sees a DMA complete and a reset could
 333          * occur before the increment leaving the kernel to believe
 334          * the command arrived but the sequencer to not.
 335          */
 336         ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
 337
 338         /* Always allow reselection */
 339         ahd_outb(ahd, SCSISEQ1,
 340                  ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 341         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 342
 343         /*
 344          * Clear any pending sequencer interrupt.  It is no
 345          * longer relevant since we're resetting the Program
 346          * Counter.
 347          */
 348         ahd_outb(ahd, CLRINT, CLRSEQINT);
 349
 350         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
 351         ahd_unpause(ahd);
 352 }
 353
 354 void
 355 ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
 356 {
 357         ahd_mode_state   saved_modes;
 358
 359 #ifdef AHD_DEBUG
 360         if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
 361                 printf("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
 362 #endif
 363         saved_modes = ahd_save_modes(ahd);
 364         ahd_set_modes(ahd, fifo, fifo);
 365         ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
 366         if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 367                 ahd_outb(ahd, CCSGCTL, CCSGRESET);
 368         ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 369         ahd_outb(ahd, SG_STATE, 0);
 370         ahd_restore_modes(ahd, saved_modes);
 371 }
 372
 373 /************************* Input/Output Queues ********************************/
 374 /*
 375  * Flush and completed commands that are sitting in the command
 376  * complete queues down on the chip but have yet to be dma'ed back up.
 377  */
 378 void
 379 ahd_flush_qoutfifo(struct ahd_softc *ahd)
 380 {
 381         struct          scb *scb;
 382         ahd_mode_state  saved_modes;
 383         u_int           saved_scbptr;
 384         u_int           ccscbctl;
 385         u_int           scbid;
 386         u_int           next_scbid;
 387
 388         saved_modes = ahd_save_modes(ahd);
 389
 390         /*
 391          * Flush the good status FIFO for completed packetized commands.
 392          */
 393         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 394         saved_scbptr = ahd_get_scbptr(ahd);
 395         while ((ahd_inb(ahd, LQISTAT2) & LQIGSAVAIL) != 0) {
 396                 u_int fifo_mode;
 397                 u_int i;
 398
 399                 scbid = ahd_inw(ahd, GSFIFO);
 400                 scb = ahd_lookup_scb(ahd, scbid);
 401                 if (scb == NULL) {
 402                         printf("%s: Warning - GSFIFO SCB %d invalid\n",
 403                                ahd_name(ahd), scbid);
 404                         AHD_CORRECTABLE_ERROR(ahd);
 405                         continue;
 406                 }
 407                 /*
 408                  * Determine if this transaction is still active in
 409                  * any FIFO.  If it is, we must flush that FIFO to
 410                  * the host before completing the  command.
 411                  */
 412                 fifo_mode = 0;
 413 rescan_fifos:
 414                 for (i = 0; i < 2; i++) {
 415                         /* Toggle to the other mode. */
 416                         fifo_mode ^= 1;
 417                         ahd_set_modes(ahd, fifo_mode, fifo_mode);
 418
 419                         if (ahd_scb_active_in_fifo(ahd, scb) == 0)
 420                                 continue;
 421
 422                         ahd_run_data_fifo(ahd, scb);
 423
 424                         /*
 425                          * Running this FIFO may cause a CFG4DATA for
 426                          * this same transaction to assert in the other
 427                          * FIFO or a new snapshot SAVEPTRS interrupt
 428                          * in this FIFO.  Even running a FIFO may not
 429                          * clear the transaction if we are still waiting
 430                          * for data to drain to the host. We must loop
 431                          * until the transaction is not active in either
 432                          * FIFO just to be sure.  Reset our loop counter
 433                          * so we will visit both FIFOs again before
 434                          * declaring this transaction finished.  We
 435                          * also delay a bit so that status has a chance
 436                          * to change before we look at this FIFO again.
 437                          */
 438                         aic_delay(200);
 439                         goto rescan_fifos;
 440                 }
 441                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 442                 ahd_set_scbptr(ahd, scbid);
 443                 if ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_LIST_NULL) == 0
 444                  && ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_FULL_RESID) != 0
 445                   || (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR)
 446                       & SG_LIST_NULL) != 0)) {
 447                         u_int comp_head;
 448
 449                         /*
 450                          * The transfer completed with a residual.
 451                          * Place this SCB on the complete DMA list
 452                          * so that we update our in-core copy of the
 453                          * SCB before completing the command.
 454                          */
 455                         ahd_outb(ahd, SCB_SCSI_STATUS, 0);
 456                         ahd_outb(ahd, SCB_SGPTR,
 457                                  ahd_inb_scbram(ahd, SCB_SGPTR)
 458                                  | SG_STATUS_VALID);
 459                         ahd_outw(ahd, SCB_TAG, scbid);
 460                         ahd_outw(ahd, SCB_NEXT_COMPLETE, SCB_LIST_NULL);
 461                         comp_head = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
 462                         if (SCBID_IS_NULL(comp_head)) {
 463                                 ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, scbid);
 464                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
 465                         } else {
 466                                 u_int tail;
 467
 468                                 tail = ahd_inw(ahd, COMPLETE_DMA_SCB_TAIL);
 469                                 ahd_set_scbptr(ahd, tail);
 470                                 ahd_outw(ahd, SCB_NEXT_COMPLETE, scbid);
 471                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
 472                                 ahd_set_scbptr(ahd, scbid);
 473                         }
 474                 } else
 475                         ahd_complete_scb(ahd, scb);
 476         }
 477         ahd_set_scbptr(ahd, saved_scbptr);
 478
 479         /*
 480          * Setup for command channel portion of flush.
 481          */
 482         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 483
 484         /*
 485          * Wait for any inprogress DMA to complete and clear DMA state
 486          * if this if for an SCB in the qinfifo.
 487          */
 488         while (((ccscbctl = ahd_inb(ahd, CCSCBCTL)) & (CCARREN|CCSCBEN)) != 0) {
 489
 490                 if ((ccscbctl & (CCSCBDIR|CCARREN)) == (CCSCBDIR|CCARREN)) {
 491                         if ((ccscbctl & ARRDONE) != 0)
 492                                 break;
 493                 } else if ((ccscbctl & CCSCBDONE) != 0)
 494                         break;
 495                 aic_delay(200);
 496         }
 497         /*
 498          * We leave the sequencer to cleanup in the case of DMA's to
 499          * update the qoutfifo.  In all other cases (DMA's to the
 500          * chip or a push of an SCB from the COMPLETE_DMA_SCB list),
 501          * we disable the DMA engine so that the sequencer will not
 502          * attempt to handle the DMA completion.
 503          */
 504         if ((ccscbctl & CCSCBDIR) != 0 || (ccscbctl & ARRDONE) != 0)
 505                 ahd_outb(ahd, CCSCBCTL, ccscbctl & ~(CCARREN|CCSCBEN));
 506
 507         /*
 508          * Complete any SCBs that just finished
 509          * being DMA'ed into the qoutfifo.
 510          */
 511         ahd_run_qoutfifo(ahd);
 512
 513         saved_scbptr = ahd_get_scbptr(ahd);
 514         /*
 515          * Manually update/complete any completed SCBs that are waiting to be
 516          * DMA'ed back up to the host.
 517          */
 518         scbid = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
 519         while (!SCBID_IS_NULL(scbid)) {
 520                 uint8_t *hscb_ptr;
 521                 u_int    i;
 522
 523                 ahd_set_scbptr(ahd, scbid);
 524                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 525                 scb = ahd_lookup_scb(ahd, scbid);
 526                 if (scb == NULL) {
 527                         printf("%s: Warning - DMA-up and complete "
 528                                "SCB %d invalid\n", ahd_name(ahd), scbid);
 529                         AHD_CORRECTABLE_ERROR(ahd);
 530                         continue;
 531                 }
 532                 hscb_ptr = (uint8_t *)scb->hscb;
 533                 for (i = 0; i < sizeof(struct hardware_scb); i++)
 534                         *hscb_ptr++ = ahd_inb_scbram(ahd, SCB_BASE + i);
 535
 536                 ahd_complete_scb(ahd, scb);
 537                 scbid = next_scbid;
 538         }
 539         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
 540         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
 541
 542         scbid = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
 543         while (!SCBID_IS_NULL(scbid)) {
 544
 545                 ahd_set_scbptr(ahd, scbid);
 546                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 547                 scb = ahd_lookup_scb(ahd, scbid);
 548                 if (scb == NULL) {
 549                         printf("%s: Warning - Complete Qfrz SCB %d invalid\n",
 550                                ahd_name(ahd), scbid);
 551                         AHD_CORRECTABLE_ERROR(ahd);
 552                         continue;
 553                 }
 554
 555                 ahd_complete_scb(ahd, scb);
 556                 scbid = next_scbid;
 557         }
 558         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
 559
 560         scbid = ahd_inw(ahd, COMPLETE_SCB_HEAD);
 561         while (!SCBID_IS_NULL(scbid)) {
 562
 563                 ahd_set_scbptr(ahd, scbid);
 564                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 565                 scb = ahd_lookup_scb(ahd, scbid);
 566                 if (scb == NULL) {
 567                         printf("%s: Warning - Complete SCB %d invalid\n",
 568                                ahd_name(ahd), scbid);
 569                         AHD_CORRECTABLE_ERROR(ahd);
 570                         continue;
 571                 }
 572
 573                 ahd_complete_scb(ahd, scb);
 574                 scbid = next_scbid;
 575         }
 576         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
 577
 578         /*
 579          * Restore state.
 580          */
 581         ahd_set_scbptr(ahd, saved_scbptr);
 582         ahd_restore_modes(ahd, saved_modes);
 583         ahd->flags |= AHD_UPDATE_PEND_CMDS;
 584 }
 585
 586 /*
 587  * Determine if an SCB for a packetized transaction
 588  * is active in a FIFO.
 589  */
 590 static int
 591 ahd_scb_active_in_fifo(struct ahd_softc *ahd, struct scb *scb)
 592 {
 593
 594         /*
 595          * The FIFO is only active for our transaction if
 596          * the SCBPTR matches the SCB's ID and the firmware
 597          * has installed a handler for the FIFO or we have
 598          * a pending SAVEPTRS or CFG4DATA interrupt.
 599          */
 600         if (ahd_get_scbptr(ahd) != SCB_GET_TAG(scb)
 601          || ((ahd_inb(ahd, LONGJMP_ADDR+1) & INVALID_ADDR) != 0
 602           && (ahd_inb(ahd, SEQINTSRC) & (CFG4DATA|SAVEPTRS)) == 0))
 603                 return (0);
 604
 605         return (1);
 606 }
 607
 608 /*
 609  * Run a data fifo to completion for a transaction we know
 610  * has completed across the SCSI bus (good status has been
 611  * received).  We are already set to the correct FIFO mode
 612  * on entry to this routine.
 613  *
 614  * This function attempts to operate exactly as the firmware
 615  * would when running this FIFO.  Care must be taken to update
 616  * this routine any time the firmware's FIFO algorithm is
 617  * changed.
 618  */
 619 static void
 620 ahd_run_data_fifo(struct ahd_softc *ahd, struct scb *scb)
 621 {
 622         u_int seqintsrc;
 623
 624         seqintsrc = ahd_inb(ahd, SEQINTSRC);
 625         if ((seqintsrc & CFG4DATA) != 0) {
 626                 uint32_t datacnt;
 627                 uint32_t sgptr;
 628
 629                 /*
 630                  * Clear full residual flag.
 631                  */
 632                 sgptr = ahd_inl_scbram(ahd, SCB_SGPTR) & ~SG_FULL_RESID;
 633                 ahd_outb(ahd, SCB_SGPTR, sgptr);
 634
 635                 /*
 636                  * Load datacnt and address.
 637                  */
 638                 datacnt = ahd_inl_scbram(ahd, SCB_DATACNT);
 639                 if ((datacnt & AHD_DMA_LAST_SEG) != 0) {
 640                         sgptr |= LAST_SEG;
 641                         ahd_outb(ahd, SG_STATE, 0);
 642                 } else
 643                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
 644                 ahd_outq(ahd, HADDR, ahd_inq_scbram(ahd, SCB_DATAPTR));
 645                 ahd_outl(ahd, HCNT, datacnt & AHD_SG_LEN_MASK);
 646                 ahd_outb(ahd, SG_CACHE_PRE, sgptr);
 647                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
 648
 649                 /*
 650                  * Initialize Residual Fields.
 651                  */
 652                 ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, datacnt >> 24);
 653                 ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr & SG_PTR_MASK);
 654
 655                 /*
 656                  * Mark the SCB as having a FIFO in use.
 657                  */
 658                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
 659                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) + 1);
 660
 661                 /*
 662                  * Install a "fake" handler for this FIFO.
 663                  */
 664                 ahd_outw(ahd, LONGJMP_ADDR, 0);
 665
 666                 /*
 667                  * Notify the hardware that we have satisfied
 668                  * this sequencer interrupt.
 669                  */
 670                 ahd_outb(ahd, CLRSEQINTSRC, CLRCFG4DATA);
 671         } else if ((seqintsrc & SAVEPTRS) != 0) {
 672                 uint32_t sgptr;
 673                 uint32_t resid;
 674
 675                 if ((ahd_inb(ahd, LONGJMP_ADDR+1)&INVALID_ADDR) != 0) {
 676                         /*
 677                          * Snapshot Save Pointers.  All that
 678                          * is necessary to clear the snapshot
 679                          * is a CLRCHN.
 680                          */
 681                         goto clrchn;
 682                 }
 683
 684                 /*
 685                  * Disable S/G fetch so the DMA engine
 686                  * is available to future users.
 687                  */
 688                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 689                         ahd_outb(ahd, CCSGCTL, 0);
 690                 ahd_outb(ahd, SG_STATE, 0);
 691
 692                 /*
 693                  * Flush the data FIFO.  Strickly only
 694                  * necessary for Rev A parts.
 695                  */
 696                 ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) | FIFOFLUSH);
 697
 698                 /*
 699                  * Calculate residual.
 700                  */
 701                 sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 702                 resid = ahd_inl(ahd, SHCNT);
 703                 resid |= ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24;
 704                 ahd_outl(ahd, SCB_RESIDUAL_DATACNT, resid);
 705                 if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG) == 0) {
 706                         /*
 707                          * Must back up to the correct S/G element.
 708                          * Typically this just means resetting our
 709                          * low byte to the offset in the SG_CACHE,
 710                          * but if we wrapped, we have to correct
 711                          * the other bytes of the sgptr too.
 712                          */
 713                         if ((ahd_inb(ahd, SG_CACHE_SHADOW) & 0x80) != 0
 714                          && (sgptr & 0x80) == 0)
 715                                 sgptr -= 0x100;
 716                         sgptr &= ~0xFF;
 717                         sgptr |= ahd_inb(ahd, SG_CACHE_SHADOW)
 718                                & SG_ADDR_MASK;
 719                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 720                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, 0);
 721                 } else if ((resid & AHD_SG_LEN_MASK) == 0) {
 722                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR,
 723                                  sgptr | SG_LIST_NULL);
 724                 }
 725                 /*
 726                  * Save Pointers.
 727                  */
 728                 ahd_outq(ahd, SCB_DATAPTR, ahd_inq(ahd, SHADDR));
 729                 ahd_outl(ahd, SCB_DATACNT, resid);
 730                 ahd_outl(ahd, SCB_SGPTR, sgptr);
 731                 ahd_outb(ahd, CLRSEQINTSRC, CLRSAVEPTRS);
 732                 ahd_outb(ahd, SEQIMODE,
 733                          ahd_inb(ahd, SEQIMODE) | ENSAVEPTRS);
 734                 /*
 735                  * If the data is to the SCSI bus, we are
 736                  * done, otherwise wait for FIFOEMP.
 737                  */
 738                 if ((ahd_inb(ahd, DFCNTRL) & DIRECTION) != 0)
 739                         goto clrchn;
 740         } else if ((ahd_inb(ahd, SG_STATE) & LOADING_NEEDED) != 0) {
 741                 uint32_t sgptr;
 742                 uint64_t data_addr;
 743                 uint32_t data_len;
 744                 u_int    dfcntrl;
 745
 746                 /*
 747                  * Disable S/G fetch so the DMA engine
 748                  * is available to future users.  We won't
 749                  * be using the DMA engine to load segments.
 750                  */
 751                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0) {
 752                         ahd_outb(ahd, CCSGCTL, 0);
 753                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
 754                 }
 755
 756                 /*
 757                  * Wait for the DMA engine to notice that the
 758                  * host transfer is enabled and that there is
 759                  * space in the S/G FIFO for new segments before
 760                  * loading more segments.
 761                  */
 762                 if ((ahd_inb(ahd, DFSTATUS) & PRELOAD_AVAIL) != 0
 763                  && (ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0) {
 764
 765                         /*
 766                          * Determine the offset of the next S/G
 767                          * element to load.
 768                          */
 769                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 770                         sgptr &= SG_PTR_MASK;
 771                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 772                                 struct ahd_dma64_seg *sg;
 773
 774                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 775                                 data_addr = sg->addr;
 776                                 data_len = sg->len;
 777                                 sgptr += sizeof(*sg);
 778                         } else {
 779                                 struct  ahd_dma_seg *sg;
 780
 781                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 782                                 data_addr = sg->len & AHD_SG_HIGH_ADDR_MASK;
 783                                 data_addr <<= 8;
 784                                 data_addr |= sg->addr;
 785                                 data_len = sg->len;
 786                                 sgptr += sizeof(*sg);
 787                         }
 788
 789                         /*
 790                          * Update residual information.
 791                          */
 792                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, data_len >> 24);
 793                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 794
 795                         /*
 796                          * Load the S/G.
 797                          */
 798                         if (data_len & AHD_DMA_LAST_SEG) {
 799                                 sgptr |= LAST_SEG;
 800                                 ahd_outb(ahd, SG_STATE, 0);
 801                         }
 802                         ahd_outq(ahd, HADDR, data_addr);
 803                         ahd_outl(ahd, HCNT, data_len & AHD_SG_LEN_MASK);
 804                         ahd_outb(ahd, SG_CACHE_PRE, sgptr & 0xFF);
 805
 806                         /*
 807                          * Advertise the segment to the hardware.
 808                          */
 809                         dfcntrl = ahd_inb(ahd, DFCNTRL)|PRELOADEN|HDMAEN;
 810                         if ((ahd->features & AHD_NEW_DFCNTRL_OPTS) != 0) {
 811                                 /*
 812                                  * Use SCSIENWRDIS so that SCSIEN
 813                                  * is never modified by this
 814                                  * operation.
 815                                  */
 816                                 dfcntrl |= SCSIENWRDIS;
 817                         }
 818                         ahd_outb(ahd, DFCNTRL, dfcntrl);
 819                 }
 820         } else if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG_DONE) != 0) {
 821
 822                 /*
 823                  * Transfer completed to the end of SG list
 824                  * and has flushed to the host.
 825                  */
 826                 ahd_outb(ahd, SCB_SGPTR,
 827                          ahd_inb_scbram(ahd, SCB_SGPTR) | SG_LIST_NULL);
 828                 goto clrchn;
 829         } else if ((ahd_inb(ahd, DFSTATUS) & FIFOEMP) != 0) {
 830 clrchn:
 831                 /*
 832                  * Clear any handler for this FIFO, decrement
 833                  * the FIFO use count for the SCB, and release
 834                  * the FIFO.
 835                  */
 836                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 837                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
 838                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) - 1);
 839                 ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
 840         }
 841 }
 842
 843 /*
 844  * Look for entries in the QoutFIFO that have completed.
 845  * The valid_tag completion field indicates the validity
 846  * of the entry - the valid value toggles each time through
 847  * the queue. We use the sg_status field in the completion
 848  * entry to avoid referencing the hscb if the completion
 849  * occurred with no errors and no residual.  sg_status is
 850  * a copy of the first byte (little endian) of the sgptr
 851  * hscb field.
 852  */
 853 void
 854 ahd_run_qoutfifo(struct ahd_softc *ahd)
 855 {
 856         struct ahd_completion *completion;
 857         struct scb *scb;
 858         u_int  scb_index;
 859
 860         if ((ahd->flags & AHD_RUNNING_QOUTFIFO) != 0)
 861                 panic("ahd_run_qoutfifo recursion");
 862         ahd->flags |= AHD_RUNNING_QOUTFIFO;
 863         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
 864         for (;;) {
 865                 completion = &ahd->qoutfifo[ahd->qoutfifonext];
 866
 867                 if (completion->valid_tag != ahd->qoutfifonext_valid_tag)
 868                         break;
 869
 870                 scb_index = aic_le16toh(completion->tag);
 871                 scb = ahd_lookup_scb(ahd, scb_index);
 872                 if (scb == NULL) {
 873                         printf("%s: WARNING no command for scb %d "
 874                                "(cmdcmplt)\nQOUTPOS = %d\n",
 875                                ahd_name(ahd), scb_index,
 876                                ahd->qoutfifonext);
 877                         AHD_CORRECTABLE_ERROR(ahd);
 878                         ahd_dump_card_state(ahd);
 879                 } else if ((completion->sg_status & SG_STATUS_VALID) != 0) {
 880                         ahd_handle_scb_status(ahd, scb);
 881                 } else {
 882                         ahd_done(ahd, scb);
 883                 }
 884
 885                 ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
 886                 if (ahd->qoutfifonext == 0)
 887                         ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID;
 888         }
 889         ahd->flags &= ~AHD_RUNNING_QOUTFIFO;
 890 }
 891
 892 /************************* Interrupt Handling *********************************/
 893 void
 894 ahd_handle_hwerrint(struct ahd_softc *ahd)
 895 {
 896         /*
 897          * Some catastrophic hardware error has occurred.
 898          * Print it for the user and disable the controller.
 899          */
 900         int i;
 901         int error;
 902
 903         error = ahd_inb(ahd, ERROR);
 904         for (i = 0; i < num_errors; i++) {
 905                 if ((error & ahd_hard_errors[i].errno) != 0) {
 906                         printf("%s: hwerrint, %s\n",
 907                                ahd_name(ahd), ahd_hard_errors[i].errmesg);
 908                         AHD_UNCORRECTABLE_ERROR(ahd);
 909                 }
 910         }
 911
 912         ahd_dump_card_state(ahd);
 913         panic("BRKADRINT");
 914
 915         /* Tell everyone that this HBA is no longer available */
 916         ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 917                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 918                        CAM_NO_HBA);
 919
 920         /* Tell the system that this controller has gone away. */
 921         ahd_free(ahd);
 922 }
 923
 924 void
 925 ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
 926 {
 927         u_int seqintcode;
 928
 929         /*
 930          * Save the sequencer interrupt code and clear the SEQINT
 931          * bit. We will unpause the sequencer, if appropriate,
 932          * after servicing the request.
 933          */
 934         seqintcode = ahd_inb(ahd, SEQINTCODE);
 935         ahd_outb(ahd, CLRINT, CLRSEQINT);
 936         if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
 937                 /*
 938                  * Unpause the sequencer and let it clear
 939                  * SEQINT by writing NO_SEQINT to it.  This
 940                  * will cause the sequencer to be paused again,
 941                  * which is the expected state of this routine.
 942                  */
 943                 ahd_unpause(ahd);
 944                 while (!ahd_is_paused(ahd))
 945                         ;
 946                 ahd_outb(ahd, CLRINT, CLRSEQINT);
 947         }
 948         ahd_update_modes(ahd);
 949 #ifdef AHD_DEBUG
 950         if ((ahd_debug & AHD_SHOW_MISC) != 0)
 951                 printf("%s: Handle Seqint Called for code %d\n",
 952                        ahd_name(ahd), seqintcode);
 953 #endif
 954         switch (seqintcode) {
 955         case ENTERING_NONPACK:
 956         {
 957                 struct  scb *scb;
 958                 u_int   scbid;
 959
 960                 AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 961                                  ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 962                 scbid = ahd_get_scbptr(ahd);
 963                 scb = ahd_lookup_scb(ahd, scbid);
 964                 if (scb == NULL) {
 965                         /*
 966                          * Somehow need to know if this
 967                          * is from a selection or reselection.
 968                          * From that, we can determine target
 969                          * ID so we at least have an I_T nexus.
 970                          */
 971                 } else {
 972                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
 973                         ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
 974                         ahd_outb(ahd, SEQ_FLAGS, 0x0);
 975                 }
 976                 if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
 977                  && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
 978                         /*
 979                          * Phase change after read stream with
 980                          * CRC error with P0 asserted on last
 981                          * packet.
 982                          */
 983 #ifdef AHD_DEBUG
 984                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 985                                 printf("%s: Assuming LQIPHASE_NLQ with "
 986                                        "P0 assertion\n", ahd_name(ahd));
 987 #endif
 988                 }
 989 #ifdef AHD_DEBUG
 990                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 991                         printf("%s: Entering NONPACK\n", ahd_name(ahd));
 992 #endif
 993                 break;
 994         }
 995         case INVALID_SEQINT:
 996                 printf("%s: Invalid Sequencer interrupt occurred.\n",
 997                        ahd_name(ahd));
 998                 ahd_dump_card_state(ahd);
 999                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1000                 AHD_UNCORRECTABLE_ERROR(ahd);
1001                 break;
1002         case STATUS_OVERRUN:
1003         {
1004                 struct  scb *scb;
1005                 u_int   scbid;
1006
1007                 scbid = ahd_get_scbptr(ahd);
1008                 scb = ahd_lookup_scb(ahd, scbid);
1009                 if (scb != NULL)
1010                         ahd_print_path(ahd, scb);
1011                 else
1012                         printf("%s: ", ahd_name(ahd));
1013                 printf("SCB %d Packetized Status Overrun", scbid);
1014                 ahd_dump_card_state(ahd);
1015                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1016                 AHD_UNCORRECTABLE_ERROR(ahd);
1017                 break;
1018         }
1019         case CFG4ISTAT_INTR:
1020         {
1021                 struct  scb *scb;
1022                 u_int   scbid;
1023
1024                 scbid = ahd_get_scbptr(ahd);
1025                 scb = ahd_lookup_scb(ahd, scbid);
1026                 if (scb == NULL) {
1027                         ahd_dump_card_state(ahd);
1028                         printf("CFG4ISTAT: Free SCB %d referenced", scbid);
1029                         AHD_FATAL_ERROR(ahd);
1030                         panic("For safety");
1031                 }
1032                 ahd_outq(ahd, HADDR, scb->sense_busaddr);
1033                 ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
1034                 ahd_outb(ahd, HCNT + 2, 0);
1035                 ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
1036                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1037                 break;
1038         }
1039         case ILLEGAL_PHASE:
1040         {
1041                 u_int bus_phase;
1042
1043                 bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1044                 printf("%s: ILLEGAL_PHASE 0x%x\n",
1045                        ahd_name(ahd), bus_phase);
1046
1047                 switch (bus_phase) {
1048                 case P_DATAOUT:
1049                 case P_DATAIN:
1050                 case P_DATAOUT_DT:
1051                 case P_DATAIN_DT:
1052                 case P_MESGOUT:
1053                 case P_STATUS:
1054                 case P_MESGIN:
1055                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1056                         printf("%s: Issued Bus Reset.\n", ahd_name(ahd));
1057                         AHD_UNCORRECTABLE_ERROR(ahd);
1058                         break;
1059                 case P_COMMAND:
1060                 {
1061                         struct  ahd_devinfo devinfo;
1062                         struct  scb *scb;
1063                         struct  ahd_initiator_tinfo *targ_info;
1064                         struct  ahd_tmode_tstate *tstate;
1065                         struct  ahd_transinfo *tinfo;
1066                         u_int   scbid;
1067
1068                         /*
1069                          * If a target takes us into the command phase
1070                          * assume that it has been externally reset and
1071                          * has thus lost our previous packetized negotiation
1072                          * agreement.  Since we have not sent an identify
1073                          * message and may not have fully qualified the
1074                          * connection, we change our command to TUR, assert
1075                          * ATN and ABORT the task when we go to message in
1076                          * phase.  The OSM will see the REQUEUE_REQUEST
1077                          * status and retry the command.
1078                          */
1079                         scbid = ahd_get_scbptr(ahd);
1080                         scb = ahd_lookup_scb(ahd, scbid);
1081                         if (scb == NULL) {
1082                                 AHD_CORRECTABLE_ERROR(ahd);
1083                                 printf("Invalid phase with no valid SCB.  "
1084                                        "Resetting bus.\n");
1085                                 ahd_reset_channel(ahd, 'A',
1086                                                   /*Initiate Reset*/TRUE);
1087                                 break;
1088                         }
1089                         ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
1090                                             SCB_GET_TARGET(ahd, scb),
1091                                             SCB_GET_LUN(scb),
1092                                             SCB_GET_CHANNEL(ahd, scb),
1093                                             ROLE_INITIATOR);
1094                         targ_info = ahd_fetch_transinfo(ahd,
1095                                                         devinfo.channel,
1096                                                         devinfo.our_scsiid,
1097                                                         devinfo.target,
1098                                                         &tstate);
1099                         tinfo = &targ_info->curr;
1100                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
1101                                       AHD_TRANS_ACTIVE, /*paused*/TRUE);
1102                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
1103                                          /*offset*/0, /*ppr_options*/0,
1104                                          AHD_TRANS_ACTIVE, /*paused*/TRUE);
1105                         ahd_outb(ahd, SCB_CDB_STORE, 0);
1106                         ahd_outb(ahd, SCB_CDB_STORE+1, 0);
1107                         ahd_outb(ahd, SCB_CDB_STORE+2, 0);
1108                         ahd_outb(ahd, SCB_CDB_STORE+3, 0);
1109                         ahd_outb(ahd, SCB_CDB_STORE+4, 0);
1110                         ahd_outb(ahd, SCB_CDB_STORE+5, 0);
1111                         ahd_outb(ahd, SCB_CDB_LEN, 6);
1112                         scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
1113                         scb->hscb->control |= MK_MESSAGE;
1114                         ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
1115                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1116                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1117                         /*
1118                          * The lun is 0, regardless of the SCB's lun
1119                          * as we have not sent an identify message.
1120                          */
1121                         ahd_outb(ahd, SAVED_LUN, 0);
1122                         ahd_outb(ahd, SEQ_FLAGS, 0);
1123                         ahd_assert_atn(ahd);
1124                         scb->flags &= ~SCB_PACKETIZED;
1125                         scb->flags |= SCB_ABORT|SCB_CMDPHASE_ABORT;
1126                         ahd_freeze_devq(ahd, scb);
1127                         aic_set_transaction_status(scb, CAM_REQUEUE_REQ);
1128                         aic_freeze_scb(scb);
1129
1130                         /*
1131                          * Allow the sequencer to continue with
1132                          * non-pack processing.
1133                          */
1134                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1135                         ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
1136                         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
1137                                 ahd_outb(ahd, CLRLQOINT1, 0);
1138                         }
1139 #ifdef AHD_DEBUG
1140                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1141                                 ahd_print_path(ahd, scb);
1142                                 AHD_CORRECTABLE_ERROR(ahd);
1143                                 printf("Unexpected command phase from "
1144                                        "packetized target\n");
1145                         }
1146 #endif
1147                         break;
1148                 }
1149                 }
1150                 break;
1151         }
1152         case CFG4OVERRUN:
1153         {
1154                 struct  scb *scb;
1155                 u_int   scb_index;
1156
1157 #ifdef AHD_DEBUG
1158                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1159                         printf("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
1160                                ahd_inb(ahd, MODE_PTR));
1161                 }
1162 #endif
1163                 scb_index = ahd_get_scbptr(ahd);
1164                 scb = ahd_lookup_scb(ahd, scb_index);
1165                 if (scb == NULL) {
1166                         /*
1167                          * Attempt to transfer to an SCB that is
1168                          * not outstanding.
1169                          */
1170                         ahd_assert_atn(ahd);
1171                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1172                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
1173                         ahd->msgout_len = 1;
1174                         ahd->msgout_index = 0;
1175                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1176                         /*
1177                          * Clear status received flag to prevent any
1178                          * attempt to complete this bogus SCB.
1179                          */
1180                         ahd_outb(ahd, SCB_CONTROL,
1181                                  ahd_inb_scbram(ahd, SCB_CONTROL)
1182                                  & ~STATUS_RCVD);
1183                 }
1184                 break;
1185         }
1186         case DUMP_CARD_STATE:
1187         {
1188                 ahd_dump_card_state(ahd);
1189                 break;
1190         }
1191         case PDATA_REINIT:
1192         {
1193 #ifdef AHD_DEBUG
1194                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1195                         printf("%s: PDATA_REINIT - DFCNTRL = 0x%x "
1196                                "SG_CACHE_SHADOW = 0x%x\n",
1197                                ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
1198                                ahd_inb(ahd, SG_CACHE_SHADOW));
1199                 }
1200 #endif
1201                 ahd_reinitialize_dataptrs(ahd);
1202                 break;
1203         }
1204         case HOST_MSG_LOOP:
1205         {
1206                 struct ahd_devinfo devinfo;
1207
1208                 /*
1209                  * The sequencer has encountered a message phase
1210                  * that requires host assistance for completion.
1211                  * While handling the message phase(s), we will be
1212                  * notified by the sequencer after each byte is
1213                  * transfered so we can track bus phase changes.
1214                  *
1215                  * If this is the first time we've seen a HOST_MSG_LOOP
1216                  * interrupt, initialize the state of the host message
1217                  * loop.
1218                  */
1219                 ahd_fetch_devinfo(ahd, &devinfo);
1220                 if (ahd->msg_type == MSG_TYPE_NONE) {
1221                         struct scb *scb;
1222                         u_int scb_index;
1223                         u_int bus_phase;
1224
1225                         bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1226                         if (bus_phase != P_MESGIN
1227                          && bus_phase != P_MESGOUT) {
1228                                 printf("ahd_intr: HOST_MSG_LOOP bad "
1229                                        "phase 0x%x\n", bus_phase);
1230                                 AHD_CORRECTABLE_ERROR(ahd);
1231                                 /*
1232                                  * Probably transitioned to bus free before
1233                                  * we got here.  Just punt the message.
1234                                  */
1235                                 ahd_dump_card_state(ahd);
1236                                 ahd_clear_intstat(ahd);
1237                                 ahd_restart(ahd);
1238                                 return;
1239                         }
1240
1241                         scb_index = ahd_get_scbptr(ahd);
1242                         scb = ahd_lookup_scb(ahd, scb_index);
1243                         if (devinfo.role == ROLE_INITIATOR) {
1244                                 if (bus_phase == P_MESGOUT)
1245                                         ahd_setup_initiator_msgout(ahd,
1246                                                                    &devinfo,
1247                                                                    scb);
1248                                 else {
1249                                         ahd->msg_type =
1250                                             MSG_TYPE_INITIATOR_MSGIN;
1251                                         ahd->msgin_index = 0;
1252                                 }
1253                         }
1254 #ifdef AHD_TARGET_MODE
1255                         else {
1256                                 if (bus_phase == P_MESGOUT) {
1257                                         ahd->msg_type =
1258                                             MSG_TYPE_TARGET_MSGOUT;
1259                                         ahd->msgin_index = 0;
1260                                 }
1261                                 else
1262                                         ahd_setup_target_msgin(ahd,
1263                                                                &devinfo,
1264                                                                scb);
1265                         }
1266 #endif
1267                 }
1268
1269                 ahd_handle_message_phase(ahd);
1270                 break;
1271         }
1272         case NO_MATCH:
1273         {
1274                 /* Ensure we don't leave the selection hardware on */
1275                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
1276                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
1277
1278                 printf("%s:%c:%d: no active SCB for reconnecting "
1279                        "target - issuing BUS DEVICE RESET\n",
1280                        ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
1281                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1282                        "REG0 == 0x%x ACCUM = 0x%x\n",
1283                        ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
1284                        ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
1285                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1286                        "SINDEX == 0x%x\n",
1287                        ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
1288                        ahd_find_busy_tcl(ahd,
1289                                          BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
1290                                                    ahd_inb(ahd, SAVED_LUN))),
1291                        ahd_inw(ahd, SINDEX));
1292                 printf("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1293                        "SCB_CONTROL == 0x%x\n",
1294                        ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
1295                        ahd_inb_scbram(ahd, SCB_LUN),
1296                        ahd_inb_scbram(ahd, SCB_CONTROL));
1297                 printf("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
1298                        ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
1299                 printf("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
1300                 printf("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
1301                 ahd_dump_card_state(ahd);
1302                 ahd->msgout_buf[0] = MSG_BUS_DEV_RESET;
1303                 ahd->msgout_len = 1;
1304                 ahd->msgout_index = 0;
1305                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1306                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
1307                 ahd_assert_atn(ahd);
1308                 break;
1309         }
1310         case PROTO_VIOLATION:
1311         {
1312                 ahd_handle_proto_violation(ahd);
1313                 break;
1314         }
1315         case IGN_WIDE_RES:
1316         {
1317                 struct ahd_devinfo devinfo;
1318
1319                 ahd_fetch_devinfo(ahd, &devinfo);
1320                 ahd_handle_ign_wide_residue(ahd, &devinfo);
1321                 break;
1322         }
1323         case BAD_PHASE:
1324         {
1325                 u_int lastphase;
1326
1327                 lastphase = ahd_inb(ahd, LASTPHASE);
1328                 printf("%s:%c:%d: unknown scsi bus phase %x, "
1329                        "lastphase = 0x%x.  Attempting to continue\n",
1330                        ahd_name(ahd), 'A',
1331                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
1332                        lastphase, ahd_inb(ahd, SCSISIGI));
1333                 AHD_CORRECTABLE_ERROR(ahd);
1334                 break;
1335         }
1336         case MISSED_BUSFREE:
1337         {
1338                 u_int lastphase;
1339
1340                 lastphase = ahd_inb(ahd, LASTPHASE);
1341                 printf("%s:%c:%d: Missed busfree. "
1342                        "Lastphase = 0x%x, Curphase = 0x%x\n",
1343                        ahd_name(ahd), 'A',
1344                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
1345                        lastphase, ahd_inb(ahd, SCSISIGI));
1346                 AHD_CORRECTABLE_ERROR(ahd);
1347                 ahd_restart(ahd);
1348                 return;
1349         }
1350         case DATA_OVERRUN:
1351         {
1352                 /*
1353                  * When the sequencer detects an overrun, it
1354                  * places the controller in "BITBUCKET" mode
1355                  * and allows the target to complete its transfer.
1356                  * Unfortunately, none of the counters get updated
1357                  * when the controller is in this mode, so we have
1358                  * no way of knowing how large the overrun was.
1359                  */
1360                 struct  scb *scb;
1361                 u_int   scbindex;
1362 #ifdef AHD_DEBUG
1363                 u_int   lastphase;
1364 #endif
1365
1366                 scbindex = ahd_get_scbptr(ahd);
1367                 scb = ahd_lookup_scb(ahd, scbindex);
1368 #ifdef AHD_DEBUG
1369                 lastphase = ahd_inb(ahd, LASTPHASE);
1370                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1371                         ahd_print_path(ahd, scb);
1372                         printf("data overrun detected %s.  Tag == 0x%x.\n",
1373                                ahd_lookup_phase_entry(lastphase)->phasemsg,
1374                                SCB_GET_TAG(scb));
1375                         ahd_print_path(ahd, scb);
1376                         printf("%s seen Data Phase.  Length = %ld.  "
1377                                "NumSGs = %d.\n",
1378                                ahd_inb(ahd, SEQ_FLAGS) & DPHASE
1379                                ? "Have" : "Haven't",
1380                                aic_get_transfer_length(scb), scb->sg_count);
1381                         ahd_dump_sglist(scb);
1382                 }
1383 #endif
1384
1385                 /*
1386                  * Set this and it will take effect when the
1387                  * target does a command complete.
1388                  */
1389                 ahd_freeze_devq(ahd, scb);
1390                 aic_set_transaction_status(scb, CAM_DATA_RUN_ERR);
1391                 aic_freeze_scb(scb);
1392                 break;
1393         }
1394         case MKMSG_FAILED:
1395         {
1396                 struct ahd_devinfo devinfo;
1397                 struct scb *scb;
1398                 u_int scbid;
1399
1400                 ahd_fetch_devinfo(ahd, &devinfo);
1401                 printf("%s:%c:%d:%d: Attempt to issue message failed\n",
1402                        ahd_name(ahd), devinfo.channel, devinfo.target,
1403                        devinfo.lun);
1404                 scbid = ahd_get_scbptr(ahd);
1405                 scb = ahd_lookup_scb(ahd, scbid);
1406                 AHD_CORRECTABLE_ERROR(ahd);
1407                 if (scb != NULL
1408                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
1409                         /*
1410                          * Ensure that we didn't put a second instance of this
1411                          * SCB into the QINFIFO.
1412                          */
1413                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
1414                                            SCB_GET_CHANNEL(ahd, scb),
1415                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
1416                                            ROLE_INITIATOR, /*status*/0,
1417                                            SEARCH_REMOVE);
1418                 ahd_outb(ahd, SCB_CONTROL,
1419                          ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
1420                 break;
1421         }
1422         case TASKMGMT_FUNC_COMPLETE:
1423         {
1424                 u_int   scbid;
1425                 struct  scb *scb;
1426
1427                 scbid = ahd_get_scbptr(ahd);
1428                 scb = ahd_lookup_scb(ahd, scbid);
1429                 if (scb != NULL) {
1430                         u_int      lun;
1431                         u_int      tag;
1432                         cam_status error;
1433
1434                         ahd_print_path(ahd, scb);
1435                         printf("Task Management Func 0x%x Complete\n",
1436                                scb->hscb->task_management);
1437                         lun = CAM_LUN_WILDCARD;
1438                         tag = SCB_LIST_NULL;
1439
1440                         switch (scb->hscb->task_management) {
1441                         case SIU_TASKMGMT_ABORT_TASK:
1442                                 tag = SCB_GET_TAG(scb);
1443                         case SIU_TASKMGMT_ABORT_TASK_SET:
1444                         case SIU_TASKMGMT_CLEAR_TASK_SET:
1445                                 lun = scb->hscb->lun;
1446                                 error = CAM_REQ_ABORTED;
1447                                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
1448                                                'A', lun, tag, ROLE_INITIATOR,
1449                                                error);
1450                                 break;
1451                         case SIU_TASKMGMT_LUN_RESET:
1452                                 lun = scb->hscb->lun;
1453                         case SIU_TASKMGMT_TARGET_RESET:
1454                         {
1455                                 struct ahd_devinfo devinfo;
1456
1457                                 ahd_scb_devinfo(ahd, &devinfo, scb);
1458                                 error = CAM_BDR_SENT;
1459                                 ahd_handle_devreset(ahd, &devinfo, lun,
1460                                                     CAM_BDR_SENT,
1461                                                     lun != CAM_LUN_WILDCARD
1462                                                     ? "Lun Reset"
1463                                                     : "Target Reset",
1464                                                     /*verbose_level*/0);
1465                                 break;
1466                         }
1467                         default:
1468                                 panic("Unexpected TaskMgmt Func\n");
1469                                 break;
1470                         }
1471                 }
1472                 break;
1473         }
1474         case TASKMGMT_CMD_CMPLT_OKAY:
1475         {
1476                 u_int   scbid;
1477                 struct  scb *scb;
1478
1479                 /*
1480                  * An ABORT TASK TMF failed to be delivered before
1481                  * the targeted command completed normally.
1482                  */
1483                 scbid = ahd_get_scbptr(ahd);
1484                 scb = ahd_lookup_scb(ahd, scbid);
1485                 if (scb != NULL) {
1486                         /*
1487                          * Remove the second instance of this SCB from
1488                          * the QINFIFO if it is still there.
1489                          */
1490                         ahd_print_path(ahd, scb);
1491                         printf("SCB completes before TMF\n");
1492                         /*
1493                          * Handle losing the race.  Wait until any
1494                          * current selection completes.  We will then
1495                          * set the TMF back to zero in this SCB so that
1496                          * the sequencer doesn't bother to issue another
1497                          * sequencer interrupt for its completion.
1498                          */
1499                         while ((ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
1500                             && (ahd_inb(ahd, SSTAT0) & SELDO) == 0
1501                             && (ahd_inb(ahd, SSTAT1) & SELTO) == 0)
1502                                 ;
1503                         ahd_outb(ahd, SCB_TASK_MANAGEMENT, 0);
1504                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
1505                                            SCB_GET_CHANNEL(ahd, scb),
1506                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
1507                                            ROLE_INITIATOR, /*status*/0,
1508                                            SEARCH_REMOVE);
1509                 }
1510                 break;
1511         }
1512         case TRACEPOINT0:
1513         case TRACEPOINT1:
1514         case TRACEPOINT2:
1515         case TRACEPOINT3:
1516                 printf("%s: Tracepoint %d\n", ahd_name(ahd),
1517                        seqintcode - TRACEPOINT0);
1518                 break;
1519         case NO_SEQINT:
1520                 break;
1521         case SAW_HWERR:
1522                 ahd_handle_hwerrint(ahd);
1523                 break;
1524         default:
1525                 printf("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
1526                        seqintcode);
1527                 break;
1528         }
1529         /*
1530          *  The sequencer is paused immediately on
1531          *  a SEQINT, so we should restart it when
1532          *  we're done.
1533          */
1534         ahd_unpause(ahd);
1535 }
1536
1537 void
1538 ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
1539 {
1540         struct scb      *scb;
1541         u_int            status0;
1542         u_int            status3;
1543         u_int            status;
1544         u_int            lqistat1;
1545         u_int            lqostat0;
1546         u_int            scbid;
1547         u_int            busfreetime;
1548
1549         ahd_update_modes(ahd);
1550         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1551
1552         status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
1553         status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
1554         status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1555         lqistat1 = ahd_inb(ahd, LQISTAT1);
1556         lqostat0 = ahd_inb(ahd, LQOSTAT0);
1557         busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
1558         if ((status0 & (SELDI|SELDO)) != 0) {
1559                 u_int simode0;
1560
1561                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1562                 simode0 = ahd_inb(ahd, SIMODE0);
1563                 status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
1564                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1565         }
1566         scbid = ahd_get_scbptr(ahd);
1567         scb = ahd_lookup_scb(ahd, scbid);
1568         if (scb != NULL
1569          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1570                 scb = NULL;
1571
1572         if ((status0 & IOERR) != 0) {
1573                 u_int now_lvd;
1574
1575                 now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
1576                 printf("%s: Transceiver State Has Changed to %s mode\n",
1577                        ahd_name(ahd), now_lvd ? "LVD" : "SE");
1578                 ahd_outb(ahd, CLRSINT0, CLRIOERR);
1579                 /*
1580                  * A change in I/O mode is equivalent to a bus reset.
1581                  */
1582                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1583                 ahd_pause(ahd);
1584                 ahd_setup_iocell_workaround(ahd);
1585                 ahd_unpause(ahd);
1586         } else if ((status0 & OVERRUN) != 0) {
1587
1588                 printf("%s: SCSI offset overrun detected.  Resetting bus.\n",
1589                        ahd_name(ahd));
1590                 AHD_CORRECTABLE_ERROR(ahd);
1591                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1592         } else if ((status & SCSIRSTI) != 0) {
1593
1594                 printf("%s: Someone reset channel A\n", ahd_name(ahd));
1595                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
1596                 AHD_UNCORRECTABLE_ERROR(ahd);
1597         } else if ((status & SCSIPERR) != 0) {
1598
1599                 /* Make sure the sequencer is in a safe location. */
1600                 ahd_clear_critical_section(ahd);
1601
1602                 ahd_handle_transmission_error(ahd);
1603         } else if (lqostat0 != 0) {
1604
1605                 printf("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
1606                 ahd_outb(ahd, CLRLQOINT0, lqostat0);
1607                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
1608                         ahd_outb(ahd, CLRLQOINT1, 0);
1609         } else if ((status & SELTO) != 0) {
1610                 u_int  scbid;
1611
1612                 /* Stop the selection */
1613                 ahd_outb(ahd, SCSISEQ0, 0);
1614
1615                 /* Make sure the sequencer is in a safe location. */
1616                 ahd_clear_critical_section(ahd);
1617
1618                 /* No more pending messages */
1619                 ahd_clear_msg_state(ahd);
1620
1621                 /* Clear interrupt state */
1622                 ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1623
1624                 /*
1625                  * Although the driver does not care about the
1626                  * 'Selection in Progress' status bit, the busy
1627                  * LED does.  SELINGO is only cleared by a sucessfull
1628                  * selection, so we must manually clear it to insure
1629                  * the LED turns off just incase no future successful
1630                  * selections occur (e.g. no devices on the bus).
1631                  */
1632                 ahd_outb(ahd, CLRSINT0, CLRSELINGO);
1633
1634                 scbid = ahd_inw(ahd, WAITING_TID_HEAD);
1635                 scb = ahd_lookup_scb(ahd, scbid);
1636                 if (scb == NULL) {
1637                         printf("%s: ahd_intr - referenced scb not "
1638                                "valid during SELTO scb(0x%x)\n",
1639                                ahd_name(ahd), scbid);
1640                         ahd_dump_card_state(ahd);
1641                         AHD_UNCORRECTABLE_ERROR(ahd);
1642                 } else {
1643                         struct ahd_devinfo devinfo;
1644 #ifdef AHD_DEBUG
1645                         if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
1646                                 ahd_print_path(ahd, scb);
1647                                 printf("Saw Selection Timeout for SCB 0x%x\n",
1648                                        scbid);
1649                         }
1650 #endif
1651                         ahd_scb_devinfo(ahd, &devinfo, scb);
1652                         aic_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1653                         ahd_freeze_devq(ahd, scb);
1654
1655                         /*
1656                          * Cancel any pending transactions on the device
1657                          * now that it seems to be missing.  This will
1658                          * also revert us to async/narrow transfers until
1659                          * we can renegotiate with the device.
1660                          */
1661                         ahd_handle_devreset(ahd, &devinfo,
1662                                             CAM_LUN_WILDCARD,
1663                                             CAM_SEL_TIMEOUT,
1664                                             "Selection Timeout",
1665                                             /*verbose_level*/1);
1666                 }
1667                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
1668                 ahd_iocell_first_selection(ahd);
1669                 ahd_unpause(ahd);
1670         } else if ((status0 & (SELDI|SELDO)) != 0) {
1671
1672                 ahd_iocell_first_selection(ahd);
1673                 ahd_unpause(ahd);
1674         } else if (status3 != 0) {
1675                 printf("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
1676                        ahd_name(ahd), status3);
1677                 AHD_CORRECTABLE_ERROR(ahd);
1678                 ahd_outb(ahd, CLRSINT3, status3);
1679         } else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
1680
1681                 /* Make sure the sequencer is in a safe location. */
1682                 ahd_clear_critical_section(ahd);
1683
1684                 ahd_handle_lqiphase_error(ahd, lqistat1);
1685         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
1686                 /*
1687                  * This status can be delayed during some
1688                  * streaming operations.  The SCSIPHASE
1689                  * handler has already dealt with this case
1690                  * so just clear the error.
1691                  */
1692                 ahd_outb(ahd, CLRLQIINT1, CLRLQICRCI_NLQ);
1693         } else if ((status & BUSFREE) != 0
1694                 || (lqistat1 & LQOBUSFREE) != 0) {
1695                 u_int lqostat1;
1696                 int   restart;
1697                 int   clear_fifo;
1698                 int   packetized;
1699                 u_int mode;
1700
1701                 /*
1702                  * Clear our selection hardware as soon as possible.
1703                  * We may have an entry in the waiting Q for this target,
1704                  * that is affected by this busfree and we don't want to
1705                  * go about selecting the target while we handle the event.
1706                  */
1707                 ahd_outb(ahd, SCSISEQ0, 0);
1708
1709                 /* Make sure the sequencer is in a safe location. */
1710                 ahd_clear_critical_section(ahd);
1711
1712                 /*
1713                  * Determine what we were up to at the time of
1714                  * the busfree.
1715                  */
1716                 mode = AHD_MODE_SCSI;
1717                 busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
1718                 lqostat1 = ahd_inb(ahd, LQOSTAT1);
1719                 switch (busfreetime) {
1720                 case BUSFREE_DFF0:
1721                 case BUSFREE_DFF1:
1722                 {
1723                         u_int   scbid;
1724                         struct  scb *scb;
1725
1726                         mode = busfreetime == BUSFREE_DFF0
1727                              ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
1728                         ahd_set_modes(ahd, mode, mode);
1729                         scbid = ahd_get_scbptr(ahd);
1730                         scb = ahd_lookup_scb(ahd, scbid);
1731                         if (scb == NULL) {
1732                                 printf("%s: Invalid SCB %d in DFF%d "
1733                                        "during unexpected busfree\n",
1734                                        ahd_name(ahd), scbid, mode);
1735                                 packetized = 0;
1736                                 AHD_CORRECTABLE_ERROR(ahd);
1737                         } else
1738                                 packetized = (scb->flags & SCB_PACKETIZED) != 0;
1739                         clear_fifo = 1;
1740                         break;
1741                 }
1742                 case BUSFREE_LQO:
1743                         clear_fifo = 0;
1744                         packetized = 1;
1745                         break;
1746                 default:
1747                         clear_fifo = 0;
1748                         packetized =  (lqostat1 & LQOBUSFREE) != 0;
1749                         if (!packetized
1750                          && ahd_inb(ahd, LASTPHASE) == P_BUSFREE
1751                          && (ahd_inb(ahd, SSTAT0) & SELDI) == 0
1752                          && ((ahd_inb(ahd, SSTAT0) & SELDO) == 0
1753                           || (ahd_inb(ahd, SCSISEQ0) & ENSELO) == 0))
1754                                 /*
1755                                  * Assume packetized if we are not
1756                                  * on the bus in a non-packetized
1757                                  * capacity and any pending selection
1758                                  * was a packetized selection.
1759                                  */
1760                                 packetized = 1;
1761                         break;
1762                 }
1763
1764 #ifdef AHD_DEBUG
1765                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
1766                         printf("Saw Busfree.  Busfreetime = 0x%x.\n",
1767                                busfreetime);
1768 #endif
1769                 /*
1770                  * Busfrees that occur in non-packetized phases are
1771                  * handled by the nonpkt_busfree handler.
1772                  */
1773                 if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
1774                         restart = ahd_handle_pkt_busfree(ahd, busfreetime);
1775                 } else {
1776                         packetized = 0;
1777                         restart = ahd_handle_nonpkt_busfree(ahd);
1778                 }
1779                 /*
1780                  * Clear the busfree interrupt status.  The setting of
1781                  * the interrupt is a pulse, so in a perfect world, we
1782                  * would not need to muck with the ENBUSFREE logic.  This
1783                  * would ensure that if the bus moves on to another
1784                  * connection, busfree protection is still in force.  If
1785                  * BUSFREEREV is broken, however, we must manually clear
1786                  * the ENBUSFREE if the busfree occurred during a non-pack
1787                  * connection so that we don't get false positives during
1788                  * future, packetized, connections.
1789                  */
1790                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
1791                 if (packetized == 0
1792                  && (ahd->bugs & AHD_BUSFREEREV_BUG) != 0)
1793                         ahd_outb(ahd, SIMODE1,
1794                                  ahd_inb(ahd, SIMODE1) & ~ENBUSFREE);
1795
1796                 if (clear_fifo)
1797                         ahd_clear_fifo(ahd, mode);
1798
1799                 ahd_clear_msg_state(ahd);
1800                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
1801                 if (restart) {
1802                         ahd_restart(ahd);
1803                 } else {
1804                         ahd_unpause(ahd);
1805                 }
1806         } else {
1807                 printf("%s: Missing case in ahd_handle_scsiint. status = %x\n",
1808                        ahd_name(ahd), status);
1809                 ahd_dump_card_state(ahd);
1810                 ahd_clear_intstat(ahd);
1811                 ahd_unpause(ahd);
1812         }
1813 }
1814
1815 static void
1816 ahd_handle_transmission_error(struct ahd_softc *ahd)
1817 {
1818         struct  scb *scb;
1819         u_int   scbid;
1820         u_int   lqistat1;
1821         u_int   lqistat2;
1822         u_int   msg_out;
1823         u_int   curphase;
1824         u_int   lastphase;
1825         u_int   perrdiag;
1826         u_int   cur_col;
1827         int     silent;
1828
1829         scb = NULL;
1830         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1831         lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
1832         lqistat2 = ahd_inb(ahd, LQISTAT2);
1833         if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
1834          && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
1835                 u_int lqistate;
1836
1837                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1838                 lqistate = ahd_inb(ahd, LQISTATE);
1839                 if ((lqistate >= 0x1E && lqistate <= 0x24)
1840                  || (lqistate == 0x29)) {
1841 #ifdef AHD_DEBUG
1842                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1843                                 printf("%s: NLQCRC found via LQISTATE\n",
1844                                        ahd_name(ahd));
1845                         }
1846 #endif
1847                         lqistat1 |= LQICRCI_NLQ;
1848                 }
1849                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1850         }
1851
1852         ahd_outb(ahd, CLRLQIINT1, lqistat1);
1853         lastphase = ahd_inb(ahd, LASTPHASE);
1854         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1855         perrdiag = ahd_inb(ahd, PERRDIAG);
1856         msg_out = MSG_INITIATOR_DET_ERR;
1857         ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
1858
1859         /*
1860          * Try to find the SCB associated with this error.
1861          */
1862         silent = FALSE;
1863         if (lqistat1 == 0
1864          || (lqistat1 & LQICRCI_NLQ) != 0) {
1865                 if ((lqistat1 & (LQICRCI_NLQ|LQIOVERI_NLQ)) != 0)
1866                         ahd_set_active_fifo(ahd);
1867                 scbid = ahd_get_scbptr(ahd);
1868                 scb = ahd_lookup_scb(ahd, scbid);
1869                 if (scb != NULL && SCB_IS_SILENT(scb))
1870                         silent = TRUE;
1871         }
1872
1873         cur_col = 0;
1874         if (silent == FALSE) {
1875                 printf("%s: Transmission error detected\n", ahd_name(ahd));
1876                 ahd_lqistat1_print(lqistat1, &cur_col, 50);
1877                 ahd_lastphase_print(lastphase, &cur_col, 50);
1878                 ahd_scsisigi_print(curphase, &cur_col, 50);
1879                 ahd_perrdiag_print(perrdiag, &cur_col, 50);
1880                 printf("\n");
1881                 AHD_CORRECTABLE_ERROR(ahd);
1882                 ahd_dump_card_state(ahd);
1883         }
1884
1885         if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
1886                 if (silent == FALSE) {
1887                         printf("%s: Gross protocol error during incoming "
1888                                "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
1889                                ahd_name(ahd), lqistat1);
1890                         AHD_UNCORRECTABLE_ERROR(ahd);
1891                 }
1892                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1893                 return;
1894         } else if ((lqistat1 & LQICRCI_LQ) != 0) {
1895                 /*
1896                  * A CRC error has been detected on an incoming LQ.
1897                  * The bus is currently hung on the last ACK.
1898                  * Hit LQIRETRY to release the last ack, and
1899                  * wait for the sequencer to determine that ATNO
1900                  * is asserted while in message out to take us
1901                  * to our host message loop.  No NONPACKREQ or
1902                  * LQIPHASE type errors will occur in this
1903                  * scenario.  After this first LQIRETRY, the LQI
1904                  * manager will be in ISELO where it will
1905                  * happily sit until another packet phase begins.
1906                  * Unexpected bus free detection is enabled
1907                  * through any phases that occur after we release
1908                  * this last ack until the LQI manager sees a
1909                  * packet phase.  This implies we may have to
1910                  * ignore a perfectly valid "unexected busfree"
1911                  * after our "initiator detected error" message is
1912                  * sent.  A busfree is the expected response after
1913                  * we tell the target that it's L_Q was corrupted.
1914                  * (SPI4R09 10.7.3.3.3)
1915                  */
1916                 ahd_outb(ahd, LQCTL2, LQIRETRY);
1917                 printf("LQIRetry for LQICRCI_LQ to release ACK\n");
1918                 AHD_CORRECTABLE_ERROR(ahd);
1919         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
1920                 /*
1921                  * We detected a CRC error in a NON-LQ packet.
1922                  * The hardware has varying behavior in this situation
1923                  * depending on whether this packet was part of a
1924                  * stream or not.
1925                  *
1926                  * PKT by PKT mode:
1927                  * The hardware has already acked the complete packet.
1928                  * If the target honors our outstanding ATN condition,
1929                  * we should be (or soon will be) in MSGOUT phase.
1930                  * This will trigger the LQIPHASE_LQ status bit as the
1931                  * hardware was expecting another LQ.  Unexpected
1932                  * busfree detection is enabled.  Once LQIPHASE_LQ is
1933                  * true (first entry into host message loop is much
1934                  * the same), we must clear LQIPHASE_LQ and hit
1935                  * LQIRETRY so the hardware is ready to handle
1936                  * a future LQ.  NONPACKREQ will not be asserted again
1937                  * once we hit LQIRETRY until another packet is
1938                  * processed.  The target may either go busfree
1939                  * or start another packet in response to our message.
1940                  *
1941                  * Read Streaming P0 asserted:
1942                  * If we raise ATN and the target completes the entire
1943                  * stream (P0 asserted during the last packet), the
1944                  * hardware will ack all data and return to the ISTART
1945                  * state.  When the target reponds to our ATN condition,
1946                  * LQIPHASE_LQ will be asserted.  We should respond to
1947                  * this with an LQIRETRY to prepare for any future
1948                  * packets.  NONPACKREQ will not be asserted again
1949                  * once we hit LQIRETRY until another packet is
1950                  * processed.  The target may either go busfree or
1951                  * start another packet in response to our message.
1952                  * Busfree detection is enabled.
1953                  *
1954                  * Read Streaming P0 not asserted:
1955                  * If we raise ATN and the target transitions to
1956                  * MSGOUT in or after a packet where P0 is not
1957                  * asserted, the hardware will assert LQIPHASE_NLQ.
1958                  * We should respond to the LQIPHASE_NLQ with an
1959                  * LQIRETRY.  Should the target stay in a non-pkt
1960                  * phase after we send our message, the hardware
1961                  * will assert LQIPHASE_LQ.  Recovery is then just as
1962                  * listed above for the read streaming with P0 asserted.
1963                  * Busfree detection is enabled.
1964                  */
1965                 if (silent == FALSE)
1966                         printf("LQICRC_NLQ\n");
1967                 if (scb == NULL) {
1968                         printf("%s: No SCB valid for LQICRC_NLQ.  "
1969                                "Resetting bus\n", ahd_name(ahd));
1970                         AHD_UNCORRECTABLE_ERROR(ahd);
1971                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1972                         return;
1973                 }
1974         } else if ((lqistat1 & LQIBADLQI) != 0) {
1975                 printf("Need to handle BADLQI!\n");
1976                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1977                 return;
1978         } else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
1979                 if ((curphase & ~P_DATAIN_DT) != 0) {
1980                         /* Ack the byte.  So we can continue. */
1981                         if (silent == FALSE)
1982                                 printf("Acking %s to clear perror\n",
1983                                     ahd_lookup_phase_entry(curphase)->phasemsg);
1984                         ahd_inb(ahd, SCSIDAT);
1985                 }
1986
1987                 if (curphase == P_MESGIN)
1988                         msg_out = MSG_PARITY_ERROR;
1989         }
1990
1991         /*
1992          * We've set the hardware to assert ATN if we
1993          * get a parity error on "in" phases, so all we
1994          * need to do is stuff the message buffer with
1995          * the appropriate message.  "In" phases have set
1996          * mesg_out to something other than MSG_NOP.
1997          */
1998         ahd->send_msg_perror = msg_out;
1999         if (scb != NULL && msg_out == MSG_INITIATOR_DET_ERR)
2000                 scb->flags |= SCB_TRANSMISSION_ERROR;
2001         ahd_outb(ahd, MSG_OUT, HOST_MSG);
2002         ahd_outb(ahd, CLRINT, CLRSCSIINT);
2003         ahd_unpause(ahd);
2004 }
2005
2006 static void
2007 ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
2008 {
2009         /*
2010          * Clear the sources of the interrupts.
2011          */
2012         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2013         ahd_outb(ahd, CLRLQIINT1, lqistat1);
2014
2015         /*
2016          * If the "illegal" phase changes were in response
2017          * to our ATN to flag a CRC error, AND we ended up
2018          * on packet boundaries, clear the error, restart the
2019          * LQI manager as appropriate, and go on our merry
2020          * way toward sending the message.  Otherwise, reset
2021          * the bus to clear the error.
2022          */
2023         ahd_set_active_fifo(ahd);
2024         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
2025          && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
2026                 if ((lqistat1 & LQIPHASE_LQ) != 0) {
2027                         printf("LQIRETRY for LQIPHASE_LQ\n");
2028                         AHD_CORRECTABLE_ERROR(ahd);
2029                         ahd_outb(ahd, LQCTL2, LQIRETRY);
2030                 } else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
2031                         printf("LQIRETRY for LQIPHASE_NLQ\n");
2032                         AHD_CORRECTABLE_ERROR(ahd);
2033                         ahd_outb(ahd, LQCTL2, LQIRETRY);
2034                 } else
2035                         panic("ahd_handle_lqiphase_error: No phase errors\n");
2036                 ahd_dump_card_state(ahd);
2037                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2038                 ahd_unpause(ahd);
2039         } else {
2040                 printf("Reseting Channel for LQI Phase error\n");
2041                 AHD_CORRECTABLE_ERROR(ahd);
2042                 ahd_dump_card_state(ahd);
2043                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2044         }
2045 }
2046
2047 /*
2048  * Packetized unexpected or expected busfree.
2049  * Entered in mode based on busfreetime.
2050  */
2051 static int
2052 ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
2053 {
2054         u_int lqostat1;
2055
2056         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2057                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2058         lqostat1 = ahd_inb(ahd, LQOSTAT1);
2059         if ((lqostat1 & LQOBUSFREE) != 0) {
2060                 struct scb *scb;
2061                 u_int scbid;
2062                 u_int saved_scbptr;
2063                 u_int waiting_h;
2064                 u_int waiting_t;
2065                 u_int next;
2066
2067                 /*
2068                  * The LQO manager detected an unexpected busfree
2069                  * either:
2070                  *
2071                  * 1) During an outgoing LQ.
2072                  * 2) After an outgoing LQ but before the first
2073                  *    REQ of the command packet.
2074                  * 3) During an outgoing command packet.
2075                  *
2076                  * In all cases, CURRSCB is pointing to the
2077                  * SCB that encountered the failure.  Clean
2078                  * up the queue, clear SELDO and LQOBUSFREE,
2079                  * and allow the sequencer to restart the select
2080                  * out at its lesure.
2081                  */
2082                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2083                 scbid = ahd_inw(ahd, CURRSCB);
2084                 scb = ahd_lookup_scb(ahd, scbid);
2085                 if (scb == NULL)
2086                        panic("SCB not valid during LQOBUSFREE");
2087                 /*
2088                  * Clear the status.
2089                  */
2090                 ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
2091                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2092                         ahd_outb(ahd, CLRLQOINT1, 0);
2093                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2094                 ahd_flush_device_writes(ahd);
2095                 ahd_outb(ahd, CLRSINT0, CLRSELDO);
2096
2097                 /*
2098                  * Return the LQO manager to its idle loop.  It will
2099                  * not do this automatically if the busfree occurs
2100                  * after the first REQ of either the LQ or command
2101                  * packet or between the LQ and command packet.
2102                  */
2103                 ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
2104
2105                 /*
2106                  * Update the waiting for selection queue so
2107                  * we restart on the correct SCB.
2108                  */
2109                 waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
2110                 saved_scbptr = ahd_get_scbptr(ahd);
2111                 if (waiting_h != scbid) {
2112
2113                         ahd_outw(ahd, WAITING_TID_HEAD, scbid);
2114                         waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
2115                         if (waiting_t == waiting_h) {
2116                                 ahd_outw(ahd, WAITING_TID_TAIL, scbid);
2117                                 next = SCB_LIST_NULL;
2118                         } else {
2119                                 ahd_set_scbptr(ahd, waiting_h);
2120                                 next = ahd_inw_scbram(ahd, SCB_NEXT2);
2121                         }
2122                         ahd_set_scbptr(ahd, scbid);
2123                         ahd_outw(ahd, SCB_NEXT2, next);
2124                 }
2125                 ahd_set_scbptr(ahd, saved_scbptr);
2126                 if (scb->crc_retry_count < AHD_MAX_LQ_CRC_ERRORS) {
2127                         if (SCB_IS_SILENT(scb) == FALSE) {
2128                                 ahd_print_path(ahd, scb);
2129                                 printf("Probable outgoing LQ CRC error.  "
2130                                        "Retrying command\n");
2131                                 AHD_CORRECTABLE_ERROR(ahd);
2132                         }
2133                         scb->crc_retry_count++;
2134                 } else {
2135                         aic_set_transaction_status(scb, CAM_UNCOR_PARITY);
2136                         aic_freeze_scb(scb);
2137                         ahd_freeze_devq(ahd, scb);
2138                 }
2139                 /* Return unpausing the sequencer. */
2140                 return (0);
2141         } else if ((ahd_inb(ahd, PERRDIAG) & PARITYERR) != 0) {
2142                 /*
2143                  * Ignore what are really parity errors that
2144                  * occur on the last REQ of a free running
2145                  * clock prior to going busfree.  Some drives
2146                  * do not properly active negate just before
2147                  * going busfree resulting in a parity glitch.
2148                  */
2149                 ahd_outb(ahd, CLRSINT1, CLRSCSIPERR|CLRBUSFREE);
2150 #ifdef AHD_DEBUG
2151                 if ((ahd_debug & AHD_SHOW_MASKED_ERRORS) != 0)
2152                         printf("%s: Parity on last REQ detected "
2153                                "during busfree phase.\n",
2154                                ahd_name(ahd));
2155 #endif
2156                 /* Return unpausing the sequencer. */
2157                 return (0);
2158         }
2159         if (ahd->src_mode != AHD_MODE_SCSI) {
2160                 u_int   scbid;
2161                 struct  scb *scb;
2162
2163                 scbid = ahd_get_scbptr(ahd);
2164                 scb = ahd_lookup_scb(ahd, scbid);
2165                 ahd_print_path(ahd, scb);
2166                 printf("Unexpected PKT busfree condition\n");
2167                 AHD_UNCORRECTABLE_ERROR(ahd);
2168                 ahd_dump_card_state(ahd);
2169                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
2170                                SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2171                                ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
2172
2173                 /* Return restarting the sequencer. */
2174                 return (1);
2175         }
2176         printf("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
2177         AHD_UNCORRECTABLE_ERROR(ahd);
2178         ahd_dump_card_state(ahd);
2179         /* Restart the sequencer. */
2180         return (1);
2181 }
2182
2183 /*
2184  * Non-packetized unexpected or expected busfree.
2185  */
2186 static int
2187 ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
2188 {
2189         struct  ahd_devinfo devinfo;
2190         struct  scb *scb;
2191         u_int   lastphase;
2192         u_int   saved_scsiid;
2193         u_int   saved_lun;
2194         u_int   target;
2195         u_int   initiator_role_id;
2196         u_int   scbid;
2197         u_int   ppr_busfree;
2198         int     printerror;
2199
2200         /*
2201          * Look at what phase we were last in.  If its message out,
2202          * chances are pretty good that the busfree was in response
2203          * to one of our abort requests.
2204          */
2205         lastphase = ahd_inb(ahd, LASTPHASE);
2206         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
2207         saved_lun = ahd_inb(ahd, SAVED_LUN);
2208         target = SCSIID_TARGET(ahd, saved_scsiid);
2209         initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
2210         ahd_compile_devinfo(&devinfo, initiator_role_id,
2211                             target, saved_lun, 'A', ROLE_INITIATOR);
2212         printerror = 1;
2213
2214         scbid = ahd_get_scbptr(ahd);
2215         scb = ahd_lookup_scb(ahd, scbid);
2216         if (scb != NULL
2217          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
2218                 scb = NULL;
2219
2220         ppr_busfree = (ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0;
2221         if (lastphase == P_MESGOUT) {
2222                 u_int tag;
2223
2224                 tag = SCB_LIST_NULL;
2225                 if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT_TAG, TRUE)
2226                  || ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT, TRUE)) {
2227                         int found;
2228                         int sent_msg;
2229
2230                         if (scb == NULL) {
2231                                 ahd_print_devinfo(ahd, &devinfo);
2232                                 printf("Abort for unidentified "
2233                                        "connection completed.\n");
2234                                 /* restart the sequencer. */
2235                                 return (1);
2236                         }
2237                         sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
2238                         ahd_print_path(ahd, scb);
2239                         printf("SCB %d - Abort%s Completed.\n",
2240                                SCB_GET_TAG(scb),
2241                                sent_msg == MSG_ABORT_TAG ? "" : " Tag");
2242
2243                         if (sent_msg == MSG_ABORT_TAG)
2244                                 tag = SCB_GET_TAG(scb);
2245
2246                         if ((scb->flags & SCB_CMDPHASE_ABORT) != 0) {
2247                                 /*
2248                                  * This abort is in response to an
2249                                  * unexpected switch to command phase
2250                                  * for a packetized connection.  Since
2251                                  * the identify message was never sent,
2252                                  * "saved lun" is 0.  We really want to
2253                                  * abort only the SCB that encountered
2254                                  * this error, which could have a different
2255                                  * lun.  The SCB will be retried so the OS
2256                                  * will see the UA after renegotiating to
2257                                  * packetized.
2258                                  */
2259                                 tag = SCB_GET_TAG(scb);
2260                                 saved_lun = scb->hscb->lun;
2261                         }
2262                         found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
2263                                                tag, ROLE_INITIATOR,
2264                                                CAM_REQ_ABORTED);
2265                         printf("found == 0x%x\n", found);
2266                         printerror = 0;
2267                 } else if (ahd_sent_msg(ahd, AHDMSG_1B,
2268                                         MSG_BUS_DEV_RESET, TRUE)) {
2269 #ifdef __FreeBSD__
2270                         /*
2271                          * Don't mark the user's request for this BDR
2272                          * as completing with CAM_BDR_SENT.  CAM3
2273                          * specifies CAM_REQ_CMP.
2274                          */
2275                         if (scb != NULL
2276                          && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
2277                          && ahd_match_scb(ahd, scb, target, 'A',
2278                                           CAM_LUN_WILDCARD, SCB_LIST_NULL,
2279                                           ROLE_INITIATOR))
2280                                 aic_set_transaction_status(scb, CAM_REQ_CMP);
2281 #endif
2282                         ahd_handle_devreset(ahd, &devinfo, CAM_LUN_WILDCARD,
2283                                             CAM_BDR_SENT, "Bus Device Reset",
2284                                             /*verbose_level*/0);
2285                         printerror = 0;
2286                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, FALSE)
2287                         && ppr_busfree == 0) {
2288                         struct ahd_initiator_tinfo *tinfo;
2289                         struct ahd_tmode_tstate *tstate;
2290
2291                         /*
2292                          * PPR Rejected.
2293                          *
2294                          * If the previous negotiation was packetized,
2295                          * this could be because the device has been
2296                          * reset without our knowledge.  Force our
2297                          * current negotiation to async and retry the
2298                          * negotiation.  Otherwise retry the command
2299                          * with non-ppr negotiation.
2300                          */
2301 #ifdef AHD_DEBUG
2302                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2303                                 printf("PPR negotiation rejected busfree.\n");
2304 #endif
2305                         tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
2306                                                     devinfo.our_scsiid,
2307                                                     devinfo.target, &tstate);
2308                         if ((tinfo->curr.ppr_options & MSG_EXT_PPR_IU_REQ)!=0) {
2309                                 ahd_set_width(ahd, &devinfo,
2310                                               MSG_EXT_WDTR_BUS_8_BIT,
2311                                               AHD_TRANS_CUR,
2312                                               /*paused*/TRUE);
2313                                 ahd_set_syncrate(ahd, &devinfo,
2314                                                 /*period*/0, /*offset*/0,
2315                                                 /*ppr_options*/0,
2316                                                 AHD_TRANS_CUR,
2317                                                 /*paused*/TRUE);
2318                                 /*
2319                                  * The expect PPR busfree handler below
2320                                  * will effect the retry and necessary
2321                                  * abort.
2322                                  */
2323                         } else {
2324                                 tinfo->curr.transport_version = 2;
2325                                 tinfo->goal.transport_version = 2;
2326                                 tinfo->goal.ppr_options = 0;
2327                                 /*
2328                                  * Remove any SCBs in the waiting for selection
2329                                  * queue that may also be for this target so
2330                                  * that command ordering is preserved.
2331                                  */
2332                                 ahd_freeze_devq(ahd, scb);
2333                                 ahd_qinfifo_requeue_tail(ahd, scb);
2334                                 printerror = 0;
2335                         }
2336                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, FALSE)
2337                         && ppr_busfree == 0) {
2338                         /*
2339                          * Negotiation Rejected.  Go-narrow and
2340                          * retry command.
2341                          */
2342 #ifdef AHD_DEBUG
2343                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2344                                 printf("WDTR negotiation rejected busfree.\n");
2345 #endif
2346                         ahd_set_width(ahd, &devinfo,
2347                                       MSG_EXT_WDTR_BUS_8_BIT,
2348                                       AHD_TRANS_CUR|AHD_TRANS_GOAL,
2349                                       /*paused*/TRUE);
2350                         /*
2351                          * Remove any SCBs in the waiting for selection
2352                          * queue that may also be for this target so that
2353                          * command ordering is preserved.
2354                          */
2355                         ahd_freeze_devq(ahd, scb);
2356                         ahd_qinfifo_requeue_tail(ahd, scb);
2357                         printerror = 0;
2358                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, FALSE)
2359                         && ppr_busfree == 0) {
2360                         /*
2361                          * Negotiation Rejected.  Go-async and
2362                          * retry command.
2363                          */
2364 #ifdef AHD_DEBUG
2365                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2366                                 printf("SDTR negotiation rejected busfree.\n");
2367 #endif
2368                         ahd_set_syncrate(ahd, &devinfo,
2369                                         /*period*/0, /*offset*/0,
2370                                         /*ppr_options*/0,
2371                                         AHD_TRANS_CUR|AHD_TRANS_GOAL,
2372                                         /*paused*/TRUE);
2373                         /*
2374                          * Remove any SCBs in the waiting for selection
2375                          * queue that may also be for this target so that
2376                          * command ordering is preserved.
2377                          */
2378                         ahd_freeze_devq(ahd, scb);
2379                         ahd_qinfifo_requeue_tail(ahd, scb);
2380                         printerror = 0;
2381                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
2382                         && ahd_sent_msg(ahd, AHDMSG_1B,
2383                                          MSG_INITIATOR_DET_ERR, TRUE)) {
2384
2385 #ifdef AHD_DEBUG
2386                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2387                                 printf("Expected IDE Busfree\n");
2388 #endif
2389                         printerror = 0;
2390                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_QASREJ_BUSFREE)
2391                         && ahd_sent_msg(ahd, AHDMSG_1B,
2392                                         MSG_MESSAGE_REJECT, TRUE)) {
2393
2394 #ifdef AHD_DEBUG
2395                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2396                                 printf("Expected QAS Reject Busfree\n");
2397 #endif
2398                         printerror = 0;
2399                 }
2400         }
2401
2402         /*
2403          * The busfree required flag is honored at the end of
2404          * the message phases.  We check it last in case we
2405          * had to send some other message that caused a busfree.
2406          */
2407         if (printerror != 0
2408          && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
2409          && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
2410
2411                 ahd_freeze_devq(ahd, scb);
2412                 aic_set_transaction_status(scb, CAM_REQUEUE_REQ);
2413                 aic_freeze_scb(scb);
2414                 if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
2415                         ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
2416                                        SCB_GET_CHANNEL(ahd, scb),
2417                                        SCB_GET_LUN(scb), SCB_LIST_NULL,
2418                                        ROLE_INITIATOR, CAM_REQ_ABORTED);
2419                 } else {
2420 #ifdef AHD_DEBUG
2421                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2422                                 printf("PPR Negotiation Busfree.\n");
2423 #endif
2424                         ahd_done(ahd, scb);
2425                 }
2426                 printerror = 0;
2427         }
2428         if (printerror != 0) {
2429                 int aborted;
2430
2431                 aborted = 0;
2432                 if (scb != NULL) {
2433                         u_int tag;
2434
2435                         if ((scb->hscb->control & TAG_ENB) != 0)
2436                                 tag = SCB_GET_TAG(scb);
2437                         else
2438                                 tag = SCB_LIST_NULL;
2439                         ahd_print_path(ahd, scb);
2440                         aborted = ahd_abort_scbs(ahd, target, 'A',
2441                                        SCB_GET_LUN(scb), tag,
2442                                        ROLE_INITIATOR,
2443                                        CAM_UNEXP_BUSFREE);
2444                 } else {
2445                         /*
2446                          * We had not fully identified this connection,
2447                          * so we cannot abort anything.
2448                          */
2449                         printf("%s: ", ahd_name(ahd));
2450                 }
2451                 printf("Unexpected busfree %s, %d SCBs aborted, "
2452                        "PRGMCNT == 0x%x\n",
2453                        ahd_lookup_phase_entry(lastphase)->phasemsg,
2454                        aborted,
2455                        ahd_inw(ahd, PRGMCNT));
2456                 AHD_UNCORRECTABLE_ERROR(ahd);
2457                 ahd_dump_card_state(ahd);
2458                 if (lastphase != P_BUSFREE)
2459                         ahd_force_renegotiation(ahd, &devinfo);
2460         }
2461         /* Always restart the sequencer. */
2462         return (1);
2463 }
2464
2465 static void
2466 ahd_handle_proto_violation(struct ahd_softc *ahd)
2467 {
2468         struct  ahd_devinfo devinfo;
2469         struct  scb *scb;
2470         u_int   scbid;
2471         u_int   seq_flags;
2472         u_int   curphase;
2473         u_int   lastphase;
2474         int     found;
2475
2476         ahd_fetch_devinfo(ahd, &devinfo);
2477         scbid = ahd_get_scbptr(ahd);
2478         scb = ahd_lookup_scb(ahd, scbid);
2479         seq_flags = ahd_inb(ahd, SEQ_FLAGS);
2480         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2481         lastphase = ahd_inb(ahd, LASTPHASE);
2482         if ((seq_flags & NOT_IDENTIFIED) != 0) {
2483
2484                 /*
2485                  * The reconnecting target either did not send an
2486                  * identify message, or did, but we didn't find an SCB
2487                  * to match.
2488                  */
2489                 ahd_print_devinfo(ahd, &devinfo);
2490                 printf("Target did not send an IDENTIFY message. "
2491                        "LASTPHASE = 0x%x.\n", lastphase);
2492                 AHD_UNCORRECTABLE_ERROR(ahd);
2493                 scb = NULL;
2494         } else if (scb == NULL) {
2495                 /*
2496                  * We don't seem to have an SCB active for this
2497                  * transaction.  Print an error and reset the bus.
2498                  */
2499                 ahd_print_devinfo(ahd, &devinfo);
2500                 printf("No SCB found during protocol violation\n");
2501                 AHD_UNCORRECTABLE_ERROR(ahd);
2502                 goto proto_violation_reset;
2503         } else {
2504                 aic_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
2505                 if ((seq_flags & NO_CDB_SENT) != 0) {
2506                         ahd_print_path(ahd, scb);
2507                         printf("No or incomplete CDB sent to device.\n");
2508                         AHD_UNCORRECTABLE_ERROR(ahd);
2509                 } else if ((ahd_inb_scbram(ahd, SCB_CONTROL)
2510                           & STATUS_RCVD) == 0) {
2511                         /*
2512                          * The target never bothered to provide status to
2513                          * us prior to completing the command.  Since we don't
2514                          * know the disposition of this command, we must attempt
2515                          * to abort it.  Assert ATN and prepare to send an abort
2516                          * message.
2517                          */
2518                         ahd_print_path(ahd, scb);
2519                         printf("Completed command without status.\n");
2520                 } else {
2521                         ahd_print_path(ahd, scb);
2522                         printf("Unknown protocol violation.\n");
2523                         AHD_UNCORRECTABLE_ERROR(ahd);
2524                         ahd_dump_card_state(ahd);
2525                 }
2526         }
2527         if ((lastphase & ~P_DATAIN_DT) == 0
2528          || lastphase == P_COMMAND) {
2529 proto_violation_reset:
2530                 /*
2531                  * Target either went directly to data
2532                  * phase or didn't respond to our ATN.
2533                  * The only safe thing to do is to blow
2534                  * it away with a bus reset.
2535                  */
2536                 found = ahd_reset_channel(ahd, 'A', TRUE);
2537                 printf("%s: Issued Channel %c Bus Reset. "
2538                        "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
2539                 AHD_UNCORRECTABLE_ERROR(ahd);
2540         } else {
2541                 /*
2542                  * Leave the selection hardware off in case
2543                  * this abort attempt will affect yet to
2544                  * be sent commands.
2545                  */
2546                 ahd_outb(ahd, SCSISEQ0,
2547                          ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2548                 ahd_assert_atn(ahd);
2549                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
2550                 if (scb == NULL) {
2551                         ahd_print_devinfo(ahd, &devinfo);
2552                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
2553                         ahd->msgout_len = 1;
2554                         ahd->msgout_index = 0;
2555                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2556                 } else {
2557                         ahd_print_path(ahd, scb);
2558                         scb->flags |= SCB_ABORT;
2559                 }
2560                 printf("Protocol violation %s.  Attempting to abort.\n",
2561                        ahd_lookup_phase_entry(curphase)->phasemsg);
2562                 AHD_UNCORRECTABLE_ERROR(ahd);
2563         }
2564 }
2565
2566 /*
2567  * Force renegotiation to occur the next time we initiate
2568  * a command to the current device.
2569  */
2570 static void
2571 ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
2572 {
2573         struct  ahd_initiator_tinfo *targ_info;
2574         struct  ahd_tmode_tstate *tstate;
2575
2576 #ifdef AHD_DEBUG
2577         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
2578                 ahd_print_devinfo(ahd, devinfo);
2579                 printf("Forcing renegotiation\n");
2580         }
2581 #endif
2582         targ_info = ahd_fetch_transinfo(ahd,
2583                                         devinfo->channel,
2584                                         devinfo->our_scsiid,
2585                                         devinfo->target,
2586                                         &tstate);
2587         ahd_update_neg_request(ahd, devinfo, tstate,
2588                                targ_info, AHD_NEG_IF_NON_ASYNC);
2589 }
2590
2591 #define AHD_MAX_STEPS 2000
2592 void
2593 ahd_clear_critical_section(struct ahd_softc *ahd)
2594 {
2595         ahd_mode_state  saved_modes;
2596         int             stepping;
2597         int             steps;
2598         int             first_instr;
2599         u_int           simode0;
2600         u_int           simode1;
2601         u_int           simode3;
2602         u_int           lqimode0;
2603         u_int           lqimode1;
2604         u_int           lqomode0;
2605         u_int           lqomode1;
2606
2607         if (ahd->num_critical_sections == 0)
2608                 return;
2609
2610         stepping = FALSE;
2611         steps = 0;
2612         first_instr = 0;
2613         simode0 = 0;
2614         simode1 = 0;
2615         simode3 = 0;
2616         lqimode0 = 0;
2617         lqimode1 = 0;
2618         lqomode0 = 0;
2619         lqomode1 = 0;
2620         saved_modes = ahd_save_modes(ahd);
2621         for (;;) {
2622                 struct  cs *cs;
2623                 u_int   seqaddr;
2624                 u_int   i;
2625
2626                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2627                 seqaddr = ahd_inw(ahd, CURADDR);
2628
2629                 cs = ahd->critical_sections;
2630                 for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
2631
2632                         if (cs->begin < seqaddr && cs->end >= seqaddr)
2633                                 break;
2634                 }
2635
2636                 if (i == ahd->num_critical_sections)
2637                         break;
2638
2639                 if (steps > AHD_MAX_STEPS) {
2640                         printf("%s: Infinite loop in critical section\n"
2641                                "%s: First Instruction 0x%x now 0x%x\n",
2642                                ahd_name(ahd), ahd_name(ahd), first_instr,
2643                                seqaddr);
2644                         AHD_FATAL_ERROR(ahd);
2645                         ahd_dump_card_state(ahd);
2646                         panic("critical section loop");
2647                 }
2648
2649                 steps++;
2650 #ifdef AHD_DEBUG
2651                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
2652                         printf("%s: Single stepping at 0x%x\n", ahd_name(ahd),
2653                                seqaddr);
2654 #endif
2655                 if (stepping == FALSE) {
2656
2657                         first_instr = seqaddr;
2658                         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2659                         simode0 = ahd_inb(ahd, SIMODE0);
2660                         simode3 = ahd_inb(ahd, SIMODE3);
2661                         lqimode0 = ahd_inb(ahd, LQIMODE0);
2662                         lqimode1 = ahd_inb(ahd, LQIMODE1);
2663                         lqomode0 = ahd_inb(ahd, LQOMODE0);
2664                         lqomode1 = ahd_inb(ahd, LQOMODE1);
2665                         ahd_outb(ahd, SIMODE0, 0);
2666                         ahd_outb(ahd, SIMODE3, 0);
2667                         ahd_outb(ahd, LQIMODE0, 0);
2668                         ahd_outb(ahd, LQIMODE1, 0);
2669                         ahd_outb(ahd, LQOMODE0, 0);
2670                         ahd_outb(ahd, LQOMODE1, 0);
2671                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2672                         simode1 = ahd_inb(ahd, SIMODE1);
2673                         /*
2674                          * We don't clear ENBUSFREE.  Unfortunately
2675                          * we cannot re-enable busfree detection within
2676                          * the current connection, so we must leave it
2677                          * on while single stepping.
2678                          */
2679                         ahd_outb(ahd, SIMODE1, simode1 & ENBUSFREE);
2680                         ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
2681                         stepping = TRUE;
2682                 }
2683                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
2684                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2685                 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
2686                 ahd_outb(ahd, HCNTRL, ahd->unpause);
2687                 while (!ahd_is_paused(ahd))
2688                         aic_delay(200);
2689                 ahd_update_modes(ahd);
2690         }
2691         if (stepping) {
2692                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2693                 ahd_outb(ahd, SIMODE0, simode0);
2694                 ahd_outb(ahd, SIMODE3, simode3);
2695                 ahd_outb(ahd, LQIMODE0, lqimode0);
2696                 ahd_outb(ahd, LQIMODE1, lqimode1);
2697                 ahd_outb(ahd, LQOMODE0, lqomode0);
2698                 ahd_outb(ahd, LQOMODE1, lqomode1);
2699                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2700                 ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
2701                 ahd_outb(ahd, SIMODE1, simode1);
2702                 /*
2703                  * SCSIINT seems to glitch occassionally when
2704                  * the interrupt masks are restored.  Clear SCSIINT
2705                  * one more time so that only persistent errors
2706                  * are seen as a real interrupt.
2707                  */
2708                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2709         }
2710         ahd_restore_modes(ahd, saved_modes);
2711 }
2712
2713 /*
2714  * Clear any pending interrupt status.
2715  */
2716 void
2717 ahd_clear_intstat(struct ahd_softc *ahd)
2718 {
2719         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2720                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2721         /* Clear any interrupt conditions this may have caused */
2722         ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
2723                                  |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
2724         ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
2725                                  |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
2726                                  |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
2727         ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
2728                                  |CLRLQOATNPKT|CLRLQOTCRC);
2729         ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
2730                                  |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
2731         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
2732                 ahd_outb(ahd, CLRLQOINT0, 0);
2733                 ahd_outb(ahd, CLRLQOINT1, 0);
2734         }
2735         ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
2736         ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
2737                                 |CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
2738         ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO
2739                                 |CLRIOERR|CLROVERRUN);
2740         ahd_outb(ahd, CLRINT, CLRSCSIINT);
2741 }
2742
2743 /**************************** Debugging Routines ******************************/
2744 #ifdef AHD_DEBUG
2745 uint32_t ahd_debug = AHD_DEBUG_OPTS;
2746 #endif
2747 void
2748 ahd_print_scb(struct scb *scb)
2749 {
2750         struct hardware_scb *hscb;
2751         int i;
2752
2753         hscb = scb->hscb;
2754         printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
2755                (void *)scb,
2756                hscb->control,
2757                hscb->scsiid,
2758                hscb->lun,
2759                hscb->cdb_len);
2760         printf("Shared Data: ");
2761         for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
2762                 printf("%#02x", hscb->shared_data.idata.cdb[i]);
2763         printf("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
2764                (uint32_t)((aic_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
2765                (uint32_t)(aic_le64toh(hscb->dataptr) & 0xFFFFFFFF),
2766                aic_le32toh(hscb->datacnt),
2767                aic_le32toh(hscb->sgptr),
2768                SCB_GET_TAG(scb));
2769         ahd_dump_sglist(scb);
2770 }
2771
2772 void
2773 ahd_dump_sglist(struct scb *scb)
2774 {
2775         int i;
2776
2777         if (scb->sg_count > 0) {
2778                 if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
2779                         struct ahd_dma64_seg *sg_list;
2780
2781                         sg_list = (struct ahd_dma64_seg*)scb->sg_list;
2782                         for (i = 0; i < scb->sg_count; i++) {
2783                                 uint64_t addr;
2784                                 uint32_t len;
2785
2786                                 addr = aic_le64toh(sg_list[i].addr);
2787                                 len = aic_le32toh(sg_list[i].len);
2788                                 printf("sg[%d] - Addr 0x%x%x : Length %d%s\n",
2789                                        i,
2790                                        (uint32_t)((addr >> 32) & 0xFFFFFFFF),
2791                                        (uint32_t)(addr & 0xFFFFFFFF),
2792                                        sg_list[i].len & AHD_SG_LEN_MASK,
2793                                        (sg_list[i].len & AHD_DMA_LAST_SEG)
2794                                      ? " Last" : "");
2795                         }
2796                 } else {
2797                         struct ahd_dma_seg *sg_list;
2798
2799                         sg_list = (struct ahd_dma_seg*)scb->sg_list;
2800                         for (i = 0; i < scb->sg_count; i++) {
2801                                 uint32_t len;
2802
2803                                 len = aic_le32toh(sg_list[i].len);
2804                                 printf("sg[%d] - Addr 0x%x%x : Length %d%s\n",
2805                                        i,
2806                                        (len & AHD_SG_HIGH_ADDR_MASK) >> 24,
2807                                        aic_le32toh(sg_list[i].addr),
2808                                        len & AHD_SG_LEN_MASK,
2809                                        len & AHD_DMA_LAST_SEG ? " Last" : "");
2810                         }
2811                 }
2812         }
2813 }
2814
2815 /************************* Transfer Negotiation *******************************/
2816 /*
2817  * Allocate per target mode instance (ID we respond to as a target)
2818  * transfer negotiation data structures.
2819  */
2820 static struct ahd_tmode_tstate *
2821 ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
2822 {
2823         struct ahd_tmode_tstate *master_tstate;
2824         struct ahd_tmode_tstate *tstate;
2825         int i;
2826
2827         master_tstate = ahd->enabled_targets[ahd->our_id];
2828         if (ahd->enabled_targets[scsi_id] != NULL
2829          && ahd->enabled_targets[scsi_id] != master_tstate)
2830                 panic("%s: ahd_alloc_tstate - Target already allocated",
2831                       ahd_name(ahd));
2832         tstate = malloc(sizeof(*tstate), M_DEVBUF, M_NOWAIT);
2833         if (tstate == NULL)
2834                 return (NULL);
2835
2836         /*
2837          * If we have allocated a master tstate, copy user settings from
2838          * the master tstate (taken from SRAM or the EEPROM) for this
2839          * channel, but reset our current and goal settings to async/narrow
2840          * until an initiator talks to us.
2841          */
2842         if (master_tstate != NULL) {
2843                 memcpy(tstate, master_tstate, sizeof(*tstate));
2844                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
2845                 for (i = 0; i < 16; i++) {
2846                         memset(&tstate->transinfo[i].curr, 0,
2847                               sizeof(tstate->transinfo[i].curr));
2848                         memset(&tstate->transinfo[i].goal, 0,
2849                               sizeof(tstate->transinfo[i].goal));
2850                 }
2851         } else
2852                 memset(tstate, 0, sizeof(*tstate));
2853         ahd->enabled_targets[scsi_id] = tstate;
2854         return (tstate);
2855 }
2856
2857 #ifdef AHD_TARGET_MODE
2858 /*
2859  * Free per target mode instance (ID we respond to as a target)
2860  * transfer negotiation data structures.
2861  */
2862 static void
2863 ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
2864 {
2865         struct ahd_tmode_tstate *tstate;
2866
2867         /*
2868          * Don't clean up our "master" tstate.
2869          * It has our default user settings.
2870          */
2871         if (scsi_id == ahd->our_id
2872          && force == FALSE)
2873                 return;
2874
2875         tstate = ahd->enabled_targets[scsi_id];
2876         if (tstate != NULL)
2877                 free(tstate, M_DEVBUF);
2878         ahd->enabled_targets[scsi_id] = NULL;
2879 }
2880 #endif
2881
2882 /*
2883  * Called when we have an active connection to a target on the bus,
2884  * this function finds the nearest period to the input period limited
2885  * by the capabilities of the bus connectivity of and sync settings for
2886  * the target.
2887  */
2888 void
2889 ahd_devlimited_syncrate(struct ahd_softc *ahd,
2890                         struct ahd_initiator_tinfo *tinfo,
2891                         u_int *period, u_int *ppr_options, role_t role)
2892 {
2893         struct  ahd_transinfo *transinfo;
2894         u_int   maxsync;
2895
2896         if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
2897          && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
2898                 maxsync = AHD_SYNCRATE_PACED;
2899         } else {
2900                 maxsync = AHD_SYNCRATE_ULTRA;
2901                 /* Can't do DT related options on an SE bus */
2902                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
2903         }
2904         /*
2905          * Never allow a value higher than our current goal
2906          * period otherwise we may allow a target initiated
2907          * negotiation to go above the limit as set by the
2908          * user.  In the case of an initiator initiated
2909          * sync negotiation, we limit based on the user
2910          * setting.  This allows the system to still accept
2911          * incoming negotiations even if target initiated
2912          * negotiation is not performed.
2913          */
2914         if (role == ROLE_TARGET)
2915                 transinfo = &tinfo->user;
2916         else
2917                 transinfo = &tinfo->goal;
2918         *ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
2919         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
2920                 maxsync = MAX(maxsync, AHD_SYNCRATE_ULTRA2);
2921                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2922         }
2923         if (transinfo->period == 0) {
2924                 *period = 0;
2925                 *ppr_options = 0;
2926         } else {
2927                 *period = MAX(*period, transinfo->period);
2928                 ahd_find_syncrate(ahd, period, ppr_options, maxsync);
2929         }
2930 }
2931
2932 /*
2933  * Look up the valid period to SCSIRATE conversion in our table.
2934  * Return the period and offset that should be sent to the target
2935  * if this was the beginning of an SDTR.
2936  */
2937 void
2938 ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
2939                   u_int *ppr_options, u_int maxsync)
2940 {
2941         if (*period < maxsync)
2942                 *period = maxsync;
2943
2944         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
2945          && *period > AHD_SYNCRATE_MIN_DT)
2946                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2947
2948         if (*period > AHD_SYNCRATE_MIN)
2949                 *period = 0;
2950
2951         /* Honor PPR option conformance rules. */
2952         if (*period > AHD_SYNCRATE_PACED)
2953                 *ppr_options &= ~MSG_EXT_PPR_RTI;
2954
2955         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
2956                 *ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
2957
2958         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
2959                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
2960
2961         /* Skip all PACED only entries if IU is not available */
2962         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
2963          && *period < AHD_SYNCRATE_DT)
2964                 *period = AHD_SYNCRATE_DT;
2965
2966         /* Skip all DT only entries if DT is not available */
2967         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
2968          && *period < AHD_SYNCRATE_ULTRA2)
2969                 *period = AHD_SYNCRATE_ULTRA2;
2970 }
2971
2972 /*
2973  * Truncate the given synchronous offset to a value the
2974  * current adapter type and syncrate are capable of.
2975  */
2976 void
2977 ahd_validate_offset(struct ahd_softc *ahd,
2978                     struct ahd_initiator_tinfo *tinfo,
2979                     u_int period, u_int *offset, int wide,
2980                     role_t role)
2981 {
2982         u_int maxoffset;
2983
2984         /* Limit offset to what we can do */
2985         if (period == 0)
2986                 maxoffset = 0;
2987         else if (period <= AHD_SYNCRATE_PACED) {
2988                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0)
2989                         maxoffset = MAX_OFFSET_PACED_BUG;
2990                 else
2991                         maxoffset = MAX_OFFSET_PACED;
2992         } else
2993                 maxoffset = MAX_OFFSET_NON_PACED;
2994         *offset = MIN(*offset, maxoffset);
2995         if (tinfo != NULL) {
2996                 if (role == ROLE_TARGET)
2997                         *offset = MIN(*offset, tinfo->user.offset);
2998                 else
2999                         *offset = MIN(*offset, tinfo->goal.offset);
3000         }
3001 }
3002
3003 /*
3004  * Truncate the given transfer width parameter to a value the
3005  * current adapter type is capable of.
3006  */
3007 void
3008 ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
3009                    u_int *bus_width, role_t role)
3010 {
3011         switch (*bus_width) {
3012         default:
3013                 if (ahd->features & AHD_WIDE) {
3014                         /* Respond Wide */
3015                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
3016                         break;
3017                 }
3018                 /* FALLTHROUGH */
3019         case MSG_EXT_WDTR_BUS_8_BIT:
3020                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
3021                 break;
3022         }
3023         if (tinfo != NULL) {
3024                 if (role == ROLE_TARGET)
3025                         *bus_width = MIN(tinfo->user.width, *bus_width);
3026                 else
3027                         *bus_width = MIN(tinfo->goal.width, *bus_width);
3028         }
3029 }
3030
3031 /*
3032  * Update the bitmask of targets for which the controller should
3033  * negotiate with at the next convenient oportunity.  This currently
3034  * means the next time we send the initial identify messages for
3035  * a new transaction.
3036  */
3037 int
3038 ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3039                        struct ahd_tmode_tstate *tstate,
3040                        struct ahd_initiator_tinfo *tinfo, ahd_neg_type neg_type)
3041 {
3042         u_int auto_negotiate_orig;
3043
3044         auto_negotiate_orig = tstate->auto_negotiate;
3045         if (neg_type == AHD_NEG_ALWAYS) {
3046                 /*
3047                  * Force our "current" settings to be
3048                  * unknown so that unless a bus reset
3049                  * occurs the need to renegotiate is
3050                  * recorded persistently.
3051                  */
3052                 if ((ahd->features & AHD_WIDE) != 0)
3053                         tinfo->curr.width = AHD_WIDTH_UNKNOWN;
3054                 tinfo->curr.period = AHD_PERIOD_UNKNOWN;
3055                 tinfo->curr.offset = AHD_OFFSET_UNKNOWN;
3056         }
3057         if (tinfo->curr.period != tinfo->goal.period
3058          || tinfo->curr.width != tinfo->goal.width
3059          || tinfo->curr.offset != tinfo->goal.offset
3060          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
3061          || (neg_type == AHD_NEG_IF_NON_ASYNC
3062           && (tinfo->goal.offset != 0
3063            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
3064            || tinfo->goal.ppr_options != 0)))
3065                 tstate->auto_negotiate |= devinfo->target_mask;
3066         else
3067                 tstate->auto_negotiate &= ~devinfo->target_mask;
3068
3069         return (auto_negotiate_orig != tstate->auto_negotiate);
3070 }
3071
3072 /*
3073  * Update the user/goal/curr tables of synchronous negotiation
3074  * parameters as well as, in the case of a current or active update,
3075  * any data structures on the host controller.  In the case of an
3076  * active update, the specified target is currently talking to us on
3077  * the bus, so the transfer parameter update must take effect
3078  * immediately.
3079  */
3080 void
3081 ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3082                  u_int period, u_int offset, u_int ppr_options,
3083                  u_int type, int paused)
3084 {
3085         struct  ahd_initiator_tinfo *tinfo;
3086         struct  ahd_tmode_tstate *tstate;
3087         u_int   old_period;
3088         u_int   old_offset;
3089         u_int   old_ppr;
3090         int     active;
3091         int     update_needed;
3092
3093         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3094         update_needed = 0;
3095
3096         if (period == 0 || offset == 0) {
3097                 period = 0;
3098                 offset = 0;
3099         }
3100
3101         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3102                                     devinfo->target, &tstate);
3103
3104         if ((type & AHD_TRANS_USER) != 0) {
3105                 tinfo->user.period = period;
3106                 tinfo->user.offset = offset;
3107                 tinfo->user.ppr_options = ppr_options;
3108         }
3109
3110         if ((type & AHD_TRANS_GOAL) != 0) {
3111                 tinfo->goal.period = period;
3112                 tinfo->goal.offset = offset;
3113                 tinfo->goal.ppr_options = ppr_options;
3114         }
3115
3116         old_period = tinfo->curr.period;
3117         old_offset = tinfo->curr.offset;
3118         old_ppr    = tinfo->curr.ppr_options;
3119
3120         if ((type & AHD_TRANS_CUR) != 0
3121          && (old_period != period
3122           || old_offset != offset
3123           || old_ppr != ppr_options)) {
3124
3125                 update_needed++;
3126
3127                 tinfo->curr.period = period;
3128                 tinfo->curr.offset = offset;
3129                 tinfo->curr.ppr_options = ppr_options;
3130
3131                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
3132                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
3133                 if (bootverbose) {
3134                         if (offset != 0) {
3135                                 int options;
3136
3137                                 printf("%s: target %d synchronous with "
3138                                        "period = 0x%x, offset = 0x%x",
3139                                        ahd_name(ahd), devinfo->target,
3140                                        period, offset);
3141                                 options = 0;
3142                                 if ((ppr_options & MSG_EXT_PPR_RD_STRM) != 0) {
3143                                         printf("(RDSTRM");
3144                                         options++;
3145                                 }
3146                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0) {
3147                                         printf("%s", options ? "|DT" : "(DT");
3148                                         options++;
3149                                 }
3150                                 if ((ppr_options & MSG_EXT_PPR_IU_REQ) != 0) {
3151                                         printf("%s", options ? "|IU" : "(IU");
3152                                         options++;
3153                                 }
3154                                 if ((ppr_options & MSG_EXT_PPR_RTI) != 0) {
3155                                         printf("%s", options ? "|RTI" : "(RTI");
3156                                         options++;
3157                                 }
3158                                 if ((ppr_options & MSG_EXT_PPR_QAS_REQ) != 0) {
3159                                         printf("%s", options ? "|QAS" : "(QAS");
3160                                         options++;
3161                                 }
3162                                 if (options != 0)
3163                                         printf(")\n");
3164                                 else
3165                                         printf("\n");
3166                         } else {
3167                                 printf("%s: target %d using "
3168                                        "asynchronous transfers%s\n",
3169                                        ahd_name(ahd), devinfo->target,
3170                                        (ppr_options & MSG_EXT_PPR_QAS_REQ) != 0
3171                                      ?  "(QAS)" : "");
3172                         }
3173                 }
3174         }
3175         /*
3176          * Always refresh the neg-table to handle the case of the
3177          * sequencer setting the ENATNO bit for a MK_MESSAGE request.
3178          * We will always renegotiate in that case if this is a
3179          * packetized request.  Also manage the busfree expected flag
3180          * from this common routine so that we catch changes due to
3181          * WDTR or SDTR messages.
3182          */
3183         if ((type & AHD_TRANS_CUR) != 0) {
3184                 if (!paused)
3185                         ahd_pause(ahd);
3186                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3187                 if (!paused)
3188                         ahd_unpause(ahd);
3189                 if (ahd->msg_type != MSG_TYPE_NONE) {
3190                         if ((old_ppr & MSG_EXT_PPR_IU_REQ)
3191                          != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
3192 #ifdef AHD_DEBUG
3193                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3194                                         ahd_print_devinfo(ahd, devinfo);
3195                                         printf("Expecting IU Change busfree\n");
3196                                 }
3197 #endif
3198                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
3199                                                |  MSG_FLAG_IU_REQ_CHANGED;
3200                         }
3201                         if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
3202 #ifdef AHD_DEBUG
3203                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3204                                         printf("PPR with IU_REQ outstanding\n");
3205 #endif
3206                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
3207                         }
3208                 }
3209         }
3210
3211         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3212                                                 tinfo, AHD_NEG_TO_GOAL);
3213
3214         if (update_needed && active)
3215                 ahd_update_pending_scbs(ahd);
3216 }
3217
3218 /*
3219  * Update the user/goal/curr tables of wide negotiation
3220  * parameters as well as, in the case of a current or active update,
3221  * any data structures on the host controller.  In the case of an
3222  * active update, the specified target is currently talking to us on
3223  * the bus, so the transfer parameter update must take effect
3224  * immediately.
3225  */
3226 void
3227 ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3228               u_int width, u_int type, int paused)
3229 {
3230         struct  ahd_initiator_tinfo *tinfo;
3231         struct  ahd_tmode_tstate *tstate;
3232         u_int   oldwidth;
3233         int     active;
3234         int     update_needed;
3235
3236         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3237         update_needed = 0;
3238         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3239                                     devinfo->target, &tstate);
3240
3241         if ((type & AHD_TRANS_USER) != 0)
3242                 tinfo->user.width = width;
3243
3244         if ((type & AHD_TRANS_GOAL) != 0)
3245                 tinfo->goal.width = width;
3246
3247         oldwidth = tinfo->curr.width;
3248         if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
3249
3250                 update_needed++;
3251
3252                 tinfo->curr.width = width;
3253                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
3254                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
3255                 if (bootverbose) {
3256                         printf("%s: target %d using %dbit transfers\n",
3257                                ahd_name(ahd), devinfo->target,
3258                                8 * (0x01 << width));
3259                 }
3260         }
3261
3262         if ((type & AHD_TRANS_CUR) != 0) {
3263                 if (!paused)
3264                         ahd_pause(ahd);
3265                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3266                 if (!paused)
3267                         ahd_unpause(ahd);
3268         }
3269
3270         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3271                                                 tinfo, AHD_NEG_TO_GOAL);
3272         if (update_needed && active)
3273                 ahd_update_pending_scbs(ahd);
3274
3275 }
3276
3277 /*
3278  * Update the current state of tagged queuing for a given target.
3279  */
3280 void
3281 ahd_set_tags(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3282              ahd_queue_alg alg)
3283 {
3284         ahd_platform_set_tags(ahd, devinfo, alg);
3285         ahd_send_async(ahd, devinfo->channel, devinfo->target,
3286                        devinfo->lun, AC_TRANSFER_NEG, &alg);
3287 }
3288
3289 static void
3290 ahd_update_neg_table(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3291                      struct ahd_transinfo *tinfo)
3292 {
3293         ahd_mode_state  saved_modes;
3294         u_int           period;
3295         u_int           ppr_opts;
3296         u_int           con_opts;
3297         u_int           offset;
3298         u_int           saved_negoaddr;
3299         uint8_t         iocell_opts[sizeof(ahd->iocell_opts)];
3300
3301         saved_modes = ahd_save_modes(ahd);
3302         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3303
3304         saved_negoaddr = ahd_inb(ahd, NEGOADDR);
3305         ahd_outb(ahd, NEGOADDR, devinfo->target);
3306         period = tinfo->period;
3307         offset = tinfo->offset;
3308         memcpy(iocell_opts, ahd->iocell_opts, sizeof(ahd->iocell_opts));
3309         ppr_opts = tinfo->ppr_options & (MSG_EXT_PPR_QAS_REQ|MSG_EXT_PPR_DT_REQ
3310                                         |MSG_EXT_PPR_IU_REQ|MSG_EXT_PPR_RTI);
3311         con_opts = 0;
3312         if (period == 0)
3313                 period = AHD_SYNCRATE_ASYNC;
3314         if (period == AHD_SYNCRATE_160) {
3315
3316                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
3317                         /*
3318                          * When the SPI4 spec was finalized, PACE transfers
3319                          * was not made a configurable option in the PPR
3320                          * message.  Instead it is assumed to be enabled for
3321                          * any syncrate faster than 80MHz.  Nevertheless,
3322                          * Harpoon2A4 allows this to be configurable.
3323                          *
3324                          * Harpoon2A4 also assumes at most 2 data bytes per
3325                          * negotiated REQ/ACK offset.  Paced transfers take
3326                          * 4, so we must adjust our offset.
3327                          */
3328                         ppr_opts |= PPROPT_PACE;
3329                         offset *= 2;
3330
3331                         /*
3332                          * Harpoon2A assumed that there would be a
3333                          * fallback rate between 160MHz and 80Mhz,
3334                          * so 7 is used as the period factor rather
3335                          * than 8 for 160MHz.
3336                          */
3337                         period = AHD_SYNCRATE_REVA_160;
3338                 }
3339                 if ((tinfo->ppr_options & MSG_EXT_PPR_PCOMP_EN) == 0)
3340                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
3341                             ~AHD_PRECOMP_MASK;
3342         } else {
3343                 /*
3344                  * Precomp should be disabled for non-paced transfers.
3345                  */
3346                 iocell_opts[AHD_PRECOMP_SLEW_INDEX] &= ~AHD_PRECOMP_MASK;
3347
3348                 if ((ahd->features & AHD_NEW_IOCELL_OPTS) != 0
3349                  && (ppr_opts & MSG_EXT_PPR_DT_REQ) != 0
3350                  && (ppr_opts & MSG_EXT_PPR_IU_REQ) == 0) {
3351                         /*
3352                          * Slow down our CRC interval to be
3353                          * compatible with non-packetized
3354                          * U160 devices that can't handle a
3355                          * CRC at full speed.
3356                          */
3357                         con_opts |= ENSLOWCRC;
3358                 }
3359
3360                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
3361                         /*
3362                          * On H2A4, revert to a slower slewrate
3363                          * on non-paced transfers.
3364                          */
3365                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
3366                             ~AHD_SLEWRATE_MASK;
3367                 }
3368         }
3369
3370         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP_SLEW);
3371         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_PRECOMP_SLEW_INDEX]);
3372         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_AMPLITUDE);
3373         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_AMPLITUDE_INDEX]);
3374
3375         ahd_outb(ahd, NEGPERIOD, period);
3376         ahd_outb(ahd, NEGPPROPTS, ppr_opts);
3377         ahd_outb(ahd, NEGOFFSET, offset);
3378
3379         if (tinfo->width == MSG_EXT_WDTR_BUS_16_BIT)
3380                 con_opts |= WIDEXFER;
3381
3382         /*
3383          * During packetized transfers, the target will
3384          * give us the oportunity to send command packets
3385          * without us asserting attention.
3386          */
3387         if ((tinfo->ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
3388                 con_opts |= ENAUTOATNO;
3389         ahd_outb(ahd, NEGCONOPTS, con_opts);
3390         ahd_outb(ahd, NEGOADDR, saved_negoaddr);
3391         ahd_restore_modes(ahd, saved_modes);
3392 }
3393
3394 /*
3395  * When the transfer settings for a connection change, setup for
3396  * negotiation in pending SCBs to effect the change as quickly as
3397  * possible.  We also cancel any negotiations that are scheduled
3398  * for inflight SCBs that have not been started yet.
3399  */
3400 static void
3401 ahd_update_pending_scbs(struct ahd_softc *ahd)
3402 {
3403         struct          scb *pending_scb;
3404         int             pending_scb_count;
3405         int             paused;
3406         u_int           saved_scbptr;
3407         ahd_mode_state  saved_modes;
3408
3409         /*
3410          * Traverse the pending SCB list and ensure that all of the
3411          * SCBs there have the proper settings.  We can only safely
3412          * clear the negotiation required flag (setting requires the
3413          * execution queue to be modified) and this is only possible
3414          * if we are not already attempting to select out for this
3415          * SCB.  For this reason, all callers only call this routine
3416          * if we are changing the negotiation settings for the currently
3417          * active transaction on the bus.
3418          */
3419         pending_scb_count = 0;
3420         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
3421                 struct ahd_devinfo devinfo;
3422                 struct ahd_initiator_tinfo *tinfo;
3423                 struct ahd_tmode_tstate *tstate;
3424
3425                 ahd_scb_devinfo(ahd, &devinfo, pending_scb);
3426                 tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
3427                                             devinfo.our_scsiid,
3428                                             devinfo.target, &tstate);
3429                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
3430                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
3431                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
3432                         pending_scb->hscb->control &= ~MK_MESSAGE;
3433                 }
3434                 ahd_sync_scb(ahd, pending_scb,
3435                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
3436                 pending_scb_count++;
3437         }
3438
3439         if (pending_scb_count == 0)
3440                 return;
3441
3442         if (ahd_is_paused(ahd)) {
3443                 paused = 1;
3444         } else {
3445                 paused = 0;
3446                 ahd_pause(ahd);
3447         }
3448
3449         /*
3450          * Force the sequencer to reinitialize the selection for
3451          * the command at the head of the execution queue if it
3452          * has already been setup.  The negotiation changes may
3453          * effect whether we select-out with ATN.  It is only
3454          * safe to clear ENSELO when the bus is not free and no
3455          * selection is in progres or completed.
3456          */
3457         saved_modes = ahd_save_modes(ahd);
3458         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3459         if ((ahd_inb(ahd, SCSISIGI) & BSYI) != 0
3460          && (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) == 0)
3461                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
3462         saved_scbptr = ahd_get_scbptr(ahd);
3463         /* Ensure that the hscbs down on the card match the new information */
3464         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
3465                 u_int   scb_tag;
3466                 u_int   control;
3467
3468                 scb_tag = SCB_GET_TAG(pending_scb);
3469                 ahd_set_scbptr(ahd, scb_tag);
3470                 control = ahd_inb_scbram(ahd, SCB_CONTROL);
3471                 control &= ~MK_MESSAGE;
3472                 control |= pending_scb->hscb->control & MK_MESSAGE;
3473                 ahd_outb(ahd, SCB_CONTROL, control);
3474         }
3475         ahd_set_scbptr(ahd, saved_scbptr);
3476         ahd_restore_modes(ahd, saved_modes);
3477
3478         if (paused == 0)
3479                 ahd_unpause(ahd);
3480 }
3481
3482 /**************************** Pathing Information *****************************/
3483 static void
3484 ahd_fetch_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3485 {
3486         ahd_mode_state  saved_modes;
3487         u_int           saved_scsiid;
3488         role_t          role;
3489         int             our_id;
3490
3491         saved_modes = ahd_save_modes(ahd);
3492         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3493
3494         if (ahd_inb(ahd, SSTAT0) & TARGET)
3495                 role = ROLE_TARGET;
3496         else
3497                 role = ROLE_INITIATOR;
3498
3499         if (role == ROLE_TARGET
3500          && (ahd_inb(ahd, SEQ_FLAGS) & CMDPHASE_PENDING) != 0) {
3501                 /* We were selected, so pull our id from TARGIDIN */
3502                 our_id = ahd_inb(ahd, TARGIDIN) & OID;
3503         } else if (role == ROLE_TARGET)
3504                 our_id = ahd_inb(ahd, TOWNID);
3505         else
3506                 our_id = ahd_inb(ahd, IOWNID);
3507
3508         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
3509         ahd_compile_devinfo(devinfo,
3510                             our_id,
3511                             SCSIID_TARGET(ahd, saved_scsiid),
3512                             ahd_inb(ahd, SAVED_LUN),
3513                             SCSIID_CHANNEL(ahd, saved_scsiid),
3514                             role);
3515         ahd_restore_modes(ahd, saved_modes);
3516 }
3517
3518 void
3519 ahd_print_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3520 {
3521         printf("%s:%c:%d:%d: ", ahd_name(ahd), 'A',
3522                devinfo->target, devinfo->lun);
3523 }
3524
3525 struct ahd_phase_table_entry*
3526 ahd_lookup_phase_entry(int phase)
3527 {
3528         struct ahd_phase_table_entry *entry;
3529         struct ahd_phase_table_entry *last_entry;
3530
3531         /*
3532          * num_phases doesn't include the default entry which
3533          * will be returned if the phase doesn't match.
3534          */
3535         last_entry = &ahd_phase_table[num_phases];
3536         for (entry = ahd_phase_table; entry < last_entry; entry++) {
3537                 if (phase == entry->phase)
3538                         break;
3539         }
3540         return (entry);
3541 }
3542
3543 void
3544 ahd_compile_devinfo(struct ahd_devinfo *devinfo, u_int our_id, u_int target,
3545                     u_int lun, char channel, role_t role)
3546 {
3547         devinfo->our_scsiid = our_id;
3548         devinfo->target = target;
3549         devinfo->lun = lun;
3550         devinfo->target_offset = target;
3551         devinfo->channel = channel;
3552         devinfo->role = role;
3553         if (channel == 'B')
3554                 devinfo->target_offset += 8;
3555         devinfo->target_mask = (0x01 << devinfo->target_offset);
3556 }
3557
3558 static void
3559 ahd_scb_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3560                 struct scb *scb)
3561 {
3562         role_t  role;
3563         int     our_id;
3564
3565         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
3566         role = ROLE_INITIATOR;
3567         if ((scb->hscb->control & TARGET_SCB) != 0)
3568                 role = ROLE_TARGET;
3569         ahd_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahd, scb),
3570                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahd, scb), role);
3571 }
3572
3573
3574 /************************ Message Phase Processing ****************************/
3575 /*
3576  * When an initiator transaction with the MK_MESSAGE flag either reconnects
3577  * or enters the initial message out phase, we are interrupted.  Fill our
3578  * outgoing message buffer with the appropriate message and beging handing
3579  * the message phase(s) manually.
3580  */
3581 static void
3582 ahd_setup_initiator_msgout(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3583                            struct scb *scb)
3584 {
3585         /*
3586          * To facilitate adding multiple messages together,
3587          * each routine should increment the index and len
3588          * variables instead of setting them explicitly.
3589          */
3590         ahd->msgout_index = 0;
3591         ahd->msgout_len = 0;
3592
3593         if (ahd_currently_packetized(ahd))
3594                 ahd->msg_flags |= MSG_FLAG_PACKETIZED;
3595
3596         if (ahd->send_msg_perror
3597          && ahd_inb(ahd, MSG_OUT) == HOST_MSG) {
3598                 ahd->msgout_buf[ahd->msgout_index++] = ahd->send_msg_perror;
3599                 ahd->msgout_len++;
3600                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3601 #ifdef AHD_DEBUG
3602                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3603                         printf("Setting up for Parity Error delivery\n");
3604 #endif
3605                 return;
3606         } else if (scb == NULL) {
3607                 printf("%s: WARNING. No pending message for "
3608                        "I_T msgin.  Issuing NO-OP\n", ahd_name(ahd));
3609                 AHD_CORRECTABLE_ERROR(ahd);
3610                 ahd->msgout_buf[ahd->msgout_index++] = MSG_NOOP;
3611                 ahd->msgout_len++;
3612                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3613                 return;
3614         }
3615
3616         if ((scb->flags & SCB_DEVICE_RESET) == 0
3617          && (scb->flags & SCB_PACKETIZED) == 0
3618          && ahd_inb(ahd, MSG_OUT) == MSG_IDENTIFYFLAG) {
3619                 u_int identify_msg;
3620
3621                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
3622                 if ((scb->hscb->control & DISCENB) != 0)
3623                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
3624                 ahd->msgout_buf[ahd->msgout_index++] = identify_msg;
3625                 ahd->msgout_len++;
3626
3627                 if ((scb->hscb->control & TAG_ENB) != 0) {
3628                         ahd->msgout_buf[ahd->msgout_index++] =
3629                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
3630                         ahd->msgout_buf[ahd->msgout_index++] = SCB_GET_TAG(scb);
3631                         ahd->msgout_len += 2;
3632                 }
3633         }
3634
3635         if (scb->flags & SCB_DEVICE_RESET) {
3636                 ahd->msgout_buf[ahd->msgout_index++] = MSG_BUS_DEV_RESET;
3637                 ahd->msgout_len++;
3638                 ahd_print_path(ahd, scb);
3639                 printf("Bus Device Reset Message Sent\n");
3640                 AHD_CORRECTABLE_ERROR(ahd);
3641                 /*
3642                  * Clear our selection hardware in advance of
3643                  * the busfree.  We may have an entry in the waiting
3644                  * Q for this target, and we don't want to go about
3645                  * selecting while we handle the busfree and blow it
3646                  * away.
3647                  */
3648                 ahd_outb(ahd, SCSISEQ0, 0);
3649         } else if ((scb->flags & SCB_ABORT) != 0) {
3650
3651                 if ((scb->hscb->control & TAG_ENB) != 0) {
3652                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT_TAG;
3653                 } else {
3654                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT;
3655                 }
3656                 ahd->msgout_len++;
3657                 ahd_print_path(ahd, scb);
3658                 printf("Abort%s Message Sent\n",
3659                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
3660                 AHD_CORRECTABLE_ERROR(ahd);
3661                 /*
3662                  * Clear our selection hardware in advance of
3663                  * the busfree.  We may have an entry in the waiting
3664                  * Q for this target, and we don't want to go about
3665                  * selecting while we handle the busfree and blow it
3666                  * away.
3667                  */
3668                 ahd_outb(ahd, SCSISEQ0, 0);
3669         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
3670                 ahd_build_transfer_msg(ahd, devinfo);
3671                 /*
3672                  * Clear our selection hardware in advance of potential
3673                  * PPR IU status change busfree.  We may have an entry in
3674                  * the waiting Q for this target, and we don't want to go
3675                  * about selecting while we handle the busfree and blow
3676                  * it away.
3677                  */
3678                 ahd_outb(ahd, SCSISEQ0, 0);
3679         } else {
3680                 printf("ahd_intr: AWAITING_MSG for an SCB that "
3681                        "does not have a waiting message\n");
3682                 printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
3683                        devinfo->target_mask);
3684                 AHD_FATAL_ERROR(ahd);
3685                 panic("SCB = %d, SCB Control = %x:%x, MSG_OUT = %x "
3686                       "SCB flags = %x", SCB_GET_TAG(scb), scb->hscb->control,
3687                       ahd_inb_scbram(ahd, SCB_CONTROL), ahd_inb(ahd, MSG_OUT),
3688                       scb->flags);
3689         }
3690
3691         /*
3692          * Clear the MK_MESSAGE flag from the SCB so we aren't
3693          * asked to send this message again.
3694          */
3695         ahd_outb(ahd, SCB_CONTROL,
3696                  ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
3697         scb->hscb->control &= ~MK_MESSAGE;
3698         ahd->msgout_index = 0;
3699         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3700 }
3701
3702 /*
3703  * Build an appropriate transfer negotiation message for the
3704  * currently active target.
3705  */
3706 static void
3707 ahd_build_transfer_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3708 {
3709         /*
3710          * We need to initiate transfer negotiations.
3711          * If our current and goal settings are identical,
3712          * we want to renegotiate due to a check condition.
3713          */
3714         struct  ahd_initiator_tinfo *tinfo;
3715         struct  ahd_tmode_tstate *tstate;
3716         int     dowide;
3717         int     dosync;
3718         int     doppr;
3719         u_int   period;
3720         u_int   ppr_options;
3721         u_int   offset;
3722
3723         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3724                                     devinfo->target, &tstate);
3725         /*
3726          * Filter our period based on the current connection.
3727          * If we can't perform DT transfers on this segment (not in LVD
3728          * mode for instance), then our decision to issue a PPR message
3729          * may change.
3730          */
3731         period = tinfo->goal.period;
3732         offset = tinfo->goal.offset;
3733         ppr_options = tinfo->goal.ppr_options;
3734         /* Target initiated PPR is not allowed in the SCSI spec */
3735         if (devinfo->role == ROLE_TARGET)
3736                 ppr_options = 0;
3737         ahd_devlimited_syncrate(ahd, tinfo, &period,
3738                                 &ppr_options, devinfo->role);
3739         dowide = tinfo->curr.width != tinfo->goal.width;
3740         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
3741         /*
3742          * Only use PPR if we have options that need it, even if the device
3743          * claims to support it.  There might be an expander in the way
3744          * that doesn't.
3745          */
3746         doppr = ppr_options != 0;
3747
3748         if (!dowide && !dosync && !doppr) {
3749                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
3750                 dosync = tinfo->goal.offset != 0;
3751         }
3752
3753         if (!dowide && !dosync && !doppr) {
3754                 /*
3755                  * Force async with a WDTR message if we have a wide bus,
3756                  * or just issue an SDTR with a 0 offset.
3757                  */
3758                 if ((ahd->features & AHD_WIDE) != 0)
3759                         dowide = 1;
3760                 else
3761                         dosync = 1;
3762
3763                 if (bootverbose) {
3764                         ahd_print_devinfo(ahd, devinfo);
3765                         printf("Ensuring async\n");
3766                 }
3767         }
3768         /* Target initiated PPR is not allowed in the SCSI spec */
3769         if (devinfo->role == ROLE_TARGET)
3770                 doppr = 0;
3771
3772         /*
3773          * Both the PPR message and SDTR message require the
3774          * goal syncrate to be limited to what the target device
3775          * is capable of handling (based on whether an LVD->SE
3776          * expander is on the bus), so combine these two cases.
3777          * Regardless, guarantee that if we are using WDTR and SDTR
3778          * messages that WDTR comes first.
3779          */
3780         if (doppr || (dosync && !dowide)) {
3781
3782                 offset = tinfo->goal.offset;
3783                 ahd_validate_offset(ahd, tinfo, period, &offset,
3784                                     doppr ? tinfo->goal.width
3785                                           : tinfo->curr.width,
3786                                     devinfo->role);
3787                 if (doppr) {
3788                         ahd_construct_ppr(ahd, devinfo, period, offset,
3789                                           tinfo->goal.width, ppr_options);
3790                 } else {
3791                         ahd_construct_sdtr(ahd, devinfo, period, offset);
3792                 }
3793         } else {
3794                 ahd_construct_wdtr(ahd, devinfo, tinfo->goal.width);
3795         }
3796 }
3797
3798 /*
3799  * Build a synchronous negotiation message in our message
3800  * buffer based on the input parameters.
3801  */
3802 static void
3803 ahd_construct_sdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3804                    u_int period, u_int offset)
3805 {
3806         if (offset == 0)
3807                 period = AHD_ASYNC_XFER_PERIOD;
3808         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3809         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_SDTR_LEN;
3810         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_SDTR;
3811         ahd->msgout_buf[ahd->msgout_index++] = period;
3812         ahd->msgout_buf[ahd->msgout_index++] = offset;
3813         ahd->msgout_len += 5;
3814         if (bootverbose) {
3815                 printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
3816                        ahd_name(ahd), devinfo->channel, devinfo->target,
3817                        devinfo->lun, period, offset);
3818         }
3819 }
3820
3821 /*
3822  * Build a wide negotiateion message in our message
3823  * buffer based on the input parameters.
3824  */
3825 static void
3826 ahd_construct_wdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3827                    u_int bus_width)
3828 {
3829         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3830         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_WDTR_LEN;
3831         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_WDTR;
3832         ahd->msgout_buf[ahd->msgout_index++] = bus_width;
3833         ahd->msgout_len += 4;
3834         if (bootverbose) {
3835                 printf("(%s:%c:%d:%d): Sending WDTR %x\n",
3836                        ahd_name(ahd), devinfo->channel, devinfo->target,
3837                        devinfo->lun, bus_width);
3838         }
3839 }
3840
3841 /*
3842  * Build a parallel protocol request message in our message
3843  * buffer based on the input parameters.
3844  */
3845 static void
3846 ahd_construct_ppr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3847                   u_int period, u_int offset, u_int bus_width,
3848                   u_int ppr_options)
3849 {
3850         /*
3851          * Always request precompensation from
3852          * the other target if we are running
3853          * at paced syncrates.
3854          */
3855         if (period <= AHD_SYNCRATE_PACED)
3856                 ppr_options |= MSG_EXT_PPR_PCOMP_EN;
3857         if (offset == 0)
3858                 period = AHD_ASYNC_XFER_PERIOD;
3859         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3860         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_PPR_LEN;
3861         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_PPR;
3862         ahd->msgout_buf[ahd->msgout_index++] = period;
3863         ahd->msgout_buf[ahd->msgout_index++] = 0;
3864         ahd->msgout_buf[ahd->msgout_index++] = offset;
3865         ahd->msgout_buf[ahd->msgout_index++] = bus_width;
3866         ahd->msgout_buf[ahd->msgout_index++] = ppr_options;
3867         ahd->msgout_len += 8;
3868         if (bootverbose) {
3869                 printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
3870                        "offset %x, ppr_options %x\n", ahd_name(ahd),
3871                        devinfo->channel, devinfo->target, devinfo->lun,
3872                        bus_width, period, offset, ppr_options);
3873         }
3874 }
3875
3876 /*
3877  * Clear any active message state.
3878  */
3879 static void
3880 ahd_clear_msg_state(struct ahd_softc *ahd)
3881 {
3882         ahd_mode_state saved_modes;
3883
3884         saved_modes = ahd_save_modes(ahd);
3885         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3886         ahd->send_msg_perror = 0;
3887         ahd->msg_flags = MSG_FLAG_NONE;
3888         ahd->msgout_len = 0;
3889         ahd->msgin_index = 0;
3890         ahd->msg_type = MSG_TYPE_NONE;
3891         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
3892                 /*
3893                  * The target didn't care to respond to our
3894                  * message request, so clear ATN.
3895                  */
3896                 ahd_outb(ahd, CLRSINT1, CLRATNO);
3897         }
3898         ahd_outb(ahd, MSG_OUT, MSG_NOOP);
3899         ahd_outb(ahd, SEQ_FLAGS2,
3900                  ahd_inb(ahd, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
3901         ahd_restore_modes(ahd, saved_modes);
3902 }
3903
3904 /*
3905  * Manual message loop handler.
3906  */
3907 static void
3908 ahd_handle_message_phase(struct ahd_softc *ahd)
3909 {
3910         struct  ahd_devinfo devinfo;
3911         u_int   bus_phase;
3912         int     end_session;
3913
3914         ahd_fetch_devinfo(ahd, &devinfo);
3915         end_session = FALSE;
3916         bus_phase = ahd_inb(ahd, LASTPHASE);
3917
3918         if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0) {
3919                 printf("LQIRETRY for LQIPHASE_OUTPKT\n");
3920                 ahd_outb(ahd, LQCTL2, LQIRETRY);
3921         }
3922 reswitch:
3923         switch (ahd->msg_type) {
3924         case MSG_TYPE_INITIATOR_MSGOUT:
3925         {
3926                 int lastbyte;
3927                 int phasemis;
3928                 int msgdone;
3929
3930                 if (ahd->msgout_len == 0 && ahd->send_msg_perror == 0)
3931                         panic("HOST_MSG_LOOP interrupt with no active message");
3932
3933 #ifdef AHD_DEBUG
3934                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3935                         ahd_print_devinfo(ahd, &devinfo);
3936                         printf("INITIATOR_MSG_OUT");
3937                 }
3938 #endif
3939                 phasemis = bus_phase != P_MESGOUT;
3940                 if (phasemis) {
3941 #ifdef AHD_DEBUG
3942                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3943                                 printf(" PHASEMIS %s\n",
3944                                        ahd_lookup_phase_entry(bus_phase)
3945                                                              ->phasemsg);
3946                         }
3947 #endif
3948                         if (bus_phase == P_MESGIN) {
3949                                 /*
3950                                  * Change gears and see if
3951                                  * this messages is of interest to
3952                                  * us or should be passed back to
3953                                  * the sequencer.
3954                                  */
3955                                 ahd_outb(ahd, CLRSINT1, CLRATNO);
3956                                 ahd->send_msg_perror = 0;
3957                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGIN;
3958                                 ahd->msgin_index = 0;
3959                                 goto reswitch;
3960                         }
3961                         end_session = TRUE;
3962                         break;
3963                 }
3964
3965                 if (ahd->send_msg_perror) {
3966                         ahd_outb(ahd, CLRSINT1, CLRATNO);
3967                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
3968 #ifdef AHD_DEBUG
3969                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3970                                 printf(" byte 0x%x\n", ahd->send_msg_perror);
3971 #endif
3972                         /*
3973                          * If we are notifying the target of a CRC error
3974                          * during packetized operations, the target is
3975                          * within its rights to acknowledge our message
3976                          * with a busfree.
3977                          */
3978                         if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0
3979                          && ahd->send_msg_perror == MSG_INITIATOR_DET_ERR)
3980                                 ahd->msg_flags |= MSG_FLAG_EXPECT_IDE_BUSFREE;
3981
3982                         ahd_outb(ahd, RETURN_2, ahd->send_msg_perror);
3983                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
3984                         break;
3985                 }
3986
3987                 msgdone = ahd->msgout_index == ahd->msgout_len;
3988                 if (msgdone) {
3989                         /*
3990                          * The target has requested a retry.
3991                          * Re-assert ATN, reset our message index to
3992                          * 0, and try again.
3993                          */
3994                         ahd->msgout_index = 0;
3995                         ahd_assert_atn(ahd);
3996                 }
3997
3998                 lastbyte = ahd->msgout_index == (ahd->msgout_len - 1);
3999                 if (lastbyte) {
4000                         /* Last byte is signified by dropping ATN */
4001                         ahd_outb(ahd, CLRSINT1, CLRATNO);
4002                 }
4003
4004                 /*
4005                  * Clear our interrupt status and present
4006                  * the next byte on the bus.
4007                  */
4008                 ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4009 #ifdef AHD_DEBUG
4010                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4011                         printf(" byte 0x%x\n",
4012                                ahd->msgout_buf[ahd->msgout_index]);
4013 #endif
4014                 ahd_outb(ahd, RETURN_2, ahd->msgout_buf[ahd->msgout_index++]);
4015                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
4016                 break;
4017         }
4018         case MSG_TYPE_INITIATOR_MSGIN:
4019         {
4020                 int phasemis;
4021                 int message_done;
4022
4023 #ifdef AHD_DEBUG
4024                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4025                         ahd_print_devinfo(ahd, &devinfo);
4026                         printf("INITIATOR_MSG_IN");
4027                 }
4028 #endif
4029                 phasemis = bus_phase != P_MESGIN;
4030                 if (phasemis) {
4031 #ifdef AHD_DEBUG
4032                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4033                                 printf(" PHASEMIS %s\n",
4034                                        ahd_lookup_phase_entry(bus_phase)
4035                                                              ->phasemsg);
4036                         }
4037 #endif
4038                         ahd->msgin_index = 0;
4039                         if (bus_phase == P_MESGOUT
4040                          && (ahd->send_msg_perror != 0
4041                           || (ahd->msgout_len != 0
4042                            && ahd->msgout_index == 0))) {
4043                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4044                                 goto reswitch;
4045                         }
4046                         end_session = TRUE;
4047                         break;
4048                 }
4049
4050                 /* Pull the byte in without acking it */
4051                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIBUS);
4052 #ifdef AHD_DEBUG
4053                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4054                         printf(" byte 0x%x\n",
4055                                ahd->msgin_buf[ahd->msgin_index]);
4056 #endif
4057
4058                 message_done = ahd_parse_msg(ahd, &devinfo);
4059
4060                 if (message_done) {
4061                         /*
4062                          * Clear our incoming message buffer in case there
4063                          * is another message following this one.
4064                          */
4065                         ahd->msgin_index = 0;
4066
4067                         /*
4068                          * If this message illicited a response,
4069                          * assert ATN so the target takes us to the
4070                          * message out phase.
4071                          */
4072                         if (ahd->msgout_len != 0) {
4073 #ifdef AHD_DEBUG
4074                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4075                                         ahd_print_devinfo(ahd, &devinfo);
4076                                         printf("Asserting ATN for response\n");
4077                                 }
4078 #endif
4079                                 ahd_assert_atn(ahd);
4080                         }
4081                 } else
4082                         ahd->msgin_index++;
4083
4084                 if (message_done == MSGLOOP_TERMINATED) {
4085                         end_session = TRUE;
4086                 } else {
4087                         /* Ack the byte */
4088                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4089                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_READ);
4090                 }
4091                 break;
4092         }
4093         case MSG_TYPE_TARGET_MSGIN:
4094         {
4095                 int msgdone;
4096                 int msgout_request;
4097
4098                 /*
4099                  * By default, the message loop will continue.
4100                  */
4101                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4102
4103                 if (ahd->msgout_len == 0)
4104                         panic("Target MSGIN with no active message");
4105
4106                 /*
4107                  * If we interrupted a mesgout session, the initiator
4108                  * will not know this until our first REQ.  So, we
4109                  * only honor mesgout requests after we've sent our
4110                  * first byte.
4111                  */
4112                 if ((ahd_inb(ahd, SCSISIGI) & ATNI) != 0
4113                  && ahd->msgout_index > 0)
4114                         msgout_request = TRUE;
4115                 else
4116                         msgout_request = FALSE;
4117
4118                 if (msgout_request) {
4119
4120                         /*
4121                          * Change gears and see if
4122                          * this messages is of interest to
4123                          * us or should be passed back to
4124                          * the sequencer.
4125                          */
4126                         ahd->msg_type = MSG_TYPE_TARGET_MSGOUT;
4127                         ahd_outb(ahd, SCSISIGO, P_MESGOUT | BSYO);
4128                         ahd->msgin_index = 0;
4129                         /* Dummy read to REQ for first byte */
4130                         ahd_inb(ahd, SCSIDAT);
4131                         ahd_outb(ahd, SXFRCTL0,
4132                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4133                         break;
4134                 }
4135
4136                 msgdone = ahd->msgout_index == ahd->msgout_len;
4137                 if (msgdone) {
4138                         ahd_outb(ahd, SXFRCTL0,
4139                                  ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4140                         end_session = TRUE;
4141                         break;
4142                 }
4143
4144                 /*
4145                  * Present the next byte on the bus.
4146                  */
4147                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4148                 ahd_outb(ahd, SCSIDAT, ahd->msgout_buf[ahd->msgout_index++]);
4149                 break;
4150         }
4151         case MSG_TYPE_TARGET_MSGOUT:
4152         {
4153                 int lastbyte;
4154                 int msgdone;
4155
4156                 /*
4157                  * By default, the message loop will continue.
4158                  */
4159                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4160
4161                 /*
4162                  * The initiator signals that this is
4163                  * the last byte by dropping ATN.
4164                  */
4165                 lastbyte = (ahd_inb(ahd, SCSISIGI) & ATNI) == 0;
4166
4167                 /*
4168                  * Read the latched byte, but turn off SPIOEN first
4169                  * so that we don't inadvertently cause a REQ for the
4170                  * next byte.
4171                  */
4172                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4173                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIDAT);
4174                 msgdone = ahd_parse_msg(ahd, &devinfo);
4175                 if (msgdone == MSGLOOP_TERMINATED) {
4176                         /*
4177                          * The message is *really* done in that it caused
4178                          * us to go to bus free.  The sequencer has already
4179                          * been reset at this point, so pull the ejection
4180                          * handle.
4181                          */
4182                         return;
4183                 }
4184
4185                 ahd->msgin_index++;
4186
4187                 /*
4188                  * XXX Read spec about initiator dropping ATN too soon
4189                  *     and use msgdone to detect it.
4190                  */
4191                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
4192                         ahd->msgin_index = 0;
4193
4194                         /*
4195                          * If this message illicited a response, transition
4196                          * to the Message in phase and send it.
4197                          */
4198                         if (ahd->msgout_len != 0) {
4199                                 ahd_outb(ahd, SCSISIGO, P_MESGIN | BSYO);
4200                                 ahd_outb(ahd, SXFRCTL0,
4201                                          ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4202                                 ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
4203                                 ahd->msgin_index = 0;
4204                                 break;
4205                         }
4206                 }
4207
4208                 if (lastbyte)
4209                         end_session = TRUE;
4210                 else {
4211                         /* Ask for the next byte. */
4212                         ahd_outb(ahd, SXFRCTL0,
4213                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4214                 }
4215
4216                 break;
4217         }
4218         default:
4219                 panic("Unknown REQINIT message type");
4220         }
4221
4222         if (end_session) {
4223                 if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0) {
4224                         printf("%s: Returning to Idle Loop\n",
4225                                ahd_name(ahd));
4226                         ahd_clear_msg_state(ahd);
4227
4228                         /*
4229                          * Perform the equivalent of a clear_target_state.
4230                          */
4231                         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
4232                         ahd_outb(ahd, SEQ_FLAGS, NOT_IDENTIFIED|NO_CDB_SENT);
4233                         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
4234                 } else {
4235                         ahd_clear_msg_state(ahd);
4236                         ahd_outb(ahd, RETURN_1, EXIT_MSG_LOOP);
4237                 }
4238         }
4239 }
4240
4241 /*
4242  * See if we sent a particular extended message to the target.
4243  * If "full" is true, return true only if the target saw the full
4244  * message.  If "full" is false, return true if the target saw at
4245  * least the first byte of the message.
4246  */
4247 static int
4248 ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type, u_int msgval, int full)
4249 {
4250         int found;
4251         u_int index;
4252
4253         found = FALSE;
4254         index = 0;
4255
4256         while (index < ahd->msgout_len) {
4257                 if (ahd->msgout_buf[index] == MSG_EXTENDED) {
4258                         u_int end_index;
4259
4260                         end_index = index + 1 + ahd->msgout_buf[index + 1];
4261                         if (ahd->msgout_buf[index+2] == msgval
4262                          && type == AHDMSG_EXT) {
4263
4264                                 if (full) {
4265                                         if (ahd->msgout_index > end_index)
4266                                                 found = TRUE;
4267                                 } else if (ahd->msgout_index > index)
4268                                         found = TRUE;
4269                         }
4270                         index = end_index;
4271                 } else if (ahd->msgout_buf[index] >= MSG_SIMPLE_TASK
4272                         && ahd->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
4273
4274                         /* Skip tag type and tag id or residue param*/
4275                         index += 2;
4276                 } else {
4277                         /* Single byte message */
4278                         if (type == AHDMSG_1B
4279                          && ahd->msgout_index > index
4280                          && (ahd->msgout_buf[index] == msgval
4281                           || ((ahd->msgout_buf[index] & MSG_IDENTIFYFLAG) != 0
4282                            && msgval == MSG_IDENTIFYFLAG)))
4283                                 found = TRUE;
4284                         index++;
4285                 }
4286
4287                 if (found)
4288                         break;
4289         }
4290         return (found);
4291 }
4292
4293 /*
4294  * Wait for a complete incoming message, parse it, and respond accordingly.
4295  */
4296 static int
4297 ahd_parse_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4298 {
4299         struct  ahd_initiator_tinfo *tinfo;
4300         struct  ahd_tmode_tstate *tstate;
4301         int     reject;
4302         int     done;
4303         int     response;
4304
4305         done = MSGLOOP_IN_PROG;
4306         response = FALSE;
4307         reject = FALSE;
4308         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4309                                     devinfo->target, &tstate);
4310
4311         /*
4312          * Parse as much of the message as is available,
4313          * rejecting it if we don't support it.  When
4314          * the entire message is available and has been
4315          * handled, return MSGLOOP_MSGCOMPLETE, indicating
4316          * that we have parsed an entire message.
4317          *
4318          * In the case of extended messages, we accept the length
4319          * byte outright and perform more checking once we know the
4320          * extended message type.
4321          */
4322         switch (ahd->msgin_buf[0]) {
4323         case MSG_DISCONNECT:
4324         case MSG_SAVEDATAPOINTER:
4325         case MSG_CMDCOMPLETE:
4326         case MSG_RESTOREPOINTERS:
4327         case MSG_IGN_WIDE_RESIDUE:
4328                 /*
4329                  * End our message loop as these are messages
4330                  * the sequencer handles on its own.
4331                  */
4332                 done = MSGLOOP_TERMINATED;
4333                 break;
4334         case MSG_MESSAGE_REJECT:
4335                 response = ahd_handle_msg_reject(ahd, devinfo);
4336                 /* FALLTHROUGH */
4337         case MSG_NOOP:
4338                 done = MSGLOOP_MSGCOMPLETE;
4339                 break;
4340         case MSG_EXTENDED:
4341         {
4342                 /* Wait for enough of the message to begin validation */
4343                 if (ahd->msgin_index < 2)
4344                         break;
4345                 switch (ahd->msgin_buf[2]) {
4346                 case MSG_EXT_SDTR:
4347                 {
4348                         u_int    period;
4349                         u_int    ppr_options;
4350                         u_int    offset;
4351                         u_int    saved_offset;
4352
4353                         if (ahd->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
4354                                 reject = TRUE;
4355                                 break;
4356                         }
4357
4358                         /*
4359                          * Wait until we have both args before validating
4360                          * and acting on this message.
4361                          *
4362                          * Add one to MSG_EXT_SDTR_LEN to account for
4363                          * the extended message preamble.
4364                          */
4365                         if (ahd->msgin_index < (MSG_EXT_SDTR_LEN + 1))
4366                                 break;
4367
4368                         period = ahd->msgin_buf[3];
4369                         ppr_options = 0;
4370                         saved_offset = offset = ahd->msgin_buf[4];
4371                         ahd_devlimited_syncrate(ahd, tinfo, &period,
4372                                                 &ppr_options, devinfo->role);
4373                         ahd_validate_offset(ahd, tinfo, period, &offset,
4374                                             tinfo->curr.width, devinfo->role);
4375                         if (bootverbose) {
4376                                 printf("(%s:%c:%d:%d): Received "
4377                                        "SDTR period %x, offset %x\n\t"
4378                                        "Filtered to period %x, offset %x\n",
4379                                        ahd_name(ahd), devinfo->channel,
4380                                        devinfo->target, devinfo->lun,
4381                                        ahd->msgin_buf[3], saved_offset,
4382                                        period, offset);
4383                         }
4384                         ahd_set_syncrate(ahd, devinfo, period,
4385                                          offset, ppr_options,
4386                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4387                                          /*paused*/TRUE);
4388
4389                         /*
4390                          * See if we initiated Sync Negotiation
4391                          * and didn't have to fall down to async
4392                          * transfers.
4393                          */
4394                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, TRUE)) {
4395                                 /* We started it */
4396                                 if (saved_offset != offset) {
4397                                         /* Went too low - force async */
4398                                         reject = TRUE;
4399                                 }
4400                         } else {
4401                                 /*
4402                                  * Send our own SDTR in reply
4403                                  */
4404                                 if (bootverbose
4405                                  && devinfo->role == ROLE_INITIATOR) {
4406                                         printf("(%s:%c:%d:%d): Target "
4407                                                "Initiated SDTR\n",
4408                                                ahd_name(ahd), devinfo->channel,
4409                                                devinfo->target, devinfo->lun);
4410                                 }
4411                                 ahd->msgout_index = 0;
4412                                 ahd->msgout_len = 0;
4413                                 ahd_construct_sdtr(ahd, devinfo,
4414                                                    period, offset);
4415                                 ahd->msgout_index = 0;
4416                                 response = TRUE;
4417                         }
4418                         done = MSGLOOP_MSGCOMPLETE;
4419                         break;
4420                 }
4421                 case MSG_EXT_WDTR:
4422                 {
4423                         u_int bus_width;
4424                         u_int saved_width;
4425                         u_int sending_reply;
4426
4427                         sending_reply = FALSE;
4428                         if (ahd->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
4429                                 reject = TRUE;
4430                                 break;
4431                         }
4432
4433                         /*
4434                          * Wait until we have our arg before validating
4435                          * and acting on this message.
4436                          *
4437                          * Add one to MSG_EXT_WDTR_LEN to account for
4438                          * the extended message preamble.
4439                          */
4440                         if (ahd->msgin_index < (MSG_EXT_WDTR_LEN + 1))
4441                                 break;
4442
4443                         bus_width = ahd->msgin_buf[3];
4444                         saved_width = bus_width;
4445                         ahd_validate_width(ahd, tinfo, &bus_width,
4446                                            devinfo->role);
4447                         if (bootverbose) {
4448                                 printf("(%s:%c:%d:%d): Received WDTR "
4449                                        "%x filtered to %x\n",
4450                                        ahd_name(ahd), devinfo->channel,
4451                                        devinfo->target, devinfo->lun,
4452                                        saved_width, bus_width);
4453                         }
4454
4455                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, TRUE)) {
4456                                 /*
4457                                  * Don't send a WDTR back to the
4458                                  * target, since we asked first.
4459                                  * If the width went higher than our
4460                                  * request, reject it.
4461                                  */
4462                                 if (saved_width > bus_width) {
4463                                         reject = TRUE;
4464                                         printf("(%s:%c:%d:%d): requested %dBit "
4465                                                "transfers.  Rejecting...\n",
4466                                                ahd_name(ahd), devinfo->channel,
4467                                                devinfo->target, devinfo->lun,
4468                                                8 * (0x01 << bus_width));
4469                                         bus_width = 0;
4470                                 }
4471                         } else {
4472                                 /*
4473                                  * Send our own WDTR in reply
4474                                  */
4475                                 if (bootverbose
4476                                  && devinfo->role == ROLE_INITIATOR) {
4477                                         printf("(%s:%c:%d:%d): Target "
4478                                                "Initiated WDTR\n",
4479                                                ahd_name(ahd), devinfo->channel,
4480                                                devinfo->target, devinfo->lun);
4481                                 }
4482                                 ahd->msgout_index = 0;
4483                                 ahd->msgout_len = 0;
4484                                 ahd_construct_wdtr(ahd, devinfo, bus_width);
4485                                 ahd->msgout_index = 0;
4486                                 response = TRUE;
4487                                 sending_reply = TRUE;
4488                         }
4489                         /*
4490                          * After a wide message, we are async, but
4491                          * some devices don't seem to honor this portion
4492                          * of the spec.  Force a renegotiation of the
4493                          * sync component of our transfer agreement even
4494                          * if our goal is async.  By updating our width
4495                          * after forcing the negotiation, we avoid
4496                          * renegotiating for width.
4497                          */
4498                         ahd_update_neg_request(ahd, devinfo, tstate,
4499                                                tinfo, AHD_NEG_ALWAYS);
4500                         ahd_set_width(ahd, devinfo, bus_width,
4501                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4502                                       /*paused*/TRUE);
4503                         if (sending_reply == FALSE && reject == FALSE) {
4504
4505                                 /*
4506                                  * We will always have an SDTR to send.
4507                                  */
4508                                 ahd->msgout_index = 0;
4509                                 ahd->msgout_len = 0;
4510                                 ahd_build_transfer_msg(ahd, devinfo);
4511                                 ahd->msgout_index = 0;
4512                                 response = TRUE;
4513                         }
4514                         done = MSGLOOP_MSGCOMPLETE;
4515                         break;
4516                 }
4517                 case MSG_EXT_PPR:
4518                 {
4519                         u_int   period;
4520                         u_int   offset;
4521                         u_int   bus_width;
4522                         u_int   ppr_options;
4523                         u_int   saved_width;
4524                         u_int   saved_offset;
4525                         u_int   saved_ppr_options;
4526
4527                         if (ahd->msgin_buf[1] != MSG_EXT_PPR_LEN) {
4528                                 reject = TRUE;
4529                                 break;
4530                         }
4531
4532                         /*
4533                          * Wait until we have all args before validating
4534                          * and acting on this message.
4535                          *
4536                          * Add one to MSG_EXT_PPR_LEN to account for
4537                          * the extended message preamble.
4538                          */
4539                         if (ahd->msgin_index < (MSG_EXT_PPR_LEN + 1))
4540                                 break;
4541
4542                         period = ahd->msgin_buf[3];
4543                         offset = ahd->msgin_buf[5];
4544                         bus_width = ahd->msgin_buf[6];
4545                         saved_width = bus_width;
4546                         ppr_options = ahd->msgin_buf[7];
4547                         /*
4548                          * According to the spec, a DT only
4549                          * period factor with no DT option
4550                          * set implies async.
4551                          */
4552                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
4553                          && period <= 9)
4554                                 offset = 0;
4555                         saved_ppr_options = ppr_options;
4556                         saved_offset = offset;
4557
4558                         /*
4559                          * Transfer options are only available if we
4560                          * are negotiating wide.
4561                          */
4562                         if (bus_width == 0)
4563                                 ppr_options &= MSG_EXT_PPR_QAS_REQ;
4564
4565                         ahd_validate_width(ahd, tinfo, &bus_width,
4566                                            devinfo->role);
4567                         ahd_devlimited_syncrate(ahd, tinfo, &period,
4568                                                 &ppr_options, devinfo->role);
4569                         ahd_validate_offset(ahd, tinfo, period, &offset,
4570                                             bus_width, devinfo->role);
4571
4572                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, TRUE)) {
4573                                 /*
4574                                  * If we are unable to do any of the
4575                                  * requested options (we went too low),
4576                                  * then we'll have to reject the message.
4577                                  */
4578                                 if (saved_width > bus_width
4579                                  || saved_offset != offset
4580                                  || saved_ppr_options != ppr_options) {
4581                                         reject = TRUE;
4582                                         period = 0;
4583                                         offset = 0;
4584                                         bus_width = 0;
4585                                         ppr_options = 0;
4586                                 }
4587                         } else {
4588                                 if (devinfo->role != ROLE_TARGET)
4589                                         printf("(%s:%c:%d:%d): Target "
4590                                                "Initiated PPR\n",
4591                                                ahd_name(ahd), devinfo->channel,
4592                                                devinfo->target, devinfo->lun);
4593                                 else
4594                                         printf("(%s:%c:%d:%d): Initiator "
4595                                                "Initiated PPR\n",
4596                                                ahd_name(ahd), devinfo->channel,
4597                                                devinfo->target, devinfo->lun);
4598                                 ahd->msgout_index = 0;
4599                                 ahd->msgout_len = 0;
4600                                 ahd_construct_ppr(ahd, devinfo, period, offset,
4601                                                   bus_width, ppr_options);
4602                                 ahd->msgout_index = 0;
4603                                 response = TRUE;
4604                         }
4605                         if (bootverbose) {
4606                                 printf("(%s:%c:%d:%d): Received PPR width %x, "
4607                                        "period %x, offset %x,options %x\n"
4608                                        "\tFiltered to width %x, period %x, "
4609                                        "offset %x, options %x\n",
4610                                        ahd_name(ahd), devinfo->channel,
4611                                        devinfo->target, devinfo->lun,
4612                                        saved_width, ahd->msgin_buf[3],
4613                                        saved_offset, saved_ppr_options,
4614                                        bus_width, period, offset, ppr_options);
4615                         }
4616                         ahd_set_width(ahd, devinfo, bus_width,
4617                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4618                                       /*paused*/TRUE);
4619                         ahd_set_syncrate(ahd, devinfo, period,
4620                                          offset, ppr_options,
4621                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4622                                          /*paused*/TRUE);
4623
4624                         done = MSGLOOP_MSGCOMPLETE;
4625                         break;
4626                 }
4627                 default:
4628                         /* Unknown extended message.  Reject it. */
4629                         reject = TRUE;
4630                         break;
4631                 }
4632                 break;
4633         }
4634 #ifdef AHD_TARGET_MODE
4635         case MSG_BUS_DEV_RESET:
4636                 ahd_handle_devreset(ahd, devinfo, CAM_LUN_WILDCARD,
4637                                     CAM_BDR_SENT,
4638                                     "Bus Device Reset Received",
4639                                     /*verbose_level*/0);
4640                 ahd_restart(ahd);
4641                 done = MSGLOOP_TERMINATED;
4642                 break;
4643         case MSG_ABORT_TAG:
4644         case MSG_ABORT:
4645         case MSG_CLEAR_QUEUE:
4646         {
4647                 int tag;
4648
4649                 /* Target mode messages */
4650                 if (devinfo->role != ROLE_TARGET) {
4651                         reject = TRUE;
4652                         break;
4653                 }
4654                 tag = SCB_LIST_NULL;
4655                 if (ahd->msgin_buf[0] == MSG_ABORT_TAG)
4656                         tag = ahd_inb(ahd, INITIATOR_TAG);
4657                 ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
4658                                devinfo->lun, tag, ROLE_TARGET,
4659                                CAM_REQ_ABORTED);
4660
4661                 tstate = ahd->enabled_targets[devinfo->our_scsiid];
4662                 if (tstate != NULL) {
4663                         struct ahd_tmode_lstate* lstate;
4664
4665                         lstate = tstate->enabled_luns[devinfo->lun];
4666                         if (lstate != NULL) {
4667                                 ahd_queue_lstate_event(ahd, lstate,
4668                                                        devinfo->our_scsiid,
4669                                                        ahd->msgin_buf[0],
4670                                                        /*arg*/tag);
4671                                 ahd_send_lstate_events(ahd, lstate);
4672                         }
4673                 }
4674                 ahd_restart(ahd);
4675                 done = MSGLOOP_TERMINATED;
4676                 break;
4677         }
4678 #endif
4679         case MSG_QAS_REQUEST:
4680 #ifdef AHD_DEBUG
4681                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4682                         printf("%s: QAS request.  SCSISIGI == 0x%x\n",
4683                                ahd_name(ahd), ahd_inb(ahd, SCSISIGI));
4684 #endif
4685                 ahd->msg_flags |= MSG_FLAG_EXPECT_QASREJ_BUSFREE;
4686                 /* FALLTHROUGH */
4687         case MSG_TERM_IO_PROC:
4688         default:
4689                 reject = TRUE;
4690                 break;
4691         }
4692
4693         if (reject) {
4694                 /*
4695                  * Setup to reject the message.
4696                  */
4697                 ahd->msgout_index = 0;
4698                 ahd->msgout_len = 1;
4699                 ahd->msgout_buf[0] = MSG_MESSAGE_REJECT;
4700                 done = MSGLOOP_MSGCOMPLETE;
4701                 response = TRUE;
4702         }
4703
4704         if (done != MSGLOOP_IN_PROG && !response)
4705                 /* Clear the outgoing message buffer */
4706                 ahd->msgout_len = 0;
4707
4708         return (done);
4709 }
4710
4711 /*
4712  * Process a message reject message.
4713  */
4714 static int
4715 ahd_handle_msg_reject(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4716 {
4717         /*
4718          * What we care about here is if we had an
4719          * outstanding SDTR or WDTR message for this
4720          * target.  If we did, this is a signal that
4721          * the target is refusing negotiation.
4722          */
4723         struct scb *scb;
4724         struct ahd_initiator_tinfo *tinfo;
4725         struct ahd_tmode_tstate *tstate;
4726         u_int scb_index;
4727         u_int last_msg;
4728         int   response = 0;
4729
4730         scb_index = ahd_get_scbptr(ahd);
4731         scb = ahd_lookup_scb(ahd, scb_index);
4732         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel,
4733                                     devinfo->our_scsiid,
4734                                     devinfo->target, &tstate);
4735         /* Might be necessary */
4736         last_msg = ahd_inb(ahd, LAST_MSG);
4737
4738         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
4739                 if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/TRUE)
4740                  && tinfo->goal.period <= AHD_SYNCRATE_PACED) {
4741                         /*
4742                          * Target may not like our SPI-4 PPR Options.
4743                          * Attempt to negotiate 80MHz which will turn
4744                          * off these options.
4745                          */
4746                         if (bootverbose) {
4747                                 printf("(%s:%c:%d:%d): PPR Rejected. "
4748                                        "Trying simple U160 PPR\n",
4749                                        ahd_name(ahd), devinfo->channel,
4750                                        devinfo->target, devinfo->lun);
4751                         }
4752                         tinfo->goal.period = AHD_SYNCRATE_DT;
4753                         tinfo->goal.ppr_options &= MSG_EXT_PPR_IU_REQ
4754                                                 |  MSG_EXT_PPR_QAS_REQ
4755                                                 |  MSG_EXT_PPR_DT_REQ;
4756                 } else {
4757                         /*
4758                          * Target does not support the PPR message.
4759                          * Attempt to negotiate SPI-2 style.
4760                          */
4761                         if (bootverbose) {
4762                                 printf("(%s:%c:%d:%d): PPR Rejected. "
4763                                        "Trying WDTR/SDTR\n",
4764                                        ahd_name(ahd), devinfo->channel,
4765                                        devinfo->target, devinfo->lun);
4766                         }
4767                         tinfo->goal.ppr_options = 0;
4768                         tinfo->curr.transport_version = 2;
4769                         tinfo->goal.transport_version = 2;
4770                 }
4771                 ahd->msgout_index = 0;
4772                 ahd->msgout_len = 0;
4773                 ahd_build_transfer_msg(ahd, devinfo);
4774                 ahd->msgout_index = 0;
4775                 response = 1;
4776         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
4777
4778                 /* note 8bit xfers */
4779                 printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
4780                        "8bit transfers\n", ahd_name(ahd),
4781                        devinfo->channel, devinfo->target, devinfo->lun);
4782                 ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4783                               AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4784                               /*paused*/TRUE);
4785                 /*
4786                  * No need to clear the sync rate.  If the target
4787                  * did not accept the command, our syncrate is
4788                  * unaffected.  If the target started the negotiation,
4789                  * but rejected our response, we already cleared the
4790                  * sync rate before sending our WDTR.
4791                  */
4792                 if (tinfo->goal.offset != tinfo->curr.offset) {
4793
4794                         /* Start the sync negotiation */
4795                         ahd->msgout_index = 0;
4796                         ahd->msgout_len = 0;
4797                         ahd_build_transfer_msg(ahd, devinfo);
4798                         ahd->msgout_index = 0;
4799                         response = 1;
4800                 }
4801         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
4802                 /* note asynch xfers and clear flag */
4803                 ahd_set_syncrate(ahd, devinfo, /*period*/0,
4804                                  /*offset*/0, /*ppr_options*/0,
4805                                  AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4806                                  /*paused*/TRUE);
4807                 printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
4808                        "Using asynchronous transfers\n",
4809                        ahd_name(ahd), devinfo->channel,
4810                        devinfo->target, devinfo->lun);
4811         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
4812                 int tag_type;
4813                 int mask;
4814
4815                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
4816
4817                 if (tag_type == MSG_SIMPLE_TASK) {
4818                         printf("(%s:%c:%d:%d): refuses tagged commands.  "
4819                                "Performing non-tagged I/O\n", ahd_name(ahd),
4820                                devinfo->channel, devinfo->target, devinfo->lun);
4821                         ahd_set_tags(ahd, devinfo, AHD_QUEUE_NONE);
4822                         mask = ~0x23;
4823                 } else {
4824                         printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
4825                                "Performing simple queue tagged I/O only\n",
4826                                ahd_name(ahd), devinfo->channel, devinfo->target,
4827                                devinfo->lun, tag_type == MSG_ORDERED_TASK
4828                                ? "ordered" : "head of queue");
4829                         ahd_set_tags(ahd, devinfo, AHD_QUEUE_BASIC);
4830                         mask = ~0x03;
4831                 }
4832
4833                 /*
4834                  * Resend the identify for this CCB as the target
4835                  * may believe that the selection is invalid otherwise.
4836                  */
4837                 ahd_outb(ahd, SCB_CONTROL,
4838                          ahd_inb_scbram(ahd, SCB_CONTROL) & mask);
4839                 scb->hscb->control &= mask;
4840                 aic_set_transaction_tag(scb, /*enabled*/FALSE,
4841                                         /*type*/MSG_SIMPLE_TASK);
4842                 ahd_outb(ahd, MSG_OUT, MSG_IDENTIFYFLAG);
4843                 ahd_assert_atn(ahd);
4844                 ahd_busy_tcl(ahd, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
4845                              SCB_GET_TAG(scb));
4846
4847                 /*
4848                  * Requeue all tagged commands for this target
4849                  * currently in our posession so they can be
4850                  * converted to untagged commands.
4851                  */
4852                 ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
4853                                    SCB_GET_CHANNEL(ahd, scb),
4854                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
4855                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
4856                                    SEARCH_COMPLETE);
4857         } else if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_IDENTIFYFLAG, TRUE)) {
4858                 /*
4859                  * Most likely the device believes that we had
4860                  * previously negotiated packetized.
4861                  */
4862                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
4863                                |  MSG_FLAG_IU_REQ_CHANGED;
4864
4865                 ahd_force_renegotiation(ahd, devinfo);
4866                 ahd->msgout_index = 0;
4867                 ahd->msgout_len = 0;
4868                 ahd_build_transfer_msg(ahd, devinfo);
4869                 ahd->msgout_index = 0;
4870                 response = 1;
4871         } else {
4872                 /*
4873                  * Otherwise, we ignore it.
4874                  */
4875                 printf("%s:%c:%d: Message reject for %x -- ignored\n",
4876                        ahd_name(ahd), devinfo->channel, devinfo->target,
4877                        last_msg);
4878         }
4879         return (response);
4880 }
4881
4882 /*
4883  * Process an ingnore wide residue message.
4884  */
4885 static void
4886 ahd_handle_ign_wide_residue(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4887 {
4888         u_int scb_index;
4889         struct scb *scb;
4890
4891         scb_index = ahd_get_scbptr(ahd);
4892         scb = ahd_lookup_scb(ahd, scb_index);
4893         /*
4894          * XXX Actually check data direction in the sequencer?
4895          * Perhaps add datadir to some spare bits in the hscb?
4896          */
4897         if ((ahd_inb(ahd, SEQ_FLAGS) & DPHASE) == 0
4898          || aic_get_transfer_dir(scb) != CAM_DIR_IN) {
4899                 /*
4900                  * Ignore the message if we haven't
4901                  * seen an appropriate data phase yet.
4902                  */
4903         } else {
4904                 /*
4905                  * If the residual occurred on the last
4906                  * transfer and the transfer request was
4907                  * expected to end on an odd count, do
4908                  * nothing.  Otherwise, subtract a byte
4909                  * and update the residual count accordingly.
4910                  */
4911                 uint32_t sgptr;
4912
4913                 sgptr = ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
4914                 if ((sgptr & SG_LIST_NULL) != 0
4915                  && (ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
4916                      & SCB_XFERLEN_ODD) != 0) {
4917                         /*
4918                          * If the residual occurred on the last
4919                          * transfer and the transfer request was
4920                          * expected to end on an odd count, do
4921                          * nothing.
4922                          */
4923                 } else {
4924                         uint32_t data_cnt;
4925                         uint64_t data_addr;
4926                         uint32_t sglen;
4927
4928                         /* Pull in the rest of the sgptr */
4929                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
4930                         data_cnt = ahd_inl_scbram(ahd, SCB_RESIDUAL_DATACNT);
4931                         if ((sgptr & SG_LIST_NULL) != 0) {
4932                                 /*
4933                                  * The residual data count is not updated
4934                                  * for the command run to completion case.
4935                                  * Explicitly zero the count.
4936                                  */
4937                                 data_cnt &= ~AHD_SG_LEN_MASK;
4938                         }
4939                         data_addr = ahd_inq(ahd, SHADDR);
4940                         data_cnt += 1;
4941                         data_addr -= 1;
4942                         sgptr &= SG_PTR_MASK;
4943                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
4944                                 struct ahd_dma64_seg *sg;
4945
4946                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
4947
4948                                 /*
4949                                  * The residual sg ptr points to the next S/G
4950                                  * to load so we must go back one.
4951                                  */
4952                                 sg--;
4953                                 sglen = aic_le32toh(sg->len) & AHD_SG_LEN_MASK;
4954                                 if (sg != scb->sg_list
4955                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
4956
4957                                         sg--;
4958                                         sglen = aic_le32toh(sg->len);
4959                                         /*
4960                                          * Preserve High Address and SG_LIST
4961                                          * bits while setting the count to 1.
4962                                          */
4963                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
4964                                         data_addr = aic_le64toh(sg->addr)
4965                                                   + (sglen & AHD_SG_LEN_MASK)
4966                                                   - 1;
4967
4968                                         /*
4969                                          * Increment sg so it points to the
4970                                          * "next" sg.
4971                                          */
4972                                         sg++;
4973                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
4974                                                                    sg);
4975                                 }
4976                         } else {
4977                                 struct ahd_dma_seg *sg;
4978
4979                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
4980
4981                                 /*
4982                                  * The residual sg ptr points to the next S/G
4983                                  * to load so we must go back one.
4984                                  */
4985                                 sg--;
4986                                 sglen = aic_le32toh(sg->len) & AHD_SG_LEN_MASK;
4987                                 if (sg != scb->sg_list
4988                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
4989
4990                                         sg--;
4991                                         sglen = aic_le32toh(sg->len);
4992                                         /*
4993                                          * Preserve High Address and SG_LIST
4994                                          * bits while setting the count to 1.
4995                                          */
4996                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
4997                                         data_addr = aic_le32toh(sg->addr)
4998                                                   + (sglen & AHD_SG_LEN_MASK)
4999                                                   - 1;
5000
5001                                         /*
5002                                          * Increment sg so it points to the
5003                                          * "next" sg.
5004                                          */
5005                                         sg++;
5006                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
5007                                                                   sg);
5008                                 }
5009                         }
5010                         /*
5011                          * Toggle the "oddness" of the transfer length
5012                          * to handle this mid-transfer ignore wide
5013                          * residue.  This ensures that the oddness is
5014                          * correct for subsequent data transfers.
5015                          */
5016                         ahd_outb(ahd, SCB_TASK_ATTRIBUTE,
5017                             ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
5018                             ^ SCB_XFERLEN_ODD);
5019
5020                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
5021                         ahd_outl(ahd, SCB_RESIDUAL_DATACNT, data_cnt);
5022                         /*
5023                          * The FIFO's pointers will be updated if/when the
5024                          * sequencer re-enters a data phase.
5025                          */
5026                 }
5027         }
5028 }
5029
5030
5031 /*
5032  * Reinitialize the data pointers for the active transfer
5033  * based on its current residual.
5034  */
5035 static void
5036 ahd_reinitialize_dataptrs(struct ahd_softc *ahd)
5037 {
5038         struct           scb *scb;
5039         ahd_mode_state   saved_modes;
5040         u_int            scb_index;
5041         u_int            wait;
5042         uint32_t         sgptr;
5043         uint32_t         resid;
5044         uint64_t         dataptr;
5045
5046         AHD_ASSERT_MODES(ahd, AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK,
5047                          AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK);
5048
5049         scb_index = ahd_get_scbptr(ahd);
5050         scb = ahd_lookup_scb(ahd, scb_index);
5051
5052         /*
5053          * Release and reacquire the FIFO so we
5054          * have a clean slate.
5055          */
5056         ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
5057         wait = 1000;
5058         while (--wait && !(ahd_inb(ahd, MDFFSTAT) & FIFOFREE))
5059                 aic_delay(100);
5060         if (wait == 0) {
5061                 ahd_print_path(ahd, scb);
5062                 printf("ahd_reinitialize_dataptrs: Forcing FIFO free.\n");
5063                 ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
5064         }
5065         saved_modes = ahd_save_modes(ahd);
5066         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5067         ahd_outb(ahd, DFFSTAT,
5068                  ahd_inb(ahd, DFFSTAT)
5069                 | (saved_modes == 0x11 ? CURRFIFO_1 : CURRFIFO_0));
5070
5071         /*
5072          * Determine initial values for data_addr and data_cnt
5073          * for resuming the data phase.
5074          */
5075         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5076         sgptr &= SG_PTR_MASK;
5077
5078         resid = (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 2) << 16)
5079               | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 1) << 8)
5080               | ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT);
5081
5082         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5083                 struct ahd_dma64_seg *sg;
5084
5085                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5086
5087                 /* The residual sg_ptr always points to the next sg */
5088                 sg--;
5089
5090                 dataptr = aic_le64toh(sg->addr)
5091                         + (aic_le32toh(sg->len) & AHD_SG_LEN_MASK)
5092                         - resid;
5093                 ahd_outl(ahd, HADDR + 4, dataptr >> 32);
5094         } else {
5095                 struct   ahd_dma_seg *sg;
5096
5097                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5098
5099                 /* The residual sg_ptr always points to the next sg */
5100                 sg--;
5101
5102                 dataptr = aic_le32toh(sg->addr)
5103                         + (aic_le32toh(sg->len) & AHD_SG_LEN_MASK)
5104                         - resid;
5105                 ahd_outb(ahd, HADDR + 4,
5106                          (aic_le32toh(sg->len) & ~AHD_SG_LEN_MASK) >> 24);
5107         }
5108         ahd_outl(ahd, HADDR, dataptr);
5109         ahd_outb(ahd, HCNT + 2, resid >> 16);
5110         ahd_outb(ahd, HCNT + 1, resid >> 8);
5111         ahd_outb(ahd, HCNT, resid);
5112 }
5113
5114 /*
5115  * Handle the effects of issuing a bus device reset message.
5116  */
5117 static void
5118 ahd_handle_devreset(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5119                     u_int lun, cam_status status, char *message,
5120                     int verbose_level)
5121 {
5122 #ifdef AHD_TARGET_MODE
5123         struct ahd_tmode_tstate* tstate;
5124 #endif
5125         int found;
5126
5127         found = ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5128                                lun, SCB_LIST_NULL, devinfo->role,
5129                                status);
5130
5131 #ifdef AHD_TARGET_MODE
5132         /*
5133          * Send an immediate notify ccb to all target mord peripheral
5134          * drivers affected by this action.
5135          */
5136         tstate = ahd->enabled_targets[devinfo->our_scsiid];
5137         if (tstate != NULL) {
5138                 u_int cur_lun;
5139                 u_int max_lun;
5140
5141                 if (lun != CAM_LUN_WILDCARD) {
5142                         cur_lun = 0;
5143                         max_lun = AHD_NUM_LUNS - 1;
5144                 } else {
5145                         cur_lun = lun;
5146                         max_lun = lun;
5147                 }
5148                 for (cur_lun <= max_lun; cur_lun++) {
5149                         struct ahd_tmode_lstate* lstate;
5150
5151                         lstate = tstate->enabled_luns[cur_lun];
5152                         if (lstate == NULL)
5153                                 continue;
5154
5155                         ahd_queue_lstate_event(ahd, lstate, devinfo->our_scsiid,
5156                                                MSG_BUS_DEV_RESET, /*arg*/0);
5157                         ahd_send_lstate_events(ahd, lstate);
5158                 }
5159         }
5160 #endif
5161
5162         /*
5163          * Go back to async/narrow transfers and renegotiate.
5164          */
5165         ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5166                       AHD_TRANS_CUR, /*paused*/TRUE);
5167         ahd_set_syncrate(ahd, devinfo, /*period*/0, /*offset*/0,
5168                          /*ppr_options*/0, AHD_TRANS_CUR,
5169                          /*paused*/TRUE);
5170
5171         if (status != CAM_SEL_TIMEOUT)
5172                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
5173                                lun, AC_SENT_BDR, NULL);
5174
5175         if (message != NULL
5176          && (verbose_level <= bootverbose)) {
5177                 AHD_CORRECTABLE_ERROR(ahd);
5178                 printf("%s: %s on %c:%d. %d SCBs aborted\n", ahd_name(ahd),
5179                        message, devinfo->channel, devinfo->target, found);
5180         }
5181 }
5182
5183 #ifdef AHD_TARGET_MODE
5184 static void
5185 ahd_setup_target_msgin(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5186                        struct scb *scb)
5187 {
5188
5189         /*
5190          * To facilitate adding multiple messages together,
5191          * each routine should increment the index and len
5192          * variables instead of setting them explicitly.
5193          */
5194         ahd->msgout_index = 0;
5195         ahd->msgout_len = 0;
5196
5197         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
5198                 ahd_build_transfer_msg(ahd, devinfo);
5199         else
5200                 panic("ahd_intr: AWAITING target message with no message");
5201
5202         ahd->msgout_index = 0;
5203         ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
5204 }
5205 #endif
5206 /**************************** Initialization **********************************/
5207 static u_int
5208 ahd_sglist_size(struct ahd_softc *ahd)
5209 {
5210         bus_size_t list_size;
5211
5212         list_size = sizeof(struct ahd_dma_seg) * AHD_NSEG;
5213         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
5214                 list_size = sizeof(struct ahd_dma64_seg) * AHD_NSEG;
5215         return (list_size);
5216 }
5217
5218 /*
5219  * Calculate the optimum S/G List allocation size.  S/G elements used
5220  * for a given transaction must be physically contiguous.  Assume the
5221  * OS will allocate full pages to us, so it doesn't make sense to request
5222  * less than a page.
5223  */
5224 static u_int
5225 ahd_sglist_allocsize(struct ahd_softc *ahd)
5226 {
5227         bus_size_t sg_list_increment;
5228         bus_size_t sg_list_size;
5229         bus_size_t max_list_size;
5230         bus_size_t best_list_size;
5231
5232         /* Start out with the minimum required for AHD_NSEG. */
5233         sg_list_increment = ahd_sglist_size(ahd);
5234         sg_list_size = sg_list_increment;
5235
5236         /* Get us as close as possible to a page in size. */
5237         while ((sg_list_size + sg_list_increment) <= PAGE_SIZE)
5238                 sg_list_size += sg_list_increment;
5239
5240         /*
5241          * Try to reduce the amount of wastage by allocating
5242          * multiple pages.
5243          */
5244         best_list_size = sg_list_size;
5245         max_list_size = roundup(sg_list_increment, PAGE_SIZE);
5246         if (max_list_size < 4 * PAGE_SIZE)
5247                 max_list_size = 4 * PAGE_SIZE;
5248         if (max_list_size > (AHD_SCB_MAX_ALLOC * sg_list_increment))
5249                 max_list_size = (AHD_SCB_MAX_ALLOC * sg_list_increment);
5250         while ((sg_list_size + sg_list_increment) <= max_list_size
5251            &&  (sg_list_size % PAGE_SIZE) != 0) {
5252                 bus_size_t new_mod;
5253                 bus_size_t best_mod;
5254
5255                 sg_list_size += sg_list_increment;
5256                 new_mod = sg_list_size % PAGE_SIZE;
5257                 best_mod = best_list_size % PAGE_SIZE;
5258                 if (new_mod > best_mod || new_mod == 0) {
5259                         best_list_size = sg_list_size;
5260                 }
5261         }
5262         return (best_list_size);
5263 }
5264
5265 /*
5266  * Allocate a controller structure for a new device
5267  * and perform initial initializion.
5268  */
5269 struct ahd_softc *
5270 ahd_alloc(void *platform_arg, char *name)
5271 {
5272         struct  ahd_softc *ahd;
5273
5274 #ifndef __FreeBSD__
5275         ahd = malloc(sizeof(*ahd), M_DEVBUF, M_NOWAIT);
5276         if (!ahd) {
5277                 printf("aic7xxx: cannot malloc softc!\n");
5278                 free(name, M_DEVBUF);
5279                 return NULL;
5280         }
5281 #else
5282         ahd = device_get_softc((device_t)platform_arg);
5283 #endif
5284         memset(ahd, 0, sizeof(*ahd));
5285         ahd->seep_config = malloc(sizeof(*ahd->seep_config),
5286                                   M_DEVBUF, M_NOWAIT);
5287         if (ahd->seep_config == NULL) {
5288 #ifndef __FreeBSD__
5289                 free(ahd, M_DEVBUF);
5290 #endif
5291                 free(name, M_DEVBUF);
5292                 return (NULL);
5293         }
5294         LIST_INIT(&ahd->pending_scbs);
5295         LIST_INIT(&ahd->timedout_scbs);
5296         /* We don't know our unit number until the OSM sets it */
5297         ahd->name = name;
5298         ahd->unit = -1;
5299         ahd->description = NULL;
5300         ahd->bus_description = NULL;
5301         ahd->channel = 'A';
5302         ahd->chip = AHD_NONE;
5303         ahd->features = AHD_FENONE;
5304         ahd->bugs = AHD_BUGNONE;
5305         ahd->flags = AHD_SPCHK_ENB_A|AHD_RESET_BUS_A|AHD_TERM_ENB_A
5306                    | AHD_EXTENDED_TRANS_A|AHD_STPWLEVEL_A;
5307         aic_timer_init(&ahd->reset_timer);
5308         aic_timer_init(&ahd->stat_timer);
5309         ahd->int_coalescing_timer = AHD_INT_COALESCING_TIMER_DEFAULT;
5310         ahd->int_coalescing_maxcmds = AHD_INT_COALESCING_MAXCMDS_DEFAULT;
5311         ahd->int_coalescing_mincmds = AHD_INT_COALESCING_MINCMDS_DEFAULT;
5312         ahd->int_coalescing_threshold = AHD_INT_COALESCING_THRESHOLD_DEFAULT;
5313         ahd->int_coalescing_stop_threshold =
5314             AHD_INT_COALESCING_STOP_THRESHOLD_DEFAULT;
5315
5316         if (ahd_platform_alloc(ahd, platform_arg) != 0) {
5317                 ahd_free(ahd);
5318                 ahd = NULL;
5319         }
5320         ahd_lockinit(ahd);
5321 #ifdef AHD_DEBUG
5322         if ((ahd_debug & AHD_SHOW_MEMORY) != 0) {
5323                 printf("%s: scb size = 0x%x, hscb size = 0x%x\n",
5324                        ahd_name(ahd), (u_int)sizeof(struct scb),
5325                        (u_int)sizeof(struct hardware_scb));
5326         }
5327 #endif
5328         return (ahd);
5329 }
5330
5331 int
5332 ahd_softc_init(struct ahd_softc *ahd)
5333 {
5334
5335         ahd->unpause = 0;
5336         ahd->pause = PAUSE;
5337         return (0);
5338 }
5339
5340 void
5341 ahd_softc_insert(struct ahd_softc *ahd)
5342 {
5343         struct ahd_softc *list_ahd;
5344
5345 #if AIC_PCI_CONFIG > 0
5346         /*
5347          * Second Function PCI devices need to inherit some
5348          * settings from function 0.
5349          */
5350         if ((ahd->features & AHD_MULTI_FUNC) != 0) {
5351                 TAILQ_FOREACH(list_ahd, &ahd_tailq, links) {
5352                         aic_dev_softc_t list_pci;
5353                         aic_dev_softc_t pci;
5354
5355                         list_pci = list_ahd->dev_softc;
5356                         pci = ahd->dev_softc;
5357                         if (aic_get_pci_slot(list_pci) == aic_get_pci_slot(pci)
5358                          && aic_get_pci_bus(list_pci) == aic_get_pci_bus(pci)) {
5359                                 struct ahd_softc *master;
5360                                 struct ahd_softc *slave;
5361
5362                                 if (aic_get_pci_function(list_pci) == 0) {
5363                                         master = list_ahd;
5364                                         slave = ahd;
5365                                 } else {
5366                                         master = ahd;
5367                                         slave = list_ahd;
5368                                 }
5369                                 slave->flags &= ~AHD_BIOS_ENABLED;
5370                                 slave->flags |=
5371                                     master->flags & AHD_BIOS_ENABLED;
5372                                 break;
5373                         }
5374                 }
5375         }
5376 #endif
5377
5378         /*
5379          * Insertion sort into our list of softcs.
5380          */
5381         list_ahd = TAILQ_FIRST(&ahd_tailq);
5382         while (list_ahd != NULL
5383             && ahd_softc_comp(ahd, list_ahd) <= 0)
5384                 list_ahd = TAILQ_NEXT(list_ahd, links);
5385         if (list_ahd != NULL)
5386                 TAILQ_INSERT_BEFORE(list_ahd, ahd, links);
5387         else
5388                 TAILQ_INSERT_TAIL(&ahd_tailq, ahd, links);
5389         ahd->init_level++;
5390 }
5391
5392 void
5393 ahd_set_unit(struct ahd_softc *ahd, int unit)
5394 {
5395         ahd->unit = unit;
5396 }
5397
5398 void
5399 ahd_set_name(struct ahd_softc *ahd, char *name)
5400 {
5401         if (ahd->name != NULL)
5402                 free(ahd->name, M_DEVBUF);
5403         ahd->name = name;
5404 }
5405
5406 void
5407 ahd_free(struct ahd_softc *ahd)
5408 {
5409         int i;
5410
5411         ahd_terminate_recovery_thread(ahd);
5412         switch (ahd->init_level) {
5413         default:
5414         case 5:
5415                 ahd_shutdown(ahd);
5416                 /* FALLTHROUGH */
5417         case 4:
5418                 aic_dmamap_unload(ahd, ahd->shared_data_dmat,
5419                                   ahd->shared_data_map.dmamap);
5420                 /* FALLTHROUGH */
5421         case 3:
5422                 aic_dmamem_free(ahd, ahd->shared_data_dmat, ahd->qoutfifo,
5423                                 ahd->shared_data_map.dmamap);
5424                 aic_dmamap_destroy(ahd, ahd->shared_data_dmat,
5425                                    ahd->shared_data_map.dmamap);
5426                 /* FALLTHROUGH */
5427         case 2:
5428                 aic_dma_tag_destroy(ahd, ahd->shared_data_dmat);
5429         case 1:
5430 #ifndef __linux__
5431                 aic_dma_tag_destroy(ahd, ahd->buffer_dmat);
5432 #endif
5433                 break;
5434         case 0:
5435                 break;
5436         }
5437
5438 #ifndef __linux__
5439         aic_dma_tag_destroy(ahd, ahd->parent_dmat);
5440 #endif
5441         ahd_platform_free(ahd);
5442         ahd_fini_scbdata(ahd);
5443         for (i = 0; i < AHD_NUM_TARGETS; i++) {
5444                 struct ahd_tmode_tstate *tstate;
5445
5446                 tstate = ahd->enabled_targets[i];
5447                 if (tstate != NULL) {
5448 #ifdef AHD_TARGET_MODE
5449                         int j;
5450
5451                         for (j = 0; j < AHD_NUM_LUNS; j++) {
5452                                 struct ahd_tmode_lstate *lstate;
5453
5454                                 lstate = tstate->enabled_luns[j];
5455                                 if (lstate != NULL) {
5456                                         xpt_free_path(lstate->path);
5457                                         free(lstate, M_DEVBUF);
5458                                 }
5459                         }
5460 #endif
5461                         free(tstate, M_DEVBUF);
5462                 }
5463         }
5464 #ifdef AHD_TARGET_MODE
5465         if (ahd->black_hole != NULL) {
5466                 xpt_free_path(ahd->black_hole->path);
5467                 free(ahd->black_hole, M_DEVBUF);
5468         }
5469 #endif
5470         if (ahd->name != NULL)
5471                 free(ahd->name, M_DEVBUF);
5472         if (ahd->seep_config != NULL)
5473                 free(ahd->seep_config, M_DEVBUF);
5474         if (ahd->saved_stack != NULL)
5475                 free(ahd->saved_stack, M_DEVBUF);
5476 #ifndef __FreeBSD__
5477         free(ahd, M_DEVBUF);
5478 #endif
5479         return;
5480 }
5481
5482 void
5483 ahd_shutdown(void *arg)
5484 {
5485         struct  ahd_softc *ahd;
5486
5487         ahd = (struct ahd_softc *)arg;
5488
5489         /*
5490          * Stop periodic timer callbacks.
5491          */
5492         aic_timer_stop(&ahd->reset_timer);
5493         aic_timer_stop(&ahd->stat_timer);
5494
5495         /* This will reset most registers to 0, but not all */
5496         ahd_reset(ahd, /*reinit*/FALSE);
5497 }
5498
5499 /*
5500  * Reset the controller and record some information about it
5501  * that is only available just after a reset.  If "reinit" is
5502  * non-zero, this reset occured after initial configuration
5503  * and the caller requests that the chip be fully reinitialized
5504  * to a runable state.  Chip interrupts are *not* enabled after
5505  * a reinitialization.  The caller must enable interrupts via
5506  * ahd_intr_enable().
5507  */
5508 int
5509 ahd_reset(struct ahd_softc *ahd, int reinit)
5510 {
5511         u_int    sxfrctl1;
5512         int      wait;
5513         uint32_t cmd;
5514
5515         /*
5516          * Preserve the value of the SXFRCTL1 register for all channels.
5517          * It contains settings that affect termination and we don't want
5518          * to disturb the integrity of the bus.
5519          */
5520         ahd_pause(ahd);
5521         ahd_update_modes(ahd);
5522         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5523         sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
5524
5525         cmd = aic_pci_read_config(ahd->dev_softc, PCIR_COMMAND, /*bytes*/2);
5526         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
5527                 uint32_t mod_cmd;
5528
5529                 /*
5530                  * A4 Razor #632
5531                  * During the assertion of CHIPRST, the chip
5532                  * does not disable its parity logic prior to
5533                  * the start of the reset.  This may cause a
5534                  * parity error to be detected and thus a
5535                  * spurious SERR or PERR assertion.  Disble
5536                  * PERR and SERR responses during the CHIPRST.
5537                  */
5538                 mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
5539                 aic_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
5540                                      mod_cmd, /*bytes*/2);
5541         }
5542         ahd_outb(ahd, HCNTRL, CHIPRST | ahd->pause);
5543
5544         /*
5545          * Ensure that the reset has finished.  We delay 1000us
5546          * prior to reading the register to make sure the chip
5547          * has sufficiently completed its reset to handle register
5548          * accesses.
5549          */
5550         wait = 1000;
5551         do {
5552                 aic_delay(1000);
5553         } while (--wait && !(ahd_inb(ahd, HCNTRL) & CHIPRSTACK));
5554
5555         if (wait == 0) {
5556                 printf("%s: WARNING - Failed chip reset!  "
5557                        "Trying to initialize anyway.\n", ahd_name(ahd));
5558                 AHD_FATAL_ERROR(ahd);
5559         }
5560         ahd_outb(ahd, HCNTRL, ahd->pause);
5561
5562         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
5563                 /*
5564                  * Clear any latched PCI error status and restore
5565                  * previous SERR and PERR response enables.
5566                  */
5567                 aic_pci_write_config(ahd->dev_softc, PCIR_STATUS + 1,
5568                                      0xFF, /*bytes*/1);
5569                 aic_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
5570                                      cmd, /*bytes*/2);
5571         }
5572
5573         /*
5574          * Mode should be SCSI after a chip reset, but lets
5575          * set it just to be safe.  We touch the MODE_PTR
5576          * register directly so as to bypass the lazy update
5577          * code in ahd_set_modes().
5578          */
5579         ahd_known_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5580         ahd_outb(ahd, MODE_PTR,
5581                  ahd_build_mode_state(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI));
5582
5583         /*
5584          * Restore SXFRCTL1.
5585          *
5586          * We must always initialize STPWEN to 1 before we
5587          * restore the saved values.  STPWEN is initialized
5588          * to a tri-state condition which can only be cleared
5589          * by turning it on.
5590          */
5591         ahd_outb(ahd, SXFRCTL1, sxfrctl1|STPWEN);
5592         ahd_outb(ahd, SXFRCTL1, sxfrctl1);
5593
5594         /* Determine chip configuration */
5595         ahd->features &= ~AHD_WIDE;
5596         if ((ahd_inb(ahd, SBLKCTL) & SELWIDE) != 0)
5597                 ahd->features |= AHD_WIDE;
5598
5599         /*
5600          * If a recovery action has forced a chip reset,
5601          * re-initialize the chip to our liking.
5602          */
5603         if (reinit != 0)
5604                 ahd_chip_init(ahd);
5605
5606         return (0);
5607 }
5608
5609 /*
5610  * Determine the number of SCBs available on the controller
5611  */
5612 int
5613 ahd_probe_scbs(struct ahd_softc *ahd) {
5614         int i;
5615
5616         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
5617                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
5618         for (i = 0; i < AHD_SCB_MAX; i++) {
5619                 int j;
5620
5621                 ahd_set_scbptr(ahd, i);
5622                 ahd_outw(ahd, SCB_BASE, i);
5623                 for (j = 2; j < 64; j++)
5624                         ahd_outb(ahd, SCB_BASE+j, 0);
5625                 /* Start out life as unallocated (needing an abort) */
5626                 ahd_outb(ahd, SCB_CONTROL, MK_MESSAGE);
5627                 if (ahd_inw_scbram(ahd, SCB_BASE) != i)
5628                         break;
5629                 ahd_set_scbptr(ahd, 0);
5630                 if (ahd_inw_scbram(ahd, SCB_BASE) != 0)
5631                         break;
5632         }
5633         return (i);
5634 }
5635
5636 static void
5637 ahd_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
5638 {
5639         bus_addr_t *baddr;
5640
5641         baddr = (bus_addr_t *)arg;
5642         *baddr = segs->ds_addr;
5643 }
5644
5645 static void
5646 ahd_initialize_hscbs(struct ahd_softc *ahd)
5647 {
5648         int i;
5649
5650         for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
5651                 ahd_set_scbptr(ahd, i);
5652
5653                 /* Clear the control byte. */
5654                 ahd_outb(ahd, SCB_CONTROL, 0);
5655
5656                 /* Set the next pointer */
5657                 ahd_outw(ahd, SCB_NEXT, SCB_LIST_NULL);
5658         }
5659 }
5660
5661 static int
5662 ahd_init_scbdata(struct ahd_softc *ahd)
5663 {
5664         struct  scb_data *scb_data;
5665         int     i;
5666
5667         scb_data = &ahd->scb_data;
5668         TAILQ_INIT(&scb_data->free_scbs);
5669         for (i = 0; i < AHD_NUM_TARGETS * AHD_NUM_LUNS_NONPKT; i++)
5670                 LIST_INIT(&scb_data->free_scb_lists[i]);
5671         LIST_INIT(&scb_data->any_dev_free_scb_list);
5672         SLIST_INIT(&scb_data->hscb_maps);
5673         SLIST_INIT(&scb_data->sg_maps);
5674         SLIST_INIT(&scb_data->sense_maps);
5675
5676         /* Determine the number of hardware SCBs and initialize them */
5677         scb_data->maxhscbs = ahd_probe_scbs(ahd);
5678         if (scb_data->maxhscbs == 0) {
5679                 printf("%s: No SCB space found\n", ahd_name(ahd));
5680                 AHD_FATAL_ERROR(ahd);
5681                 return (ENXIO);
5682         }
5683
5684         ahd_initialize_hscbs(ahd);
5685
5686         /*
5687          * Create our DMA tags.  These tags define the kinds of device
5688          * accessible memory allocations and memory mappings we will
5689          * need to perform during normal operation.
5690          *
5691          * Unless we need to further restrict the allocation, we rely
5692          * on the restrictions of the parent dmat, hence the common
5693          * use of MAXADDR and MAXSIZE.
5694          */
5695
5696         /* DMA tag for our hardware scb structures */
5697         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
5698                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5699                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5700                                /*highaddr*/BUS_SPACE_MAXADDR,
5701                                /*filter*/NULL, /*filterarg*/NULL,
5702                                PAGE_SIZE, /*nsegments*/1,
5703                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5704                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
5705                 goto error_exit;
5706         }
5707
5708         scb_data->init_level++;
5709
5710         /* DMA tag for our S/G structures. */
5711         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/8,
5712                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5713                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5714                                /*highaddr*/BUS_SPACE_MAXADDR,
5715                                /*filter*/NULL, /*filterarg*/NULL,
5716                                ahd_sglist_allocsize(ahd), /*nsegments*/1,
5717                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5718                                /*flags*/0, &scb_data->sg_dmat) != 0) {
5719                 goto error_exit;
5720         }
5721 #ifdef AHD_DEBUG
5722         if ((ahd_debug & AHD_SHOW_MEMORY) != 0)
5723                 printf("%s: ahd_sglist_allocsize = 0x%x\n", ahd_name(ahd),
5724                        ahd_sglist_allocsize(ahd));
5725 #endif
5726
5727         scb_data->init_level++;
5728
5729         /* DMA tag for our sense buffers.  We allocate in page sized chunks */
5730         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
5731                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5732                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5733                                /*highaddr*/BUS_SPACE_MAXADDR,
5734                                /*filter*/NULL, /*filterarg*/NULL,
5735                                PAGE_SIZE, /*nsegments*/1,
5736                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5737                                /*flags*/0, &scb_data->sense_dmat) != 0) {
5738                 goto error_exit;
5739         }
5740
5741         scb_data->init_level++;
5742
5743         /* Perform initial CCB allocation */
5744         while (ahd_alloc_scbs(ahd) != 0)
5745                 ;
5746
5747         if (scb_data->numscbs == 0) {
5748                 printf("%s: ahd_init_scbdata - "
5749                        "Unable to allocate initial scbs\n",
5750                        ahd_name(ahd));
5751                 goto error_exit;
5752         }
5753
5754         /*
5755          * Note that we were successfull
5756          */
5757         return (0);
5758
5759 error_exit:
5760
5761         return (ENOMEM);
5762 }
5763
5764 static struct scb *
5765 ahd_find_scb_by_tag(struct ahd_softc *ahd, u_int tag)
5766 {
5767         struct scb *scb;
5768
5769         /*
5770          * Look on the pending list.
5771          */
5772         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
5773                 if (SCB_GET_TAG(scb) == tag)
5774                         return (scb);
5775         }
5776
5777         /*
5778          * Then on all of the collision free lists.
5779          */
5780         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
5781                 struct scb *list_scb;
5782
5783                 list_scb = scb;
5784                 do {
5785                         if (SCB_GET_TAG(list_scb) == tag)
5786                                 return (list_scb);
5787                         list_scb = LIST_NEXT(list_scb, collision_links);
5788                 } while (list_scb);
5789         }
5790
5791         /*
5792          * And finally on the generic free list.
5793          */
5794         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
5795                 if (SCB_GET_TAG(scb) == tag)
5796                         return (scb);
5797         }
5798
5799         return (NULL);
5800 }
5801
5802 static void
5803 ahd_fini_scbdata(struct ahd_softc *ahd)
5804 {
5805         struct scb_data *scb_data;
5806
5807         scb_data = &ahd->scb_data;
5808         if (scb_data == NULL)
5809                 return;
5810
5811         switch (scb_data->init_level) {
5812         default:
5813         case 7:
5814         {
5815                 struct map_node *sns_map;
5816
5817                 while ((sns_map = SLIST_FIRST(&scb_data->sense_maps)) != NULL) {
5818                         SLIST_REMOVE_HEAD(&scb_data->sense_maps, links);
5819                         aic_dmamap_unload(ahd, scb_data->sense_dmat,
5820                                           sns_map->dmamap);
5821                         aic_dmamem_free(ahd, scb_data->sense_dmat,
5822                                         sns_map->vaddr, sns_map->dmamap);
5823                         free(sns_map, M_DEVBUF);
5824                 }
5825                 aic_dma_tag_destroy(ahd, scb_data->sense_dmat);
5826                 /* FALLTHROUGH */
5827         }
5828         case 6:
5829         {
5830                 struct map_node *sg_map;
5831
5832                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps)) != NULL) {
5833                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
5834                         aic_dmamap_unload(ahd, scb_data->sg_dmat,
5835                                           sg_map->dmamap);
5836                         aic_dmamem_free(ahd, scb_data->sg_dmat,
5837                                         sg_map->vaddr, sg_map->dmamap);
5838                         free(sg_map, M_DEVBUF);
5839                 }
5840                 aic_dma_tag_destroy(ahd, scb_data->sg_dmat);
5841                 /* FALLTHROUGH */
5842         }
5843         case 5:
5844         {
5845                 struct map_node *hscb_map;
5846
5847                 while ((hscb_map = SLIST_FIRST(&scb_data->hscb_maps)) != NULL) {
5848                         SLIST_REMOVE_HEAD(&scb_data->hscb_maps, links);
5849                         aic_dmamap_unload(ahd, scb_data->hscb_dmat,
5850                                           hscb_map->dmamap);
5851                         aic_dmamem_free(ahd, scb_data->hscb_dmat,
5852                                         hscb_map->vaddr, hscb_map->dmamap);
5853                         free(hscb_map, M_DEVBUF);
5854                 }
5855                 aic_dma_tag_destroy(ahd, scb_data->hscb_dmat);
5856                 /* FALLTHROUGH */
5857         }
5858         case 4:
5859         case 3:
5860         case 2:
5861         case 1:
5862         case 0:
5863                 break;
5864         }
5865 }
5866
5867 /*
5868  * DSP filter Bypass must be enabled until the first selection
5869  * after a change in bus mode (Razor #491 and #493).
5870  */
5871 static void
5872 ahd_setup_iocell_workaround(struct ahd_softc *ahd)
5873 {
5874         ahd_mode_state saved_modes;
5875
5876         saved_modes = ahd_save_modes(ahd);
5877         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
5878         ahd_outb(ahd, DSPDATACTL, ahd_inb(ahd, DSPDATACTL)
5879                | BYPASSENAB | RCVROFFSTDIS | XMITOFFSTDIS);
5880         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) | (ENSELDO|ENSELDI));
5881 #ifdef AHD_DEBUG
5882         if ((ahd_debug & AHD_SHOW_MISC) != 0)
5883                 printf("%s: Setting up iocell workaround\n", ahd_name(ahd));
5884 #endif
5885         ahd_restore_modes(ahd, saved_modes);
5886         ahd->flags &= ~AHD_HAD_FIRST_SEL;
5887 }
5888
5889 static void
5890 ahd_iocell_first_selection(struct ahd_softc *ahd)
5891 {
5892         ahd_mode_state  saved_modes;
5893         u_int           sblkctl;
5894
5895         if ((ahd->flags & AHD_HAD_FIRST_SEL) != 0)
5896                 return;
5897         saved_modes = ahd_save_modes(ahd);
5898         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5899         sblkctl = ahd_inb(ahd, SBLKCTL);
5900         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
5901 #ifdef AHD_DEBUG
5902         if ((ahd_debug & AHD_SHOW_MISC) != 0)
5903                 printf("%s: iocell first selection\n", ahd_name(ahd));
5904 #endif
5905         if ((sblkctl & ENAB40) != 0) {
5906                 ahd_outb(ahd, DSPDATACTL,
5907                          ahd_inb(ahd, DSPDATACTL) & ~BYPASSENAB);
5908 #ifdef AHD_DEBUG
5909                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
5910                         printf("%s: BYPASS now disabled\n", ahd_name(ahd));
5911 #endif
5912         }
5913         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) & ~(ENSELDO|ENSELDI));
5914         ahd_outb(ahd, CLRINT, CLRSCSIINT);
5915         ahd_restore_modes(ahd, saved_modes);
5916         ahd->flags |= AHD_HAD_FIRST_SEL;
5917 }
5918
5919 /*************************** SCB Management ***********************************/
5920 static void
5921 ahd_add_col_list(struct ahd_softc *ahd, struct scb *scb, u_int col_idx)
5922 {
5923         struct  scb_list *free_list;
5924         struct  scb_tailq *free_tailq;
5925         struct  scb *first_scb;
5926
5927         scb->flags |= SCB_ON_COL_LIST;
5928         AHD_SET_SCB_COL_IDX(scb, col_idx);
5929         free_list = &ahd->scb_data.free_scb_lists[col_idx];
5930         free_tailq = &ahd->scb_data.free_scbs;
5931         first_scb = LIST_FIRST(free_list);
5932         if (first_scb != NULL) {
5933                 LIST_INSERT_AFTER(first_scb, scb, collision_links);
5934         } else {
5935                 LIST_INSERT_HEAD(free_list, scb, collision_links);
5936                 TAILQ_INSERT_TAIL(free_tailq, scb, links.tqe);
5937         }
5938 }
5939
5940 static void
5941 ahd_rem_col_list(struct ahd_softc *ahd, struct scb *scb)
5942 {
5943         struct  scb_list *free_list;
5944         struct  scb_tailq *free_tailq;
5945         struct  scb *first_scb;
5946         u_int   col_idx;
5947
5948         scb->flags &= ~SCB_ON_COL_LIST;
5949         col_idx = AHD_GET_SCB_COL_IDX(ahd, scb);
5950         free_list = &ahd->scb_data.free_scb_lists[col_idx];
5951         free_tailq = &ahd->scb_data.free_scbs;
5952         first_scb = LIST_FIRST(free_list);
5953         if (first_scb == scb) {
5954                 struct scb *next_scb;
5955
5956                 /*
5957                  * Maintain order in the collision free
5958                  * lists for fairness if this device has
5959                  * other colliding tags active.
5960                  */
5961                 next_scb = LIST_NEXT(scb, collision_links);
5962                 if (next_scb != NULL) {
5963                         TAILQ_INSERT_AFTER(free_tailq, scb,
5964                                            next_scb, links.tqe);
5965                 }
5966                 TAILQ_REMOVE(free_tailq, scb, links.tqe);
5967         }
5968         LIST_REMOVE(scb, collision_links);
5969 }
5970
5971 /*
5972  * Get a free scb. If there are none, see if we can allocate a new SCB.
5973  */
5974 struct scb *
5975 ahd_get_scb(struct ahd_softc *ahd, u_int col_idx)
5976 {
5977         struct scb *scb;
5978         int tries;
5979
5980         tries = 0;
5981 look_again:
5982         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
5983                 if (AHD_GET_SCB_COL_IDX(ahd, scb) != col_idx) {
5984                         ahd_rem_col_list(ahd, scb);
5985                         goto found;
5986                 }
5987         }
5988         if ((scb = LIST_FIRST(&ahd->scb_data.any_dev_free_scb_list)) == NULL) {
5989
5990                 if (tries++ != 0)
5991                         return (NULL);
5992                 if (ahd_alloc_scbs(ahd) == 0)
5993                         return (NULL);
5994                 goto look_again;
5995         }
5996         LIST_REMOVE(scb, links.le);
5997         if (col_idx != AHD_NEVER_COL_IDX
5998          && (scb->col_scb != NULL)
5999          && (scb->col_scb->flags & SCB_ACTIVE) == 0) {
6000                 LIST_REMOVE(scb->col_scb, links.le);
6001                 ahd_add_col_list(ahd, scb->col_scb, col_idx);
6002         }
6003 found:
6004         scb->flags |= SCB_ACTIVE;
6005         return (scb);
6006 }
6007
6008 /*
6009  * Return an SCB resource to the free list.
6010  */
6011 void
6012 ahd_free_scb(struct ahd_softc *ahd, struct scb *scb)
6013 {
6014
6015         /* Clean up for the next user */
6016         scb->flags = SCB_FLAG_NONE;
6017         scb->hscb->control = 0;
6018         ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = NULL;
6019
6020         if (scb->col_scb == NULL) {
6021
6022                 /*
6023                  * No collision possible.  Just free normally.
6024                  */
6025                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6026                                  scb, links.le);
6027         } else if ((scb->col_scb->flags & SCB_ON_COL_LIST) != 0) {
6028
6029                 /*
6030                  * The SCB we might have collided with is on
6031                  * a free collision list.  Put both SCBs on
6032                  * the generic list.
6033                  */
6034                 ahd_rem_col_list(ahd, scb->col_scb);
6035                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6036                                  scb, links.le);
6037                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6038                                  scb->col_scb, links.le);
6039         } else if ((scb->col_scb->flags
6040                   & (SCB_PACKETIZED|SCB_ACTIVE)) == SCB_ACTIVE
6041                 && (scb->col_scb->hscb->control & TAG_ENB) != 0) {
6042
6043                 /*
6044                  * The SCB we might collide with on the next allocation
6045                  * is still active in a non-packetized, tagged, context.
6046                  * Put us on the SCB collision list.
6047                  */
6048                 ahd_add_col_list(ahd, scb,
6049                                  AHD_GET_SCB_COL_IDX(ahd, scb->col_scb));
6050         } else {
6051                 /*
6052                  * The SCB we might collide with on the next allocation
6053                  * is either active in a packetized context, or free.
6054                  * Since we can't collide, put this SCB on the generic
6055                  * free list.
6056                  */
6057                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6058                                  scb, links.le);
6059         }
6060
6061         aic_platform_scb_free(ahd, scb);
6062 }
6063
6064 int
6065 ahd_alloc_scbs(struct ahd_softc *ahd)
6066 {
6067         struct scb_data *scb_data;
6068         struct scb      *next_scb;
6069         struct hardware_scb *hscb;
6070         struct map_node *hscb_map;
6071         struct map_node *sg_map;
6072         struct map_node *sense_map;
6073         uint8_t         *segs;
6074         uint8_t         *sense_data;
6075         bus_addr_t       hscb_busaddr;
6076         bus_addr_t       sg_busaddr;
6077         bus_addr_t       sense_busaddr;
6078         int              newcount;
6079         int              i;
6080
6081         scb_data = &ahd->scb_data;
6082         if (scb_data->numscbs >= AHD_SCB_MAX_ALLOC)
6083                 /* Can't allocate any more */
6084                 return (0);
6085
6086         if (scb_data->scbs_left != 0) {
6087                 int offset;
6088
6089                 offset = (PAGE_SIZE / sizeof(*hscb)) - scb_data->scbs_left;
6090                 hscb_map = SLIST_FIRST(&scb_data->hscb_maps);
6091                 hscb = &((struct hardware_scb *)hscb_map->vaddr)[offset];
6092                 hscb_busaddr = hscb_map->busaddr + (offset * sizeof(*hscb));
6093         } else {
6094                 hscb_map = malloc(sizeof(*hscb_map), M_DEVBUF, M_NOWAIT);
6095
6096                 if (hscb_map == NULL)
6097                         return (0);
6098
6099                 /* Allocate the next batch of hardware SCBs */
6100                 if (aic_dmamem_alloc(ahd, scb_data->hscb_dmat,
6101                                      (void **)&hscb_map->vaddr,
6102                                      BUS_DMA_NOWAIT, &hscb_map->dmamap) != 0) {
6103                         free(hscb_map, M_DEVBUF);
6104                         return (0);
6105                 }
6106
6107                 SLIST_INSERT_HEAD(&scb_data->hscb_maps, hscb_map, links);
6108
6109                 aic_dmamap_load(ahd, scb_data->hscb_dmat, hscb_map->dmamap,
6110                                 hscb_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6111                                 &hscb_map->busaddr, /*flags*/0);
6112
6113                 hscb = (struct hardware_scb *)hscb_map->vaddr;
6114                 hscb_busaddr = hscb_map->busaddr;
6115                 scb_data->scbs_left = PAGE_SIZE / sizeof(*hscb);
6116         }
6117
6118         if (scb_data->sgs_left != 0) {
6119                 int offset;
6120
6121                 offset = ((ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd))
6122                        - scb_data->sgs_left) * ahd_sglist_size(ahd);
6123                 sg_map = SLIST_FIRST(&scb_data->sg_maps);
6124                 segs = sg_map->vaddr + offset;
6125                 sg_busaddr = sg_map->busaddr + offset;
6126         } else {
6127                 sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
6128
6129                 if (sg_map == NULL)
6130                         return (0);
6131
6132                 /* Allocate the next batch of S/G lists */
6133                 if (aic_dmamem_alloc(ahd, scb_data->sg_dmat,
6134                                      (void **)&sg_map->vaddr,
6135                                      BUS_DMA_NOWAIT, &sg_map->dmamap) != 0) {
6136                         free(sg_map, M_DEVBUF);
6137                         return (0);
6138                 }
6139
6140                 SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
6141
6142                 aic_dmamap_load(ahd, scb_data->sg_dmat, sg_map->dmamap,
6143                                 sg_map->vaddr, ahd_sglist_allocsize(ahd),
6144                                 ahd_dmamap_cb, &sg_map->busaddr, /*flags*/0);
6145
6146                 segs = sg_map->vaddr;
6147                 sg_busaddr = sg_map->busaddr;
6148                 scb_data->sgs_left =
6149                     ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd);
6150 #ifdef AHD_DEBUG
6151                 if (ahd_debug & AHD_SHOW_MEMORY)
6152                         printf("Mapped SG data\n");
6153 #endif
6154         }
6155
6156         if (scb_data->sense_left != 0) {
6157                 int offset;
6158
6159                 offset = PAGE_SIZE - (AHD_SENSE_BUFSIZE * scb_data->sense_left);
6160                 sense_map = SLIST_FIRST(&scb_data->sense_maps);
6161                 sense_data = sense_map->vaddr + offset;
6162                 sense_busaddr = sense_map->busaddr + offset;
6163         } else {
6164                 sense_map = malloc(sizeof(*sense_map), M_DEVBUF, M_NOWAIT);
6165
6166                 if (sense_map == NULL)
6167                         return (0);
6168
6169                 /* Allocate the next batch of sense buffers */
6170                 if (aic_dmamem_alloc(ahd, scb_data->sense_dmat,
6171                                      (void **)&sense_map->vaddr,
6172                                      BUS_DMA_NOWAIT, &sense_map->dmamap) != 0) {
6173                         free(sense_map, M_DEVBUF);
6174                         return (0);
6175                 }
6176
6177                 SLIST_INSERT_HEAD(&scb_data->sense_maps, sense_map, links);
6178
6179                 aic_dmamap_load(ahd, scb_data->sense_dmat, sense_map->dmamap,
6180                                 sense_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6181                                 &sense_map->busaddr, /*flags*/0);
6182
6183                 sense_data = sense_map->vaddr;
6184                 sense_busaddr = sense_map->busaddr;
6185                 scb_data->sense_left = PAGE_SIZE / AHD_SENSE_BUFSIZE;
6186 #ifdef AHD_DEBUG
6187                 if (ahd_debug & AHD_SHOW_MEMORY)
6188                         printf("Mapped sense data\n");
6189 #endif
6190         }
6191
6192         newcount = MIN(scb_data->sense_left, scb_data->scbs_left);
6193         newcount = MIN(newcount, scb_data->sgs_left);
6194         newcount = MIN(newcount, (AHD_SCB_MAX_ALLOC - scb_data->numscbs));
6195         scb_data->sense_left -= newcount;
6196         scb_data->scbs_left -= newcount;
6197         scb_data->sgs_left -= newcount;
6198         for (i = 0; i < newcount; i++) {
6199                 struct scb_platform_data *pdata;
6200                 u_int col_tag;
6201 #ifndef __linux__
6202                 int error;
6203 #endif
6204
6205                 next_scb = (struct scb *)malloc(sizeof(*next_scb),
6206                                                 M_DEVBUF, M_NOWAIT);
6207                 if (next_scb == NULL)
6208                         break;
6209
6210                 pdata = (struct scb_platform_data *)malloc(sizeof(*pdata),
6211                                                            M_DEVBUF, M_NOWAIT);
6212                 if (pdata == NULL) {
6213                         free(next_scb, M_DEVBUF);
6214                         break;
6215                 }
6216                 next_scb->platform_data = pdata;
6217                 next_scb->hscb_map = hscb_map;
6218                 next_scb->sg_map = sg_map;
6219                 next_scb->sense_map = sense_map;
6220                 next_scb->sg_list = segs;
6221                 next_scb->sense_data = sense_data;
6222                 next_scb->sense_busaddr = sense_busaddr;
6223                 memset(hscb, 0, sizeof(*hscb));
6224                 next_scb->hscb = hscb;
6225                 hscb->hscb_busaddr = aic_htole32(hscb_busaddr);
6226
6227                 /*
6228                  * The sequencer always starts with the second entry.
6229                  * The first entry is embedded in the scb.
6230                  */
6231                 next_scb->sg_list_busaddr = sg_busaddr;
6232                 if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
6233                         next_scb->sg_list_busaddr
6234                             += sizeof(struct ahd_dma64_seg);
6235                 else
6236                         next_scb->sg_list_busaddr += sizeof(struct ahd_dma_seg);
6237                 next_scb->ahd_softc = ahd;
6238                 next_scb->flags = SCB_FLAG_NONE;
6239 #ifndef __linux__
6240                 error = aic_dmamap_create(ahd, ahd->buffer_dmat, /*flags*/0,
6241                                           &next_scb->dmamap);
6242                 if (error != 0) {
6243                         free(next_scb, M_DEVBUF);
6244                         free(pdata, M_DEVBUF);
6245                         break;
6246                 }
6247 #endif
6248                 next_scb->hscb->tag = aic_htole16(scb_data->numscbs);
6249                 col_tag = scb_data->numscbs ^ 0x100;
6250                 next_scb->col_scb = ahd_find_scb_by_tag(ahd, col_tag);
6251                 if (next_scb->col_scb != NULL)
6252                         next_scb->col_scb->col_scb = next_scb;
6253                 aic_timer_init(&next_scb->io_timer);
6254                 ahd_free_scb(ahd, next_scb);
6255                 hscb++;
6256                 hscb_busaddr += sizeof(*hscb);
6257                 segs += ahd_sglist_size(ahd);
6258                 sg_busaddr += ahd_sglist_size(ahd);
6259                 sense_data += AHD_SENSE_BUFSIZE;
6260                 sense_busaddr += AHD_SENSE_BUFSIZE;
6261                 scb_data->numscbs++;
6262         }
6263         return (i);
6264 }
6265
6266 void
6267 ahd_controller_info(struct ahd_softc *ahd, char *buf)
6268 {
6269         const char *speed;
6270         const char *type;
6271         int len;
6272
6273         len = sprintf(buf, "%s: ", ahd_chip_names[ahd->chip & AHD_CHIPID_MASK]);
6274         buf += len;
6275
6276         speed = "Ultra320 ";
6277         if ((ahd->features & AHD_WIDE) != 0) {
6278                 type = "Wide ";
6279         } else {
6280                 type = "Single ";
6281         }
6282         len = sprintf(buf, "%s%sChannel %c, SCSI Id=%d, ",
6283                       speed, type, ahd->channel, ahd->our_id);
6284         buf += len;
6285
6286         sprintf(buf, "%s, %d SCBs", ahd->bus_description,
6287                 ahd->scb_data.maxhscbs);
6288 }
6289
6290 static const char *channel_strings[] = {
6291         "Primary Low",
6292         "Primary High",
6293         "Secondary Low",
6294         "Secondary High"
6295 };
6296
6297 static const char *termstat_strings[] = {
6298         "Terminated Correctly",
6299         "Over Terminated",
6300         "Under Terminated",
6301         "Not Configured"
6302 };
6303
6304 /*
6305  * Start the board, ready for normal operation
6306  */
6307 int
6308 ahd_init(struct ahd_softc *ahd)
6309 {
6310         uint8_t         *next_vaddr;
6311         bus_addr_t       next_baddr;
6312         size_t           driver_data_size;
6313         int              i;
6314         int              error;
6315         u_int            warn_user;
6316         uint8_t          current_sensing;
6317         uint8_t          fstat;
6318
6319         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
6320
6321         ahd->stack_size = ahd_probe_stack_size(ahd);
6322         ahd->saved_stack = malloc(ahd->stack_size * sizeof(uint16_t),
6323                                   M_DEVBUF, M_NOWAIT);
6324         if (ahd->saved_stack == NULL)
6325                 return (ENOMEM);
6326
6327         /*
6328          * Verify that the compiler hasn't over-agressively
6329          * padded important structures.
6330          */
6331         if (sizeof(struct hardware_scb) != 64)
6332                 panic("Hardware SCB size is incorrect");
6333
6334 #ifdef AHD_DEBUG
6335         if ((ahd_debug & AHD_DEBUG_SEQUENCER) != 0)
6336                 ahd->flags |= AHD_SEQUENCER_DEBUG;
6337 #endif
6338
6339         /*
6340          * Default to allowing initiator operations.
6341          */
6342         ahd->flags |= AHD_INITIATORROLE;
6343
6344         /*
6345          * Only allow target mode features if this unit has them enabled.
6346          */
6347         if ((AHD_TMODE_ENABLE & (0x1 << ahd->unit)) == 0)
6348                 ahd->features &= ~AHD_TARGETMODE;
6349
6350 #ifndef __linux__
6351         /* DMA tag for mapping buffers into device visible space. */
6352         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6353                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6354                                /*lowaddr*/ahd->flags & AHD_39BIT_ADDRESSING
6355                                         ? (bus_addr_t)0x7FFFFFFFFFULL
6356                                         : BUS_SPACE_MAXADDR_32BIT,
6357                                /*highaddr*/BUS_SPACE_MAXADDR,
6358                                /*filter*/NULL, /*filterarg*/NULL,
6359                                /*maxsize*/(AHD_NSEG - 1) * PAGE_SIZE,
6360                                /*nsegments*/AHD_NSEG,
6361                                /*maxsegsz*/AHD_MAXTRANSFER_SIZE,
6362                                /*flags*/BUS_DMA_ALLOCNOW,
6363                                &ahd->buffer_dmat) != 0) {
6364                 return (ENOMEM);
6365         }
6366 #endif
6367
6368         ahd->init_level++;
6369
6370         /*
6371          * DMA tag for our command fifos and other data in system memory
6372          * the card's sequencer must be able to access.  For initiator
6373          * roles, we need to allocate space for the qoutfifo.  When providing
6374          * for the target mode role, we must additionally provide space for
6375          * the incoming target command fifo.
6376          */
6377         driver_data_size = AHD_SCB_MAX * sizeof(*ahd->qoutfifo)
6378                          + sizeof(struct hardware_scb);
6379         if ((ahd->features & AHD_TARGETMODE) != 0)
6380                 driver_data_size += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6381         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0)
6382                 driver_data_size += PKT_OVERRUN_BUFSIZE;
6383         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6384                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6385                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6386                                /*highaddr*/BUS_SPACE_MAXADDR,
6387                                /*filter*/NULL, /*filterarg*/NULL,
6388                                driver_data_size,
6389                                /*nsegments*/1,
6390                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6391                                /*flags*/0, &ahd->shared_data_dmat) != 0) {
6392                 return (ENOMEM);
6393         }
6394
6395         ahd->init_level++;
6396
6397         /* Allocation of driver data */
6398         if (aic_dmamem_alloc(ahd, ahd->shared_data_dmat,
6399                              (void **)&ahd->shared_data_map.vaddr,
6400                              BUS_DMA_NOWAIT,
6401                              &ahd->shared_data_map.dmamap) != 0) {
6402                 return (ENOMEM);
6403         }
6404
6405         ahd->init_level++;
6406
6407         /* And permanently map it in */
6408         aic_dmamap_load(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
6409                         ahd->shared_data_map.vaddr, driver_data_size,
6410                         ahd_dmamap_cb, &ahd->shared_data_map.busaddr,
6411                         /*flags*/0);
6412         ahd->qoutfifo = (struct ahd_completion *)ahd->shared_data_map.vaddr;
6413         next_vaddr = (uint8_t *)&ahd->qoutfifo[AHD_QOUT_SIZE];
6414         next_baddr = ahd->shared_data_map.busaddr
6415                    + AHD_QOUT_SIZE*sizeof(struct ahd_completion);
6416         if ((ahd->features & AHD_TARGETMODE) != 0) {
6417                 ahd->targetcmds = (struct target_cmd *)next_vaddr;
6418                 next_vaddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6419                 next_baddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6420         }
6421
6422         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0) {
6423                 ahd->overrun_buf = next_vaddr;
6424                 next_vaddr += PKT_OVERRUN_BUFSIZE;
6425                 next_baddr += PKT_OVERRUN_BUFSIZE;
6426         }
6427
6428         /*
6429          * We need one SCB to serve as the "next SCB".  Since the
6430          * tag identifier in this SCB will never be used, there is
6431          * no point in using a valid HSCB tag from an SCB pulled from
6432          * the standard free pool.  So, we allocate this "sentinel"
6433          * specially from the DMA safe memory chunk used for the QOUTFIFO.
6434          */
6435         ahd->next_queued_hscb = (struct hardware_scb *)next_vaddr;
6436         ahd->next_queued_hscb_map = &ahd->shared_data_map;
6437         ahd->next_queued_hscb->hscb_busaddr = aic_htole32(next_baddr);
6438
6439         ahd->init_level++;
6440
6441         /* Allocate SCB data now that buffer_dmat is initialized */
6442         if (ahd_init_scbdata(ahd) != 0)
6443                 return (ENOMEM);
6444
6445         if ((ahd->flags & AHD_INITIATORROLE) == 0)
6446                 ahd->flags &= ~AHD_RESET_BUS_A;
6447
6448         /*
6449          * Before committing these settings to the chip, give
6450          * the OSM one last chance to modify our configuration.
6451          */
6452         ahd_platform_init(ahd);
6453
6454         /* Bring up the chip. */
6455         ahd_chip_init(ahd);
6456
6457         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
6458
6459         if ((ahd->flags & AHD_CURRENT_SENSING) == 0)
6460                 goto init_done;
6461
6462         /*
6463          * Verify termination based on current draw and
6464          * warn user if the bus is over/under terminated.
6465          */
6466         error = ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL,
6467                                    CURSENSE_ENB);
6468         if (error != 0) {
6469                 printf("%s: current sensing timeout 1\n", ahd_name(ahd));
6470                 goto init_done;
6471         }
6472         for (i = 20, fstat = FLX_FSTAT_BUSY;
6473              (fstat & FLX_FSTAT_BUSY) != 0 && i; i--) {
6474                 error = ahd_read_flexport(ahd, FLXADDR_FLEXSTAT, &fstat);
6475                 if (error != 0) {
6476                         printf("%s: current sensing timeout 2\n",
6477                                ahd_name(ahd));
6478                         goto init_done;
6479                 }
6480         }
6481         if (i == 0) {
6482                 printf("%s: Timedout during current-sensing test\n",
6483                        ahd_name(ahd));
6484                 goto init_done;
6485         }
6486
6487         /* Latch Current Sensing status. */
6488         error = ahd_read_flexport(ahd, FLXADDR_CURRENT_STAT, &current_sensing);
6489         if (error != 0) {
6490                 printf("%s: current sensing timeout 3\n", ahd_name(ahd));
6491                 goto init_done;
6492         }
6493
6494         /* Diable current sensing. */
6495         ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, 0);
6496
6497 #ifdef AHD_DEBUG
6498         if ((ahd_debug & AHD_SHOW_TERMCTL) != 0) {
6499                 printf("%s: current_sensing == 0x%x\n",
6500                        ahd_name(ahd), current_sensing);
6501         }
6502 #endif
6503         warn_user = 0;
6504         for (i = 0; i < 4; i++, current_sensing >>= FLX_CSTAT_SHIFT) {
6505                 u_int term_stat;
6506
6507                 term_stat = (current_sensing & FLX_CSTAT_MASK);
6508                 switch (term_stat) {
6509                 case FLX_CSTAT_OVER:
6510                 case FLX_CSTAT_UNDER:
6511                         warn_user++;
6512                 case FLX_CSTAT_INVALID:
6513                 case FLX_CSTAT_OKAY:
6514                         if (warn_user == 0 && bootverbose == 0)
6515                                 break;
6516                         printf("%s: %s Channel %s\n", ahd_name(ahd),
6517                                channel_strings[i], termstat_strings[term_stat]);
6518                         break;
6519                 }
6520         }
6521         if (warn_user) {
6522                 printf("%s: WARNING. Termination is not configured correctly.\n"
6523                        "%s: WARNING. SCSI bus operations may FAIL.\n",
6524                        ahd_name(ahd), ahd_name(ahd));
6525                 AHD_CORRECTABLE_ERROR(ahd);
6526         }
6527 init_done:
6528         ahd_restart(ahd);
6529         aic_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_MS,
6530                         ahd_stat_timer, ahd);
6531         return (0);
6532 }
6533
6534 /*
6535  * (Re)initialize chip state after a chip reset.
6536  */
6537 static void
6538 ahd_chip_init(struct ahd_softc *ahd)
6539 {
6540         uint32_t busaddr;
6541         u_int    sxfrctl1;
6542         u_int    scsiseq_template;
6543         u_int    wait;
6544         u_int    i;
6545         u_int    target;
6546
6547         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6548         /*
6549          * Take the LED out of diagnostic mode
6550          */
6551         ahd_outb(ahd, SBLKCTL, ahd_inb(ahd, SBLKCTL) & ~(DIAGLEDEN|DIAGLEDON));
6552
6553         /*
6554          * Return HS_MAILBOX to its default value.
6555          */
6556         ahd->hs_mailbox = 0;
6557         ahd_outb(ahd, HS_MAILBOX, 0);
6558
6559         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1. */
6560         ahd_outb(ahd, IOWNID, ahd->our_id);
6561         ahd_outb(ahd, TOWNID, ahd->our_id);
6562         sxfrctl1 = (ahd->flags & AHD_TERM_ENB_A) != 0 ? STPWEN : 0;
6563         sxfrctl1 |= (ahd->flags & AHD_SPCHK_ENB_A) != 0 ? ENSPCHK : 0;
6564         if ((ahd->bugs & AHD_LONG_SETIMO_BUG)
6565          && (ahd->seltime != STIMESEL_MIN)) {
6566                 /*
6567                  * The selection timer duration is twice as long
6568                  * as it should be.  Halve it by adding "1" to
6569                  * the user specified setting.
6570                  */
6571                 sxfrctl1 |= ahd->seltime + STIMESEL_BUG_ADJ;
6572         } else {
6573                 sxfrctl1 |= ahd->seltime;
6574         }
6575
6576         ahd_outb(ahd, SXFRCTL0, DFON);
6577         ahd_outb(ahd, SXFRCTL1, sxfrctl1|ahd->seltime|ENSTIMER|ACTNEGEN);
6578         ahd_outb(ahd, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
6579
6580         /*
6581          * Now that termination is set, wait for up
6582          * to 500ms for our transceivers to settle.  If
6583          * the adapter does not have a cable attached,
6584          * the transceivers may never settle, so don't
6585          * complain if we fail here.
6586          */
6587         for (wait = 10000;
6588              (ahd_inb(ahd, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
6589              wait--)
6590                 aic_delay(100);
6591
6592         /* Clear any false bus resets due to the transceivers settling */
6593         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
6594         ahd_outb(ahd, CLRINT, CLRSCSIINT);
6595
6596         /* Initialize mode specific S/G state. */
6597         for (i = 0; i < 2; i++) {
6598                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
6599                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
6600                 ahd_outb(ahd, SG_STATE, 0);
6601                 ahd_outb(ahd, CLRSEQINTSRC, 0xFF);
6602                 ahd_outb(ahd, SEQIMODE,
6603                          ENSAVEPTRS|ENCFG4DATA|ENCFG4ISTAT
6604                         |ENCFG4TSTAT|ENCFG4ICMD|ENCFG4TCMD);
6605         }
6606
6607         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6608         ahd_outb(ahd, DSCOMMAND0, ahd_inb(ahd, DSCOMMAND0)|MPARCKEN|CACHETHEN);
6609         ahd_outb(ahd, DFF_THRSH, RD_DFTHRSH_75|WR_DFTHRSH_75);
6610         ahd_outb(ahd, SIMODE0, ENIOERR|ENOVERRUN);
6611         ahd_outb(ahd, SIMODE3, ENNTRAMPERR|ENOSRAMPERR);
6612         if ((ahd->bugs & AHD_BUSFREEREV_BUG) != 0) {
6613                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|AUTO_MSGOUT_DE);
6614         } else {
6615                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|BUSFREEREV|AUTO_MSGOUT_DE);
6616         }
6617         ahd_outb(ahd, SCSCHKN, CURRFIFODEF|WIDERESEN|SHVALIDSTDIS);
6618         if ((ahd->chip & AHD_BUS_MASK) == AHD_PCIX)
6619                 /*
6620                  * Do not issue a target abort when a split completion
6621                  * error occurs.  Let our PCIX interrupt handler deal
6622                  * with it instead. H2A4 Razor #625
6623                  */
6624                 ahd_outb(ahd, PCIXCTL, ahd_inb(ahd, PCIXCTL) | SPLTSTADIS);
6625
6626         if ((ahd->bugs & AHD_LQOOVERRUN_BUG) != 0)
6627                 ahd_outb(ahd, LQOSCSCTL, LQONOCHKOVER);
6628
6629         /*
6630          * Tweak IOCELL settings.
6631          */
6632         if ((ahd->flags & AHD_HP_BOARD) != 0) {
6633                 for (i = 0; i < NUMDSPS; i++) {
6634                         ahd_outb(ahd, DSPSELECT, i);
6635                         ahd_outb(ahd, WRTBIASCTL, WRTBIASCTL_HP_DEFAULT);
6636                 }
6637 #ifdef AHD_DEBUG
6638                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
6639                         printf("%s: WRTBIASCTL now 0x%x\n", ahd_name(ahd),
6640                                WRTBIASCTL_HP_DEFAULT);
6641 #endif
6642         }
6643         ahd_setup_iocell_workaround(ahd);
6644
6645         /*
6646          * Enable LQI Manager interrupts.
6647          */
6648         ahd_outb(ahd, LQIMODE1, ENLQIPHASE_LQ|ENLQIPHASE_NLQ|ENLIQABORT
6649                               | ENLQICRCI_LQ|ENLQICRCI_NLQ|ENLQIBADLQI
6650                               | ENLQIOVERI_LQ|ENLQIOVERI_NLQ);
6651         ahd_outb(ahd, LQOMODE0, ENLQOATNLQ|ENLQOATNPKT|ENLQOTCRC);
6652         /*
6653          * We choose to have the sequencer catch LQOPHCHGINPKT errors
6654          * manually for the command phase at the start of a packetized
6655          * selection case.  ENLQOBUSFREE should be made redundant by
6656          * the BUSFREE interrupt, but it seems that some LQOBUSFREE
6657          * events fail to assert the BUSFREE interrupt so we must
6658          * also enable LQOBUSFREE interrupts.
6659          */
6660         ahd_outb(ahd, LQOMODE1, ENLQOBUSFREE);
6661
6662         /*
6663          * Setup sequencer interrupt handlers.
6664          */
6665         ahd_outw(ahd, INTVEC1_ADDR, ahd_resolve_seqaddr(ahd, LABEL_seq_isr));
6666         ahd_outw(ahd, INTVEC2_ADDR, ahd_resolve_seqaddr(ahd, LABEL_timer_isr));
6667
6668         /*
6669          * Setup SCB Offset registers.
6670          */
6671         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
6672                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb,
6673                          pkt_long_lun));
6674         } else {
6675                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb, lun));
6676         }
6677         ahd_outb(ahd, CMDLENPTR, offsetof(struct hardware_scb, cdb_len));
6678         ahd_outb(ahd, ATTRPTR, offsetof(struct hardware_scb, task_attribute));
6679         ahd_outb(ahd, FLAGPTR, offsetof(struct hardware_scb, task_management));
6680         ahd_outb(ahd, CMDPTR, offsetof(struct hardware_scb,
6681                                        shared_data.idata.cdb));
6682         ahd_outb(ahd, QNEXTPTR,
6683                  offsetof(struct hardware_scb, next_hscb_busaddr));
6684         ahd_outb(ahd, ABRTBITPTR, MK_MESSAGE_BIT_OFFSET);
6685         ahd_outb(ahd, ABRTBYTEPTR, offsetof(struct hardware_scb, control));
6686         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
6687                 ahd_outb(ahd, LUNLEN,
6688                          sizeof(ahd->next_queued_hscb->pkt_long_lun) - 1);
6689         } else {
6690                 ahd_outb(ahd, LUNLEN, LUNLEN_SINGLE_LEVEL_LUN);
6691         }
6692         ahd_outb(ahd, CDBLIMIT, SCB_CDB_LEN_PTR - 1);
6693         ahd_outb(ahd, MAXCMD, 0xFF);
6694         ahd_outb(ahd, SCBAUTOPTR,
6695                  AUSCBPTR_EN | offsetof(struct hardware_scb, tag));
6696
6697         /* We haven't been enabled for target mode yet. */
6698         ahd_outb(ahd, MULTARGID, 0);
6699         ahd_outb(ahd, MULTARGID + 1, 0);
6700
6701         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6702         /* Initialize the negotiation table. */
6703         if ((ahd->features & AHD_NEW_IOCELL_OPTS) == 0) {
6704                 /*
6705                  * Clear the spare bytes in the neg table to avoid
6706                  * spurious parity errors.
6707                  */
6708                 for (target = 0; target < AHD_NUM_TARGETS; target++) {
6709                         ahd_outb(ahd, NEGOADDR, target);
6710                         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PER_DEV0);
6711                         for (i = 0; i < AHD_NUM_PER_DEV_ANNEXCOLS; i++)
6712                                 ahd_outb(ahd, ANNEXDAT, 0);
6713                 }
6714         }
6715         for (target = 0; target < AHD_NUM_TARGETS; target++) {
6716                 struct   ahd_devinfo devinfo;
6717                 struct   ahd_initiator_tinfo *tinfo;
6718                 struct   ahd_tmode_tstate *tstate;
6719
6720                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6721                                             target, &tstate);
6722                 ahd_compile_devinfo(&devinfo, ahd->our_id,
6723                                     target, CAM_LUN_WILDCARD,
6724                                     'A', ROLE_INITIATOR);
6725                 ahd_update_neg_table(ahd, &devinfo, &tinfo->curr);
6726         }
6727
6728         ahd_outb(ahd, CLRSINT3, NTRAMPERR|OSRAMPERR);
6729         ahd_outb(ahd, CLRINT, CLRSCSIINT);
6730
6731 #ifdef NEEDS_MORE_TESTING
6732         /*
6733          * Always enable abort on incoming L_Qs if this feature is
6734          * supported.  We use this to catch invalid SCB references.
6735          */
6736         if ((ahd->bugs & AHD_ABORT_LQI_BUG) == 0)
6737                 ahd_outb(ahd, LQCTL1, ABORTPENDING);
6738         else
6739 #endif
6740                 ahd_outb(ahd, LQCTL1, 0);
6741
6742         /* All of our queues are empty */
6743         ahd->qoutfifonext = 0;
6744         ahd->qoutfifonext_valid_tag = QOUTFIFO_ENTRY_VALID;
6745         ahd_outb(ahd, QOUTFIFO_ENTRY_VALID_TAG, QOUTFIFO_ENTRY_VALID);
6746         for (i = 0; i < AHD_QOUT_SIZE; i++)
6747                 ahd->qoutfifo[i].valid_tag = 0;
6748         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_PREREAD);
6749
6750         ahd->qinfifonext = 0;
6751         for (i = 0; i < AHD_QIN_SIZE; i++)
6752                 ahd->qinfifo[i] = SCB_LIST_NULL;
6753
6754         if ((ahd->features & AHD_TARGETMODE) != 0) {
6755                 /* All target command blocks start out invalid. */
6756                 for (i = 0; i < AHD_TMODE_CMDS; i++)
6757                         ahd->targetcmds[i].cmd_valid = 0;
6758                 ahd_sync_tqinfifo(ahd, BUS_DMASYNC_PREREAD);
6759                 ahd->tqinfifonext = 1;
6760                 ahd_outb(ahd, KERNEL_TQINPOS, ahd->tqinfifonext - 1);
6761                 ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
6762         }
6763
6764         /* Initialize Scratch Ram. */
6765         ahd_outb(ahd, SEQ_FLAGS, 0);
6766         ahd_outb(ahd, SEQ_FLAGS2, 0);
6767
6768         /* We don't have any waiting selections */
6769         ahd_outw(ahd, WAITING_TID_HEAD, SCB_LIST_NULL);
6770         ahd_outw(ahd, WAITING_TID_TAIL, SCB_LIST_NULL);
6771         ahd_outw(ahd, MK_MESSAGE_SCB, SCB_LIST_NULL);
6772         ahd_outw(ahd, MK_MESSAGE_SCSIID, 0xFF);
6773         for (i = 0; i < AHD_NUM_TARGETS; i++)
6774                 ahd_outw(ahd, WAITING_SCB_TAILS + (2 * i), SCB_LIST_NULL);
6775
6776         /*
6777          * Nobody is waiting to be DMAed into the QOUTFIFO.
6778          */
6779         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
6780         ahd_outw(ahd, COMPLETE_SCB_DMAINPROG_HEAD, SCB_LIST_NULL);
6781         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
6782         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
6783         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
6784
6785         /*
6786          * The Freeze Count is 0.
6787          */
6788         ahd->qfreeze_cnt = 0;
6789         ahd_outw(ahd, QFREEZE_COUNT, 0);
6790         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, 0);
6791
6792         /*
6793          * Tell the sequencer where it can find our arrays in memory.
6794          */
6795         busaddr = ahd->shared_data_map.busaddr;
6796         ahd_outl(ahd, SHARED_DATA_ADDR, busaddr);
6797         ahd_outl(ahd, QOUTFIFO_NEXT_ADDR, busaddr);
6798
6799         /*
6800          * Setup the allowed SCSI Sequences based on operational mode.
6801          * If we are a target, we'll enable select in operations once
6802          * we've had a lun enabled.
6803          */
6804         scsiseq_template = ENAUTOATNP;
6805         if ((ahd->flags & AHD_INITIATORROLE) != 0)
6806                 scsiseq_template |= ENRSELI;
6807         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq_template);
6808
6809         /* There are no busy SCBs yet. */
6810         for (target = 0; target < AHD_NUM_TARGETS; target++) {
6811                 int lun;
6812
6813                 for (lun = 0; lun < AHD_NUM_LUNS_NONPKT; lun++)
6814                         ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(target, 'A', lun));
6815         }
6816
6817         /*
6818          * Initialize the group code to command length table.
6819          * Vendor Unique codes are set to 0 so we only capture
6820          * the first byte of the cdb.  These can be overridden
6821          * when target mode is enabled.
6822          */
6823         ahd_outb(ahd, CMDSIZE_TABLE, 5);
6824         ahd_outb(ahd, CMDSIZE_TABLE + 1, 9);
6825         ahd_outb(ahd, CMDSIZE_TABLE + 2, 9);
6826         ahd_outb(ahd, CMDSIZE_TABLE + 3, 0);
6827         ahd_outb(ahd, CMDSIZE_TABLE + 4, 15);
6828         ahd_outb(ahd, CMDSIZE_TABLE + 5, 11);
6829         ahd_outb(ahd, CMDSIZE_TABLE + 6, 0);
6830         ahd_outb(ahd, CMDSIZE_TABLE + 7, 0);
6831
6832         /* Tell the sequencer of our initial queue positions */
6833         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
6834         ahd_outb(ahd, QOFF_CTLSTA, SCB_QSIZE_512);
6835         ahd->qinfifonext = 0;
6836         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
6837         ahd_set_hescb_qoff(ahd, 0);
6838         ahd_set_snscb_qoff(ahd, 0);
6839         ahd_set_sescb_qoff(ahd, 0);
6840         ahd_set_sdscb_qoff(ahd, 0);
6841
6842         /*
6843          * Tell the sequencer which SCB will be the next one it receives.
6844          */
6845         busaddr = aic_le32toh(ahd->next_queued_hscb->hscb_busaddr);
6846         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
6847
6848         /*
6849          * Default to coalescing disabled.
6850          */
6851         ahd_outw(ahd, INT_COALESCING_CMDCOUNT, 0);
6852         ahd_outw(ahd, CMDS_PENDING, 0);
6853         ahd_update_coalescing_values(ahd, ahd->int_coalescing_timer,
6854                                      ahd->int_coalescing_maxcmds,
6855                                      ahd->int_coalescing_mincmds);
6856         ahd_enable_coalescing(ahd, FALSE);
6857
6858         ahd_loadseq(ahd);
6859         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6860 }
6861
6862 /*
6863  * Setup default device and controller settings.
6864  * This should only be called if our probe has
6865  * determined that no configuration data is available.
6866  */
6867 int
6868 ahd_default_config(struct ahd_softc *ahd)
6869 {
6870         int     targ;
6871
6872         ahd->our_id = 7;
6873
6874         /*
6875          * Allocate a tstate to house information for our
6876          * initiator presence on the bus as well as the user
6877          * data for any target mode initiator.
6878          */
6879         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
6880                 printf("%s: unable to allocate ahd_tmode_tstate.  "
6881                        "Failing attach\n", ahd_name(ahd));
6882                 AHD_FATAL_ERROR(ahd);
6883                 return (ENOMEM);
6884         }
6885
6886         for (targ = 0; targ < AHD_NUM_TARGETS; targ++) {
6887                 struct   ahd_devinfo devinfo;
6888                 struct   ahd_initiator_tinfo *tinfo;
6889                 struct   ahd_tmode_tstate *tstate;
6890                 uint16_t target_mask;
6891
6892                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6893                                             targ, &tstate);
6894                 /*
6895                  * We support SPC2 and SPI4.
6896                  */
6897                 tinfo->user.protocol_version = 4;
6898                 tinfo->user.transport_version = 4;
6899
6900                 target_mask = 0x01 << targ;
6901                 ahd->user_discenable |= target_mask;
6902                 tstate->discenable |= target_mask;
6903                 ahd->user_tagenable |= target_mask;
6904 #ifdef AHD_FORCE_160
6905                 tinfo->user.period = AHD_SYNCRATE_DT;
6906 #else
6907                 tinfo->user.period = AHD_SYNCRATE_160;
6908 #endif
6909                 tinfo->user.offset = MAX_OFFSET;
6910                 tinfo->user.ppr_options = MSG_EXT_PPR_RD_STRM
6911                                         | MSG_EXT_PPR_WR_FLOW
6912                                         | MSG_EXT_PPR_HOLD_MCS
6913                                         | MSG_EXT_PPR_IU_REQ
6914                                         | MSG_EXT_PPR_QAS_REQ
6915                                         | MSG_EXT_PPR_DT_REQ;
6916                 if ((ahd->features & AHD_RTI) != 0)
6917                         tinfo->user.ppr_options |= MSG_EXT_PPR_RTI;
6918
6919                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
6920
6921                 /*
6922                  * Start out Async/Narrow/Untagged and with
6923                  * conservative protocol support.
6924                  */
6925                 tinfo->goal.protocol_version = 2;
6926                 tinfo->goal.transport_version = 2;
6927                 tinfo->curr.protocol_version = 2;
6928                 tinfo->curr.transport_version = 2;
6929                 ahd_compile_devinfo(&devinfo, ahd->our_id,
6930                                     targ, CAM_LUN_WILDCARD,
6931                                     'A', ROLE_INITIATOR);
6932                 tstate->tagenable &= ~target_mask;
6933                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6934                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
6935                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
6936                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
6937                                  /*paused*/TRUE);
6938         }
6939         return (0);
6940 }
6941
6942 /*
6943  * Parse device configuration information.
6944  */
6945 int
6946 ahd_parse_cfgdata(struct ahd_softc *ahd, struct seeprom_config *sc)
6947 {
6948         int targ;
6949         int max_targ;
6950
6951         max_targ = sc->max_targets & CFMAXTARG;
6952         ahd->our_id = sc->brtime_id & CFSCSIID;
6953
6954         /*
6955          * Allocate a tstate to house information for our
6956          * initiator presence on the bus as well as the user
6957          * data for any target mode initiator.
6958          */
6959         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
6960                 printf("%s: unable to allocate ahd_tmode_tstate.  "
6961                        "Failing attach\n", ahd_name(ahd));
6962                 AHD_FATAL_ERROR(ahd);
6963                 return (ENOMEM);
6964         }
6965
6966         for (targ = 0; targ < max_targ; targ++) {
6967                 struct   ahd_devinfo devinfo;
6968                 struct   ahd_initiator_tinfo *tinfo;
6969                 struct   ahd_transinfo *user_tinfo;
6970                 struct   ahd_tmode_tstate *tstate;
6971                 uint16_t target_mask;
6972
6973                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6974                                             targ, &tstate);
6975                 user_tinfo = &tinfo->user;
6976
6977                 /*
6978                  * We support SPC2 and SPI4.
6979                  */
6980                 tinfo->user.protocol_version = 4;
6981                 tinfo->user.transport_version = 4;
6982
6983                 target_mask = 0x01 << targ;
6984                 ahd->user_discenable &= ~target_mask;
6985                 tstate->discenable &= ~target_mask;
6986                 ahd->user_tagenable &= ~target_mask;
6987                 if (sc->device_flags[targ] & CFDISC) {
6988                         tstate->discenable |= target_mask;
6989                         ahd->user_discenable |= target_mask;
6990                         ahd->user_tagenable |= target_mask;
6991                 } else {
6992                         /*
6993                          * Cannot be packetized without disconnection.
6994                          */
6995                         sc->device_flags[targ] &= ~CFPACKETIZED;
6996                 }
6997
6998                 user_tinfo->ppr_options = 0;
6999                 user_tinfo->period = (sc->device_flags[targ] & CFXFER);
7000                 if (user_tinfo->period < CFXFER_ASYNC) {
7001                         if (user_tinfo->period <= AHD_PERIOD_10MHz)
7002                                 user_tinfo->ppr_options |= MSG_EXT_PPR_DT_REQ;
7003                         user_tinfo->offset = MAX_OFFSET;
7004                 } else  {
7005                         user_tinfo->offset = 0;
7006                         user_tinfo->period = AHD_ASYNC_XFER_PERIOD;
7007                 }
7008 #ifdef AHD_FORCE_160
7009                 if (user_tinfo->period <= AHD_SYNCRATE_160)
7010                         user_tinfo->period = AHD_SYNCRATE_DT;
7011 #endif
7012
7013                 if ((sc->device_flags[targ] & CFPACKETIZED) != 0) {
7014                         user_tinfo->ppr_options |= MSG_EXT_PPR_RD_STRM
7015                                                 |  MSG_EXT_PPR_WR_FLOW
7016                                                 |  MSG_EXT_PPR_HOLD_MCS
7017                                                 |  MSG_EXT_PPR_IU_REQ;
7018                         if ((ahd->features & AHD_RTI) != 0)
7019                                 user_tinfo->ppr_options |= MSG_EXT_PPR_RTI;
7020                 }
7021
7022                 if ((sc->device_flags[targ] & CFQAS) != 0)
7023                         user_tinfo->ppr_options |= MSG_EXT_PPR_QAS_REQ;
7024
7025                 if ((sc->device_flags[targ] & CFWIDEB) != 0)
7026                         user_tinfo->width = MSG_EXT_WDTR_BUS_16_BIT;
7027                 else
7028                         user_tinfo->width = MSG_EXT_WDTR_BUS_8_BIT;
7029 #ifdef AHD_DEBUG
7030                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
7031                         printf("(%d): %x:%x:%x:%x\n", targ, user_tinfo->width,
7032                                user_tinfo->period, user_tinfo->offset,
7033                                user_tinfo->ppr_options);
7034 #endif
7035                 /*
7036                  * Start out Async/Narrow/Untagged and with
7037                  * conservative protocol support.
7038                  */
7039                 tstate->tagenable &= ~target_mask;
7040                 tinfo->goal.protocol_version = 2;
7041                 tinfo->goal.transport_version = 2;
7042                 tinfo->curr.protocol_version = 2;
7043                 tinfo->curr.transport_version = 2;
7044                 ahd_compile_devinfo(&devinfo, ahd->our_id,
7045                                     targ, CAM_LUN_WILDCARD,
7046                                     'A', ROLE_INITIATOR);
7047                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7048                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
7049                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
7050                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
7051                                  /*paused*/TRUE);
7052         }
7053
7054         ahd->flags &= ~AHD_SPCHK_ENB_A;
7055         if (sc->bios_control & CFSPARITY)
7056                 ahd->flags |= AHD_SPCHK_ENB_A;
7057
7058         ahd->flags &= ~AHD_RESET_BUS_A;
7059         if (sc->bios_control & CFRESETB)
7060                 ahd->flags |= AHD_RESET_BUS_A;
7061
7062         ahd->flags &= ~AHD_EXTENDED_TRANS_A;
7063         if (sc->bios_control & CFEXTEND)
7064                 ahd->flags |= AHD_EXTENDED_TRANS_A;
7065
7066         ahd->flags &= ~AHD_BIOS_ENABLED;
7067         if ((sc->bios_control & CFBIOSSTATE) == CFBS_ENABLED)
7068                 ahd->flags |= AHD_BIOS_ENABLED;
7069
7070         ahd->flags &= ~AHD_STPWLEVEL_A;
7071         if ((sc->adapter_control & CFSTPWLEVEL) != 0)
7072                 ahd->flags |= AHD_STPWLEVEL_A;
7073
7074         return (0);
7075 }
7076
7077 /*
7078  * Parse device configuration information.
7079  */
7080 int
7081 ahd_parse_vpddata(struct ahd_softc *ahd, struct vpd_config *vpd)
7082 {
7083         int error;
7084
7085         error = ahd_verify_vpd_cksum(vpd);
7086         if (error == 0)
7087                 return (EINVAL);
7088         if ((vpd->bios_flags & VPDBOOTHOST) != 0)
7089                 ahd->flags |= AHD_BOOT_CHANNEL;
7090         return (0);
7091 }
7092
7093 void
7094 ahd_intr_enable(struct ahd_softc *ahd, int enable)
7095 {
7096         u_int hcntrl;
7097
7098         hcntrl = ahd_inb(ahd, HCNTRL);
7099         hcntrl &= ~INTEN;
7100         ahd->pause &= ~INTEN;
7101         ahd->unpause &= ~INTEN;
7102         if (enable) {
7103                 hcntrl |= INTEN;
7104                 ahd->pause |= INTEN;
7105                 ahd->unpause |= INTEN;
7106         }
7107         ahd_outb(ahd, HCNTRL, hcntrl);
7108 }
7109
7110 void
7111 ahd_update_coalescing_values(struct ahd_softc *ahd, u_int timer, u_int maxcmds,
7112                              u_int mincmds)
7113 {
7114         if (timer > AHD_TIMER_MAX_US)
7115                 timer = AHD_TIMER_MAX_US;
7116         ahd->int_coalescing_timer = timer;
7117
7118         if (maxcmds > AHD_INT_COALESCING_MAXCMDS_MAX)
7119                 maxcmds = AHD_INT_COALESCING_MAXCMDS_MAX;
7120         if (mincmds > AHD_INT_COALESCING_MINCMDS_MAX)
7121                 mincmds = AHD_INT_COALESCING_MINCMDS_MAX;
7122         ahd->int_coalescing_maxcmds = maxcmds;
7123         ahd_outw(ahd, INT_COALESCING_TIMER, timer / AHD_TIMER_US_PER_TICK);
7124         ahd_outb(ahd, INT_COALESCING_MAXCMDS, -maxcmds);
7125         ahd_outb(ahd, INT_COALESCING_MINCMDS, -mincmds);
7126 }
7127
7128 void
7129 ahd_enable_coalescing(struct ahd_softc *ahd, int enable)
7130 {
7131
7132         ahd->hs_mailbox &= ~ENINT_COALESCE;
7133         if (enable)
7134                 ahd->hs_mailbox |= ENINT_COALESCE;
7135         ahd_outb(ahd, HS_MAILBOX, ahd->hs_mailbox);
7136         ahd_flush_device_writes(ahd);
7137         ahd_run_qoutfifo(ahd);
7138 }
7139
7140 /*
7141  * Ensure that the card is paused in a location
7142  * outside of all critical sections and that all
7143  * pending work is completed prior to returning.
7144  * This routine should only be called from outside
7145  * an interrupt context.
7146  */
7147 void
7148 ahd_pause_and_flushwork(struct ahd_softc *ahd)
7149 {
7150         u_int intstat;
7151         u_int maxloops;
7152
7153         maxloops = 1000;
7154         ahd->flags |= AHD_ALL_INTERRUPTS;
7155         ahd_pause(ahd);
7156         /*
7157          * Freeze the outgoing selections.  We do this only
7158          * until we are safely paused without further selections
7159          * pending.
7160          */
7161         ahd->qfreeze_cnt--;
7162         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7163         ahd_outb(ahd, SEQ_FLAGS2, ahd_inb(ahd, SEQ_FLAGS2) | SELECTOUT_QFROZEN);
7164         do {
7165
7166                 ahd_unpause(ahd);
7167                 /*
7168                  * Give the sequencer some time to service
7169                  * any active selections.
7170                  */
7171                 aic_delay(500);
7172
7173                 ahd_intr(ahd);
7174                 ahd_pause(ahd);
7175                 intstat = ahd_inb(ahd, INTSTAT);
7176                 if ((intstat & INT_PEND) == 0) {
7177                         ahd_clear_critical_section(ahd);
7178                         intstat = ahd_inb(ahd, INTSTAT);
7179                 }
7180         } while (--maxloops
7181               && (intstat != 0xFF || (ahd->features & AHD_REMOVABLE) == 0)
7182               && ((intstat & INT_PEND) != 0
7183                || (ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
7184                || (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) != 0));
7185
7186         if (maxloops == 0) {
7187                 printf("Infinite interrupt loop, INTSTAT = %x",
7188                       ahd_inb(ahd, INTSTAT));
7189                 AHD_FATAL_ERROR(ahd);
7190         }
7191         ahd->qfreeze_cnt++;
7192         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7193
7194         ahd_flush_qoutfifo(ahd);
7195
7196         ahd_platform_flushwork(ahd);
7197         ahd->flags &= ~AHD_ALL_INTERRUPTS;
7198 }
7199
7200 int
7201 ahd_suspend(struct ahd_softc *ahd)
7202 {
7203
7204         ahd_pause_and_flushwork(ahd);
7205
7206         if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
7207                 ahd_unpause(ahd);
7208                 return (EBUSY);
7209         }
7210         ahd_shutdown(ahd);
7211         return (0);
7212 }
7213
7214 int
7215 ahd_resume(struct ahd_softc *ahd)
7216 {
7217
7218         ahd_reset(ahd, /*reinit*/TRUE);
7219         ahd_intr_enable(ahd, TRUE);
7220         ahd_restart(ahd);
7221         return (0);
7222 }
7223
7224 /************************** Busy Target Table *********************************/
7225 /*
7226  * Set SCBPTR to the SCB that contains the busy
7227  * table entry for TCL.  Return the offset into
7228  * the SCB that contains the entry for TCL.
7229  * saved_scbid is dereferenced and set to the
7230  * scbid that should be restored once manipualtion
7231  * of the TCL entry is complete.
7232  */
7233 static __inline u_int
7234 ahd_index_busy_tcl(struct ahd_softc *ahd, u_int *saved_scbid, u_int tcl)
7235 {
7236         /*
7237          * Index to the SCB that contains the busy entry.
7238          */
7239         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7240         *saved_scbid = ahd_get_scbptr(ahd);
7241         ahd_set_scbptr(ahd, TCL_LUN(tcl)
7242                      | ((TCL_TARGET_OFFSET(tcl) & 0xC) << 4));
7243
7244         /*
7245          * And now calculate the SCB offset to the entry.
7246          * Each entry is 2 bytes wide, hence the
7247          * multiplication by 2.
7248          */
7249         return (((TCL_TARGET_OFFSET(tcl) & 0x3) << 1) + SCB_DISCONNECTED_LISTS);
7250 }
7251
7252 /*
7253  * Return the untagged transaction id for a given target/channel lun.
7254  */
7255 u_int
7256 ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl)
7257 {
7258         u_int scbid;
7259         u_int scb_offset;
7260         u_int saved_scbptr;
7261
7262         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7263         scbid = ahd_inw_scbram(ahd, scb_offset);
7264         ahd_set_scbptr(ahd, saved_scbptr);
7265         return (scbid);
7266 }
7267
7268 void
7269 ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl, u_int scbid)
7270 {
7271         u_int scb_offset;
7272         u_int saved_scbptr;
7273
7274         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7275         ahd_outw(ahd, scb_offset, scbid);
7276         ahd_set_scbptr(ahd, saved_scbptr);
7277 }
7278
7279 /************************** SCB and SCB queue management **********************/
7280 int
7281 ahd_match_scb(struct ahd_softc *ahd, struct scb *scb, int target,
7282               char channel, int lun, u_int tag, role_t role)
7283 {
7284         int targ = SCB_GET_TARGET(ahd, scb);
7285         char chan = SCB_GET_CHANNEL(ahd, scb);
7286         int slun = SCB_GET_LUN(scb);
7287         int match;
7288
7289         match = ((chan == channel) || (channel == ALL_CHANNELS));
7290         if (match != 0)
7291                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
7292         if (match != 0)
7293                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
7294         if (match != 0) {
7295 #ifdef AHD_TARGET_MODE
7296                 int group;
7297
7298                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
7299                 if (role == ROLE_INITIATOR) {
7300                         match = (group != XPT_FC_GROUP_TMODE)
7301                               && ((tag == SCB_GET_TAG(scb))
7302                                || (tag == SCB_LIST_NULL));
7303                 } else if (role == ROLE_TARGET) {
7304                         match = (group == XPT_FC_GROUP_TMODE)
7305                               && ((tag == scb->io_ctx->csio.tag_id)
7306                                || (tag == SCB_LIST_NULL));
7307                 }
7308 #else /* !AHD_TARGET_MODE */
7309                 match = ((tag == SCB_GET_TAG(scb)) || (tag == SCB_LIST_NULL));
7310 #endif /* AHD_TARGET_MODE */
7311         }
7312
7313         return match;
7314 }
7315
7316 void
7317 ahd_freeze_devq(struct ahd_softc *ahd, struct scb *scb)
7318 {
7319         int     target;
7320         char    channel;
7321         int     lun;
7322
7323         target = SCB_GET_TARGET(ahd, scb);
7324         lun = SCB_GET_LUN(scb);
7325         channel = SCB_GET_CHANNEL(ahd, scb);
7326
7327         ahd_search_qinfifo(ahd, target, channel, lun,
7328                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
7329                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
7330
7331         ahd_platform_freeze_devq(ahd, scb);
7332 }
7333
7334 void
7335 ahd_qinfifo_requeue_tail(struct ahd_softc *ahd, struct scb *scb)
7336 {
7337         struct scb      *prev_scb;
7338         ahd_mode_state   saved_modes;
7339
7340         saved_modes = ahd_save_modes(ahd);
7341         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7342         prev_scb = NULL;
7343         if (ahd_qinfifo_count(ahd) != 0) {
7344                 u_int prev_tag;
7345                 u_int prev_pos;
7346
7347                 prev_pos = AHD_QIN_WRAP(ahd->qinfifonext - 1);
7348                 prev_tag = ahd->qinfifo[prev_pos];
7349                 prev_scb = ahd_lookup_scb(ahd, prev_tag);
7350         }
7351         ahd_qinfifo_requeue(ahd, prev_scb, scb);
7352         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7353         ahd_restore_modes(ahd, saved_modes);
7354 }
7355
7356 static void
7357 ahd_qinfifo_requeue(struct ahd_softc *ahd, struct scb *prev_scb,
7358                     struct scb *scb)
7359 {
7360         if (prev_scb == NULL) {
7361                 uint32_t busaddr;
7362
7363                 busaddr = aic_le32toh(scb->hscb->hscb_busaddr);
7364                 ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7365         } else {
7366                 prev_scb->hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
7367                 ahd_sync_scb(ahd, prev_scb,
7368                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
7369         }
7370         ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
7371         ahd->qinfifonext++;
7372         scb->hscb->next_hscb_busaddr = ahd->next_queued_hscb->hscb_busaddr;
7373         ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
7374 }
7375
7376 static int
7377 ahd_qinfifo_count(struct ahd_softc *ahd)
7378 {
7379         u_int qinpos;
7380         u_int wrap_qinpos;
7381         u_int wrap_qinfifonext;
7382
7383         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
7384         qinpos = ahd_get_snscb_qoff(ahd);
7385         wrap_qinpos = AHD_QIN_WRAP(qinpos);
7386         wrap_qinfifonext = AHD_QIN_WRAP(ahd->qinfifonext);
7387         if (wrap_qinfifonext >= wrap_qinpos)
7388                 return (wrap_qinfifonext - wrap_qinpos);
7389         else
7390                 return (wrap_qinfifonext
7391                       + NUM_ELEMENTS(ahd->qinfifo) - wrap_qinpos);
7392 }
7393
7394 void
7395 ahd_reset_cmds_pending(struct ahd_softc *ahd)
7396 {
7397         struct          scb *scb;
7398         ahd_mode_state  saved_modes;
7399         u_int           pending_cmds;
7400
7401         saved_modes = ahd_save_modes(ahd);
7402         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7403
7404         /*
7405          * Don't count any commands as outstanding that the
7406          * sequencer has already marked for completion.
7407          */
7408         ahd_flush_qoutfifo(ahd);
7409
7410         pending_cmds = 0;
7411         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
7412                 pending_cmds++;
7413         }
7414         ahd_outw(ahd, CMDS_PENDING, pending_cmds - ahd_qinfifo_count(ahd));
7415         ahd_restore_modes(ahd, saved_modes);
7416         ahd->flags &= ~AHD_UPDATE_PEND_CMDS;
7417 }
7418
7419 void
7420 ahd_done_with_status(struct ahd_softc *ahd, struct scb *scb, uint32_t status)
7421 {
7422         cam_status ostat;
7423         cam_status cstat;
7424
7425         ostat = aic_get_transaction_status(scb);
7426         if (ostat == CAM_REQ_INPROG)
7427                 aic_set_transaction_status(scb, status);
7428         cstat = aic_get_transaction_status(scb);
7429         if (cstat != CAM_REQ_CMP)
7430                 aic_freeze_scb(scb);
7431         ahd_done(ahd, scb);
7432 }
7433
7434 int
7435 ahd_search_qinfifo(struct ahd_softc *ahd, int target, char channel,
7436                    int lun, u_int tag, role_t role, uint32_t status,
7437                    ahd_search_action action)
7438 {
7439         struct scb      *scb;
7440         struct scb      *mk_msg_scb;
7441         struct scb      *prev_scb;
7442         ahd_mode_state   saved_modes;
7443         u_int            qinstart;
7444         u_int            qinpos;
7445         u_int            qintail;
7446         u_int            tid_next;
7447         u_int            tid_prev;
7448         u_int            scbid;
7449         u_int            seq_flags2;
7450         u_int            savedscbptr;
7451         uint32_t         busaddr;
7452         int              found;
7453         int              targets;
7454
7455         /* Must be in CCHAN mode */
7456         saved_modes = ahd_save_modes(ahd);
7457         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7458
7459         /*
7460          * Halt any pending SCB DMA.  The sequencer will reinitiate
7461          * this dma if the qinfifo is not empty once we unpause.
7462          */
7463         if ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN|CCSCBDIR))
7464          == (CCARREN|CCSCBEN|CCSCBDIR)) {
7465                 ahd_outb(ahd, CCSCBCTL,
7466                          ahd_inb(ahd, CCSCBCTL) & ~(CCARREN|CCSCBEN));
7467                 while ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN)) != 0)
7468                         ;
7469         }
7470         /* Determine sequencer's position in the qinfifo. */
7471         qintail = AHD_QIN_WRAP(ahd->qinfifonext);
7472         qinstart = ahd_get_snscb_qoff(ahd);
7473         qinpos = AHD_QIN_WRAP(qinstart);
7474         found = 0;
7475         prev_scb = NULL;
7476
7477         if (action == SEARCH_PRINT) {
7478                 printf("qinstart = %d qinfifonext = %d\nQINFIFO:",
7479                        qinstart, ahd->qinfifonext);
7480         }
7481
7482         /*
7483          * Start with an empty queue.  Entries that are not chosen
7484          * for removal will be re-added to the queue as we go.
7485          */
7486         ahd->qinfifonext = qinstart;
7487         busaddr = aic_le32toh(ahd->next_queued_hscb->hscb_busaddr);
7488         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7489
7490         while (qinpos != qintail) {
7491                 scb = ahd_lookup_scb(ahd, ahd->qinfifo[qinpos]);
7492                 if (scb == NULL) {
7493                         printf("qinpos = %d, SCB index = %d\n",
7494                                 qinpos, ahd->qinfifo[qinpos]);
7495                         AHD_FATAL_ERROR(ahd);
7496                         panic("Loop 1\n");
7497                 }
7498
7499                 if (ahd_match_scb(ahd, scb, target, channel, lun, tag, role)) {
7500                         /*
7501                          * We found an scb that needs to be acted on.
7502                          */
7503                         found++;
7504                         switch (action) {
7505                         case SEARCH_COMPLETE:
7506                                 if ((scb->flags & SCB_ACTIVE) == 0)
7507                                         printf("Inactive SCB in qinfifo\n");
7508                                 ahd_done_with_status(ahd, scb, status);
7509                                 /* FALLTHROUGH */
7510                         case SEARCH_REMOVE:
7511                                 break;
7512                         case SEARCH_PRINT:
7513                                 printf(" 0x%x", ahd->qinfifo[qinpos]);
7514                                 /* FALLTHROUGH */
7515                         case SEARCH_COUNT:
7516                                 ahd_qinfifo_requeue(ahd, prev_scb, scb);
7517                                 prev_scb = scb;
7518                                 break;
7519                         }
7520                 } else {
7521                         ahd_qinfifo_requeue(ahd, prev_scb, scb);
7522                         prev_scb = scb;
7523                 }
7524                 qinpos = AHD_QIN_WRAP(qinpos+1);
7525         }
7526
7527         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7528
7529         if (action == SEARCH_PRINT)
7530                 printf("\nWAITING_TID_QUEUES:\n");
7531
7532         /*
7533          * Search waiting for selection lists.  We traverse the
7534          * list of "their ids" waiting for selection and, if
7535          * appropriate, traverse the SCBs of each "their id"
7536          * looking for matches.
7537          */
7538         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7539         seq_flags2 = ahd_inb(ahd, SEQ_FLAGS2);
7540         if ((seq_flags2 & PENDING_MK_MESSAGE) != 0) {
7541                 scbid = ahd_inw(ahd, MK_MESSAGE_SCB);
7542                 mk_msg_scb = ahd_lookup_scb(ahd, scbid);
7543         } else
7544                 mk_msg_scb = NULL;
7545         savedscbptr = ahd_get_scbptr(ahd);
7546         tid_next = ahd_inw(ahd, WAITING_TID_HEAD);
7547         tid_prev = SCB_LIST_NULL;
7548         targets = 0;
7549         for (scbid = tid_next; !SCBID_IS_NULL(scbid); scbid = tid_next) {
7550                 u_int tid_head;
7551                 u_int tid_tail;
7552
7553                 targets++;
7554                 if (targets > AHD_NUM_TARGETS)
7555                         panic("TID LIST LOOP");
7556
7557                 if (scbid >= ahd->scb_data.numscbs) {
7558                         printf("%s: Waiting TID List inconsistency. "
7559                                "SCB index == 0x%x, yet numscbs == 0x%x.",
7560                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
7561                         ahd_dump_card_state(ahd);
7562                         panic("for safety");
7563                 }
7564                 scb = ahd_lookup_scb(ahd, scbid);
7565                 if (scb == NULL) {
7566                         printf("%s: SCB = 0x%x Not Active!\n",
7567                                ahd_name(ahd), scbid);
7568                         panic("Waiting TID List traversal\n");
7569                 }
7570                 ahd_set_scbptr(ahd, scbid);
7571                 tid_next = ahd_inw_scbram(ahd, SCB_NEXT2);
7572                 if (ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
7573                                   SCB_LIST_NULL, ROLE_UNKNOWN) == 0) {
7574                         tid_prev = scbid;
7575                         continue;
7576                 }
7577
7578                 /*
7579                  * We found a list of scbs that needs to be searched.
7580                  */
7581                 if (action == SEARCH_PRINT)
7582                         printf("       %d ( ", SCB_GET_TARGET(ahd, scb));
7583                 tid_head = scbid;
7584                 found += ahd_search_scb_list(ahd, target, channel,
7585                                              lun, tag, role, status,
7586                                              action, &tid_head, &tid_tail,
7587                                              SCB_GET_TARGET(ahd, scb));
7588                 /*
7589                  * Check any MK_MESSAGE SCB that is still waiting to
7590                  * enter this target's waiting for selection queue.
7591                  */
7592                 if (mk_msg_scb != NULL
7593                  && ahd_match_scb(ahd, mk_msg_scb, target, channel,
7594                                   lun, tag, role)) {
7595
7596                         /*
7597                          * We found an scb that needs to be acted on.
7598                          */
7599                         found++;
7600                         switch (action) {
7601                         case SEARCH_COMPLETE:
7602                                 if ((mk_msg_scb->flags & SCB_ACTIVE) == 0)
7603                                         printf("Inactive SCB pending MK_MSG\n");
7604                                 ahd_done_with_status(ahd, mk_msg_scb, status);
7605                                 /* FALLTHROUGH */
7606                         case SEARCH_REMOVE:
7607                         {
7608                                 u_int tail_offset;
7609
7610                                 printf("Removing MK_MSG scb\n");
7611
7612                                 /*
7613                                  * Reset our tail to the tail of the
7614                                  * main per-target list.
7615                                  */
7616                                 tail_offset = WAITING_SCB_TAILS
7617                                     + (2 * SCB_GET_TARGET(ahd, mk_msg_scb));
7618                                 ahd_outw(ahd, tail_offset, tid_tail);
7619
7620                                 seq_flags2 &= ~PENDING_MK_MESSAGE;
7621                                 ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
7622                                 ahd_outw(ahd, CMDS_PENDING,
7623                                          ahd_inw(ahd, CMDS_PENDING)-1);
7624                                 mk_msg_scb = NULL;
7625                                 break;
7626                         }
7627                         case SEARCH_PRINT:
7628                                 printf(" 0x%x", SCB_GET_TAG(scb));
7629                                 /* FALLTHROUGH */
7630                         case SEARCH_COUNT:
7631                                 break;
7632                         }
7633                 }
7634
7635                 if (mk_msg_scb != NULL
7636                  && SCBID_IS_NULL(tid_head)
7637                  && ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
7638                                   SCB_LIST_NULL, ROLE_UNKNOWN)) {
7639
7640                         /*
7641                          * When removing the last SCB for a target
7642                          * queue with a pending MK_MESSAGE scb, we
7643                          * must queue the MK_MESSAGE scb.
7644                          */
7645                         printf("Queueing mk_msg_scb\n");
7646                         tid_head = ahd_inw(ahd, MK_MESSAGE_SCB);
7647                         seq_flags2 &= ~PENDING_MK_MESSAGE;
7648                         ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
7649                         mk_msg_scb = NULL;
7650                 }
7651                 if (tid_head != scbid)
7652                         ahd_stitch_tid_list(ahd, tid_prev, tid_head, tid_next);
7653                 if (!SCBID_IS_NULL(tid_head))
7654                         tid_prev = tid_head;
7655                 if (action == SEARCH_PRINT)
7656                         printf(")\n");
7657         }
7658
7659         /* Restore saved state. */
7660         ahd_set_scbptr(ahd, savedscbptr);
7661         ahd_restore_modes(ahd, saved_modes);
7662         return (found);
7663 }
7664
7665 static int
7666 ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel,
7667                     int lun, u_int tag, role_t role, uint32_t status,
7668                     ahd_search_action action, u_int *list_head,
7669                     u_int *list_tail, u_int tid)
7670 {
7671         struct  scb *scb;
7672         u_int   scbid;
7673         u_int   next;
7674         u_int   prev;
7675         int     found;
7676
7677         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7678         found = 0;
7679         prev = SCB_LIST_NULL;
7680         next = *list_head;
7681         *list_tail = SCB_LIST_NULL;
7682         for (scbid = next; !SCBID_IS_NULL(scbid); scbid = next) {
7683                 if (scbid >= ahd->scb_data.numscbs) {
7684                         printf("%s:SCB List inconsistency. "
7685                                "SCB == 0x%x, yet numscbs == 0x%x.",
7686                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
7687                         ahd_dump_card_state(ahd);
7688                         panic("for safety");
7689                 }
7690                 scb = ahd_lookup_scb(ahd, scbid);
7691                 if (scb == NULL) {
7692                         printf("%s: SCB = %d Not Active!\n",
7693                                ahd_name(ahd), scbid);
7694                         panic("Waiting List traversal\n");
7695                 }
7696                 ahd_set_scbptr(ahd, scbid);
7697                 *list_tail = scbid;
7698                 next = ahd_inw_scbram(ahd, SCB_NEXT);
7699                 if (ahd_match_scb(ahd, scb, target, channel,
7700                                   lun, SCB_LIST_NULL, role) == 0) {
7701                         prev = scbid;
7702                         continue;
7703                 }
7704                 found++;
7705                 switch (action) {
7706                 case SEARCH_COMPLETE:
7707                         if ((scb->flags & SCB_ACTIVE) == 0)
7708                                 printf("Inactive SCB in Waiting List\n");
7709                         ahd_done_with_status(ahd, scb, status);
7710                         /* FALLTHROUGH */
7711                 case SEARCH_REMOVE:
7712                         ahd_rem_wscb(ahd, scbid, prev, next, tid);
7713                         *list_tail = prev;
7714                         if (SCBID_IS_NULL(prev))
7715                                 *list_head = next;
7716                         break;
7717                 case SEARCH_PRINT:
7718                         printf("0x%x ", scbid);
7719                 case SEARCH_COUNT:
7720                         prev = scbid;
7721                         break;
7722                 }
7723                 if (found > AHD_SCB_MAX)
7724                         panic("SCB LIST LOOP");
7725         }
7726         if (action == SEARCH_COMPLETE
7727          || action == SEARCH_REMOVE)
7728                 ahd_outw(ahd, CMDS_PENDING, ahd_inw(ahd, CMDS_PENDING) - found);
7729         return (found);
7730 }
7731
7732 static void
7733 ahd_stitch_tid_list(struct ahd_softc *ahd, u_int tid_prev,
7734                     u_int tid_cur, u_int tid_next)
7735 {
7736         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7737
7738         if (SCBID_IS_NULL(tid_cur)) {
7739
7740                 /* Bypass current TID list */
7741                 if (SCBID_IS_NULL(tid_prev)) {
7742                         ahd_outw(ahd, WAITING_TID_HEAD, tid_next);
7743                 } else {
7744                         ahd_set_scbptr(ahd, tid_prev);
7745                         ahd_outw(ahd, SCB_NEXT2, tid_next);
7746                 }
7747                 if (SCBID_IS_NULL(tid_next))
7748                         ahd_outw(ahd, WAITING_TID_TAIL, tid_prev);
7749         } else {
7750
7751                 /* Stitch through tid_cur */
7752                 if (SCBID_IS_NULL(tid_prev)) {
7753                         ahd_outw(ahd, WAITING_TID_HEAD, tid_cur);
7754                 } else {
7755                         ahd_set_scbptr(ahd, tid_prev);
7756                         ahd_outw(ahd, SCB_NEXT2, tid_cur);
7757                 }
7758                 ahd_set_scbptr(ahd, tid_cur);
7759                 ahd_outw(ahd, SCB_NEXT2, tid_next);
7760
7761                 if (SCBID_IS_NULL(tid_next))
7762                         ahd_outw(ahd, WAITING_TID_TAIL, tid_cur);
7763         }
7764 }
7765
7766 /*
7767  * Manipulate the waiting for selection list and return the
7768  * scb that follows the one that we remove.
7769  */
7770 static u_int
7771 ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
7772              u_int prev, u_int next, u_int tid)
7773 {
7774         u_int tail_offset;
7775
7776         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7777         if (!SCBID_IS_NULL(prev)) {
7778                 ahd_set_scbptr(ahd, prev);
7779                 ahd_outw(ahd, SCB_NEXT, next);
7780         }
7781
7782         /*
7783          * SCBs that have MK_MESSAGE set in them may
7784          * cause the tail pointer to be updated without
7785          * setting the next pointer of the previous tail.
7786          * Only clear the tail if the removed SCB was
7787          * the tail.
7788          */
7789         tail_offset = WAITING_SCB_TAILS + (2 * tid);
7790         if (SCBID_IS_NULL(next)
7791          && ahd_inw(ahd, tail_offset) == scbid)
7792                 ahd_outw(ahd, tail_offset, prev);
7793
7794         ahd_add_scb_to_free_list(ahd, scbid);
7795         return (next);
7796 }
7797
7798 /*
7799  * Add the SCB as selected by SCBPTR onto the on chip list of
7800  * free hardware SCBs.  This list is empty/unused if we are not
7801  * performing SCB paging.
7802  */
7803 static void
7804 ahd_add_scb_to_free_list(struct ahd_softc *ahd, u_int scbid)
7805 {
7806 /* XXX Need some other mechanism to designate "free". */
7807         /*
7808          * Invalidate the tag so that our abort
7809          * routines don't think it's active.
7810         ahd_outb(ahd, SCB_TAG, SCB_LIST_NULL);
7811          */
7812 }
7813
7814 /******************************** Error Handling ******************************/
7815 /*
7816  * Abort all SCBs that match the given description (target/channel/lun/tag),
7817  * setting their status to the passed in status if the status has not already
7818  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
7819  * is paused before it is called.
7820  */
7821 int
7822 ahd_abort_scbs(struct ahd_softc *ahd, int target, char channel,
7823                int lun, u_int tag, role_t role, uint32_t status)
7824 {
7825         struct          scb *scbp;
7826         struct          scb *scbp_next;
7827         u_int           i, j;
7828         u_int           maxtarget;
7829         u_int           minlun;
7830         u_int           maxlun;
7831         int             found;
7832         ahd_mode_state  saved_modes;
7833
7834         /* restore this when we're done */
7835         saved_modes = ahd_save_modes(ahd);
7836         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7837
7838         found = ahd_search_qinfifo(ahd, target, channel, lun, SCB_LIST_NULL,
7839                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
7840
7841         /*
7842          * Clean out the busy target table for any untagged commands.
7843          */
7844         i = 0;
7845         maxtarget = 16;
7846         if (target != CAM_TARGET_WILDCARD) {
7847                 i = target;
7848                 if (channel == 'B')
7849                         i += 8;
7850                 maxtarget = i + 1;
7851         }
7852
7853         if (lun == CAM_LUN_WILDCARD) {
7854                 minlun = 0;
7855                 maxlun = AHD_NUM_LUNS_NONPKT;
7856         } else if (lun >= AHD_NUM_LUNS_NONPKT) {
7857                 minlun = maxlun = 0;
7858         } else {
7859                 minlun = lun;
7860                 maxlun = lun + 1;
7861         }
7862
7863         if (role != ROLE_TARGET) {
7864                 for (;i < maxtarget; i++) {
7865                         for (j = minlun;j < maxlun; j++) {
7866                                 u_int scbid;
7867                                 u_int tcl;
7868
7869                                 tcl = BUILD_TCL_RAW(i, 'A', j);
7870                                 scbid = ahd_find_busy_tcl(ahd, tcl);
7871                                 scbp = ahd_lookup_scb(ahd, scbid);
7872                                 if (scbp == NULL
7873                                  || ahd_match_scb(ahd, scbp, target, channel,
7874                                                   lun, tag, role) == 0)
7875                                         continue;
7876                                 ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(i, 'A', j));
7877                         }
7878                 }
7879         }
7880
7881         /*
7882          * Don't abort commands that have already completed,
7883          * but haven't quite made it up to the host yet.
7884          */
7885         ahd_flush_qoutfifo(ahd);
7886
7887         /*
7888          * Go through the pending CCB list and look for
7889          * commands for this target that are still active.
7890          * These are other tagged commands that were
7891          * disconnected when the reset occurred.
7892          */
7893         scbp_next = LIST_FIRST(&ahd->pending_scbs);
7894         while (scbp_next != NULL) {
7895                 scbp = scbp_next;
7896                 scbp_next = LIST_NEXT(scbp, pending_links);
7897                 if (ahd_match_scb(ahd, scbp, target, channel, lun, tag, role)) {
7898                         cam_status ostat;
7899
7900                         ostat = aic_get_transaction_status(scbp);
7901                         if (ostat == CAM_REQ_INPROG)
7902                                 aic_set_transaction_status(scbp, status);
7903                         if (aic_get_transaction_status(scbp) != CAM_REQ_CMP)
7904                                 aic_freeze_scb(scbp);
7905                         if ((scbp->flags & SCB_ACTIVE) == 0)
7906                                 printf("Inactive SCB on pending list\n");
7907                         ahd_done(ahd, scbp);
7908                         found++;
7909                 }
7910         }
7911         ahd_restore_modes(ahd, saved_modes);
7912         ahd_platform_abort_scbs(ahd, target, channel, lun, tag, role, status);
7913         ahd->flags |= AHD_UPDATE_PEND_CMDS;
7914         return found;
7915 }
7916
7917 static void
7918 ahd_reset_current_bus(struct ahd_softc *ahd)
7919 {
7920         uint8_t scsiseq;
7921
7922         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7923         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) & ~ENSCSIRST);
7924         scsiseq = ahd_inb(ahd, SCSISEQ0) & ~(ENSELO|ENARBO|SCSIRSTO);
7925         ahd_outb(ahd, SCSISEQ0, scsiseq | SCSIRSTO);
7926         ahd_flush_device_writes(ahd);
7927         aic_delay(AHD_BUSRESET_DELAY);
7928         /* Turn off the bus reset */
7929         ahd_outb(ahd, SCSISEQ0, scsiseq);
7930         ahd_flush_device_writes(ahd);
7931         aic_delay(AHD_BUSRESET_DELAY);
7932         if ((ahd->bugs & AHD_SCSIRST_BUG) != 0) {
7933                 /*
7934                  * 2A Razor #474
7935                  * Certain chip state is not cleared for
7936                  * SCSI bus resets that we initiate, so
7937                  * we must reset the chip.
7938                  */
7939                 ahd_reset(ahd, /*reinit*/TRUE);
7940                 ahd_intr_enable(ahd, /*enable*/TRUE);
7941                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7942         }
7943
7944         ahd_clear_intstat(ahd);
7945 }
7946
7947 int
7948 ahd_reset_channel(struct ahd_softc *ahd, char channel, int initiate_reset)
7949 {
7950         struct  ahd_devinfo devinfo;
7951         u_int   initiator;
7952         u_int   target;
7953         u_int   max_scsiid;
7954         int     found;
7955         u_int   fifo;
7956         u_int   next_fifo;
7957
7958         ahd->pending_device = NULL;
7959
7960         ahd_compile_devinfo(&devinfo,
7961                             CAM_TARGET_WILDCARD,
7962                             CAM_TARGET_WILDCARD,
7963                             CAM_LUN_WILDCARD,
7964                             channel, ROLE_UNKNOWN);
7965         ahd_pause(ahd);
7966
7967         /* Make sure the sequencer is in a safe location. */
7968         ahd_clear_critical_section(ahd);
7969
7970 #ifdef AHD_TARGET_MODE
7971         if ((ahd->flags & AHD_TARGETROLE) != 0) {
7972                 ahd_run_tqinfifo(ahd, /*paused*/TRUE);
7973         }
7974 #endif
7975         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7976
7977         /*
7978          * Disable selections so no automatic hardware
7979          * functions will modify chip state.
7980          */
7981         ahd_outb(ahd, SCSISEQ0, 0);
7982         ahd_outb(ahd, SCSISEQ1, 0);
7983
7984         /*
7985          * Safely shut down our DMA engines.  Always start with
7986          * the FIFO that is not currently active (if any are
7987          * actively connected).
7988          */
7989         next_fifo = fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
7990         if (next_fifo > CURRFIFO_1)
7991                 /* If disconneced, arbitrarily start with FIFO1. */
7992                 next_fifo = fifo = 0;
7993         do {
7994                 next_fifo ^= CURRFIFO_1;
7995                 ahd_set_modes(ahd, next_fifo, next_fifo);
7996                 ahd_outb(ahd, DFCNTRL,
7997                          ahd_inb(ahd, DFCNTRL) & ~(SCSIEN|HDMAEN));
7998                 while ((ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0)
7999                         aic_delay(10);
8000                 /*
8001                  * Set CURRFIFO to the now inactive channel.
8002                  */
8003                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8004                 ahd_outb(ahd, DFFSTAT, next_fifo);
8005         } while (next_fifo != fifo);
8006
8007         /*
8008          * Reset the bus if we are initiating this reset
8009          */
8010         ahd_clear_msg_state(ahd);
8011         ahd_outb(ahd, SIMODE1,
8012                  ahd_inb(ahd, SIMODE1) & ~(ENBUSFREE|ENSCSIRST));
8013
8014         if (initiate_reset)
8015                 ahd_reset_current_bus(ahd);
8016
8017         ahd_clear_intstat(ahd);
8018
8019         /*
8020          * Clean up all the state information for the
8021          * pending transactions on this bus.
8022          */
8023         found = ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, channel,
8024                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
8025                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
8026
8027         /*
8028          * Cleanup anything left in the FIFOs.
8029          */
8030         ahd_clear_fifo(ahd, 0);
8031         ahd_clear_fifo(ahd, 1);
8032
8033         /*
8034          * Revert to async/narrow transfers until we renegotiate.
8035          */
8036         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
8037         for (target = 0; target <= max_scsiid; target++) {
8038
8039                 if (ahd->enabled_targets[target] == NULL)
8040                         continue;
8041                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
8042                         struct ahd_devinfo devinfo;
8043
8044                         ahd_compile_devinfo(&devinfo, target, initiator,
8045                                             CAM_LUN_WILDCARD,
8046                                             'A', ROLE_UNKNOWN);
8047                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
8048                                       AHD_TRANS_CUR, /*paused*/TRUE);
8049                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
8050                                          /*offset*/0, /*ppr_options*/0,
8051                                          AHD_TRANS_CUR, /*paused*/TRUE);
8052                 }
8053         }
8054
8055 #ifdef AHD_TARGET_MODE
8056         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
8057
8058         /*
8059          * Send an immediate notify ccb to all target more peripheral
8060          * drivers affected by this action.
8061          */
8062         for (target = 0; target <= max_scsiid; target++) {
8063                 struct ahd_tmode_tstate* tstate;
8064                 u_int lun;
8065
8066                 tstate = ahd->enabled_targets[target];
8067                 if (tstate == NULL)
8068                         continue;
8069                 for (lun = 0; lun < AHD_NUM_LUNS; lun++) {
8070                         struct ahd_tmode_lstate* lstate;
8071
8072                         lstate = tstate->enabled_luns[lun];
8073                         if (lstate == NULL)
8074                                 continue;
8075
8076                         ahd_queue_lstate_event(ahd, lstate, CAM_TARGET_WILDCARD,
8077                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
8078                         ahd_send_lstate_events(ahd, lstate);
8079                 }
8080         }
8081 #endif
8082         /* Notify the XPT that a bus reset occurred */
8083         ahd_send_async(ahd, devinfo.channel, CAM_TARGET_WILDCARD,
8084                        CAM_LUN_WILDCARD, AC_BUS_RESET, NULL);
8085         ahd_restart(ahd);
8086         /*
8087          * Freeze the SIMQ until our poller can determine that
8088          * the bus reset has really gone away.  We set the initial
8089          * timer to 0 to have the check performed as soon as possible
8090          * from the timer context.
8091          */
8092         if ((ahd->flags & AHD_RESET_POLL_ACTIVE) == 0) {
8093                 ahd->flags |= AHD_RESET_POLL_ACTIVE;
8094                 aic_freeze_simq(ahd);
8095                 aic_timer_reset(&ahd->reset_timer, 0, ahd_reset_poll, ahd);
8096         }
8097         return (found);
8098 }
8099
8100
8101 #define AHD_RESET_POLL_MS 1
8102 static void
8103 ahd_reset_poll(void *arg)
8104 {
8105         struct  ahd_softc *ahd = (struct ahd_softc *)arg;
8106         u_int   scsiseq1;
8107
8108         ahd_lock(ahd);
8109         ahd_pause(ahd);
8110         ahd_update_modes(ahd);
8111         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8112         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
8113         if ((ahd_inb(ahd, SSTAT1) & SCSIRSTI) != 0) {
8114                 aic_timer_reset(&ahd->reset_timer, AHD_RESET_POLL_MS,
8115                                 ahd_reset_poll, ahd);
8116                 ahd_unpause(ahd);
8117                 ahd_unlock(ahd);
8118                 return;
8119         }
8120
8121         /* Reset is now low.  Complete chip reinitialization. */
8122         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) | ENSCSIRST);
8123         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
8124         ahd_outb(ahd, SCSISEQ1, scsiseq1 & (ENSELI|ENRSELI|ENAUTOATNP));
8125         ahd_unpause(ahd);
8126         ahd->flags &= ~AHD_RESET_POLL_ACTIVE;
8127         aic_release_simq(ahd);
8128         ahd_unlock(ahd);
8129 }
8130
8131 /**************************** Statistics Processing ***************************/
8132 static void
8133 ahd_stat_timer(void *arg)
8134 {
8135         struct  ahd_softc *ahd = (struct ahd_softc *)arg;
8136         int     enint_coal;
8137
8138         ahd_lock(ahd);
8139         enint_coal = ahd->hs_mailbox & ENINT_COALESCE;
8140         if (ahd->cmdcmplt_total > ahd->int_coalescing_threshold)
8141                 enint_coal |= ENINT_COALESCE;
8142         else if (ahd->cmdcmplt_total < ahd->int_coalescing_stop_threshold)
8143                 enint_coal &= ~ENINT_COALESCE;
8144
8145         if (enint_coal != (ahd->hs_mailbox & ENINT_COALESCE)) {
8146                 ahd_enable_coalescing(ahd, enint_coal);
8147 #ifdef AHD_DEBUG
8148                 if ((ahd_debug & AHD_SHOW_INT_COALESCING) != 0)
8149                         printf("%s: Interrupt coalescing "
8150                                "now %sabled. Cmds %d\n",
8151                                ahd_name(ahd),
8152                                (enint_coal & ENINT_COALESCE) ? "en" : "dis",
8153                                ahd->cmdcmplt_total);
8154 #endif
8155         }
8156
8157         ahd->cmdcmplt_bucket = (ahd->cmdcmplt_bucket+1) & (AHD_STAT_BUCKETS-1);
8158         ahd->cmdcmplt_total -= ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket];
8159         ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket] = 0;
8160         aic_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_MS,
8161                         ahd_stat_timer, ahd);
8162         ahd_unlock(ahd);
8163 }
8164
8165 /****************************** Status Processing *****************************/
8166 void
8167 ahd_handle_scb_status(struct ahd_softc *ahd, struct scb *scb)
8168 {
8169         if (scb->hscb->shared_data.istatus.scsi_status != 0) {
8170                 ahd_handle_scsi_status(ahd, scb);
8171         } else {
8172                 ahd_calc_residual(ahd, scb);
8173                 ahd_done(ahd, scb);
8174         }
8175 }
8176
8177 void
8178 ahd_handle_scsi_status(struct ahd_softc *ahd, struct scb *scb)
8179 {
8180         struct  hardware_scb *hscb;
8181         int     paused;
8182
8183         /*
8184          * The sequencer freezes its select-out queue
8185          * anytime a SCSI status error occurs.  We must
8186          * handle the error and increment our qfreeze count
8187          * to allow the sequencer to continue.  We don't
8188          * bother clearing critical sections here since all
8189          * operations are on data structures that the sequencer
8190          * is not touching once the queue is frozen.
8191          */
8192         hscb = scb->hscb;
8193
8194         if (ahd_is_paused(ahd)) {
8195                 paused = 1;
8196         } else {
8197                 paused = 0;
8198                 ahd_pause(ahd);
8199         }
8200
8201         /* Freeze the queue until the client sees the error. */
8202         ahd_freeze_devq(ahd, scb);
8203         aic_freeze_scb(scb);
8204         ahd->qfreeze_cnt++;
8205         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
8206
8207         if (paused == 0)
8208                 ahd_unpause(ahd);
8209
8210         /* Don't want to clobber the original sense code */
8211         if ((scb->flags & SCB_SENSE) != 0) {
8212                 /*
8213                  * Clear the SCB_SENSE Flag and perform
8214                  * a normal command completion.
8215                  */
8216                 scb->flags &= ~SCB_SENSE;
8217                 aic_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
8218                 ahd_done(ahd, scb);
8219                 return;
8220         }
8221         aic_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
8222         aic_set_scsi_status(scb, hscb->shared_data.istatus.scsi_status);
8223         switch (hscb->shared_data.istatus.scsi_status) {
8224         case STATUS_PKT_SENSE:
8225         {
8226                 struct scsi_status_iu_header *siu;
8227
8228                 ahd_sync_sense(ahd, scb, BUS_DMASYNC_POSTREAD);
8229                 siu = (struct scsi_status_iu_header *)scb->sense_data;
8230                 aic_set_scsi_status(scb, siu->status);
8231 #ifdef AHD_DEBUG
8232                 if ((ahd_debug & AHD_SHOW_SENSE) != 0) {
8233                         ahd_print_path(ahd, scb);
8234                         printf("SCB 0x%x Received PKT Status of 0x%x\n",
8235                                SCB_GET_TAG(scb), siu->status);
8236                         printf("\tflags = 0x%x, sense len = 0x%x, "
8237                                "pktfail = 0x%x\n",
8238                                siu->flags, scsi_4btoul(siu->sense_length),
8239                                scsi_4btoul(siu->pkt_failures_length));
8240                 }
8241 #endif
8242                 if ((siu->flags & SIU_RSPVALID) != 0) {
8243                         ahd_print_path(ahd, scb);
8244                         if (scsi_4btoul(siu->pkt_failures_length) < 4) {
8245                                 printf("Unable to parse pkt_failures\n");
8246                         } else {
8247
8248                                 switch (SIU_PKTFAIL_CODE(siu)) {
8249                                 case SIU_PFC_NONE:
8250                                         printf("No packet failure found\n");
8251                                         AHD_UNCORRECTABLE_ERROR(ahd);
8252                                         break;
8253                                 case SIU_PFC_CIU_FIELDS_INVALID:
8254                                         printf("Invalid Command IU Field\n");
8255                                         AHD_UNCORRECTABLE_ERROR(ahd);
8256                                         break;
8257                                 case SIU_PFC_TMF_NOT_SUPPORTED:
8258                                         printf("TMF not supportd\n");
8259                                         AHD_UNCORRECTABLE_ERROR(ahd);
8260                                         break;
8261                                 case SIU_PFC_TMF_FAILED:
8262                                         printf("TMF failed\n");
8263                                         AHD_UNCORRECTABLE_ERROR(ahd);
8264                                         break;
8265                                 case SIU_PFC_INVALID_TYPE_CODE:
8266                                         printf("Invalid L_Q Type code\n");
8267                                         AHD_UNCORRECTABLE_ERROR(ahd);
8268                                         break;
8269                                 case SIU_PFC_ILLEGAL_REQUEST:
8270                                         AHD_UNCORRECTABLE_ERROR(ahd);
8271                                         printf("Illegal request\n");
8272                                 default:
8273                                         break;
8274                                 }
8275                         }
8276                         if (siu->status == SCSI_STATUS_OK)
8277                                 aic_set_transaction_status(scb,
8278                                                            CAM_REQ_CMP_ERR);
8279                 }
8280                 if ((siu->flags & SIU_SNSVALID) != 0) {
8281                         scb->flags |= SCB_PKT_SENSE;
8282 #ifdef AHD_DEBUG
8283                         if ((ahd_debug & AHD_SHOW_SENSE) != 0)
8284                                 printf("Sense data available\n");
8285 #endif
8286                 }
8287                 ahd_done(ahd, scb);
8288                 break;
8289         }
8290         case SCSI_STATUS_CMD_TERMINATED:
8291         case SCSI_STATUS_CHECK_COND:
8292         {
8293                 struct ahd_devinfo devinfo;
8294                 struct ahd_dma_seg *sg;
8295                 struct scsi_sense *sc;
8296                 struct ahd_initiator_tinfo *targ_info;
8297                 struct ahd_tmode_tstate *tstate;
8298                 struct ahd_transinfo *tinfo;
8299 #ifdef AHD_DEBUG
8300                 if (ahd_debug & AHD_SHOW_SENSE) {
8301                         ahd_print_path(ahd, scb);
8302                         printf("SCB %d: requests Check Status\n",
8303                                SCB_GET_TAG(scb));
8304                 }
8305 #endif
8306
8307                 if (aic_perform_autosense(scb) == 0)
8308                         break;
8309
8310                 ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
8311                                     SCB_GET_TARGET(ahd, scb),
8312                                     SCB_GET_LUN(scb),
8313                                     SCB_GET_CHANNEL(ahd, scb),
8314                                     ROLE_INITIATOR);
8315                 targ_info = ahd_fetch_transinfo(ahd,
8316                                                 devinfo.channel,
8317                                                 devinfo.our_scsiid,
8318                                                 devinfo.target,
8319                                                 &tstate);
8320                 tinfo = &targ_info->curr;
8321                 sg = scb->sg_list;
8322                 sc = (struct scsi_sense *)hscb->shared_data.idata.cdb;
8323                 /*
8324                  * Save off the residual if there is one.
8325                  */
8326                 ahd_update_residual(ahd, scb);
8327 #ifdef AHD_DEBUG
8328                 if (ahd_debug & AHD_SHOW_SENSE) {
8329                         ahd_print_path(ahd, scb);
8330                         printf("Sending Sense\n");
8331                 }
8332 #endif
8333                 scb->sg_count = 0;
8334                 sg = ahd_sg_setup(ahd, scb, sg, ahd_get_sense_bufaddr(ahd, scb),
8335                                   aic_get_sense_bufsize(ahd, scb),
8336                                   /*last*/TRUE);
8337                 sc->opcode = REQUEST_SENSE;
8338                 sc->byte2 = 0;
8339                 if (tinfo->protocol_version <= SCSI_REV_2
8340                  && SCB_GET_LUN(scb) < 8)
8341                         sc->byte2 = SCB_GET_LUN(scb) << 5;
8342                 sc->unused[0] = 0;
8343                 sc->unused[1] = 0;
8344                 sc->length = aic_get_sense_bufsize(ahd, scb);
8345                 sc->control = 0;
8346
8347                 /*
8348                  * We can't allow the target to disconnect.
8349                  * This will be an untagged transaction and
8350                  * having the target disconnect will make this
8351                  * transaction indestinguishable from outstanding
8352                  * tagged transactions.
8353                  */
8354                 hscb->control = 0;
8355
8356                 /*
8357                  * This request sense could be because the
8358                  * the device lost power or in some other
8359                  * way has lost our transfer negotiations.
8360                  * Renegotiate if appropriate.  Unit attention
8361                  * errors will be reported before any data
8362                  * phases occur.
8363                  */
8364                 if (aic_get_residual(scb) == aic_get_transfer_length(scb)) {
8365                         ahd_update_neg_request(ahd, &devinfo,
8366                                                tstate, targ_info,
8367                                                AHD_NEG_IF_NON_ASYNC);
8368                 }
8369                 if (tstate->auto_negotiate & devinfo.target_mask) {
8370                         hscb->control |= MK_MESSAGE;
8371                         scb->flags &=
8372                             ~(SCB_NEGOTIATE|SCB_ABORT|SCB_DEVICE_RESET);
8373                         scb->flags |= SCB_AUTO_NEGOTIATE;
8374                 }
8375                 hscb->cdb_len = sizeof(*sc);
8376                 ahd_setup_data_scb(ahd, scb);
8377                 scb->flags |= SCB_SENSE;
8378                 ahd_queue_scb(ahd, scb);
8379                 /*
8380                  * Ensure we have enough time to actually
8381                  * retrieve the sense, but only schedule
8382                  * the timer if we are not in recovery or
8383                  * this is a recovery SCB that is allowed
8384                  * to have an active timer.
8385                  */
8386                 if (ahd->scb_data.recovery_scbs == 0
8387                  || (scb->flags & SCB_RECOVERY_SCB) != 0)
8388                         aic_scb_timer_reset(scb, 5 * 1000);
8389                 break;
8390         }
8391         case SCSI_STATUS_OK:
8392                 printf("%s: Interrupted for staus of 0???\n",
8393                        ahd_name(ahd));
8394                 /* FALLTHROUGH */
8395         default:
8396                 ahd_done(ahd, scb);
8397                 break;
8398         }
8399 }
8400
8401 /*
8402  * Calculate the residual for a just completed SCB.
8403  */
8404 void
8405 ahd_calc_residual(struct ahd_softc *ahd, struct scb *scb)
8406 {
8407         struct hardware_scb *hscb;
8408         struct initiator_status *spkt;
8409         uint32_t sgptr;
8410         uint32_t resid_sgptr;
8411         uint32_t resid;
8412
8413         /*
8414          * 5 cases.
8415          * 1) No residual.
8416          *    SG_STATUS_VALID clear in sgptr.
8417          * 2) Transferless command
8418          * 3) Never performed any transfers.
8419          *    sgptr has SG_FULL_RESID set.
8420          * 4) No residual but target did not
8421          *    save data pointers after the
8422          *    last transfer, so sgptr was
8423          *    never updated.
8424          * 5) We have a partial residual.
8425          *    Use residual_sgptr to determine
8426          *    where we are.
8427          */
8428
8429         hscb = scb->hscb;
8430         sgptr = aic_le32toh(hscb->sgptr);
8431         if ((sgptr & SG_STATUS_VALID) == 0)
8432                 /* Case 1 */
8433                 return;
8434         sgptr &= ~SG_STATUS_VALID;
8435
8436         if ((sgptr & SG_LIST_NULL) != 0)
8437                 /* Case 2 */
8438                 return;
8439
8440         /*
8441          * Residual fields are the same in both
8442          * target and initiator status packets,
8443          * so we can always use the initiator fields
8444          * regardless of the role for this SCB.
8445          */
8446         spkt = &hscb->shared_data.istatus;
8447         resid_sgptr = aic_le32toh(spkt->residual_sgptr);
8448         if ((sgptr & SG_FULL_RESID) != 0) {
8449                 /* Case 3 */
8450                 resid = aic_get_transfer_length(scb);
8451         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
8452                 /* Case 4 */
8453                 return;
8454         } else if ((resid_sgptr & SG_OVERRUN_RESID) != 0) {
8455                 ahd_print_path(ahd, scb);
8456                 printf("data overrun detected Tag == 0x%x.\n",
8457                        SCB_GET_TAG(scb));
8458                 ahd_freeze_devq(ahd, scb);
8459                 aic_set_transaction_status(scb, CAM_DATA_RUN_ERR);
8460                 aic_freeze_scb(scb);
8461                 return;
8462         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
8463                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
8464                 /* NOTREACHED */
8465         } else {
8466                 struct ahd_dma_seg *sg;
8467
8468                 /*
8469                  * Remainder of the SG where the transfer
8470                  * stopped.
8471                  */
8472                 resid = aic_le32toh(spkt->residual_datacnt) & AHD_SG_LEN_MASK;
8473                 sg = ahd_sg_bus_to_virt(ahd, scb, resid_sgptr & SG_PTR_MASK);
8474
8475                 /* The residual sg_ptr always points to the next sg */
8476                 sg--;
8477
8478                 /*
8479                  * Add up the contents of all residual
8480                  * SG segments that are after the SG where
8481                  * the transfer stopped.
8482                  */
8483                 while ((aic_le32toh(sg->len) & AHD_DMA_LAST_SEG) == 0) {
8484                         sg++;
8485                         resid += aic_le32toh(sg->len) & AHD_SG_LEN_MASK;
8486                 }
8487         }
8488         if ((scb->flags & SCB_SENSE) == 0)
8489                 aic_set_residual(scb, resid);
8490         else
8491                 aic_set_sense_residual(scb, resid);
8492
8493 #ifdef AHD_DEBUG
8494         if ((ahd_debug & AHD_SHOW_MISC) != 0) {
8495                 ahd_print_path(ahd, scb);
8496                 printf("Handled %sResidual of %d bytes\n",
8497                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
8498         }
8499 #endif
8500 }
8501
8502 /******************************* Target Mode **********************************/
8503 #ifdef AHD_TARGET_MODE
8504 /*
8505  * Add a target mode event to this lun's queue
8506  */
8507 static void
8508 ahd_queue_lstate_event(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate,
8509                        u_int initiator_id, u_int event_type, u_int event_arg)
8510 {
8511         struct ahd_tmode_event *event;
8512         int pending;
8513
8514         xpt_freeze_devq(lstate->path, /*count*/1);
8515         if (lstate->event_w_idx >= lstate->event_r_idx)
8516                 pending = lstate->event_w_idx - lstate->event_r_idx;
8517         else
8518                 pending = AHD_TMODE_EVENT_BUFFER_SIZE + 1
8519                         - (lstate->event_r_idx - lstate->event_w_idx);
8520
8521         if (event_type == EVENT_TYPE_BUS_RESET
8522          || event_type == MSG_BUS_DEV_RESET) {
8523                 /*
8524                  * Any earlier events are irrelevant, so reset our buffer.
8525                  * This has the effect of allowing us to deal with reset
8526                  * floods (an external device holding down the reset line)
8527                  * without losing the event that is really interesting.
8528                  */
8529                 lstate->event_r_idx = 0;
8530                 lstate->event_w_idx = 0;
8531                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
8532         }
8533
8534         if (pending == AHD_TMODE_EVENT_BUFFER_SIZE) {
8535                 xpt_print_path(lstate->path);
8536                 printf("immediate event %x:%x lost\n",
8537                        lstate->event_buffer[lstate->event_r_idx].event_type,
8538                        lstate->event_buffer[lstate->event_r_idx].event_arg);
8539                 lstate->event_r_idx++;
8540                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8541                         lstate->event_r_idx = 0;
8542                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
8543         }
8544
8545         event = &lstate->event_buffer[lstate->event_w_idx];
8546         event->initiator_id = initiator_id;
8547         event->event_type = event_type;
8548         event->event_arg = event_arg;
8549         lstate->event_w_idx++;
8550         if (lstate->event_w_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8551                 lstate->event_w_idx = 0;
8552 }
8553
8554 /*
8555  * Send any target mode events queued up waiting
8556  * for immediate notify resources.
8557  */
8558 void
8559 ahd_send_lstate_events(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate)
8560 {
8561         struct ccb_hdr *ccbh;
8562         struct ccb_immed_notify *inot;
8563
8564         while (lstate->event_r_idx != lstate->event_w_idx
8565             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
8566                 struct ahd_tmode_event *event;
8567
8568                 event = &lstate->event_buffer[lstate->event_r_idx];
8569                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
8570                 inot = (struct ccb_immed_notify *)ccbh;
8571                 switch (event->event_type) {
8572                 case EVENT_TYPE_BUS_RESET:
8573                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
8574                         break;
8575                 default:
8576                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
8577                         inot->message_args[0] = event->event_type;
8578                         inot->message_args[1] = event->event_arg;
8579                         break;
8580                 }
8581                 inot->initiator_id = event->initiator_id;
8582                 inot->sense_len = 0;
8583                 xpt_done((union ccb *)inot);
8584                 lstate->event_r_idx++;
8585                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8586                         lstate->event_r_idx = 0;
8587         }
8588 }
8589 #endif
8590
8591 /******************** Sequencer Program Patching/Download *********************/
8592
8593 #ifdef AHD_DUMP_SEQ
8594 void
8595 ahd_dumpseq(struct ahd_softc* ahd)
8596 {
8597         int i;
8598         int max_prog;
8599
8600         max_prog = 2048;
8601
8602         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
8603         ahd_outw(ahd, PRGMCNT, 0);
8604         for (i = 0; i < max_prog; i++) {
8605                 uint8_t ins_bytes[4];
8606
8607                 ahd_insb(ahd, SEQRAM, ins_bytes, 4);
8608                 printf("0x%08x\n", ins_bytes[0] << 24
8609                                  | ins_bytes[1] << 16
8610                                  | ins_bytes[2] << 8
8611                                  | ins_bytes[3]);
8612         }
8613 }
8614 #endif
8615
8616 static void
8617 ahd_loadseq(struct ahd_softc *ahd)
8618 {
8619         struct  cs cs_table[num_critical_sections];
8620         u_int   begin_set[num_critical_sections];
8621         u_int   end_set[num_critical_sections];
8622         struct  patch *cur_patch;
8623         u_int   cs_count;
8624         u_int   cur_cs;
8625         u_int   i;
8626         int     downloaded;
8627         u_int   skip_addr;
8628         u_int   sg_prefetch_cnt;
8629         u_int   sg_prefetch_cnt_limit;
8630         u_int   sg_prefetch_align;
8631         u_int   sg_size;
8632         u_int   cacheline_mask;
8633         uint8_t download_consts[DOWNLOAD_CONST_COUNT];
8634
8635         if (bootverbose)
8636                 printf("%s: Downloading Sequencer Program...",
8637                        ahd_name(ahd));
8638
8639 #if DOWNLOAD_CONST_COUNT != 8
8640 #error "Download Const Mismatch"
8641 #endif
8642         /*
8643          * Start out with 0 critical sections
8644          * that apply to this firmware load.
8645          */
8646         cs_count = 0;
8647         cur_cs = 0;
8648         memset(begin_set, 0, sizeof(begin_set));
8649         memset(end_set, 0, sizeof(end_set));
8650
8651         /*
8652          * Setup downloadable constant table.
8653          *
8654          * The computation for the S/G prefetch variables is
8655          * a bit complicated.  We would like to always fetch
8656          * in terms of cachelined sized increments.  However,
8657          * if the cacheline is not an even multiple of the
8658          * SG element size or is larger than our SG RAM, using
8659          * just the cache size might leave us with only a portion
8660          * of an SG element at the tail of a prefetch.  If the
8661          * cacheline is larger than our S/G prefetch buffer less
8662          * the size of an SG element, we may round down to a cacheline
8663          * that doesn't contain any or all of the S/G of interest
8664          * within the bounds of our S/G ram.  Provide variables to
8665          * the sequencer that will allow it to handle these edge
8666          * cases.
8667          */
8668         /* Start by aligning to the nearest cacheline. */
8669         sg_prefetch_align = ahd->pci_cachesize;
8670         if (sg_prefetch_align == 0)
8671                 sg_prefetch_align = 8;
8672         /* Round down to the nearest power of 2. */
8673         while (powerof2(sg_prefetch_align) == 0)
8674                 sg_prefetch_align--;
8675
8676         cacheline_mask = sg_prefetch_align - 1;
8677
8678         /*
8679          * If the cacheline boundary is greater than half our prefetch RAM
8680          * we risk not being able to fetch even a single complete S/G
8681          * segment if we align to that boundary.
8682          */
8683         if (sg_prefetch_align > CCSGADDR_MAX/2)
8684                 sg_prefetch_align = CCSGADDR_MAX/2;
8685         /* Start by fetching a single cacheline. */
8686         sg_prefetch_cnt = sg_prefetch_align;
8687         /*
8688          * Increment the prefetch count by cachelines until
8689          * at least one S/G element will fit.
8690          */
8691         sg_size = sizeof(struct ahd_dma_seg);
8692         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
8693                 sg_size = sizeof(struct ahd_dma64_seg);
8694         while (sg_prefetch_cnt < sg_size)
8695                 sg_prefetch_cnt += sg_prefetch_align;
8696         /*
8697          * If the cacheline is not an even multiple of
8698          * the S/G size, we may only get a partial S/G when
8699          * we align. Add a cacheline if this is the case.
8700          */
8701         if ((sg_prefetch_align % sg_size) != 0
8702          && (sg_prefetch_cnt < CCSGADDR_MAX))
8703                 sg_prefetch_cnt += sg_prefetch_align;
8704         /*
8705          * Lastly, compute a value that the sequencer can use
8706          * to determine if the remainder of the CCSGRAM buffer
8707          * has a full S/G element in it.
8708          */
8709         sg_prefetch_cnt_limit = -(sg_prefetch_cnt - sg_size + 1);
8710         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
8711         download_consts[SG_PREFETCH_CNT_LIMIT] = sg_prefetch_cnt_limit;
8712         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_align - 1);
8713         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_align - 1);
8714         download_consts[SG_SIZEOF] = sg_size;
8715         download_consts[PKT_OVERRUN_BUFOFFSET] =
8716                 (ahd->overrun_buf - (uint8_t *)ahd->qoutfifo) / 256;
8717         download_consts[SCB_TRANSFER_SIZE] = SCB_TRANSFER_SIZE_1BYTE_LUN;
8718         download_consts[CACHELINE_MASK] = cacheline_mask;
8719         cur_patch = patches;
8720         downloaded = 0;
8721         skip_addr = 0;
8722         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
8723         ahd_outw(ahd, PRGMCNT, 0);
8724
8725         for (i = 0; i < sizeof(seqprog)/4; i++) {
8726                 if (ahd_check_patch(ahd, &cur_patch, i, &skip_addr) == 0) {
8727                         /*
8728                          * Don't download this instruction as it
8729                          * is in a patch that was removed.
8730                          */
8731                         continue;
8732                 }
8733                 /*
8734                  * Move through the CS table until we find a CS
8735                  * that might apply to this instruction.
8736                  */
8737                 for (; cur_cs < num_critical_sections; cur_cs++) {
8738                         if (critical_sections[cur_cs].end <= i) {
8739                                 if (begin_set[cs_count] == TRUE
8740                                  && end_set[cs_count] == FALSE) {
8741                                         cs_table[cs_count].end = downloaded;
8742                                         end_set[cs_count] = TRUE;
8743                                         cs_count++;
8744                                 }
8745                                 continue;
8746                         }
8747                         if (critical_sections[cur_cs].begin <= i
8748                          && begin_set[cs_count] == FALSE) {
8749                                 cs_table[cs_count].begin = downloaded;
8750                                 begin_set[cs_count] = TRUE;
8751                         }
8752                         break;
8753                 }
8754                 ahd_download_instr(ahd, i, download_consts);
8755                 downloaded++;
8756         }
8757
8758         ahd->num_critical_sections = cs_count;
8759         if (cs_count != 0) {
8760
8761                 cs_count *= sizeof(struct cs);
8762                 ahd->critical_sections = malloc(cs_count, M_DEVBUF, M_NOWAIT);
8763                 if (ahd->critical_sections == NULL)
8764                         panic("ahd_loadseq: Could not malloc");
8765                 memcpy(ahd->critical_sections, cs_table, cs_count);
8766         }
8767         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE);
8768
8769         if (bootverbose) {
8770                 printf(" %d instructions downloaded\n", downloaded);
8771                 printf("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
8772                        ahd_name(ahd), ahd->features, ahd->bugs, ahd->flags);
8773         }
8774 }
8775
8776 static int
8777 ahd_check_patch(struct ahd_softc *ahd, struct patch **start_patch,
8778                 u_int start_instr, u_int *skip_addr)
8779 {
8780         struct  patch *cur_patch;
8781         struct  patch *last_patch;
8782         u_int   num_patches;
8783
8784         num_patches = sizeof(patches)/sizeof(struct patch);
8785         last_patch = &patches[num_patches];
8786         cur_patch = *start_patch;
8787
8788         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
8789
8790                 if (cur_patch->patch_func(ahd) == 0) {
8791
8792                         /* Start rejecting code */
8793                         *skip_addr = start_instr + cur_patch->skip_instr;
8794                         cur_patch += cur_patch->skip_patch;
8795                 } else {
8796                         /* Accepted this patch.  Advance to the next
8797                          * one and wait for our intruction pointer to
8798                          * hit this point.
8799                          */
8800                         cur_patch++;
8801                 }
8802         }
8803
8804         *start_patch = cur_patch;
8805         if (start_instr < *skip_addr)
8806                 /* Still skipping */
8807                 return (0);
8808
8809         return (1);
8810 }
8811
8812 static u_int
8813 ahd_resolve_seqaddr(struct ahd_softc *ahd, u_int address)
8814 {
8815         struct patch *cur_patch;
8816         int address_offset;
8817         u_int skip_addr;
8818         u_int i;
8819
8820         address_offset = 0;
8821         cur_patch = patches;
8822         skip_addr = 0;
8823
8824         for (i = 0; i < address;) {
8825
8826                 ahd_check_patch(ahd, &cur_patch, i, &skip_addr);
8827
8828                 if (skip_addr > i) {
8829                         int end_addr;
8830
8831                         end_addr = MIN(address, skip_addr);
8832                         address_offset += end_addr - i;
8833                         i = skip_addr;
8834                 } else {
8835                         i++;
8836                 }
8837         }
8838         return (address - address_offset);
8839 }
8840
8841 static void
8842 ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts)
8843 {
8844         union   ins_formats instr;
8845         struct  ins_format1 *fmt1_ins;
8846         struct  ins_format3 *fmt3_ins;
8847         u_int   opcode;
8848
8849         /*
8850          * The firmware is always compiled into a little endian format.
8851          */
8852         instr.integer = aic_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
8853
8854         fmt1_ins = &instr.format1;
8855         fmt3_ins = NULL;
8856
8857         /* Pull the opcode */
8858         opcode = instr.format1.opcode;
8859         switch (opcode) {
8860         case AIC_OP_JMP:
8861         case AIC_OP_JC:
8862         case AIC_OP_JNC:
8863         case AIC_OP_CALL:
8864         case AIC_OP_JNE:
8865         case AIC_OP_JNZ:
8866         case AIC_OP_JE:
8867         case AIC_OP_JZ:
8868         {
8869                 fmt3_ins = &instr.format3;
8870                 fmt3_ins->address = ahd_resolve_seqaddr(ahd, fmt3_ins->address);
8871                 /* FALLTHROUGH */
8872         }
8873         case AIC_OP_OR:
8874         case AIC_OP_AND:
8875         case AIC_OP_XOR:
8876         case AIC_OP_ADD:
8877         case AIC_OP_ADC:
8878         case AIC_OP_BMOV:
8879                 if (fmt1_ins->parity != 0) {
8880                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
8881                 }
8882                 fmt1_ins->parity = 0;
8883                 /* FALLTHROUGH */
8884         case AIC_OP_ROL:
8885         {
8886                 int i, count;
8887
8888                 /* Calculate odd parity for the instruction */
8889                 for (i = 0, count = 0; i < 31; i++) {
8890                         uint32_t mask;
8891
8892                         mask = 0x01 << i;
8893                         if ((instr.integer & mask) != 0)
8894                                 count++;
8895                 }
8896                 if ((count & 0x01) == 0)
8897                         instr.format1.parity = 1;
8898
8899                 /* The sequencer is a little endian cpu */
8900                 instr.integer = aic_htole32(instr.integer);
8901                 ahd_outsb(ahd, SEQRAM, instr.bytes, 4);
8902                 break;
8903         }
8904         default:
8905                 panic("Unknown opcode encountered in seq program");
8906                 break;
8907         }
8908 }
8909
8910 static int
8911 ahd_probe_stack_size(struct ahd_softc *ahd)
8912 {
8913         int last_probe;
8914
8915         last_probe = 0;
8916         while (1) {
8917                 int i;
8918
8919                 /*
8920                  * We avoid using 0 as a pattern to avoid
8921                  * confusion if the stack implementation
8922                  * "back-fills" with zeros when "poping'
8923                  * entries.
8924                  */
8925                 for (i = 1; i <= last_probe+1; i++) {
8926                        ahd_outb(ahd, STACK, i & 0xFF);
8927                        ahd_outb(ahd, STACK, (i >> 8) & 0xFF);
8928                 }
8929
8930                 /* Verify */
8931                 for (i = last_probe+1; i > 0; i--) {
8932                         u_int stack_entry;
8933
8934                         stack_entry = ahd_inb(ahd, STACK)
8935                                     |(ahd_inb(ahd, STACK) << 8);
8936                         if (stack_entry != i)
8937                                 goto sized;
8938                 }
8939                 last_probe++;
8940         }
8941 sized:
8942         return (last_probe);
8943 }
8944
8945 void
8946 ahd_dump_all_cards_state(void)
8947 {
8948         struct ahd_softc *list_ahd;
8949
8950         TAILQ_FOREACH(list_ahd, &ahd_tailq, links) {
8951                 ahd_dump_card_state(list_ahd);
8952         }
8953 }
8954
8955 int
8956 ahd_print_register(ahd_reg_parse_entry_t *table, u_int num_entries,
8957                    const char *name, u_int address, u_int value,
8958                    u_int *cur_column, u_int wrap_point)
8959 {
8960         int     printed;
8961         u_int   printed_mask;
8962         u_int   dummy_column;
8963
8964         if (cur_column == NULL) {
8965                 dummy_column = 0;
8966                 cur_column = &dummy_column;
8967         }
8968
8969         if (cur_column != NULL && *cur_column >= wrap_point) {
8970                 printf("\n");
8971                 *cur_column = 0;
8972         }
8973         printed = printf("%s[0x%x]", name, value);
8974         if (table == NULL) {
8975                 printed += printf(" ");
8976                 *cur_column += printed;
8977                 return (printed);
8978         }
8979         printed_mask = 0;
8980         while (printed_mask != 0xFF) {
8981                 int entry;
8982
8983                 for (entry = 0; entry < num_entries; entry++) {
8984                         if (((value & table[entry].mask)
8985                           != table[entry].value)
8986                          || ((printed_mask & table[entry].mask)
8987                           == table[entry].mask))
8988                                 continue;
8989
8990                         printed += printf("%s%s",
8991                                           printed_mask == 0 ? ":(" : "|",
8992                                           table[entry].name);
8993                         printed_mask |= table[entry].mask;
8994
8995                         break;
8996                 }
8997                 if (entry >= num_entries)
8998                         break;
8999         }
9000         if (printed_mask != 0)
9001                 printed += printf(") ");
9002         else
9003                 printed += printf(" ");
9004         *cur_column += printed;
9005         return (printed);
9006 }
9007
9008 void
9009 ahd_dump_card_state(struct ahd_softc *ahd)
9010 {
9011         struct scb      *scb;
9012         ahd_mode_state   saved_modes;
9013         u_int            dffstat;
9014         int              paused;
9015         u_int            scb_index;
9016         u_int            saved_scb_index;
9017         u_int            cur_col;
9018         int              i;
9019
9020         if (ahd_is_paused(ahd)) {
9021                 paused = 1;
9022         } else {
9023                 paused = 0;
9024                 ahd_pause(ahd);
9025         }
9026         saved_modes = ahd_save_modes(ahd);
9027         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9028         printf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
9029                "%s: Dumping Card State at program address 0x%x Mode 0x%x\n",
9030                ahd_name(ahd),
9031                ahd_inw(ahd, CURADDR),
9032                ahd_build_mode_state(ahd, ahd->saved_src_mode,
9033                                     ahd->saved_dst_mode));
9034         if (paused)
9035                 printf("Card was paused\n");
9036
9037         if (ahd_check_cmdcmpltqueues(ahd))
9038                 printf("Completions are pending\n");
9039
9040         /*
9041          * Mode independent registers.
9042          */
9043         cur_col = 0;
9044         ahd_intstat_print(ahd_inb(ahd, INTSTAT), &cur_col, 50);
9045         ahd_seloid_print(ahd_inb(ahd, SELOID), &cur_col, 50);
9046         ahd_selid_print(ahd_inb(ahd, SELID), &cur_col, 50);
9047         ahd_hs_mailbox_print(ahd_inb(ahd, LOCAL_HS_MAILBOX), &cur_col, 50);
9048         ahd_intctl_print(ahd_inb(ahd, INTCTL), &cur_col, 50);
9049         ahd_seqintstat_print(ahd_inb(ahd, SEQINTSTAT), &cur_col, 50);
9050         ahd_saved_mode_print(ahd_inb(ahd, SAVED_MODE), &cur_col, 50);
9051         ahd_dffstat_print(ahd_inb(ahd, DFFSTAT), &cur_col, 50);
9052         ahd_scsisigi_print(ahd_inb(ahd, SCSISIGI), &cur_col, 50);
9053         ahd_scsiphase_print(ahd_inb(ahd, SCSIPHASE), &cur_col, 50);
9054         ahd_scsibus_print(ahd_inb(ahd, SCSIBUS), &cur_col, 50);
9055         ahd_lastphase_print(ahd_inb(ahd, LASTPHASE), &cur_col, 50);
9056         ahd_scsiseq0_print(ahd_inb(ahd, SCSISEQ0), &cur_col, 50);
9057         ahd_scsiseq1_print(ahd_inb(ahd, SCSISEQ1), &cur_col, 50);
9058         ahd_seqctl0_print(ahd_inb(ahd, SEQCTL0), &cur_col, 50);
9059         ahd_seqintctl_print(ahd_inb(ahd, SEQINTCTL), &cur_col, 50);
9060         ahd_seq_flags_print(ahd_inb(ahd, SEQ_FLAGS), &cur_col, 50);
9061         ahd_seq_flags2_print(ahd_inb(ahd, SEQ_FLAGS2), &cur_col, 50);
9062         ahd_qfreeze_count_print(ahd_inw(ahd, QFREEZE_COUNT), &cur_col, 50);
9063         ahd_kernel_qfreeze_count_print(ahd_inw(ahd, KERNEL_QFREEZE_COUNT),
9064                                        &cur_col, 50);
9065         ahd_mk_message_scb_print(ahd_inw(ahd, MK_MESSAGE_SCB), &cur_col, 50);
9066         ahd_mk_message_scsiid_print(ahd_inb(ahd, MK_MESSAGE_SCSIID),
9067                                     &cur_col, 50);
9068         ahd_sstat0_print(ahd_inb(ahd, SSTAT0), &cur_col, 50);
9069         ahd_sstat1_print(ahd_inb(ahd, SSTAT1), &cur_col, 50);
9070         ahd_sstat2_print(ahd_inb(ahd, SSTAT2), &cur_col, 50);
9071         ahd_sstat3_print(ahd_inb(ahd, SSTAT3), &cur_col, 50);
9072         ahd_perrdiag_print(ahd_inb(ahd, PERRDIAG), &cur_col, 50);
9073         ahd_simode1_print(ahd_inb(ahd, SIMODE1), &cur_col, 50);
9074         ahd_lqistat0_print(ahd_inb(ahd, LQISTAT0), &cur_col, 50);
9075         ahd_lqistat1_print(ahd_inb(ahd, LQISTAT1), &cur_col, 50);
9076         ahd_lqistat2_print(ahd_inb(ahd, LQISTAT2), &cur_col, 50);
9077         ahd_lqostat0_print(ahd_inb(ahd, LQOSTAT0), &cur_col, 50);
9078         ahd_lqostat1_print(ahd_inb(ahd, LQOSTAT1), &cur_col, 50);
9079         ahd_lqostat2_print(ahd_inb(ahd, LQOSTAT2), &cur_col, 50);
9080         printf("\n");
9081         printf("\nSCB Count = %d CMDS_PENDING = %d LASTSCB 0x%x "
9082                "CURRSCB 0x%x NEXTSCB 0x%x\n",
9083                ahd->scb_data.numscbs, ahd_inw(ahd, CMDS_PENDING),
9084                ahd_inw(ahd, LASTSCB), ahd_inw(ahd, CURRSCB),
9085                ahd_inw(ahd, NEXTSCB));
9086         cur_col = 0;
9087         /* QINFIFO */
9088         ahd_search_qinfifo(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
9089                            CAM_LUN_WILDCARD, SCB_LIST_NULL,
9090                            ROLE_UNKNOWN, /*status*/0, SEARCH_PRINT);
9091         saved_scb_index = ahd_get_scbptr(ahd);
9092         printf("Pending list:");
9093         i = 0;
9094         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
9095                 if (i++ > AHD_SCB_MAX)
9096                         break;
9097                 cur_col = printf("\n%3d FIFO_USE[0x%x] ", SCB_GET_TAG(scb),
9098                                  ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT));
9099                 ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
9100                 ahd_scb_control_print(ahd_inb_scbram(ahd, SCB_CONTROL),
9101                                       &cur_col, 60);
9102                 ahd_scb_scsiid_print(ahd_inb_scbram(ahd, SCB_SCSIID),
9103                                      &cur_col, 60);
9104         }
9105         printf("\nTotal %d\n", i);
9106
9107         printf("Kernel Free SCB lists: ");
9108         i = 0;
9109         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
9110                 struct scb *list_scb;
9111
9112                 printf("\n  COLIDX[%d]: ", AHD_GET_SCB_COL_IDX(ahd, scb));
9113                 list_scb = scb;
9114                 do {
9115                         printf("%d ", SCB_GET_TAG(list_scb));
9116                         list_scb = LIST_NEXT(list_scb, collision_links);
9117                 } while (list_scb && i++ < AHD_SCB_MAX);
9118         }
9119
9120         printf("\n  Any Device: ");
9121         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
9122                 if (i++ > AHD_SCB_MAX)
9123                         break;
9124                 printf("%d ", SCB_GET_TAG(scb));
9125         }
9126         printf("\n");
9127
9128         printf("Sequencer Complete DMA-inprog list: ");
9129         scb_index = ahd_inw(ahd, COMPLETE_SCB_DMAINPROG_HEAD);
9130         i = 0;
9131         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9132                 ahd_set_scbptr(ahd, scb_index);
9133                 printf("%d ", scb_index);
9134                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9135         }
9136         printf("\n");
9137
9138         printf("Sequencer Complete list: ");
9139         scb_index = ahd_inw(ahd, COMPLETE_SCB_HEAD);
9140         i = 0;
9141         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9142                 ahd_set_scbptr(ahd, scb_index);
9143                 printf("%d ", scb_index);
9144                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9145         }
9146         printf("\n");
9147
9148
9149         printf("Sequencer DMA-Up and Complete list: ");
9150         scb_index = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
9151         i = 0;
9152         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9153                 ahd_set_scbptr(ahd, scb_index);
9154                 printf("%d ", scb_index);
9155                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9156         }
9157         printf("\n");
9158         printf("Sequencer On QFreeze and Complete list: ");
9159         scb_index = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
9160         i = 0;
9161         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9162                 ahd_set_scbptr(ahd, scb_index);
9163                 printf("%d ", scb_index);
9164                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9165         }
9166         printf("\n");
9167         ahd_set_scbptr(ahd, saved_scb_index);
9168         dffstat = ahd_inb(ahd, DFFSTAT);
9169         for (i = 0; i < 2; i++) {
9170 #ifdef AHD_DEBUG
9171                 struct scb *fifo_scb;
9172 #endif
9173                 u_int       fifo_scbptr;
9174
9175                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
9176                 fifo_scbptr = ahd_get_scbptr(ahd);
9177                 printf("\n\n%s: FIFO%d %s, LONGJMP == 0x%x, SCB 0x%x\n",
9178                        ahd_name(ahd), i,
9179                        (dffstat & (FIFO0FREE << i)) ? "Free" : "Active",
9180                        ahd_inw(ahd, LONGJMP_ADDR), fifo_scbptr);
9181                 cur_col = 0;
9182                 ahd_seqimode_print(ahd_inb(ahd, SEQIMODE), &cur_col, 50);
9183                 ahd_seqintsrc_print(ahd_inb(ahd, SEQINTSRC), &cur_col, 50);
9184                 ahd_dfcntrl_print(ahd_inb(ahd, DFCNTRL), &cur_col, 50);
9185                 ahd_dfstatus_print(ahd_inb(ahd, DFSTATUS), &cur_col, 50);
9186                 ahd_sg_cache_shadow_print(ahd_inb(ahd, SG_CACHE_SHADOW),
9187                                           &cur_col, 50);
9188                 ahd_sg_state_print(ahd_inb(ahd, SG_STATE), &cur_col, 50);
9189                 ahd_dffsxfrctl_print(ahd_inb(ahd, DFFSXFRCTL), &cur_col, 50);
9190                 ahd_soffcnt_print(ahd_inb(ahd, SOFFCNT), &cur_col, 50);
9191                 ahd_mdffstat_print(ahd_inb(ahd, MDFFSTAT), &cur_col, 50);
9192                 if (cur_col > 50) {
9193                         printf("\n");
9194                         cur_col = 0;
9195                 }
9196                 cur_col += printf("SHADDR = 0x%x%x, SHCNT = 0x%x ",
9197                                   ahd_inl(ahd, SHADDR+4),
9198                                   ahd_inl(ahd, SHADDR),
9199                                   (ahd_inb(ahd, SHCNT)
9200                                 | (ahd_inb(ahd, SHCNT + 1) << 8)
9201                                 | (ahd_inb(ahd, SHCNT + 2) << 16)));
9202                 if (cur_col > 50) {
9203                         printf("\n");
9204                         cur_col = 0;
9205                 }
9206                 cur_col += printf("HADDR = 0x%x%x, HCNT = 0x%x ",
9207                                   ahd_inl(ahd, HADDR+4),
9208                                   ahd_inl(ahd, HADDR),
9209                                   (ahd_inb(ahd, HCNT)
9210                                 | (ahd_inb(ahd, HCNT + 1) << 8)
9211                                 | (ahd_inb(ahd, HCNT + 2) << 16)));
9212                 ahd_ccsgctl_print(ahd_inb(ahd, CCSGCTL), &cur_col, 50);
9213 #ifdef AHD_DEBUG
9214                 if ((ahd_debug & AHD_SHOW_SG) != 0) {
9215                         fifo_scb = ahd_lookup_scb(ahd, fifo_scbptr);
9216                         if (fifo_scb != NULL)
9217                                 ahd_dump_sglist(fifo_scb);
9218                 }
9219 #endif
9220         }
9221         printf("\nLQIN: ");
9222         for (i = 0; i < 20; i++)
9223                 printf("0x%x ", ahd_inb(ahd, LQIN + i));
9224         printf("\n");
9225         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
9226         printf("%s: LQISTATE = 0x%x, LQOSTATE = 0x%x, OPTIONMODE = 0x%x\n",
9227                ahd_name(ahd), ahd_inb(ahd, LQISTATE), ahd_inb(ahd, LQOSTATE),
9228                ahd_inb(ahd, OPTIONMODE));
9229         printf("%s: OS_SPACE_CNT = 0x%x MAXCMDCNT = 0x%x\n",
9230                ahd_name(ahd), ahd_inb(ahd, OS_SPACE_CNT),
9231                ahd_inb(ahd, MAXCMDCNT));
9232         printf("%s: SAVED_SCSIID = 0x%x SAVED_LUN = 0x%x\n",
9233                ahd_name(ahd), ahd_inb(ahd, SAVED_SCSIID),
9234                ahd_inb(ahd, SAVED_LUN));
9235         ahd_simode0_print(ahd_inb(ahd, SIMODE0), &cur_col, 50);
9236         printf("\n");
9237         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
9238         cur_col = 0;
9239         ahd_ccscbctl_print(ahd_inb(ahd, CCSCBCTL), &cur_col, 50);
9240         printf("\n");
9241         ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
9242         printf("%s: REG0 == 0x%x, SINDEX = 0x%x, DINDEX = 0x%x\n",
9243                ahd_name(ahd), ahd_inw(ahd, REG0), ahd_inw(ahd, SINDEX),
9244                ahd_inw(ahd, DINDEX));
9245         printf("%s: SCBPTR == 0x%x, SCB_NEXT == 0x%x, SCB_NEXT2 == 0x%x\n",
9246                ahd_name(ahd), ahd_get_scbptr(ahd),
9247                ahd_inw_scbram(ahd, SCB_NEXT),
9248                ahd_inw_scbram(ahd, SCB_NEXT2));
9249         printf("CDB %x %x %x %x %x %x\n",
9250                ahd_inb_scbram(ahd, SCB_CDB_STORE),
9251                ahd_inb_scbram(ahd, SCB_CDB_STORE+1),
9252                ahd_inb_scbram(ahd, SCB_CDB_STORE+2),
9253                ahd_inb_scbram(ahd, SCB_CDB_STORE+3),
9254                ahd_inb_scbram(ahd, SCB_CDB_STORE+4),
9255                ahd_inb_scbram(ahd, SCB_CDB_STORE+5));
9256         printf("STACK:");
9257         for (i = 0; i < ahd->stack_size; i++) {
9258                 ahd->saved_stack[i] =
9259                     ahd_inb(ahd, STACK)|(ahd_inb(ahd, STACK) << 8);
9260                 printf(" 0x%x", ahd->saved_stack[i]);
9261         }
9262         for (i = ahd->stack_size-1; i >= 0; i--) {
9263                 ahd_outb(ahd, STACK, ahd->saved_stack[i] & 0xFF);
9264                 ahd_outb(ahd, STACK, (ahd->saved_stack[i] >> 8) & 0xFF);
9265         }
9266         printf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
9267         ahd_platform_dump_card_state(ahd);
9268         ahd_restore_modes(ahd, saved_modes);
9269         if (paused == 0)
9270                 ahd_unpause(ahd);
9271 }
9272
9273 void
9274 ahd_dump_scbs(struct ahd_softc *ahd)
9275 {
9276         ahd_mode_state saved_modes;
9277         u_int          saved_scb_index;
9278         int            i;
9279
9280         saved_modes = ahd_save_modes(ahd);
9281         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9282         saved_scb_index = ahd_get_scbptr(ahd);
9283         for (i = 0; i < AHD_SCB_MAX; i++) {
9284                 ahd_set_scbptr(ahd, i);
9285                 printf("%3d", i);
9286                 printf("(CTRL 0x%x ID 0x%x N 0x%x N2 0x%x SG 0x%x, RSG 0x%x)\n",
9287                        ahd_inb_scbram(ahd, SCB_CONTROL),
9288                        ahd_inb_scbram(ahd, SCB_SCSIID),
9289                        ahd_inw_scbram(ahd, SCB_NEXT),
9290                        ahd_inw_scbram(ahd, SCB_NEXT2),
9291                        ahd_inl_scbram(ahd, SCB_SGPTR),
9292                        ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR));
9293         }
9294         printf("\n");
9295         ahd_set_scbptr(ahd, saved_scb_index);
9296         ahd_restore_modes(ahd, saved_modes);
9297 }
9298
9299
9300 /*************************** Timeout Handling *********************************/
9301 void
9302 ahd_timeout(struct scb *scb)
9303 {
9304         struct ahd_softc *ahd;
9305
9306         ahd = scb->ahd_softc;
9307         if ((scb->flags & SCB_ACTIVE) != 0) {
9308                 if ((scb->flags & SCB_TIMEDOUT) == 0) {
9309                         LIST_INSERT_HEAD(&ahd->timedout_scbs, scb,
9310                                          timedout_links);
9311                         scb->flags |= SCB_TIMEDOUT;
9312                 }
9313                 ahd_wakeup_recovery_thread(ahd);
9314         }
9315 }
9316
9317 /*
9318  * ahd_recover_commands determines if any of the commands that have currently
9319  * timedout are the root cause for this timeout.  Innocent commands are given
9320  * a new timeout while we wait for the command executing on the bus to timeout.
9321  * This routine is invoked from a thread context so we are allowed to sleep.
9322  * Our lock is not held on entry.
9323  */
9324 void
9325 ahd_recover_commands(struct ahd_softc *ahd)
9326 {
9327         struct  scb *scb;
9328         struct  scb *active_scb;
9329         int     found;
9330         int     was_paused;
9331         u_int   active_scbptr;
9332         u_int   last_phase;
9333
9334         /*
9335          * Pause the controller and manually flush any
9336          * commands that have just completed but that our
9337          * interrupt handler has yet to see.
9338          */
9339         was_paused = ahd_is_paused(ahd);
9340
9341         printf("%s: Recovery Initiated - Card was %spaused\n", ahd_name(ahd),
9342                was_paused ? "" : "not ");
9343         AHD_CORRECTABLE_ERROR(ahd);
9344         ahd_dump_card_state(ahd);
9345
9346         ahd_pause_and_flushwork(ahd);
9347
9348         if (LIST_EMPTY(&ahd->timedout_scbs) != 0) {
9349                 /*
9350                  * The timedout commands have already
9351                  * completed.  This typically means
9352                  * that either the timeout value was on
9353                  * the hairy edge of what the device
9354                  * requires or - more likely - interrupts
9355                  * are not happening.
9356                  */
9357                 printf("%s: Timedout SCBs already complete. "
9358                        "Interrupts may not be functioning.\n", ahd_name(ahd));
9359                 ahd_unpause(ahd);
9360                 return;
9361         }
9362
9363         /*
9364          * Determine identity of SCB acting on the bus.
9365          * This test only catches non-packetized transactions.
9366          * Due to the fleeting nature of packetized operations,
9367          * we can't easily determine that a packetized operation
9368          * is on the bus.
9369          */
9370         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9371         last_phase = ahd_inb(ahd, LASTPHASE);
9372         active_scbptr = ahd_get_scbptr(ahd);
9373         active_scb = NULL;
9374         if (last_phase != P_BUSFREE
9375          || (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) == 0)
9376                 active_scb = ahd_lookup_scb(ahd, active_scbptr);
9377
9378         while ((scb = LIST_FIRST(&ahd->timedout_scbs)) != NULL) {
9379                 int     target;
9380                 int     lun;
9381                 char    channel;
9382
9383                 target = SCB_GET_TARGET(ahd, scb);
9384                 channel = SCB_GET_CHANNEL(ahd, scb);
9385                 lun = SCB_GET_LUN(scb);
9386
9387                 ahd_print_path(ahd, scb);
9388                 printf("SCB %d - timed out\n", SCB_GET_TAG(scb));
9389
9390                 if (scb->flags & (SCB_DEVICE_RESET|SCB_ABORT)) {
9391                         /*
9392                          * Been down this road before.
9393                          * Do a full bus reset.
9394                          */
9395                         aic_set_transaction_status(scb, CAM_CMD_TIMEOUT);
9396 bus_reset:
9397                         found = ahd_reset_channel(ahd, channel,
9398                                                   /*Initiate Reset*/TRUE);
9399                         printf("%s: Issued Channel %c Bus Reset. "
9400                                "%d SCBs aborted\n", ahd_name(ahd), channel,
9401                                found);
9402                         continue;
9403                 }
9404
9405                 /*
9406                  * Remove the command from the timedout list in
9407                  * preparation for requeing it.
9408                  */
9409                 LIST_REMOVE(scb, timedout_links);
9410                 scb->flags &= ~SCB_TIMEDOUT;
9411
9412                 if (active_scb != NULL) {
9413
9414                         if (active_scb != scb) {
9415
9416                                 /*
9417                                  * If the active SCB is not us, assume that
9418                                  * the active SCB has a longer timeout than
9419                                  * the timedout SCB, and wait for the active
9420                                  * SCB to timeout.  As a safeguard, only
9421                                  * allow this deferral to continue if some
9422                                  * untimed-out command is outstanding.
9423                                  */
9424                                 if (ahd_other_scb_timeout(ahd, scb,
9425                                                           active_scb) == 0)
9426                                         goto bus_reset;
9427                                 continue;
9428                         }
9429
9430                         /*
9431                          * We're active on the bus, so assert ATN
9432                          * and hope that the target responds.
9433                          */
9434                         ahd_set_recoveryscb(ahd, active_scb);
9435                         active_scb->flags |= SCB_RECOVERY_SCB|SCB_DEVICE_RESET;
9436                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
9437                         ahd_outb(ahd, SCSISIGO, last_phase|ATNO);
9438                         ahd_print_path(ahd, active_scb);
9439                         printf("BDR message in message buffer\n");
9440                         aic_scb_timer_reset(scb, 2 * 1000);
9441                         break;
9442                 } else if (last_phase != P_BUSFREE
9443                         && ahd_inb(ahd, SCSIPHASE) == 0) {
9444                         /*
9445                          * SCB is not identified, there
9446                          * is no pending REQ, and the sequencer
9447                          * has not seen a busfree.  Looks like
9448                          * a stuck connection waiting to
9449                          * go busfree.  Reset the bus.
9450                          */
9451                         printf("%s: Connection stuck awaiting busfree or "
9452                                "Identify Msg.\n", ahd_name(ahd));
9453                         goto bus_reset;
9454                 } else if (ahd_search_qinfifo(ahd, target, channel, lun,
9455                                               SCB_GET_TAG(scb),
9456                                               ROLE_INITIATOR, /*status*/0,
9457                                               SEARCH_COUNT) > 0) {
9458
9459                         /*
9460                          * We haven't even gone out on the bus
9461                          * yet, so the timeout must be due to
9462                          * some other command.  Reset the timer
9463                          * and go on.
9464                          */
9465                         if (ahd_other_scb_timeout(ahd, scb, NULL) == 0)
9466                                 goto bus_reset;
9467                 } else {
9468                         /*
9469                          * This SCB is for a disconnected transaction
9470                          * and we haven't found a better candidate on
9471                          * the bus to explain this timeout.
9472                          */
9473                         ahd_set_recoveryscb(ahd, scb);
9474
9475                         /*
9476                          * Actually re-queue this SCB in an attempt
9477                          * to select the device before it reconnects.
9478                          * In either case (selection or reselection),
9479                          * we will now issue a target reset to the
9480                          * timed-out device.
9481                          */
9482                         scb->flags |= SCB_DEVICE_RESET;
9483                         scb->hscb->cdb_len = 0;
9484                         scb->hscb->task_attribute = 0;
9485                         scb->hscb->task_management = SIU_TASKMGMT_ABORT_TASK;
9486
9487                         ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
9488                         if ((scb->flags & SCB_PACKETIZED) != 0) {
9489                                 /*
9490                                  * Mark the SCB has having an outstanding
9491                                  * task management function.  Should the command
9492                                  * complete normally before the task management
9493                                  * function can be sent, the host will be
9494                                  * notified to abort our requeued SCB.
9495                                  */
9496                                 ahd_outb(ahd, SCB_TASK_MANAGEMENT,
9497                                          scb->hscb->task_management);
9498                         } else {
9499                                 /*
9500                                  * If non-packetized, set the MK_MESSAGE control
9501                                  * bit indicating that we desire to send a
9502                                  * message.  We also set the disconnected flag
9503                                  * since there is no guarantee that our SCB
9504                                  * control byte matches the version on the
9505                                  * card.  We don't want the sequencer to abort
9506                                  * the command thinking an unsolicited
9507                                  * reselection occurred.
9508                                  */
9509                                 scb->hscb->control |= MK_MESSAGE|DISCONNECTED;
9510
9511                                 /*
9512                                  * The sequencer will never re-reference the
9513                                  * in-core SCB.  To make sure we are notified
9514                                  * during reslection, set the MK_MESSAGE flag in
9515                                  * the card's copy of the SCB.
9516                                  */
9517                                 ahd_outb(ahd, SCB_CONTROL,
9518                                          ahd_inb(ahd, SCB_CONTROL)|MK_MESSAGE);
9519                         }
9520
9521                         /*
9522                          * Clear out any entries in the QINFIFO first
9523                          * so we are the next SCB for this target
9524                          * to run.
9525                          */
9526                         ahd_search_qinfifo(ahd, target, channel, lun,
9527                                            SCB_LIST_NULL, ROLE_INITIATOR,
9528                                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
9529                         ahd_qinfifo_requeue_tail(ahd, scb);
9530                         ahd_set_scbptr(ahd, active_scbptr);
9531                         ahd_print_path(ahd, scb);
9532                         printf("Queuing a BDR SCB\n");
9533                         aic_scb_timer_reset(scb, 2 * 1000);
9534                         break;
9535                 }
9536         }
9537
9538         /*
9539          * Any remaining SCBs were not the "culprit", so remove
9540          * them from the timeout list.  The timer for these commands
9541          * will be reset once the recovery SCB completes.
9542          */
9543         while ((scb = LIST_FIRST(&ahd->timedout_scbs)) != NULL) {
9544
9545                 LIST_REMOVE(scb, timedout_links);
9546                 scb->flags &= ~SCB_TIMEDOUT;
9547         }
9548
9549         ahd_unpause(ahd);
9550 }
9551
9552 /*
9553  * Re-schedule a timeout for the passed in SCB if we determine that some
9554  * other SCB is in the process of recovery or an SCB with a longer
9555  * timeout is still pending.  Limit our search to just "other_scb"
9556  * if it is non-NULL.
9557  */
9558 static int
9559 ahd_other_scb_timeout(struct ahd_softc *ahd, struct scb *scb,
9560                       struct scb *other_scb)
9561 {
9562         u_int   newtimeout;
9563         int     found;
9564
9565         ahd_print_path(ahd, scb);
9566         printf("Other SCB Timeout%s",
9567                (scb->flags & SCB_OTHERTCL_TIMEOUT) != 0
9568                ? " again\n" : "\n");
9569
9570         AHD_UNCORRECTABLE_ERROR(ahd);
9571         newtimeout = aic_get_timeout(scb);
9572         scb->flags |= SCB_OTHERTCL_TIMEOUT;
9573         found = 0;
9574         if (other_scb != NULL) {
9575                 if ((other_scb->flags
9576                    & (SCB_OTHERTCL_TIMEOUT|SCB_TIMEDOUT)) == 0
9577                  || (other_scb->flags & SCB_RECOVERY_SCB) != 0) {
9578                         found++;
9579                         newtimeout = MAX(aic_get_timeout(other_scb),
9580                                          newtimeout);
9581                 }
9582         } else {
9583                 LIST_FOREACH(other_scb, &ahd->pending_scbs, pending_links) {
9584                         if ((other_scb->flags
9585                            & (SCB_OTHERTCL_TIMEOUT|SCB_TIMEDOUT)) == 0
9586                          || (other_scb->flags & SCB_RECOVERY_SCB) != 0) {
9587                                 found++;
9588                                 newtimeout = MAX(aic_get_timeout(other_scb),
9589                                                  newtimeout);
9590                         }
9591                 }
9592         }
9593
9594         if (found != 0)
9595                 aic_scb_timer_reset(scb, newtimeout);
9596         else {
9597                 ahd_print_path(ahd, scb);
9598                 printf("No other SCB worth waiting for...\n");
9599         }
9600
9601         return (found != 0);
9602 }
9603
9604 /**************************** Flexport Logic **********************************/
9605 /*
9606  * Read count 16bit words from 16bit word address start_addr from the
9607  * SEEPROM attached to the controller, into buf, using the controller's
9608  * SEEPROM reading state machine.  Optionally treat the data as a byte
9609  * stream in terms of byte order.
9610  */
9611 int
9612 ahd_read_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9613                  u_int start_addr, u_int count, int bytestream)
9614 {
9615         u_int cur_addr;
9616         u_int end_addr;
9617         int   error;
9618
9619         /*
9620          * If we never make it through the loop even once,
9621          * we were passed invalid arguments.
9622          */
9623         error = EINVAL;
9624         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9625         end_addr = start_addr + count;
9626         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
9627
9628                 ahd_outb(ahd, SEEADR, cur_addr);
9629                 ahd_outb(ahd, SEECTL, SEEOP_READ | SEESTART);
9630
9631                 error = ahd_wait_seeprom(ahd);
9632                 if (error)
9633                         break;
9634                 if (bytestream != 0) {
9635                         uint8_t *bytestream_ptr;
9636
9637                         bytestream_ptr = (uint8_t *)buf;
9638                         *bytestream_ptr++ = ahd_inb(ahd, SEEDAT);
9639                         *bytestream_ptr = ahd_inb(ahd, SEEDAT+1);
9640                 } else {
9641                         /*
9642                          * ahd_inw() already handles machine byte order.
9643                          */
9644                         *buf = ahd_inw(ahd, SEEDAT);
9645                 }
9646                 buf++;
9647         }
9648         return (error);
9649 }
9650
9651 /*
9652  * Write count 16bit words from buf, into SEEPROM attache to the
9653  * controller starting at 16bit word address start_addr, using the
9654  * controller's SEEPROM writing state machine.
9655  */
9656 int
9657 ahd_write_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9658                   u_int start_addr, u_int count)
9659 {
9660         u_int cur_addr;
9661         u_int end_addr;
9662         int   error;
9663         int   retval;
9664
9665         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9666         error = ENOENT;
9667
9668         /* Place the chip into write-enable mode */
9669         ahd_outb(ahd, SEEADR, SEEOP_EWEN_ADDR);
9670         ahd_outb(ahd, SEECTL, SEEOP_EWEN | SEESTART);
9671         error = ahd_wait_seeprom(ahd);
9672         if (error)
9673                 return (error);
9674
9675         /*
9676          * Write the data.  If we don't get throught the loop at
9677          * least once, the arguments were invalid.
9678          */
9679         retval = EINVAL;
9680         end_addr = start_addr + count;
9681         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
9682                 ahd_outw(ahd, SEEDAT, *buf++);
9683                 ahd_outb(ahd, SEEADR, cur_addr);
9684                 ahd_outb(ahd, SEECTL, SEEOP_WRITE | SEESTART);
9685
9686                 retval = ahd_wait_seeprom(ahd);
9687                 if (retval)
9688                         break;
9689         }
9690
9691         /*
9692          * Disable writes.
9693          */
9694         ahd_outb(ahd, SEEADR, SEEOP_EWDS_ADDR);
9695         ahd_outb(ahd, SEECTL, SEEOP_EWDS | SEESTART);
9696         error = ahd_wait_seeprom(ahd);
9697         if (error)
9698                 return (error);
9699         return (retval);
9700 }
9701
9702 /*
9703  * Wait ~100us for the serial eeprom to satisfy our request.
9704  */
9705 int
9706 ahd_wait_seeprom(struct ahd_softc *ahd)
9707 {
9708         int cnt;
9709
9710         cnt = 5000;
9711         while ((ahd_inb(ahd, SEESTAT) & (SEEARBACK|SEEBUSY)) != 0 && --cnt)
9712                 aic_delay(5);
9713
9714         if (cnt == 0)
9715                 return (ETIMEDOUT);
9716         return (0);
9717 }
9718
9719 /*
9720  * Validate the two checksums in the per_channel
9721  * vital product data struct.
9722  */
9723 int
9724 ahd_verify_vpd_cksum(struct vpd_config *vpd)
9725 {
9726         int i;
9727         int maxaddr;
9728         uint32_t checksum;
9729         uint8_t *vpdarray;
9730
9731         vpdarray = (uint8_t *)vpd;
9732         maxaddr = offsetof(struct vpd_config, vpd_checksum);
9733         checksum = 0;
9734         for (i = offsetof(struct vpd_config, resource_type); i < maxaddr; i++)
9735                 checksum = checksum + vpdarray[i];
9736         if (checksum == 0
9737          || (-checksum & 0xFF) != vpd->vpd_checksum)
9738                 return (0);
9739
9740         checksum = 0;
9741         maxaddr = offsetof(struct vpd_config, checksum);
9742         for (i = offsetof(struct vpd_config, default_target_flags);
9743              i < maxaddr; i++)
9744                 checksum = checksum + vpdarray[i];
9745         if (checksum == 0
9746          || (-checksum & 0xFF) != vpd->checksum)
9747                 return (0);
9748         return (1);
9749 }
9750
9751 int
9752 ahd_verify_cksum(struct seeprom_config *sc)
9753 {
9754         int i;
9755         int maxaddr;
9756         uint32_t checksum;
9757         uint16_t *scarray;
9758
9759         maxaddr = (sizeof(*sc)/2) - 1;
9760         checksum = 0;
9761         scarray = (uint16_t *)sc;
9762
9763         for (i = 0; i < maxaddr; i++)
9764                 checksum = checksum + scarray[i];
9765         if (checksum == 0
9766          || (checksum & 0xFFFF) != sc->checksum) {
9767                 return (0);
9768         } else {
9769                 return (1);
9770         }
9771 }
9772
9773 int
9774 ahd_acquire_seeprom(struct ahd_softc *ahd)
9775 {
9776         /*
9777          * We should be able to determine the SEEPROM type
9778          * from the flexport logic, but unfortunately not
9779          * all implementations have this logic and there is
9780          * no programatic method for determining if the logic
9781          * is present.
9782          */
9783         return (1);
9784 #if 0
9785         uint8_t seetype;
9786         int     error;
9787
9788         error = ahd_read_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, &seetype);
9789         if (error != 0
9790          || ((seetype & FLX_ROMSTAT_SEECFG) == FLX_ROMSTAT_SEE_NONE))
9791                 return (0);
9792         return (1);
9793 #endif
9794 }
9795
9796 void
9797 ahd_release_seeprom(struct ahd_softc *ahd)
9798 {
9799         /* Currently a no-op */
9800 }
9801
9802 int
9803 ahd_write_flexport(struct ahd_softc *ahd, u_int addr, u_int value)
9804 {
9805         int error;
9806
9807         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9808         if (addr > 7)
9809                 panic("ahd_write_flexport: address out of range");
9810         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
9811         error = ahd_wait_flexport(ahd);
9812         if (error != 0)
9813                 return (error);
9814         ahd_outb(ahd, BRDDAT, value);
9815         ahd_flush_device_writes(ahd);
9816         ahd_outb(ahd, BRDCTL, BRDSTB|BRDEN|(addr << 3));
9817         ahd_flush_device_writes(ahd);
9818         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
9819         ahd_flush_device_writes(ahd);
9820         ahd_outb(ahd, BRDCTL, 0);
9821         ahd_flush_device_writes(ahd);
9822         return (0);
9823 }
9824
9825 int
9826 ahd_read_flexport(struct ahd_softc *ahd, u_int addr, uint8_t *value)
9827 {
9828         int     error;
9829
9830         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9831         if (addr > 7)
9832                 panic("ahd_read_flexport: address out of range");
9833         ahd_outb(ahd, BRDCTL, BRDRW|BRDEN|(addr << 3));
9834         error = ahd_wait_flexport(ahd);
9835         if (error != 0)
9836                 return (error);
9837         *value = ahd_inb(ahd, BRDDAT);
9838         ahd_outb(ahd, BRDCTL, 0);
9839         ahd_flush_device_writes(ahd);
9840         return (0);
9841 }
9842
9843 /*
9844  * Wait at most 2 seconds for flexport arbitration to succeed.
9845  */
9846 int
9847 ahd_wait_flexport(struct ahd_softc *ahd)
9848 {
9849         int cnt;
9850
9851         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9852         cnt = 1000000 * 2 / 5;
9853         while ((ahd_inb(ahd, BRDCTL) & FLXARBACK) == 0 && --cnt)
9854                 aic_delay(5);
9855
9856         if (cnt == 0)
9857                 return (ETIMEDOUT);
9858         return (0);
9859 }
9860
9861 /************************* Target Mode ****************************************/
9862 #ifdef AHD_TARGET_MODE
9863 cam_status
9864 ahd_find_tmode_devs(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb,
9865                     struct ahd_tmode_tstate **tstate,
9866                     struct ahd_tmode_lstate **lstate,
9867                     int notfound_failure)
9868 {
9869
9870         if ((ahd->features & AHD_TARGETMODE) == 0)
9871                 return (CAM_REQ_INVALID);
9872
9873         /*
9874          * Handle the 'black hole' device that sucks up
9875          * requests to unattached luns on enabled targets.
9876          */
9877         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
9878          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
9879                 *tstate = NULL;
9880                 *lstate = ahd->black_hole;
9881         } else {
9882                 u_int max_id;
9883
9884                 max_id = (ahd->features & AHD_WIDE) ? 15 : 7;
9885                 if (ccb->ccb_h.target_id > max_id)
9886                         return (CAM_TID_INVALID);
9887
9888                 if (ccb->ccb_h.target_lun >= AHD_NUM_LUNS)
9889                         return (CAM_LUN_INVALID);
9890
9891                 *tstate = ahd->enabled_targets[ccb->ccb_h.target_id];
9892                 *lstate = NULL;
9893                 if (*tstate != NULL)
9894                         *lstate =
9895                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
9896         }
9897
9898         if (notfound_failure != 0 && *lstate == NULL)
9899                 return (CAM_PATH_INVALID);
9900
9901         return (CAM_REQ_CMP);
9902 }
9903
9904 void
9905 ahd_handle_en_lun(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb)
9906 {
9907 #if NOT_YET
9908         struct     ahd_tmode_tstate *tstate;
9909         struct     ahd_tmode_lstate *lstate;
9910         struct     ccb_en_lun *cel;
9911         cam_status status;
9912         u_int      target;
9913         u_int      lun;
9914         u_int      target_mask;
9915         u_long     s;
9916         char       channel;
9917
9918         status = ahd_find_tmode_devs(ahd, sim, ccb, &tstate, &lstate,
9919                                      /*notfound_failure*/FALSE);
9920
9921         if (status != CAM_REQ_CMP) {
9922                 ccb->ccb_h.status = status;
9923                 return;
9924         }
9925
9926         if ((ahd->features & AHD_MULTIROLE) != 0) {
9927                 u_int      our_id;
9928
9929                 our_id = ahd->our_id;
9930                 if (ccb->ccb_h.target_id != our_id) {
9931                         if ((ahd->features & AHD_MULTI_TID) != 0
9932                          && (ahd->flags & AHD_INITIATORROLE) != 0) {
9933                                 /*
9934                                  * Only allow additional targets if
9935                                  * the initiator role is disabled.
9936                                  * The hardware cannot handle a re-select-in
9937                                  * on the initiator id during a re-select-out
9938                                  * on a different target id.
9939                                  */
9940                                 status = CAM_TID_INVALID;
9941                         } else if ((ahd->flags & AHD_INITIATORROLE) != 0
9942                                 || ahd->enabled_luns > 0) {
9943                                 /*
9944                                  * Only allow our target id to change
9945                                  * if the initiator role is not configured
9946                                  * and there are no enabled luns which
9947                                  * are attached to the currently registered
9948                                  * scsi id.
9949                                  */
9950                                 status = CAM_TID_INVALID;
9951                         }
9952                 }
9953         }
9954
9955         if (status != CAM_REQ_CMP) {
9956                 ccb->ccb_h.status = status;
9957                 return;
9958         }
9959
9960         /*
9961          * We now have an id that is valid.
9962          * If we aren't in target mode, switch modes.
9963          */
9964         if ((ahd->flags & AHD_TARGETROLE) == 0
9965          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
9966                 printf("Configuring Target Mode\n");
9967                 if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
9968                         ccb->ccb_h.status = CAM_BUSY;
9969                         return;
9970                 }
9971                 ahd->flags |= AHD_TARGETROLE;
9972                 if ((ahd->features & AHD_MULTIROLE) == 0)
9973                         ahd->flags &= ~AHD_INITIATORROLE;
9974                 ahd_pause(ahd);
9975                 ahd_loadseq(ahd);
9976                 ahd_restart(ahd);
9977         }
9978         cel = &ccb->cel;
9979         target = ccb->ccb_h.target_id;
9980         lun = ccb->ccb_h.target_lun;
9981         channel = SIM_CHANNEL(ahd, sim);
9982         target_mask = 0x01 << target;
9983         if (channel == 'B')
9984                 target_mask <<= 8;
9985
9986         if (cel->enable != 0) {
9987                 u_int scsiseq1;
9988
9989                 /* Are we already enabled?? */
9990                 if (lstate != NULL) {
9991                         xpt_print_path(ccb->ccb_h.path);
9992                         printf("Lun already enabled\n");
9993                         AHD_CORRECTABLE_ERROR(ahd);
9994                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
9995                         return;
9996                 }
9997
9998                 if (cel->grp6_len != 0
9999                  || cel->grp7_len != 0) {
10000                         /*
10001                          * Don't (yet?) support vendor
10002                          * specific commands.
10003                          */
10004                         ccb->ccb_h.status = CAM_REQ_INVALID;
10005                         printf("Non-zero Group Codes\n");
10006                         return;
10007                 }
10008
10009                 /*
10010                  * Seems to be okay.
10011                  * Setup our data structures.
10012                  */
10013                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
10014                         tstate = ahd_alloc_tstate(ahd, target, channel);
10015                         if (tstate == NULL) {
10016                                 xpt_print_path(ccb->ccb_h.path);
10017                                 printf("Couldn't allocate tstate\n");
10018                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10019                                 return;
10020                         }
10021                 }
10022                 lstate = malloc(sizeof(*lstate), M_DEVBUF, M_NOWAIT);
10023                 if (lstate == NULL) {
10024                         xpt_print_path(ccb->ccb_h.path);
10025                         printf("Couldn't allocate lstate\n");
10026                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10027                         return;
10028                 }
10029                 memset(lstate, 0, sizeof(*lstate));
10030                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
10031                                          xpt_path_path_id(ccb->ccb_h.path),
10032                                          xpt_path_target_id(ccb->ccb_h.path),
10033                                          xpt_path_lun_id(ccb->ccb_h.path));
10034                 if (status != CAM_REQ_CMP) {
10035                         free(lstate, M_DEVBUF);
10036                         xpt_print_path(ccb->ccb_h.path);
10037                         printf("Couldn't allocate path\n");
10038                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10039                         return;
10040                 }
10041                 SLIST_INIT(&lstate->accept_tios);
10042                 SLIST_INIT(&lstate->immed_notifies);
10043                 ahd_pause(ahd);
10044                 if (target != CAM_TARGET_WILDCARD) {
10045                         tstate->enabled_luns[lun] = lstate;
10046                         ahd->enabled_luns++;
10047
10048                         if ((ahd->features & AHD_MULTI_TID) != 0) {
10049                                 u_int targid_mask;
10050
10051                                 targid_mask = ahd_inw(ahd, TARGID);
10052                                 targid_mask |= target_mask;
10053                                 ahd_outw(ahd, TARGID, targid_mask);
10054                                 ahd_update_scsiid(ahd, targid_mask);
10055                         } else {
10056                                 u_int our_id;
10057                                 char  channel;
10058
10059                                 channel = SIM_CHANNEL(ahd, sim);
10060                                 our_id = SIM_SCSI_ID(ahd, sim);
10061
10062                                 /*
10063                                  * This can only happen if selections
10064                                  * are not enabled
10065                                  */
10066                                 if (target != our_id) {
10067                                         u_int sblkctl;
10068                                         char  cur_channel;
10069                                         int   swap;
10070
10071                                         sblkctl = ahd_inb(ahd, SBLKCTL);
10072                                         cur_channel = (sblkctl & SELBUSB)
10073                                                     ? 'B' : 'A';
10074                                         if ((ahd->features & AHD_TWIN) == 0)
10075                                                 cur_channel = 'A';
10076                                         swap = cur_channel != channel;
10077                                         ahd->our_id = target;
10078
10079                                         if (swap)
10080                                                 ahd_outb(ahd, SBLKCTL,
10081                                                          sblkctl ^ SELBUSB);
10082
10083                                         ahd_outb(ahd, SCSIID, target);
10084
10085                                         if (swap)
10086                                                 ahd_outb(ahd, SBLKCTL, sblkctl);
10087                                 }
10088                         }
10089                 } else
10090                         ahd->black_hole = lstate;
10091                 /* Allow select-in operations */
10092                 if (ahd->black_hole != NULL && ahd->enabled_luns > 0) {
10093                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10094                         scsiseq1 |= ENSELI;
10095                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10096                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10097                         scsiseq1 |= ENSELI;
10098                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
10099                 }
10100                 ahd_unpause(ahd);
10101                 ccb->ccb_h.status = CAM_REQ_CMP;
10102                 xpt_print_path(ccb->ccb_h.path);
10103                 printf("Lun now enabled for target mode\n");
10104         } else {
10105                 struct scb *scb;
10106                 int i, empty;
10107
10108                 if (lstate == NULL) {
10109                         ccb->ccb_h.status = CAM_LUN_INVALID;
10110                         return;
10111                 }
10112
10113                 ccb->ccb_h.status = CAM_REQ_CMP;
10114                 LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
10115                         struct ccb_hdr *ccbh;
10116
10117                         ccbh = &scb->io_ctx->ccb_h;
10118                         if (ccbh->func_code == XPT_CONT_TARGET_IO
10119                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
10120                                 printf("CTIO pending\n");
10121                                 ccb->ccb_h.status = CAM_REQ_INVALID;
10122                                 return;
10123                         }
10124                 }
10125
10126                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
10127                         printf("ATIOs pending\n");
10128                         ccb->ccb_h.status = CAM_REQ_INVALID;
10129                 }
10130
10131                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
10132                         printf("INOTs pending\n");
10133                         ccb->ccb_h.status = CAM_REQ_INVALID;
10134                 }
10135
10136                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
10137                         return;
10138                 }
10139
10140                 xpt_print_path(ccb->ccb_h.path);
10141                 printf("Target mode disabled\n");
10142                 xpt_free_path(lstate->path);
10143                 free(lstate, M_DEVBUF);
10144
10145                 ahd_pause(ahd);
10146                 /* Can we clean up the target too? */
10147                 if (target != CAM_TARGET_WILDCARD) {
10148                         tstate->enabled_luns[lun] = NULL;
10149                         ahd->enabled_luns--;
10150                         for (empty = 1, i = 0; i < 8; i++)
10151                                 if (tstate->enabled_luns[i] != NULL) {
10152                                         empty = 0;
10153                                         break;
10154                                 }
10155
10156                         if (empty) {
10157                                 ahd_free_tstate(ahd, target, channel,
10158                                                 /*force*/FALSE);
10159                                 if (ahd->features & AHD_MULTI_TID) {
10160                                         u_int targid_mask;
10161
10162                                         targid_mask = ahd_inw(ahd, TARGID);
10163                                         targid_mask &= ~target_mask;
10164                                         ahd_outw(ahd, TARGID, targid_mask);
10165                                         ahd_update_scsiid(ahd, targid_mask);
10166                                 }
10167                         }
10168                 } else {
10169
10170                         ahd->black_hole = NULL;
10171
10172                         /*
10173                          * We can't allow selections without
10174                          * our black hole device.
10175                          */
10176                         empty = TRUE;
10177                 }
10178                 if (ahd->enabled_luns == 0) {
10179                         /* Disallow select-in */
10180                         u_int scsiseq1;
10181
10182                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10183                         scsiseq1 &= ~ENSELI;
10184                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10185                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10186                         scsiseq1 &= ~ENSELI;
10187                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
10188
10189                         if ((ahd->features & AHD_MULTIROLE) == 0) {
10190                                 printf("Configuring Initiator Mode\n");
10191                                 ahd->flags &= ~AHD_TARGETROLE;
10192                                 ahd->flags |= AHD_INITIATORROLE;
10193                                 ahd_pause(ahd);
10194                                 ahd_loadseq(ahd);
10195                                 ahd_restart(ahd);
10196                                 /*
10197                                  * Unpaused.  The extra unpause
10198                                  * that follows is harmless.
10199                                  */
10200                         }
10201                 }
10202                 ahd_unpause(ahd);
10203         }
10204 #endif
10205 }
10206
10207 static void
10208 ahd_update_scsiid(struct ahd_softc *ahd, u_int targid_mask)
10209 {
10210 #if NOT_YET
10211         u_int scsiid_mask;
10212         u_int scsiid;
10213
10214         if ((ahd->features & AHD_MULTI_TID) == 0)
10215                 panic("ahd_update_scsiid called on non-multitid unit\n");
10216
10217         /*
10218          * Since we will rely on the TARGID mask
10219          * for selection enables, ensure that OID
10220          * in SCSIID is not set to some other ID
10221          * that we don't want to allow selections on.
10222          */
10223         if ((ahd->features & AHD_ULTRA2) != 0)
10224                 scsiid = ahd_inb(ahd, SCSIID_ULTRA2);
10225         else
10226                 scsiid = ahd_inb(ahd, SCSIID);
10227         scsiid_mask = 0x1 << (scsiid & OID);
10228         if ((targid_mask & scsiid_mask) == 0) {
10229                 u_int our_id;
10230
10231                 /* ffs counts from 1 */
10232                 our_id = ffs(targid_mask);
10233                 if (our_id == 0)
10234                         our_id = ahd->our_id;
10235                 else
10236                         our_id--;
10237                 scsiid &= TID;
10238                 scsiid |= our_id;
10239         }
10240         if ((ahd->features & AHD_ULTRA2) != 0)
10241                 ahd_outb(ahd, SCSIID_ULTRA2, scsiid);
10242         else
10243                 ahd_outb(ahd, SCSIID, scsiid);
10244 #endif
10245 }
10246
10247 void
10248 ahd_run_tqinfifo(struct ahd_softc *ahd, int paused)
10249 {
10250         struct target_cmd *cmd;
10251
10252         ahd_sync_tqinfifo(ahd, BUS_DMASYNC_POSTREAD);
10253         while ((cmd = &ahd->targetcmds[ahd->tqinfifonext])->cmd_valid != 0) {
10254
10255                 /*
10256                  * Only advance through the queue if we
10257                  * have the resources to process the command.
10258                  */
10259                 if (ahd_handle_target_cmd(ahd, cmd) != 0)
10260                         break;
10261
10262                 cmd->cmd_valid = 0;
10263                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
10264                                 ahd->shared_data_dmamap,
10265                                 ahd_targetcmd_offset(ahd, ahd->tqinfifonext),
10266                                 sizeof(struct target_cmd),
10267                                 BUS_DMASYNC_PREREAD);
10268                 ahd->tqinfifonext++;
10269
10270                 /*
10271                  * Lazily update our position in the target mode incoming
10272                  * command queue as seen by the sequencer.
10273                  */
10274                 if ((ahd->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
10275                         u_int hs_mailbox;
10276
10277                         hs_mailbox = ahd_inb(ahd, HS_MAILBOX);
10278                         hs_mailbox &= ~HOST_TQINPOS;
10279                         hs_mailbox |= ahd->tqinfifonext & HOST_TQINPOS;
10280                         ahd_outb(ahd, HS_MAILBOX, hs_mailbox);
10281                 }
10282         }
10283 }
10284
10285 static int
10286 ahd_handle_target_cmd(struct ahd_softc *ahd, struct target_cmd *cmd)
10287 {
10288         struct    ahd_tmode_tstate *tstate;
10289         struct    ahd_tmode_lstate *lstate;
10290         struct    ccb_accept_tio *atio;
10291         uint8_t *byte;
10292         int       initiator;
10293         int       target;
10294         int       lun;
10295
10296         initiator = SCSIID_TARGET(ahd, cmd->scsiid);
10297         target = SCSIID_OUR_ID(cmd->scsiid);
10298         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
10299
10300         byte = cmd->bytes;
10301         tstate = ahd->enabled_targets[target];
10302         lstate = NULL;
10303         if (tstate != NULL)
10304                 lstate = tstate->enabled_luns[lun];
10305
10306         /*
10307          * Commands for disabled luns go to the black hole driver.
10308          */
10309         if (lstate == NULL)
10310                 lstate = ahd->black_hole;
10311
10312         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
10313         if (atio == NULL) {
10314                 ahd->flags |= AHD_TQINFIFO_BLOCKED;
10315                 /*
10316                  * Wait for more ATIOs from the peripheral driver for this lun.
10317                  */
10318                 return (1);
10319         } else
10320                 ahd->flags &= ~AHD_TQINFIFO_BLOCKED;
10321 #ifdef AHD_DEBUG
10322         if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10323                 printf("Incoming command from %d for %d:%d%s\n",
10324                        initiator, target, lun,
10325                        lstate == ahd->black_hole ? "(Black Holed)" : "");
10326 #endif
10327         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
10328
10329         if (lstate == ahd->black_hole) {
10330                 /* Fill in the wildcards */
10331                 atio->ccb_h.target_id = target;
10332                 atio->ccb_h.target_lun = lun;
10333         }
10334
10335         /*
10336          * Package it up and send it off to
10337          * whomever has this lun enabled.
10338          */
10339         atio->sense_len = 0;
10340         atio->init_id = initiator;
10341         if (byte[0] != 0xFF) {
10342                 /* Tag was included */
10343                 atio->tag_action = *byte++;
10344                 atio->tag_id = *byte++;
10345                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
10346         } else {
10347                 atio->ccb_h.flags = 0;
10348         }
10349         byte++;
10350
10351         /* Okay.  Now determine the cdb size based on the command code */
10352         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
10353         case 0:
10354                 atio->cdb_len = 6;
10355                 break;
10356         case 1:
10357         case 2:
10358                 atio->cdb_len = 10;
10359                 break;
10360         case 4:
10361                 atio->cdb_len = 16;
10362                 break;
10363         case 5:
10364                 atio->cdb_len = 12;
10365                 break;
10366         case 3:
10367         default:
10368                 /* Only copy the opcode. */
10369                 atio->cdb_len = 1;
10370                 printf("Reserved or VU command code type encountered\n");
10371                 break;
10372         }
10373
10374         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
10375
10376         atio->ccb_h.status |= CAM_CDB_RECVD;
10377
10378         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
10379                 /*
10380                  * We weren't allowed to disconnect.
10381                  * We're hanging on the bus until a
10382                  * continue target I/O comes in response
10383                  * to this accept tio.
10384                  */
10385 #ifdef AHD_DEBUG
10386                 if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10387                         printf("Received Immediate Command %d:%d:%d - %p\n",
10388                                initiator, target, lun, ahd->pending_device);
10389 #endif
10390                 ahd->pending_device = lstate;
10391                 ahd_freeze_ccb((union ccb *)atio);
10392                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
10393         }
10394         xpt_done((union ccb*)atio);
10395         return (0);
10396 }
10397
10398 #endif