sys/dev/aic7xxx/aic79xx.c

   1 /*-
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002, 2004 Justin T. Gibbs.
   5  * Copyright (c) 2000-2003 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#246 $
  41  */
  42
  43 #ifdef __linux__
  44 #include "aic79xx_osm.h"
  45 #include "aic79xx_inline.h"
  46 #include "aicasm/aicasm_insformat.h"
  47 #else
  48 #include <sys/cdefs.h>
  49 __FBSDID("$FreeBSD$");
  50 #include <dev/aic7xxx/aic79xx_osm.h>
  51 #include <dev/aic7xxx/aic79xx_inline.h>
  52 #include <dev/aic7xxx/aicasm/aicasm_insformat.h>
  53 #endif
  54
  55 /******************************** Globals *************************************/
  56 struct ahd_softc_tailq ahd_tailq = TAILQ_HEAD_INITIALIZER(ahd_tailq);
  57 uint32_t ahd_attach_to_HostRAID_controllers = 1;
  58
  59 /***************************** Lookup Tables **********************************/
  60 char *ahd_chip_names[] =
  61 {
  62         "NONE",
  63         "aic7901",
  64         "aic7902",
  65         "aic7901A"
  66 };
  67 static const u_int num_chip_names = NUM_ELEMENTS(ahd_chip_names);
  68
  69 /*
  70  * Hardware error codes.
  71  */
  72 struct ahd_hard_error_entry {
  73         uint8_t errno;
  74         char *errmesg;
  75 };
  76
  77 static struct ahd_hard_error_entry ahd_hard_errors[] = {
  78         { DSCTMOUT,     "Discard Timer has timed out" },
  79         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  80         { SQPARERR,     "Sequencer Parity Error" },
  81         { DPARERR,      "Data-path Parity Error" },
  82         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  83         { CIOPARERR,    "CIOBUS Parity Error" },
  84 };
  85 static const u_int num_errors = NUM_ELEMENTS(ahd_hard_errors);
  86
  87 static struct ahd_phase_table_entry ahd_phase_table[] =
  88 {
  89         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  90         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  91         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  92         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  93         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
  94         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
  95         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
  96         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  97         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
  98         { 0,            MSG_NOOP,               "in unknown phase"      }
  99 };
 100
 101 /*
 102  * In most cases we only wish to itterate over real phases, so
 103  * exclude the last element from the count.
 104  */
 105 static const u_int num_phases = NUM_ELEMENTS(ahd_phase_table) - 1;
 106
 107 /* Our Sequencer Program */
 108 #include "aic79xx_seq.h"
 109
 110 /**************************** Function Declarations ***************************/
 111 static void             ahd_handle_transmission_error(struct ahd_softc *ahd);
 112 static void             ahd_handle_lqiphase_error(struct ahd_softc *ahd,
 113                                                   u_int lqistat1);
 114 static int              ahd_handle_pkt_busfree(struct ahd_softc *ahd,
 115                                                u_int busfreetime);
 116 static int              ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
 117 static void             ahd_handle_proto_violation(struct ahd_softc *ahd);
 118 static void             ahd_force_renegotiation(struct ahd_softc *ahd,
 119                                                 struct ahd_devinfo *devinfo);
 120
 121 static struct ahd_tmode_tstate*
 122                         ahd_alloc_tstate(struct ahd_softc *ahd,
 123                                          u_int scsi_id, char channel);
 124 #ifdef AHD_TARGET_MODE
 125 static void             ahd_free_tstate(struct ahd_softc *ahd,
 126                                         u_int scsi_id, char channel, int force);
 127 #endif
 128 static void             ahd_devlimited_syncrate(struct ahd_softc *ahd,
 129                                                 struct ahd_initiator_tinfo *,
 130                                                 u_int *period,
 131                                                 u_int *ppr_options,
 132                                                 role_t role);
 133 static void             ahd_update_neg_table(struct ahd_softc *ahd,
 134                                              struct ahd_devinfo *devinfo,
 135                                              struct ahd_transinfo *tinfo);
 136 static void             ahd_update_pending_scbs(struct ahd_softc *ahd);
 137 static void             ahd_fetch_devinfo(struct ahd_softc *ahd,
 138                                           struct ahd_devinfo *devinfo);
 139 static void             ahd_scb_devinfo(struct ahd_softc *ahd,
 140                                         struct ahd_devinfo *devinfo,
 141                                         struct scb *scb);
 142 static void             ahd_setup_initiator_msgout(struct ahd_softc *ahd,
 143                                                    struct ahd_devinfo *devinfo,
 144                                                    struct scb *scb);
 145 static void             ahd_build_transfer_msg(struct ahd_softc *ahd,
 146                                                struct ahd_devinfo *devinfo);
 147 static void             ahd_construct_sdtr(struct ahd_softc *ahd,
 148                                            struct ahd_devinfo *devinfo,
 149                                            u_int period, u_int offset);
 150 static void             ahd_construct_wdtr(struct ahd_softc *ahd,
 151                                            struct ahd_devinfo *devinfo,
 152                                            u_int bus_width);
 153 static void             ahd_construct_ppr(struct ahd_softc *ahd,
 154                                           struct ahd_devinfo *devinfo,
 155                                           u_int period, u_int offset,
 156                                           u_int bus_width, u_int ppr_options);
 157 static void             ahd_clear_msg_state(struct ahd_softc *ahd);
 158 static void             ahd_handle_message_phase(struct ahd_softc *ahd);
 159 typedef enum {
 160         AHDMSG_1B,
 161         AHDMSG_2B,
 162         AHDMSG_EXT
 163 } ahd_msgtype;
 164 static int              ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
 165                                      u_int msgval, int full);
 166 static int              ahd_parse_msg(struct ahd_softc *ahd,
 167                                       struct ahd_devinfo *devinfo);
 168 static int              ahd_handle_msg_reject(struct ahd_softc *ahd,
 169                                               struct ahd_devinfo *devinfo);
 170 static void             ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
 171                                                 struct ahd_devinfo *devinfo);
 172 static void             ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
 173 static void             ahd_handle_devreset(struct ahd_softc *ahd,
 174                                             struct ahd_devinfo *devinfo,
 175                                             u_int lun, cam_status status,
 176                                             char *message, int verbose_level);
 177 #ifdef AHD_TARGET_MODE
 178 static void             ahd_setup_target_msgin(struct ahd_softc *ahd,
 179                                                struct ahd_devinfo *devinfo,
 180                                                struct scb *scb);
 181 #endif
 182
 183 static u_int            ahd_sglist_size(struct ahd_softc *ahd);
 184 static u_int            ahd_sglist_allocsize(struct ahd_softc *ahd);
 185 static bus_dmamap_callback_t
 186                         ahd_dmamap_cb;
 187 static void             ahd_initialize_hscbs(struct ahd_softc *ahd);
 188 static int              ahd_init_scbdata(struct ahd_softc *ahd);
 189 static void             ahd_fini_scbdata(struct ahd_softc *ahd);
 190 static void             ahd_setup_iocell_workaround(struct ahd_softc *ahd);
 191 static void             ahd_iocell_first_selection(struct ahd_softc *ahd);
 192 static void             ahd_add_col_list(struct ahd_softc *ahd,
 193                                          struct scb *scb, u_int col_idx);
 194 static void             ahd_rem_col_list(struct ahd_softc *ahd,
 195                                          struct scb *scb);
 196 static void             ahd_chip_init(struct ahd_softc *ahd);
 197 static void             ahd_qinfifo_requeue(struct ahd_softc *ahd,
 198                                             struct scb *prev_scb,
 199                                             struct scb *scb);
 200 static int              ahd_qinfifo_count(struct ahd_softc *ahd);
 201 static int              ahd_search_scb_list(struct ahd_softc *ahd, int target,
 202                                             char channel, int lun, u_int tag,
 203                                             role_t role, uint32_t status,
 204                                             ahd_search_action action,
 205                                             u_int *list_head, u_int *list_tail,
 206                                             u_int tid);
 207 static void             ahd_stitch_tid_list(struct ahd_softc *ahd,
 208                                             u_int tid_prev, u_int tid_cur,
 209                                             u_int tid_next);
 210 static void             ahd_add_scb_to_free_list(struct ahd_softc *ahd,
 211                                                  u_int scbid);
 212 static u_int            ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
 213                                      u_int prev, u_int next, u_int tid);
 214 static void             ahd_reset_current_bus(struct ahd_softc *ahd);
 215 static ahd_callback_t   ahd_reset_poll;
 216 static ahd_callback_t   ahd_stat_timer;
 217 #ifdef AHD_DUMP_SEQ
 218 static void             ahd_dumpseq(struct ahd_softc *ahd);
 219 #endif
 220 static void             ahd_loadseq(struct ahd_softc *ahd);
 221 static int              ahd_check_patch(struct ahd_softc *ahd,
 222                                         struct patch **start_patch,
 223                                         u_int start_instr, u_int *skip_addr);
 224 static u_int            ahd_resolve_seqaddr(struct ahd_softc *ahd,
 225                                             u_int address);
 226 static void             ahd_download_instr(struct ahd_softc *ahd,
 227                                            u_int instrptr, uint8_t *dconsts);
 228 static int              ahd_probe_stack_size(struct ahd_softc *ahd);
 229 static int              ahd_other_scb_timeout(struct ahd_softc *ahd,
 230                                               struct scb *scb,
 231                                               struct scb *other_scb);
 232 static int              ahd_scb_active_in_fifo(struct ahd_softc *ahd,
 233                                                struct scb *scb);
 234 static void             ahd_run_data_fifo(struct ahd_softc *ahd,
 235                                           struct scb *scb);
 236
 237 #ifdef AHD_TARGET_MODE
 238 static void             ahd_queue_lstate_event(struct ahd_softc *ahd,
 239                                                struct ahd_tmode_lstate *lstate,
 240                                                u_int initiator_id,
 241                                                u_int event_type,
 242                                                u_int event_arg);
 243 static void             ahd_update_scsiid(struct ahd_softc *ahd,
 244                                           u_int targid_mask);
 245 static int              ahd_handle_target_cmd(struct ahd_softc *ahd,
 246                                               struct target_cmd *cmd);
 247 #endif
 248
 249 /******************************** Private Inlines *****************************/
 250 static __inline void    ahd_assert_atn(struct ahd_softc *ahd);
 251 static __inline int     ahd_currently_packetized(struct ahd_softc *ahd);
 252 static __inline int     ahd_set_active_fifo(struct ahd_softc *ahd);
 253
 254 static __inline void
 255 ahd_assert_atn(struct ahd_softc *ahd)
 256 {
 257         ahd_outb(ahd, SCSISIGO, ATNO);
 258 }
 259
 260 /*
 261  * Determine if the current connection has a packetized
 262  * agreement.  This does not necessarily mean that we
 263  * are currently in a packetized transfer.  We could
 264  * just as easily be sending or receiving a message.
 265  */
 266 static __inline int
 267 ahd_currently_packetized(struct ahd_softc *ahd)
 268 {
 269         ahd_mode_state   saved_modes;
 270         int              packetized;
 271
 272         saved_modes = ahd_save_modes(ahd);
 273         if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
 274                 /*
 275                  * The packetized bit refers to the last
 276                  * connection, not the current one.  Check
 277                  * for non-zero LQISTATE instead.
 278                  */
 279                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 280                 packetized = ahd_inb(ahd, LQISTATE) != 0;
 281         } else {
 282                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 283                 packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
 284         }
 285         ahd_restore_modes(ahd, saved_modes);
 286         return (packetized);
 287 }
 288
 289 static __inline int
 290 ahd_set_active_fifo(struct ahd_softc *ahd)
 291 {
 292         u_int active_fifo;
 293
 294         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 295         active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
 296         switch (active_fifo) {
 297         case 0:
 298         case 1:
 299                 ahd_set_modes(ahd, active_fifo, active_fifo);
 300                 return (1);
 301         default:
 302                 return (0);
 303         }
 304 }
 305
 306 /************************* Sequencer Execution Control ************************/
 307 /*
 308  * Restart the sequencer program from address zero
 309  */
 310 void
 311 ahd_restart(struct ahd_softc *ahd)
 312 {
 313
 314         ahd_pause(ahd);
 315
 316         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 317
 318         /* No more pending messages */
 319         ahd_clear_msg_state(ahd);
 320         ahd_outb(ahd, SCSISIGO, 0);             /* De-assert BSY */
 321         ahd_outb(ahd, MSG_OUT, MSG_NOOP);       /* No message to send */
 322         ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
 323         ahd_outb(ahd, SEQINTCTL, 0);
 324         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
 325         ahd_outb(ahd, SEQ_FLAGS, 0);
 326         ahd_outb(ahd, SAVED_SCSIID, 0xFF);
 327         ahd_outb(ahd, SAVED_LUN, 0xFF);
 328
 329         /*
 330          * Ensure that the sequencer's idea of TQINPOS
 331          * matches our own.  The sequencer increments TQINPOS
 332          * only after it sees a DMA complete and a reset could
 333          * occur before the increment leaving the kernel to believe
 334          * the command arrived but the sequencer to not.
 335          */
 336         ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
 337
 338         /* Always allow reselection */
 339         ahd_outb(ahd, SCSISEQ1,
 340                  ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 341         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 342
 343         /*
 344          * Clear any pending sequencer interrupt.  It is no
 345          * longer relevant since we're resetting the Program
 346          * Counter.
 347          */
 348         ahd_outb(ahd, CLRINT, CLRSEQINT);
 349
 350         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
 351         ahd_unpause(ahd);
 352 }
 353
 354 void
 355 ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
 356 {
 357         ahd_mode_state   saved_modes;
 358
 359 #ifdef AHD_DEBUG
 360         if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
 361                 printf("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
 362 #endif
 363         saved_modes = ahd_save_modes(ahd);
 364         ahd_set_modes(ahd, fifo, fifo);
 365         ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
 366         if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 367                 ahd_outb(ahd, CCSGCTL, CCSGRESET);
 368         ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 369         ahd_outb(ahd, SG_STATE, 0);
 370         ahd_restore_modes(ahd, saved_modes);
 371 }
 372
 373 /************************* Input/Output Queues ********************************/
 374 /*
 375  * Flush and completed commands that are sitting in the command
 376  * complete queues down on the chip but have yet to be dma'ed back up.
 377  */
 378 void
 379 ahd_flush_qoutfifo(struct ahd_softc *ahd)
 380 {
 381         struct          scb *scb;
 382         ahd_mode_state  saved_modes;
 383         u_int           saved_scbptr;
 384         u_int           ccscbctl;
 385         u_int           scbid;
 386         u_int           next_scbid;
 387
 388         saved_modes = ahd_save_modes(ahd);
 389
 390         /*
 391          * Flush the good status FIFO for completed packetized commands.
 392          */
 393         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 394         saved_scbptr = ahd_get_scbptr(ahd);
 395         while ((ahd_inb(ahd, LQISTAT2) & LQIGSAVAIL) != 0) {
 396                 u_int fifo_mode;
 397                 u_int i;
 398
 399                 scbid = ahd_inw(ahd, GSFIFO);
 400                 scb = ahd_lookup_scb(ahd, scbid);
 401                 if (scb == NULL) {
 402                         printf("%s: Warning - GSFIFO SCB %d invalid\n",
 403                                ahd_name(ahd), scbid);
 404                         AHD_CORRECTABLE_ERROR(ahd);
 405                         continue;
 406                 }
 407                 /*
 408                  * Determine if this transaction is still active in
 409                  * any FIFO.  If it is, we must flush that FIFO to
 410                  * the host before completing the  command.
 411                  */
 412                 fifo_mode = 0;
 413 rescan_fifos:
 414                 for (i = 0; i < 2; i++) {
 415                         /* Toggle to the other mode. */
 416                         fifo_mode ^= 1;
 417                         ahd_set_modes(ahd, fifo_mode, fifo_mode);
 418
 419                         if (ahd_scb_active_in_fifo(ahd, scb) == 0)
 420                                 continue;
 421
 422                         ahd_run_data_fifo(ahd, scb);
 423
 424                         /*
 425                          * Running this FIFO may cause a CFG4DATA for
 426                          * this same transaction to assert in the other
 427                          * FIFO or a new snapshot SAVEPTRS interrupt
 428                          * in this FIFO.  Even running a FIFO may not
 429                          * clear the transaction if we are still waiting
 430                          * for data to drain to the host. We must loop
 431                          * until the transaction is not active in either
 432                          * FIFO just to be sure.  Reset our loop counter
 433                          * so we will visit both FIFOs again before
 434                          * declaring this transaction finished.  We
 435                          * also delay a bit so that status has a chance
 436                          * to change before we look at this FIFO again.
 437                          */
 438                         aic_delay(200);
 439                         goto rescan_fifos;
 440                 }
 441                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 442                 ahd_set_scbptr(ahd, scbid);
 443                 if ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_LIST_NULL) == 0
 444                  && ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_FULL_RESID) != 0
 445                   || (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR)
 446                       & SG_LIST_NULL) != 0)) {
 447                         u_int comp_head;
 448
 449                         /*
 450                          * The transfer completed with a residual.
 451                          * Place this SCB on the complete DMA list
 452                          * so that we update our in-core copy of the
 453                          * SCB before completing the command.
 454                          */
 455                         ahd_outb(ahd, SCB_SCSI_STATUS, 0);
 456                         ahd_outb(ahd, SCB_SGPTR,
 457                                  ahd_inb_scbram(ahd, SCB_SGPTR)
 458                                  | SG_STATUS_VALID);
 459                         ahd_outw(ahd, SCB_TAG, scbid);
 460                         ahd_outw(ahd, SCB_NEXT_COMPLETE, SCB_LIST_NULL);
 461                         comp_head = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
 462                         if (SCBID_IS_NULL(comp_head)) {
 463                                 ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, scbid);
 464                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
 465                         } else {
 466                                 u_int tail;
 467
 468                                 tail = ahd_inw(ahd, COMPLETE_DMA_SCB_TAIL);
 469                                 ahd_set_scbptr(ahd, tail);
 470                                 ahd_outw(ahd, SCB_NEXT_COMPLETE, scbid);
 471                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
 472                                 ahd_set_scbptr(ahd, scbid);
 473                         }
 474                 } else
 475                         ahd_complete_scb(ahd, scb);
 476         }
 477         ahd_set_scbptr(ahd, saved_scbptr);
 478
 479         /*
 480          * Setup for command channel portion of flush.
 481          */
 482         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 483
 484         /*
 485          * Wait for any inprogress DMA to complete and clear DMA state
 486          * if this if for an SCB in the qinfifo.
 487          */
 488         while (((ccscbctl = ahd_inb(ahd, CCSCBCTL)) & (CCARREN|CCSCBEN)) != 0) {
 489
 490                 if ((ccscbctl & (CCSCBDIR|CCARREN)) == (CCSCBDIR|CCARREN)) {
 491                         if ((ccscbctl & ARRDONE) != 0)
 492                                 break;
 493                 } else if ((ccscbctl & CCSCBDONE) != 0)
 494                         break;
 495                 aic_delay(200);
 496         }
 497         /*
 498          * We leave the sequencer to cleanup in the case of DMA's to
 499          * update the qoutfifo.  In all other cases (DMA's to the
 500          * chip or a push of an SCB from the COMPLETE_DMA_SCB list),
 501          * we disable the DMA engine so that the sequencer will not
 502          * attempt to handle the DMA completion.
 503          */
 504         if ((ccscbctl & CCSCBDIR) != 0 || (ccscbctl & ARRDONE) != 0)
 505                 ahd_outb(ahd, CCSCBCTL, ccscbctl & ~(CCARREN|CCSCBEN));
 506
 507         /*
 508          * Complete any SCBs that just finished
 509          * being DMA'ed into the qoutfifo.
 510          */
 511         ahd_run_qoutfifo(ahd);
 512
 513         saved_scbptr = ahd_get_scbptr(ahd);
 514         /*
 515          * Manually update/complete any completed SCBs that are waiting to be
 516          * DMA'ed back up to the host.
 517          */
 518         scbid = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
 519         while (!SCBID_IS_NULL(scbid)) {
 520                 uint8_t *hscb_ptr;
 521                 u_int    i;
 522
 523                 ahd_set_scbptr(ahd, scbid);
 524                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 525                 scb = ahd_lookup_scb(ahd, scbid);
 526                 if (scb == NULL) {
 527                         printf("%s: Warning - DMA-up and complete "
 528                                "SCB %d invalid\n", ahd_name(ahd), scbid);
 529                         AHD_CORRECTABLE_ERROR(ahd);
 530                         continue;
 531                 }
 532                 hscb_ptr = (uint8_t *)scb->hscb;
 533                 for (i = 0; i < sizeof(struct hardware_scb); i++)
 534                         *hscb_ptr++ = ahd_inb_scbram(ahd, SCB_BASE + i);
 535
 536                 ahd_complete_scb(ahd, scb);
 537                 scbid = next_scbid;
 538         }
 539         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
 540         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
 541
 542         scbid = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
 543         while (!SCBID_IS_NULL(scbid)) {
 544
 545                 ahd_set_scbptr(ahd, scbid);
 546                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 547                 scb = ahd_lookup_scb(ahd, scbid);
 548                 if (scb == NULL) {
 549                         printf("%s: Warning - Complete Qfrz SCB %d invalid\n",
 550                                ahd_name(ahd), scbid);
 551                         AHD_CORRECTABLE_ERROR(ahd);
 552                         continue;
 553                 }
 554
 555                 ahd_complete_scb(ahd, scb);
 556                 scbid = next_scbid;
 557         }
 558         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
 559
 560         scbid = ahd_inw(ahd, COMPLETE_SCB_HEAD);
 561         while (!SCBID_IS_NULL(scbid)) {
 562
 563                 ahd_set_scbptr(ahd, scbid);
 564                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 565                 scb = ahd_lookup_scb(ahd, scbid);
 566                 if (scb == NULL) {
 567                         printf("%s: Warning - Complete SCB %d invalid\n",
 568                                ahd_name(ahd), scbid);
 569                         AHD_CORRECTABLE_ERROR(ahd);
 570                         continue;
 571                 }
 572
 573                 ahd_complete_scb(ahd, scb);
 574                 scbid = next_scbid;
 575         }
 576         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
 577
 578         /*
 579          * Restore state.
 580          */
 581         ahd_set_scbptr(ahd, saved_scbptr);
 582         ahd_restore_modes(ahd, saved_modes);
 583         ahd->flags |= AHD_UPDATE_PEND_CMDS;
 584 }
 585
 586 /*
 587  * Determine if an SCB for a packetized transaction
 588  * is active in a FIFO.
 589  */
 590 static int
 591 ahd_scb_active_in_fifo(struct ahd_softc *ahd, struct scb *scb)
 592 {
 593
 594         /*
 595          * The FIFO is only active for our transaction if
 596          * the SCBPTR matches the SCB's ID and the firmware
 597          * has installed a handler for the FIFO or we have
 598          * a pending SAVEPTRS or CFG4DATA interrupt.
 599          */
 600         if (ahd_get_scbptr(ahd) != SCB_GET_TAG(scb)
 601          || ((ahd_inb(ahd, LONGJMP_ADDR+1) & INVALID_ADDR) != 0
 602           && (ahd_inb(ahd, SEQINTSRC) & (CFG4DATA|SAVEPTRS)) == 0))
 603                 return (0);
 604
 605         return (1);
 606 }
 607
 608 /*
 609  * Run a data fifo to completion for a transaction we know
 610  * has completed across the SCSI bus (good status has been
 611  * received).  We are already set to the correct FIFO mode
 612  * on entry to this routine.
 613  *
 614  * This function attempts to operate exactly as the firmware
 615  * would when running this FIFO.  Care must be taken to update
 616  * this routine any time the firmware's FIFO algorithm is
 617  * changed.
 618  */
 619 static void
 620 ahd_run_data_fifo(struct ahd_softc *ahd, struct scb *scb)
 621 {
 622         u_int seqintsrc;
 623
 624         seqintsrc = ahd_inb(ahd, SEQINTSRC);
 625         if ((seqintsrc & CFG4DATA) != 0) {
 626                 uint32_t datacnt;
 627                 uint32_t sgptr;
 628
 629                 /*
 630                  * Clear full residual flag.
 631                  */
 632                 sgptr = ahd_inl_scbram(ahd, SCB_SGPTR) & ~SG_FULL_RESID;
 633                 ahd_outb(ahd, SCB_SGPTR, sgptr);
 634
 635                 /*
 636                  * Load datacnt and address.
 637                  */
 638                 datacnt = ahd_inl_scbram(ahd, SCB_DATACNT);
 639                 if ((datacnt & AHD_DMA_LAST_SEG) != 0) {
 640                         sgptr |= LAST_SEG;
 641                         ahd_outb(ahd, SG_STATE, 0);
 642                 } else
 643                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
 644                 ahd_outq(ahd, HADDR, ahd_inq_scbram(ahd, SCB_DATAPTR));
 645                 ahd_outl(ahd, HCNT, datacnt & AHD_SG_LEN_MASK);
 646                 ahd_outb(ahd, SG_CACHE_PRE, sgptr);
 647                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
 648
 649                 /*
 650                  * Initialize Residual Fields.
 651                  */
 652                 ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, datacnt >> 24);
 653                 ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr & SG_PTR_MASK);
 654
 655                 /*
 656                  * Mark the SCB as having a FIFO in use.
 657                  */
 658                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
 659                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) + 1);
 660
 661                 /*
 662                  * Install a "fake" handler for this FIFO.
 663                  */
 664                 ahd_outw(ahd, LONGJMP_ADDR, 0);
 665
 666                 /*
 667                  * Notify the hardware that we have satisfied
 668                  * this sequencer interrupt.
 669                  */
 670                 ahd_outb(ahd, CLRSEQINTSRC, CLRCFG4DATA);
 671         } else if ((seqintsrc & SAVEPTRS) != 0) {
 672                 uint32_t sgptr;
 673                 uint32_t resid;
 674
 675                 if ((ahd_inb(ahd, LONGJMP_ADDR+1)&INVALID_ADDR) != 0) {
 676                         /*
 677                          * Snapshot Save Pointers.  All that
 678                          * is necessary to clear the snapshot
 679                          * is a CLRCHN.
 680                          */
 681                         goto clrchn;
 682                 }
 683
 684                 /*
 685                  * Disable S/G fetch so the DMA engine
 686                  * is available to future users.
 687                  */
 688                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 689                         ahd_outb(ahd, CCSGCTL, 0);
 690                 ahd_outb(ahd, SG_STATE, 0);
 691
 692                 /*
 693                  * Flush the data FIFO.  Strickly only
 694                  * necessary for Rev A parts.
 695                  */
 696                 ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) | FIFOFLUSH);
 697
 698                 /*
 699                  * Calculate residual.
 700                  */
 701                 sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 702                 resid = ahd_inl(ahd, SHCNT);
 703                 resid |= ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24;
 704                 ahd_outl(ahd, SCB_RESIDUAL_DATACNT, resid);
 705                 if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG) == 0) {
 706                         /*
 707                          * Must back up to the correct S/G element.
 708                          * Typically this just means resetting our
 709                          * low byte to the offset in the SG_CACHE,
 710                          * but if we wrapped, we have to correct
 711                          * the other bytes of the sgptr too.
 712                          */
 713                         if ((ahd_inb(ahd, SG_CACHE_SHADOW) & 0x80) != 0
 714                          && (sgptr & 0x80) == 0)
 715                                 sgptr -= 0x100;
 716                         sgptr &= ~0xFF;
 717                         sgptr |= ahd_inb(ahd, SG_CACHE_SHADOW)
 718                                & SG_ADDR_MASK;
 719                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 720                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, 0);
 721                 } else if ((resid & AHD_SG_LEN_MASK) == 0) {
 722                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR,
 723                                  sgptr | SG_LIST_NULL);
 724                 }
 725                 /*
 726                  * Save Pointers.
 727                  */
 728                 ahd_outq(ahd, SCB_DATAPTR, ahd_inq(ahd, SHADDR));
 729                 ahd_outl(ahd, SCB_DATACNT, resid);
 730                 ahd_outl(ahd, SCB_SGPTR, sgptr);
 731                 ahd_outb(ahd, CLRSEQINTSRC, CLRSAVEPTRS);
 732                 ahd_outb(ahd, SEQIMODE,
 733                          ahd_inb(ahd, SEQIMODE) | ENSAVEPTRS);
 734                 /*
 735                  * If the data is to the SCSI bus, we are
 736                  * done, otherwise wait for FIFOEMP.
 737                  */
 738                 if ((ahd_inb(ahd, DFCNTRL) & DIRECTION) != 0)
 739                         goto clrchn;
 740         } else if ((ahd_inb(ahd, SG_STATE) & LOADING_NEEDED) != 0) {
 741                 uint32_t sgptr;
 742                 uint64_t data_addr;
 743                 uint32_t data_len;
 744                 u_int    dfcntrl;
 745
 746                 /*
 747                  * Disable S/G fetch so the DMA engine
 748                  * is available to future users.  We won't
 749                  * be using the DMA engine to load segments.
 750                  */
 751                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0) {
 752                         ahd_outb(ahd, CCSGCTL, 0);
 753                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
 754                 }
 755
 756                 /*
 757                  * Wait for the DMA engine to notice that the
 758                  * host transfer is enabled and that there is
 759                  * space in the S/G FIFO for new segments before
 760                  * loading more segments.
 761                  */
 762                 if ((ahd_inb(ahd, DFSTATUS) & PRELOAD_AVAIL) != 0
 763                  && (ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0) {
 764
 765                         /*
 766                          * Determine the offset of the next S/G
 767                          * element to load.
 768                          */
 769                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 770                         sgptr &= SG_PTR_MASK;
 771                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 772                                 struct ahd_dma64_seg *sg;
 773
 774                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 775                                 data_addr = sg->addr;
 776                                 data_len = sg->len;
 777                                 sgptr += sizeof(*sg);
 778                         } else {
 779                                 struct  ahd_dma_seg *sg;
 780
 781                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 782                                 data_addr = sg->len & AHD_SG_HIGH_ADDR_MASK;
 783                                 data_addr <<= 8;
 784                                 data_addr |= sg->addr;
 785                                 data_len = sg->len;
 786                                 sgptr += sizeof(*sg);
 787                         }
 788
 789                         /*
 790                          * Update residual information.
 791                          */
 792                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, data_len >> 24);
 793                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 794
 795                         /*
 796                          * Load the S/G.
 797                          */
 798                         if (data_len & AHD_DMA_LAST_SEG) {
 799                                 sgptr |= LAST_SEG;
 800                                 ahd_outb(ahd, SG_STATE, 0);
 801                         }
 802                         ahd_outq(ahd, HADDR, data_addr);
 803                         ahd_outl(ahd, HCNT, data_len & AHD_SG_LEN_MASK);
 804                         ahd_outb(ahd, SG_CACHE_PRE, sgptr & 0xFF);
 805
 806                         /*
 807                          * Advertise the segment to the hardware.
 808                          */
 809                         dfcntrl = ahd_inb(ahd, DFCNTRL)|PRELOADEN|HDMAEN;
 810                         if ((ahd->features & AHD_NEW_DFCNTRL_OPTS) != 0) {
 811                                 /*
 812                                  * Use SCSIENWRDIS so that SCSIEN
 813                                  * is never modified by this
 814                                  * operation.
 815                                  */
 816                                 dfcntrl |= SCSIENWRDIS;
 817                         }
 818                         ahd_outb(ahd, DFCNTRL, dfcntrl);
 819                 }
 820         } else if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG_DONE) != 0) {
 821
 822                 /*
 823                  * Transfer completed to the end of SG list
 824                  * and has flushed to the host.
 825                  */
 826                 ahd_outb(ahd, SCB_SGPTR,
 827                          ahd_inb_scbram(ahd, SCB_SGPTR) | SG_LIST_NULL);
 828                 goto clrchn;
 829         } else if ((ahd_inb(ahd, DFSTATUS) & FIFOEMP) != 0) {
 830 clrchn:
 831                 /*
 832                  * Clear any handler for this FIFO, decrement
 833                  * the FIFO use count for the SCB, and release
 834                  * the FIFO.
 835                  */
 836                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 837                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
 838                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) - 1);
 839                 ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
 840         }
 841 }
 842
 843 /*
 844  * Look for entries in the QoutFIFO that have completed.
 845  * The valid_tag completion field indicates the validity
 846  * of the entry - the valid value toggles each time through
 847  * the queue. We use the sg_status field in the completion
 848  * entry to avoid referencing the hscb if the completion
 849  * occurred with no errors and no residual.  sg_status is
 850  * a copy of the first byte (little endian) of the sgptr
 851  * hscb field.
 852  */
 853 void
 854 ahd_run_qoutfifo(struct ahd_softc *ahd)
 855 {
 856         struct ahd_completion *completion;
 857         struct scb *scb;
 858         u_int  scb_index;
 859
 860         if ((ahd->flags & AHD_RUNNING_QOUTFIFO) != 0)
 861                 panic("ahd_run_qoutfifo recursion");
 862         ahd->flags |= AHD_RUNNING_QOUTFIFO;
 863         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
 864         for (;;) {
 865                 completion = &ahd->qoutfifo[ahd->qoutfifonext];
 866
 867                 if (completion->valid_tag != ahd->qoutfifonext_valid_tag)
 868                         break;
 869
 870                 scb_index = aic_le16toh(completion->tag);
 871                 scb = ahd_lookup_scb(ahd, scb_index);
 872                 if (scb == NULL) {
 873                         printf("%s: WARNING no command for scb %d "
 874                                "(cmdcmplt)\nQOUTPOS = %d\n",
 875                                ahd_name(ahd), scb_index,
 876                                ahd->qoutfifonext);
 877                         AHD_CORRECTABLE_ERROR(ahd);
 878                         ahd_dump_card_state(ahd);
 879                 } else if ((completion->sg_status & SG_STATUS_VALID) != 0) {
 880                         ahd_handle_scb_status(ahd, scb);
 881                 } else {
 882                         ahd_done(ahd, scb);
 883                 }
 884
 885                 ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
 886                 if (ahd->qoutfifonext == 0)
 887                         ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID;
 888         }
 889         ahd->flags &= ~AHD_RUNNING_QOUTFIFO;
 890 }
 891
 892 /************************* Interrupt Handling *********************************/
 893 void
 894 ahd_handle_hwerrint(struct ahd_softc *ahd)
 895 {
 896         /*
 897          * Some catastrophic hardware error has occurred.
 898          * Print it for the user and disable the controller.
 899          */
 900         int i;
 901         int error;
 902
 903         error = ahd_inb(ahd, ERROR);
 904         for (i = 0; i < num_errors; i++) {
 905                 if ((error & ahd_hard_errors[i].errno) != 0) {
 906                         printf("%s: hwerrint, %s\n",
 907                                ahd_name(ahd), ahd_hard_errors[i].errmesg);
 908                         AHD_UNCORRECTABLE_ERROR(ahd);
 909                 }
 910         }
 911
 912         ahd_dump_card_state(ahd);
 913         panic("BRKADRINT");
 914
 915         /* Tell everyone that this HBA is no longer available */
 916         ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 917                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 918                        CAM_NO_HBA);
 919
 920         /* Tell the system that this controller has gone away. */
 921         ahd_free(ahd);
 922 }
 923
 924 void
 925 ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
 926 {
 927         u_int seqintcode;
 928
 929         /*
 930          * Save the sequencer interrupt code and clear the SEQINT
 931          * bit. We will unpause the sequencer, if appropriate,
 932          * after servicing the request.
 933          */
 934         seqintcode = ahd_inb(ahd, SEQINTCODE);
 935         ahd_outb(ahd, CLRINT, CLRSEQINT);
 936         if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
 937                 /*
 938                  * Unpause the sequencer and let it clear
 939                  * SEQINT by writing NO_SEQINT to it.  This
 940                  * will cause the sequencer to be paused again,
 941                  * which is the expected state of this routine.
 942                  */
 943                 ahd_unpause(ahd);
 944                 while (!ahd_is_paused(ahd))
 945                         ;
 946                 ahd_outb(ahd, CLRINT, CLRSEQINT);
 947         }
 948         ahd_update_modes(ahd);
 949 #ifdef AHD_DEBUG
 950         if ((ahd_debug & AHD_SHOW_MISC) != 0)
 951                 printf("%s: Handle Seqint Called for code %d\n",
 952                        ahd_name(ahd), seqintcode);
 953 #endif
 954         switch (seqintcode) {
 955         case ENTERING_NONPACK:
 956         {
 957                 struct  scb *scb;
 958                 u_int   scbid;
 959
 960                 AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 961                                  ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 962                 scbid = ahd_get_scbptr(ahd);
 963                 scb = ahd_lookup_scb(ahd, scbid);
 964                 if (scb == NULL) {
 965                         /*
 966                          * Somehow need to know if this
 967                          * is from a selection or reselection.
 968                          * From that, we can determine target
 969                          * ID so we at least have an I_T nexus.
 970                          */
 971                 } else {
 972                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
 973                         ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
 974                         ahd_outb(ahd, SEQ_FLAGS, 0x0);
 975                 }
 976                 if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
 977                  && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
 978                         /*
 979                          * Phase change after read stream with
 980                          * CRC error with P0 asserted on last
 981                          * packet.
 982                          */
 983 #ifdef AHD_DEBUG
 984                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 985                                 printf("%s: Assuming LQIPHASE_NLQ with "
 986                                        "P0 assertion\n", ahd_name(ahd));
 987 #endif
 988                 }
 989 #ifdef AHD_DEBUG
 990                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 991                         printf("%s: Entering NONPACK\n", ahd_name(ahd));
 992 #endif
 993                 break;
 994         }
 995         case INVALID_SEQINT:
 996                 printf("%s: Invalid Sequencer interrupt occurred.\n",
 997                        ahd_name(ahd));
 998                 ahd_dump_card_state(ahd);
 999                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1000                 AHD_UNCORRECTABLE_ERROR(ahd);
1001                 break;
1002         case STATUS_OVERRUN:
1003         {
1004                 struct  scb *scb;
1005                 u_int   scbid;
1006
1007                 scbid = ahd_get_scbptr(ahd);
1008                 scb = ahd_lookup_scb(ahd, scbid);
1009                 if (scb != NULL)
1010                         ahd_print_path(ahd, scb);
1011                 else
1012                         printf("%s: ", ahd_name(ahd));
1013                 printf("SCB %d Packetized Status Overrun", scbid);
1014                 ahd_dump_card_state(ahd);
1015                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1016                 AHD_UNCORRECTABLE_ERROR(ahd);
1017                 break;
1018         }
1019         case CFG4ISTAT_INTR:
1020         {
1021                 struct  scb *scb;
1022                 u_int   scbid;
1023
1024                 scbid = ahd_get_scbptr(ahd);
1025                 scb = ahd_lookup_scb(ahd, scbid);
1026                 if (scb == NULL) {
1027                         ahd_dump_card_state(ahd);
1028                         printf("CFG4ISTAT: Free SCB %d referenced", scbid);
1029                         AHD_FATAL_ERROR(ahd);
1030                         panic("For safety");
1031                 }
1032                 ahd_outq(ahd, HADDR, scb->sense_busaddr);
1033                 ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
1034                 ahd_outb(ahd, HCNT + 2, 0);
1035                 ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
1036                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1037                 break;
1038         }
1039         case ILLEGAL_PHASE:
1040         {
1041                 u_int bus_phase;
1042
1043                 bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1044                 printf("%s: ILLEGAL_PHASE 0x%x\n",
1045                        ahd_name(ahd), bus_phase);
1046
1047                 switch (bus_phase) {
1048                 case P_DATAOUT:
1049                 case P_DATAIN:
1050                 case P_DATAOUT_DT:
1051                 case P_DATAIN_DT:
1052                 case P_MESGOUT:
1053                 case P_STATUS:
1054                 case P_MESGIN:
1055                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1056                         printf("%s: Issued Bus Reset.\n", ahd_name(ahd));
1057                         AHD_UNCORRECTABLE_ERROR(ahd);
1058                         break;
1059                 case P_COMMAND:
1060                 {
1061                         struct  ahd_devinfo devinfo;
1062                         struct  scb *scb;
1063                         struct  ahd_initiator_tinfo *targ_info;
1064                         struct  ahd_tmode_tstate *tstate;
1065                         struct  ahd_transinfo *tinfo;
1066                         u_int   scbid;
1067
1068                         /*
1069                          * If a target takes us into the command phase
1070                          * assume that it has been externally reset and
1071                          * has thus lost our previous packetized negotiation
1072                          * agreement.  Since we have not sent an identify
1073                          * message and may not have fully qualified the
1074                          * connection, we change our command to TUR, assert
1075                          * ATN and ABORT the task when we go to message in
1076                          * phase.  The OSM will see the REQUEUE_REQUEST
1077                          * status and retry the command.
1078                          */
1079                         scbid = ahd_get_scbptr(ahd);
1080                         scb = ahd_lookup_scb(ahd, scbid);
1081                         if (scb == NULL) {
1082                                 AHD_CORRECTABLE_ERROR(ahd);
1083                                 printf("Invalid phase with no valid SCB.  "
1084                                        "Resetting bus.\n");
1085                                 ahd_reset_channel(ahd, 'A',
1086                                                   /*Initiate Reset*/TRUE);
1087                                 break;
1088                         }
1089                         ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
1090                                             SCB_GET_TARGET(ahd, scb),
1091                                             SCB_GET_LUN(scb),
1092                                             SCB_GET_CHANNEL(ahd, scb),
1093                                             ROLE_INITIATOR);
1094                         targ_info = ahd_fetch_transinfo(ahd,
1095                                                         devinfo.channel,
1096                                                         devinfo.our_scsiid,
1097                                                         devinfo.target,
1098                                                         &tstate);
1099                         tinfo = &targ_info->curr;
1100                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
1101                                       AHD_TRANS_ACTIVE, /*paused*/TRUE);
1102                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
1103                                          /*offset*/0, /*ppr_options*/0,
1104                                          AHD_TRANS_ACTIVE, /*paused*/TRUE);
1105                         ahd_outb(ahd, SCB_CDB_STORE, 0);
1106                         ahd_outb(ahd, SCB_CDB_STORE+1, 0);
1107                         ahd_outb(ahd, SCB_CDB_STORE+2, 0);
1108                         ahd_outb(ahd, SCB_CDB_STORE+3, 0);
1109                         ahd_outb(ahd, SCB_CDB_STORE+4, 0);
1110                         ahd_outb(ahd, SCB_CDB_STORE+5, 0);
1111                         ahd_outb(ahd, SCB_CDB_LEN, 6);
1112                         scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
1113                         scb->hscb->control |= MK_MESSAGE;
1114                         ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
1115                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1116                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1117                         /*
1118                          * The lun is 0, regardless of the SCB's lun
1119                          * as we have not sent an identify message.
1120                          */
1121                         ahd_outb(ahd, SAVED_LUN, 0);
1122                         ahd_outb(ahd, SEQ_FLAGS, 0);
1123                         ahd_assert_atn(ahd);
1124                         scb->flags &= ~SCB_PACKETIZED;
1125                         scb->flags |= SCB_ABORT|SCB_CMDPHASE_ABORT;
1126                         ahd_freeze_devq(ahd, scb);
1127                         aic_set_transaction_status(scb, CAM_REQUEUE_REQ);
1128                         aic_freeze_scb(scb);
1129
1130                         /*
1131                          * Allow the sequencer to continue with
1132                          * non-pack processing.
1133                          */
1134                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1135                         ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
1136                         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
1137                                 ahd_outb(ahd, CLRLQOINT1, 0);
1138                         }
1139 #ifdef AHD_DEBUG
1140                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1141                                 ahd_print_path(ahd, scb);
1142                                 AHD_CORRECTABLE_ERROR(ahd);
1143                                 printf("Unexpected command phase from "
1144                                        "packetized target\n");
1145                         }
1146 #endif
1147                         break;
1148                 }
1149                 }
1150                 break;
1151         }
1152         case CFG4OVERRUN:
1153         {
1154                 struct  scb *scb;
1155                 u_int   scb_index;
1156
1157 #ifdef AHD_DEBUG
1158                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1159                         printf("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
1160                                ahd_inb(ahd, MODE_PTR));
1161                 }
1162 #endif
1163                 scb_index = ahd_get_scbptr(ahd);
1164                 scb = ahd_lookup_scb(ahd, scb_index);
1165                 if (scb == NULL) {
1166                         /*
1167                          * Attempt to transfer to an SCB that is
1168                          * not outstanding.
1169                          */
1170                         ahd_assert_atn(ahd);
1171                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1172                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
1173                         ahd->msgout_len = 1;
1174                         ahd->msgout_index = 0;
1175                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1176                         /*
1177                          * Clear status received flag to prevent any
1178                          * attempt to complete this bogus SCB.
1179                          */
1180                         ahd_outb(ahd, SCB_CONTROL,
1181                                  ahd_inb_scbram(ahd, SCB_CONTROL)
1182                                  & ~STATUS_RCVD);
1183                 }
1184                 break;
1185         }
1186         case DUMP_CARD_STATE:
1187         {
1188                 ahd_dump_card_state(ahd);
1189                 break;
1190         }
1191         case PDATA_REINIT:
1192         {
1193 #ifdef AHD_DEBUG
1194                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1195                         printf("%s: PDATA_REINIT - DFCNTRL = 0x%x "
1196                                "SG_CACHE_SHADOW = 0x%x\n",
1197                                ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
1198                                ahd_inb(ahd, SG_CACHE_SHADOW));
1199                 }
1200 #endif
1201                 ahd_reinitialize_dataptrs(ahd);
1202                 break;
1203         }
1204         case HOST_MSG_LOOP:
1205         {
1206                 struct ahd_devinfo devinfo;
1207
1208                 /*
1209                  * The sequencer has encountered a message phase
1210                  * that requires host assistance for completion.
1211                  * While handling the message phase(s), we will be
1212                  * notified by the sequencer after each byte is
1213                  * transfered so we can track bus phase changes.
1214                  *
1215                  * If this is the first time we've seen a HOST_MSG_LOOP
1216                  * interrupt, initialize the state of the host message
1217                  * loop.
1218                  */
1219                 ahd_fetch_devinfo(ahd, &devinfo);
1220                 if (ahd->msg_type == MSG_TYPE_NONE) {
1221                         struct scb *scb;
1222                         u_int scb_index;
1223                         u_int bus_phase;
1224
1225                         bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1226                         if (bus_phase != P_MESGIN
1227                          && bus_phase != P_MESGOUT) {
1228                                 printf("ahd_intr: HOST_MSG_LOOP bad "
1229                                        "phase 0x%x\n", bus_phase);
1230                                 AHD_CORRECTABLE_ERROR(ahd);
1231                                 /*
1232                                  * Probably transitioned to bus free before
1233                                  * we got here.  Just punt the message.
1234                                  */
1235                                 ahd_dump_card_state(ahd);
1236                                 ahd_clear_intstat(ahd);
1237                                 ahd_restart(ahd);
1238                                 return;
1239                         }
1240
1241                         scb_index = ahd_get_scbptr(ahd);
1242                         scb = ahd_lookup_scb(ahd, scb_index);
1243                         if (devinfo.role == ROLE_INITIATOR) {
1244                                 if (bus_phase == P_MESGOUT)
1245                                         ahd_setup_initiator_msgout(ahd,
1246                                                                    &devinfo,
1247                                                                    scb);
1248                                 else {
1249                                         ahd->msg_type =
1250                                             MSG_TYPE_INITIATOR_MSGIN;
1251                                         ahd->msgin_index = 0;
1252                                 }
1253                         }
1254 #ifdef AHD_TARGET_MODE
1255                         else {
1256                                 if (bus_phase == P_MESGOUT) {
1257                                         ahd->msg_type =
1258                                             MSG_TYPE_TARGET_MSGOUT;
1259                                         ahd->msgin_index = 0;
1260                                 }
1261                                 else
1262                                         ahd_setup_target_msgin(ahd,
1263                                                                &devinfo,
1264                                                                scb);
1265                         }
1266 #endif
1267                 }
1268
1269                 ahd_handle_message_phase(ahd);
1270                 break;
1271         }
1272         case NO_MATCH:
1273         {
1274                 /* Ensure we don't leave the selection hardware on */
1275                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
1276                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
1277
1278                 printf("%s:%c:%d: no active SCB for reconnecting "
1279                        "target - issuing BUS DEVICE RESET\n",
1280                        ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
1281                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1282                        "REG0 == 0x%x ACCUM = 0x%x\n",
1283                        ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
1284                        ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
1285                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1286                        "SINDEX == 0x%x\n",
1287                        ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
1288                        ahd_find_busy_tcl(ahd,
1289                                          BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
1290                                                    ahd_inb(ahd, SAVED_LUN))),
1291                        ahd_inw(ahd, SINDEX));
1292                 printf("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1293                        "SCB_CONTROL == 0x%x\n",
1294                        ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
1295                        ahd_inb_scbram(ahd, SCB_LUN),
1296                        ahd_inb_scbram(ahd, SCB_CONTROL));
1297                 printf("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
1298                        ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
1299                 printf("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
1300                 printf("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
1301                 ahd_dump_card_state(ahd);
1302                 ahd->msgout_buf[0] = MSG_BUS_DEV_RESET;
1303                 ahd->msgout_len = 1;
1304                 ahd->msgout_index = 0;
1305                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1306                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
1307                 ahd_assert_atn(ahd);
1308                 break;
1309         }
1310         case PROTO_VIOLATION:
1311         {
1312                 ahd_handle_proto_violation(ahd);
1313                 break;
1314         }
1315         case IGN_WIDE_RES:
1316         {
1317                 struct ahd_devinfo devinfo;
1318
1319                 ahd_fetch_devinfo(ahd, &devinfo);
1320                 ahd_handle_ign_wide_residue(ahd, &devinfo);
1321                 break;
1322         }
1323         case BAD_PHASE:
1324         {
1325                 u_int lastphase;
1326
1327                 lastphase = ahd_inb(ahd, LASTPHASE);
1328                 printf("%s:%c:%d: unknown scsi bus phase %x, "
1329                        "lastphase = 0x%x.  Attempting to continue\n",
1330                        ahd_name(ahd), 'A',
1331                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
1332                        lastphase, ahd_inb(ahd, SCSISIGI));
1333                 AHD_CORRECTABLE_ERROR(ahd);
1334                 break;
1335         }
1336         case MISSED_BUSFREE:
1337         {
1338                 u_int lastphase;
1339
1340                 lastphase = ahd_inb(ahd, LASTPHASE);
1341                 printf("%s:%c:%d: Missed busfree. "
1342                        "Lastphase = 0x%x, Curphase = 0x%x\n",
1343                        ahd_name(ahd), 'A',
1344                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
1345                        lastphase, ahd_inb(ahd, SCSISIGI));
1346                 AHD_CORRECTABLE_ERROR(ahd);
1347                 ahd_restart(ahd);
1348                 return;
1349         }
1350         case DATA_OVERRUN:
1351         {
1352                 /*
1353                  * When the sequencer detects an overrun, it
1354                  * places the controller in "BITBUCKET" mode
1355                  * and allows the target to complete its transfer.
1356                  * Unfortunately, none of the counters get updated
1357                  * when the controller is in this mode, so we have
1358                  * no way of knowing how large the overrun was.
1359                  */
1360                 struct  scb *scb;
1361                 u_int   scbindex;
1362 #ifdef AHD_DEBUG
1363                 u_int   lastphase;
1364 #endif
1365
1366                 scbindex = ahd_get_scbptr(ahd);
1367                 scb = ahd_lookup_scb(ahd, scbindex);
1368 #ifdef AHD_DEBUG
1369                 lastphase = ahd_inb(ahd, LASTPHASE);
1370                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1371                         ahd_print_path(ahd, scb);
1372                         printf("data overrun detected %s.  Tag == 0x%x.\n",
1373                                ahd_lookup_phase_entry(lastphase)->phasemsg,
1374                                SCB_GET_TAG(scb));
1375                         ahd_print_path(ahd, scb);
1376                         printf("%s seen Data Phase.  Length = %ld.  "
1377                                "NumSGs = %d.\n",
1378                                ahd_inb(ahd, SEQ_FLAGS) & DPHASE
1379                                ? "Have" : "Haven't",
1380                                aic_get_transfer_length(scb), scb->sg_count);
1381                         ahd_dump_sglist(scb);
1382                 }
1383 #endif
1384
1385                 /*
1386                  * Set this and it will take effect when the
1387                  * target does a command complete.
1388                  */
1389                 ahd_freeze_devq(ahd, scb);
1390                 aic_set_transaction_status(scb, CAM_DATA_RUN_ERR);
1391                 aic_freeze_scb(scb);
1392                 break;
1393         }
1394         case MKMSG_FAILED:
1395         {
1396                 struct ahd_devinfo devinfo;
1397                 struct scb *scb;
1398                 u_int scbid;
1399
1400                 ahd_fetch_devinfo(ahd, &devinfo);
1401                 printf("%s:%c:%d:%d: Attempt to issue message failed\n",
1402                        ahd_name(ahd), devinfo.channel, devinfo.target,
1403                        devinfo.lun);
1404                 scbid = ahd_get_scbptr(ahd);
1405                 scb = ahd_lookup_scb(ahd, scbid);
1406                 AHD_CORRECTABLE_ERROR(ahd);
1407                 if (scb != NULL
1408                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
1409                         /*
1410                          * Ensure that we didn't put a second instance of this
1411                          * SCB into the QINFIFO.
1412                          */
1413                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
1414                                            SCB_GET_CHANNEL(ahd, scb),
1415                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
1416                                            ROLE_INITIATOR, /*status*/0,
1417                                            SEARCH_REMOVE);
1418                 ahd_outb(ahd, SCB_CONTROL,
1419                          ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
1420                 break;
1421         }
1422         case TASKMGMT_FUNC_COMPLETE:
1423         {
1424                 u_int   scbid;
1425                 struct  scb *scb;
1426
1427                 scbid = ahd_get_scbptr(ahd);
1428                 scb = ahd_lookup_scb(ahd, scbid);
1429                 if (scb != NULL) {
1430                         u_int      lun;
1431                         u_int      tag;
1432                         cam_status error;
1433
1434                         ahd_print_path(ahd, scb);
1435                         printf("Task Management Func 0x%x Complete\n",
1436                                scb->hscb->task_management);
1437                         lun = CAM_LUN_WILDCARD;
1438                         tag = SCB_LIST_NULL;
1439
1440                         switch (scb->hscb->task_management) {
1441                         case SIU_TASKMGMT_ABORT_TASK:
1442                                 tag = SCB_GET_TAG(scb);
1443                         case SIU_TASKMGMT_ABORT_TASK_SET:
1444                         case SIU_TASKMGMT_CLEAR_TASK_SET:
1445                                 lun = scb->hscb->lun;
1446                                 error = CAM_REQ_ABORTED;
1447                                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
1448                                                'A', lun, tag, ROLE_INITIATOR,
1449                                                error);
1450                                 break;
1451                         case SIU_TASKMGMT_LUN_RESET:
1452                                 lun = scb->hscb->lun;
1453                         case SIU_TASKMGMT_TARGET_RESET:
1454                         {
1455                                 struct ahd_devinfo devinfo;
1456
1457                                 ahd_scb_devinfo(ahd, &devinfo, scb);
1458                                 error = CAM_BDR_SENT;
1459                                 ahd_handle_devreset(ahd, &devinfo, lun,
1460                                                     CAM_BDR_SENT,
1461                                                     lun != CAM_LUN_WILDCARD
1462                                                     ? "Lun Reset"
1463                                                     : "Target Reset",
1464                                                     /*verbose_level*/0);
1465                                 break;
1466                         }
1467                         default:
1468                                 panic("Unexpected TaskMgmt Func\n");
1469                                 break;
1470                         }
1471                 }
1472                 break;
1473         }
1474         case TASKMGMT_CMD_CMPLT_OKAY:
1475         {
1476                 u_int   scbid;
1477                 struct  scb *scb;
1478
1479                 /*
1480                  * An ABORT TASK TMF failed to be delivered before
1481                  * the targeted command completed normally.
1482                  */
1483                 scbid = ahd_get_scbptr(ahd);
1484                 scb = ahd_lookup_scb(ahd, scbid);
1485                 if (scb != NULL) {
1486                         /*
1487                          * Remove the second instance of this SCB from
1488                          * the QINFIFO if it is still there.
1489                          */
1490                         ahd_print_path(ahd, scb);
1491                         printf("SCB completes before TMF\n");
1492                         /*
1493                          * Handle losing the race.  Wait until any
1494                          * current selection completes.  We will then
1495                          * set the TMF back to zero in this SCB so that
1496                          * the sequencer doesn't bother to issue another
1497                          * sequencer interrupt for its completion.
1498                          */
1499                         while ((ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
1500                             && (ahd_inb(ahd, SSTAT0) & SELDO) == 0
1501                             && (ahd_inb(ahd, SSTAT1) & SELTO) == 0)
1502                                 ;
1503                         ahd_outb(ahd, SCB_TASK_MANAGEMENT, 0);
1504                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
1505                                            SCB_GET_CHANNEL(ahd, scb),
1506                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
1507                                            ROLE_INITIATOR, /*status*/0,
1508                                            SEARCH_REMOVE);
1509                 }
1510                 break;
1511         }
1512         case TRACEPOINT0:
1513         case TRACEPOINT1:
1514         case TRACEPOINT2:
1515         case TRACEPOINT3:
1516                 printf("%s: Tracepoint %d\n", ahd_name(ahd),
1517                        seqintcode - TRACEPOINT0);
1518                 break;
1519         case NO_SEQINT:
1520                 break;
1521         case SAW_HWERR:
1522                 ahd_handle_hwerrint(ahd);
1523                 break;
1524         default:
1525                 printf("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
1526                        seqintcode);
1527                 break;
1528         }
1529         /*
1530          *  The sequencer is paused immediately on
1531          *  a SEQINT, so we should restart it when
1532          *  we're done.
1533          */
1534         ahd_unpause(ahd);
1535 }
1536
1537 void
1538 ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
1539 {
1540         struct scb      *scb;
1541         u_int            status0;
1542         u_int            status3;
1543         u_int            status;
1544         u_int            lqistat1;
1545         u_int            lqostat0;
1546         u_int            scbid;
1547         u_int            busfreetime;
1548
1549         ahd_update_modes(ahd);
1550         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1551
1552         status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
1553         status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
1554         status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1555         lqistat1 = ahd_inb(ahd, LQISTAT1);
1556         lqostat0 = ahd_inb(ahd, LQOSTAT0);
1557         busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
1558         if ((status0 & (SELDI|SELDO)) != 0) {
1559                 u_int simode0;
1560
1561                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1562                 simode0 = ahd_inb(ahd, SIMODE0);
1563                 status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
1564                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1565         }
1566         scbid = ahd_get_scbptr(ahd);
1567         scb = ahd_lookup_scb(ahd, scbid);
1568         if (scb != NULL
1569          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1570                 scb = NULL;
1571
1572         if ((status0 & IOERR) != 0) {
1573                 u_int now_lvd;
1574
1575                 now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
1576                 printf("%s: Transceiver State Has Changed to %s mode\n",
1577                        ahd_name(ahd), now_lvd ? "LVD" : "SE");
1578                 ahd_outb(ahd, CLRSINT0, CLRIOERR);
1579                 /*
1580                  * A change in I/O mode is equivalent to a bus reset.
1581                  */
1582                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1583                 ahd_pause(ahd);
1584                 ahd_setup_iocell_workaround(ahd);
1585                 ahd_unpause(ahd);
1586         } else if ((status0 & OVERRUN) != 0) {
1587
1588                 printf("%s: SCSI offset overrun detected.  Resetting bus.\n",
1589                        ahd_name(ahd));
1590                 AHD_CORRECTABLE_ERROR(ahd);
1591                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1592         } else if ((status & SCSIRSTI) != 0) {
1593
1594                 printf("%s: Someone reset channel A\n", ahd_name(ahd));
1595                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
1596                 AHD_UNCORRECTABLE_ERROR(ahd);
1597         } else if ((status & SCSIPERR) != 0) {
1598
1599                 /* Make sure the sequencer is in a safe location. */
1600                 ahd_clear_critical_section(ahd);
1601
1602                 ahd_handle_transmission_error(ahd);
1603         } else if (lqostat0 != 0) {
1604
1605                 printf("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
1606                 ahd_outb(ahd, CLRLQOINT0, lqostat0);
1607                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
1608                         ahd_outb(ahd, CLRLQOINT1, 0);
1609         } else if ((status & SELTO) != 0) {
1610                 u_int  scbid;
1611
1612                 /* Stop the selection */
1613                 ahd_outb(ahd, SCSISEQ0, 0);
1614
1615                 /* Make sure the sequencer is in a safe location. */
1616                 ahd_clear_critical_section(ahd);
1617
1618                 /* No more pending messages */
1619                 ahd_clear_msg_state(ahd);
1620
1621                 /* Clear interrupt state */
1622                 ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1623
1624                 /*
1625                  * Although the driver does not care about the
1626                  * 'Selection in Progress' status bit, the busy
1627                  * LED does.  SELINGO is only cleared by a sucessfull
1628                  * selection, so we must manually clear it to insure
1629                  * the LED turns off just incase no future successful
1630                  * selections occur (e.g. no devices on the bus).
1631                  */
1632                 ahd_outb(ahd, CLRSINT0, CLRSELINGO);
1633
1634                 scbid = ahd_inw(ahd, WAITING_TID_HEAD);
1635                 scb = ahd_lookup_scb(ahd, scbid);
1636                 if (scb == NULL) {
1637                         printf("%s: ahd_intr - referenced scb not "
1638                                "valid during SELTO scb(0x%x)\n",
1639                                ahd_name(ahd), scbid);
1640                         ahd_dump_card_state(ahd);
1641                         AHD_UNCORRECTABLE_ERROR(ahd);
1642                 } else {
1643                         struct ahd_devinfo devinfo;
1644 #ifdef AHD_DEBUG
1645                         if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
1646                                 ahd_print_path(ahd, scb);
1647                                 printf("Saw Selection Timeout for SCB 0x%x\n",
1648                                        scbid);
1649                         }
1650 #endif
1651                         ahd_scb_devinfo(ahd, &devinfo, scb);
1652                         aic_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1653                         ahd_freeze_devq(ahd, scb);
1654
1655                         /*
1656                          * Cancel any pending transactions on the device
1657                          * now that it seems to be missing.  This will
1658                          * also revert us to async/narrow transfers until
1659                          * we can renegotiate with the device.
1660                          */
1661                         ahd_handle_devreset(ahd, &devinfo,
1662                                             CAM_LUN_WILDCARD,
1663                                             CAM_SEL_TIMEOUT,
1664                                             "Selection Timeout",
1665                                             /*verbose_level*/1);
1666                 }
1667                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
1668                 ahd_iocell_first_selection(ahd);
1669                 ahd_unpause(ahd);
1670         } else if ((status0 & (SELDI|SELDO)) != 0) {
1671
1672                 ahd_iocell_first_selection(ahd);
1673                 ahd_unpause(ahd);
1674         } else if (status3 != 0) {
1675                 printf("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
1676                        ahd_name(ahd), status3);
1677                 AHD_CORRECTABLE_ERROR(ahd);
1678                 ahd_outb(ahd, CLRSINT3, status3);
1679         } else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
1680
1681                 /* Make sure the sequencer is in a safe location. */
1682                 ahd_clear_critical_section(ahd);
1683
1684                 ahd_handle_lqiphase_error(ahd, lqistat1);
1685         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
1686                 /*
1687                  * This status can be delayed during some
1688                  * streaming operations.  The SCSIPHASE
1689                  * handler has already dealt with this case
1690                  * so just clear the error.
1691                  */
1692                 ahd_outb(ahd, CLRLQIINT1, CLRLQICRCI_NLQ);
1693         } else if ((status & BUSFREE) != 0
1694                 || (lqistat1 & LQOBUSFREE) != 0) {
1695                 u_int lqostat1;
1696                 int   restart;
1697                 int   clear_fifo;
1698                 int   packetized;
1699                 u_int mode;
1700
1701                 /*
1702                  * Clear our selection hardware as soon as possible.
1703                  * We may have an entry in the waiting Q for this target,
1704                  * that is affected by this busfree and we don't want to
1705                  * go about selecting the target while we handle the event.
1706                  */
1707                 ahd_outb(ahd, SCSISEQ0, 0);
1708
1709                 /* Make sure the sequencer is in a safe location. */
1710                 ahd_clear_critical_section(ahd);
1711
1712                 /*
1713                  * Determine what we were up to at the time of
1714                  * the busfree.
1715                  */
1716                 mode = AHD_MODE_SCSI;
1717                 busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
1718                 lqostat1 = ahd_inb(ahd, LQOSTAT1);
1719                 switch (busfreetime) {
1720                 case BUSFREE_DFF0:
1721                 case BUSFREE_DFF1:
1722                 {
1723                         u_int   scbid;
1724                         struct  scb *scb;
1725
1726                         mode = busfreetime == BUSFREE_DFF0
1727                              ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
1728                         ahd_set_modes(ahd, mode, mode);
1729                         scbid = ahd_get_scbptr(ahd);
1730                         scb = ahd_lookup_scb(ahd, scbid);
1731                         if (scb == NULL) {
1732                                 printf("%s: Invalid SCB %d in DFF%d "
1733                                        "during unexpected busfree\n",
1734                                        ahd_name(ahd), scbid, mode);
1735                                 packetized = 0;
1736                                 AHD_CORRECTABLE_ERROR(ahd);
1737                         } else
1738                                 packetized = (scb->flags & SCB_PACKETIZED) != 0;
1739                         clear_fifo = 1;
1740                         break;
1741                 }
1742                 case BUSFREE_LQO:
1743                         clear_fifo = 0;
1744                         packetized = 1;
1745                         break;
1746                 default:
1747                         clear_fifo = 0;
1748                         packetized =  (lqostat1 & LQOBUSFREE) != 0;
1749                         if (!packetized
1750                          && ahd_inb(ahd, LASTPHASE) == P_BUSFREE
1751                          && (ahd_inb(ahd, SSTAT0) & SELDI) == 0
1752                          && ((ahd_inb(ahd, SSTAT0) & SELDO) == 0
1753                           || (ahd_inb(ahd, SCSISEQ0) & ENSELO) == 0))
1754                                 /*
1755                                  * Assume packetized if we are not
1756                                  * on the bus in a non-packetized
1757                                  * capacity and any pending selection
1758                                  * was a packetized selection.
1759                                  */
1760                                 packetized = 1;
1761                         break;
1762                 }
1763
1764 #ifdef AHD_DEBUG
1765                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
1766                         printf("Saw Busfree.  Busfreetime = 0x%x.\n",
1767                                busfreetime);
1768 #endif
1769                 /*
1770                  * Busfrees that occur in non-packetized phases are
1771                  * handled by the nonpkt_busfree handler.
1772                  */
1773                 if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
1774                         restart = ahd_handle_pkt_busfree(ahd, busfreetime);
1775                 } else {
1776                         packetized = 0;
1777                         restart = ahd_handle_nonpkt_busfree(ahd);
1778                 }
1779                 /*
1780                  * Clear the busfree interrupt status.  The setting of
1781                  * the interrupt is a pulse, so in a perfect world, we
1782                  * would not need to muck with the ENBUSFREE logic.  This
1783                  * would ensure that if the bus moves on to another
1784                  * connection, busfree protection is still in force.  If
1785                  * BUSFREEREV is broken, however, we must manually clear
1786                  * the ENBUSFREE if the busfree occurred during a non-pack
1787                  * connection so that we don't get false positives during
1788                  * future, packetized, connections.
1789                  */
1790                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
1791                 if (packetized == 0
1792                  && (ahd->bugs & AHD_BUSFREEREV_BUG) != 0)
1793                         ahd_outb(ahd, SIMODE1,
1794                                  ahd_inb(ahd, SIMODE1) & ~ENBUSFREE);
1795
1796                 if (clear_fifo)
1797                         ahd_clear_fifo(ahd, mode);
1798
1799                 ahd_clear_msg_state(ahd);
1800                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
1801                 if (restart) {
1802                         ahd_restart(ahd);
1803                 } else {
1804                         ahd_unpause(ahd);
1805                 }
1806         } else {
1807                 printf("%s: Missing case in ahd_handle_scsiint. status = %x\n",
1808                        ahd_name(ahd), status);
1809                 ahd_dump_card_state(ahd);
1810                 ahd_clear_intstat(ahd);
1811                 ahd_unpause(ahd);
1812         }
1813 }
1814
1815 static void
1816 ahd_handle_transmission_error(struct ahd_softc *ahd)
1817 {
1818         struct  scb *scb;
1819         u_int   scbid;
1820         u_int   lqistat1;
1821         u_int   lqistat2;
1822         u_int   msg_out;
1823         u_int   curphase;
1824         u_int   lastphase;
1825         u_int   perrdiag;
1826         u_int   cur_col;
1827         int     silent;
1828
1829         scb = NULL;
1830         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1831         lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
1832         lqistat2 = ahd_inb(ahd, LQISTAT2);
1833         if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
1834          && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
1835                 u_int lqistate;
1836
1837                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1838                 lqistate = ahd_inb(ahd, LQISTATE);
1839                 if ((lqistate >= 0x1E && lqistate <= 0x24)
1840                  || (lqistate == 0x29)) {
1841 #ifdef AHD_DEBUG
1842                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1843                                 printf("%s: NLQCRC found via LQISTATE\n",
1844                                        ahd_name(ahd));
1845                         }
1846 #endif
1847                         lqistat1 |= LQICRCI_NLQ;
1848                 }
1849                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1850         }
1851
1852         ahd_outb(ahd, CLRLQIINT1, lqistat1);
1853         lastphase = ahd_inb(ahd, LASTPHASE);
1854         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1855         perrdiag = ahd_inb(ahd, PERRDIAG);
1856         msg_out = MSG_INITIATOR_DET_ERR;
1857         ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
1858
1859         /*
1860          * Try to find the SCB associated with this error.
1861          */
1862         silent = FALSE;
1863         if (lqistat1 == 0
1864          || (lqistat1 & LQICRCI_NLQ) != 0) {
1865                 if ((lqistat1 & (LQICRCI_NLQ|LQIOVERI_NLQ)) != 0)
1866                         ahd_set_active_fifo(ahd);
1867                 scbid = ahd_get_scbptr(ahd);
1868                 scb = ahd_lookup_scb(ahd, scbid);
1869                 if (scb != NULL && SCB_IS_SILENT(scb))
1870                         silent = TRUE;
1871         }
1872
1873         cur_col = 0;
1874         if (silent == FALSE) {
1875                 printf("%s: Transmission error detected\n", ahd_name(ahd));
1876                 ahd_lqistat1_print(lqistat1, &cur_col, 50);
1877                 ahd_lastphase_print(lastphase, &cur_col, 50);
1878                 ahd_scsisigi_print(curphase, &cur_col, 50);
1879                 ahd_perrdiag_print(perrdiag, &cur_col, 50);
1880                 printf("\n");
1881                 AHD_CORRECTABLE_ERROR(ahd);
1882                 ahd_dump_card_state(ahd);
1883         }
1884
1885         if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
1886                 if (silent == FALSE) {
1887                         printf("%s: Gross protocol error during incoming "
1888                                "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
1889                                ahd_name(ahd), lqistat1);
1890                         AHD_UNCORRECTABLE_ERROR(ahd);
1891                 }
1892                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1893                 return;
1894         } else if ((lqistat1 & LQICRCI_LQ) != 0) {
1895                 /*
1896                  * A CRC error has been detected on an incoming LQ.
1897                  * The bus is currently hung on the last ACK.
1898                  * Hit LQIRETRY to release the last ack, and
1899                  * wait for the sequencer to determine that ATNO
1900                  * is asserted while in message out to take us
1901                  * to our host message loop.  No NONPACKREQ or
1902                  * LQIPHASE type errors will occur in this
1903                  * scenario.  After this first LQIRETRY, the LQI
1904                  * manager will be in ISELO where it will
1905                  * happily sit until another packet phase begins.
1906                  * Unexpected bus free detection is enabled
1907                  * through any phases that occur after we release
1908                  * this last ack until the LQI manager sees a
1909                  * packet phase.  This implies we may have to
1910                  * ignore a perfectly valid "unexected busfree"
1911                  * after our "initiator detected error" message is
1912                  * sent.  A busfree is the expected response after
1913                  * we tell the target that it's L_Q was corrupted.
1914                  * (SPI4R09 10.7.3.3.3)
1915                  */
1916                 ahd_outb(ahd, LQCTL2, LQIRETRY);
1917                 printf("LQIRetry for LQICRCI_LQ to release ACK\n");
1918                 AHD_CORRECTABLE_ERROR(ahd);
1919         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
1920                 /*
1921                  * We detected a CRC error in a NON-LQ packet.
1922                  * The hardware has varying behavior in this situation
1923                  * depending on whether this packet was part of a
1924                  * stream or not.
1925                  *
1926                  * PKT by PKT mode:
1927                  * The hardware has already acked the complete packet.
1928                  * If the target honors our outstanding ATN condition,
1929                  * we should be (or soon will be) in MSGOUT phase.
1930                  * This will trigger the LQIPHASE_LQ status bit as the
1931                  * hardware was expecting another LQ.  Unexpected
1932                  * busfree detection is enabled.  Once LQIPHASE_LQ is
1933                  * true (first entry into host message loop is much
1934                  * the same), we must clear LQIPHASE_LQ and hit
1935                  * LQIRETRY so the hardware is ready to handle
1936                  * a future LQ.  NONPACKREQ will not be asserted again
1937                  * once we hit LQIRETRY until another packet is
1938                  * processed.  The target may either go busfree
1939                  * or start another packet in response to our message.
1940                  *
1941                  * Read Streaming P0 asserted:
1942                  * If we raise ATN and the target completes the entire
1943                  * stream (P0 asserted during the last packet), the
1944                  * hardware will ack all data and return to the ISTART
1945                  * state.  When the target reponds to our ATN condition,
1946                  * LQIPHASE_LQ will be asserted.  We should respond to
1947                  * this with an LQIRETRY to prepare for any future
1948                  * packets.  NONPACKREQ will not be asserted again
1949                  * once we hit LQIRETRY until another packet is
1950                  * processed.  The target may either go busfree or
1951                  * start another packet in response to our message.
1952                  * Busfree detection is enabled.
1953                  *
1954                  * Read Streaming P0 not asserted:
1955                  * If we raise ATN and the target transitions to
1956                  * MSGOUT in or after a packet where P0 is not
1957                  * asserted, the hardware will assert LQIPHASE_NLQ.
1958                  * We should respond to the LQIPHASE_NLQ with an
1959                  * LQIRETRY.  Should the target stay in a non-pkt
1960                  * phase after we send our message, the hardware
1961                  * will assert LQIPHASE_LQ.  Recovery is then just as
1962                  * listed above for the read streaming with P0 asserted.
1963                  * Busfree detection is enabled.
1964                  */
1965                 if (silent == FALSE)
1966                         printf("LQICRC_NLQ\n");
1967                 if (scb == NULL) {
1968                         printf("%s: No SCB valid for LQICRC_NLQ.  "
1969                                "Resetting bus\n", ahd_name(ahd));
1970                         AHD_UNCORRECTABLE_ERROR(ahd);
1971                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1972                         return;
1973                 }
1974         } else if ((lqistat1 & LQIBADLQI) != 0) {
1975                 printf("Need to handle BADLQI!\n");
1976                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1977                 return;
1978         } else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
1979                 if ((curphase & ~P_DATAIN_DT) != 0) {
1980                         /* Ack the byte.  So we can continue. */
1981                         if (silent == FALSE)
1982                                 printf("Acking %s to clear perror\n",
1983                                     ahd_lookup_phase_entry(curphase)->phasemsg);
1984                         ahd_inb(ahd, SCSIDAT);
1985                 }
1986
1987                 if (curphase == P_MESGIN)
1988                         msg_out = MSG_PARITY_ERROR;
1989         }
1990
1991         /*
1992          * We've set the hardware to assert ATN if we
1993          * get a parity error on "in" phases, so all we
1994          * need to do is stuff the message buffer with
1995          * the appropriate message.  "In" phases have set
1996          * mesg_out to something other than MSG_NOP.
1997          */
1998         ahd->send_msg_perror = msg_out;
1999         if (scb != NULL && msg_out == MSG_INITIATOR_DET_ERR)
2000                 scb->flags |= SCB_TRANSMISSION_ERROR;
2001         ahd_outb(ahd, MSG_OUT, HOST_MSG);
2002         ahd_outb(ahd, CLRINT, CLRSCSIINT);
2003         ahd_unpause(ahd);
2004 }
2005
2006 static void
2007 ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
2008 {
2009         /*
2010          * Clear the sources of the interrupts.
2011          */
2012         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2013         ahd_outb(ahd, CLRLQIINT1, lqistat1);
2014
2015         /*
2016          * If the "illegal" phase changes were in response
2017          * to our ATN to flag a CRC error, AND we ended up
2018          * on packet boundaries, clear the error, restart the
2019          * LQI manager as appropriate, and go on our merry
2020          * way toward sending the message.  Otherwise, reset
2021          * the bus to clear the error.
2022          */
2023         ahd_set_active_fifo(ahd);
2024         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
2025          && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
2026                 if ((lqistat1 & LQIPHASE_LQ) != 0) {
2027                         printf("LQIRETRY for LQIPHASE_LQ\n");
2028                         AHD_CORRECTABLE_ERROR(ahd);
2029                         ahd_outb(ahd, LQCTL2, LQIRETRY);
2030                 } else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
2031                         printf("LQIRETRY for LQIPHASE_NLQ\n");
2032                         AHD_CORRECTABLE_ERROR(ahd);
2033                         ahd_outb(ahd, LQCTL2, LQIRETRY);
2034                 } else
2035                         panic("ahd_handle_lqiphase_error: No phase errors\n");
2036                 ahd_dump_card_state(ahd);
2037                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2038                 ahd_unpause(ahd);
2039         } else {
2040                 printf("Reseting Channel for LQI Phase error\n");
2041                 AHD_CORRECTABLE_ERROR(ahd);
2042                 ahd_dump_card_state(ahd);
2043                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2044         }
2045 }
2046
2047 /*
2048  * Packetized unexpected or expected busfree.
2049  * Entered in mode based on busfreetime.
2050  */
2051 static int
2052 ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
2053 {
2054         u_int lqostat1;
2055
2056         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2057                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2058         lqostat1 = ahd_inb(ahd, LQOSTAT1);
2059         if ((lqostat1 & LQOBUSFREE) != 0) {
2060                 struct scb *scb;
2061                 u_int scbid;
2062                 u_int saved_scbptr;
2063                 u_int waiting_h;
2064                 u_int waiting_t;
2065                 u_int next;
2066
2067                 /*
2068                  * The LQO manager detected an unexpected busfree
2069                  * either:
2070                  *
2071                  * 1) During an outgoing LQ.
2072                  * 2) After an outgoing LQ but before the first
2073                  *    REQ of the command packet.
2074                  * 3) During an outgoing command packet.
2075                  *
2076                  * In all cases, CURRSCB is pointing to the
2077                  * SCB that encountered the failure.  Clean
2078                  * up the queue, clear SELDO and LQOBUSFREE,
2079                  * and allow the sequencer to restart the select
2080                  * out at its lesure.
2081                  */
2082                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2083                 scbid = ahd_inw(ahd, CURRSCB);
2084                 scb = ahd_lookup_scb(ahd, scbid);
2085                 if (scb == NULL)
2086                        panic("SCB not valid during LQOBUSFREE");
2087                 /*
2088                  * Clear the status.
2089                  */
2090                 ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
2091                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2092                         ahd_outb(ahd, CLRLQOINT1, 0);
2093                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2094                 ahd_flush_device_writes(ahd);
2095                 ahd_outb(ahd, CLRSINT0, CLRSELDO);
2096
2097                 /*
2098                  * Return the LQO manager to its idle loop.  It will
2099                  * not do this automatically if the busfree occurs
2100                  * after the first REQ of either the LQ or command
2101                  * packet or between the LQ and command packet.
2102                  */
2103                 ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
2104
2105                 /*
2106                  * Update the waiting for selection queue so
2107                  * we restart on the correct SCB.
2108                  */
2109                 waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
2110                 saved_scbptr = ahd_get_scbptr(ahd);
2111                 if (waiting_h != scbid) {
2112
2113                         ahd_outw(ahd, WAITING_TID_HEAD, scbid);
2114                         waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
2115                         if (waiting_t == waiting_h) {
2116                                 ahd_outw(ahd, WAITING_TID_TAIL, scbid);
2117                                 next = SCB_LIST_NULL;
2118                         } else {
2119                                 ahd_set_scbptr(ahd, waiting_h);
2120                                 next = ahd_inw_scbram(ahd, SCB_NEXT2);
2121                         }
2122                         ahd_set_scbptr(ahd, scbid);
2123                         ahd_outw(ahd, SCB_NEXT2, next);
2124                 }
2125                 ahd_set_scbptr(ahd, saved_scbptr);
2126                 if (scb->crc_retry_count < AHD_MAX_LQ_CRC_ERRORS) {
2127                         if (SCB_IS_SILENT(scb) == FALSE) {
2128                                 ahd_print_path(ahd, scb);
2129                                 printf("Probable outgoing LQ CRC error.  "
2130                                        "Retrying command\n");
2131                                 AHD_CORRECTABLE_ERROR(ahd);
2132                         }
2133                         scb->crc_retry_count++;
2134                 } else {
2135                         aic_set_transaction_status(scb, CAM_UNCOR_PARITY);
2136                         aic_freeze_scb(scb);
2137                         ahd_freeze_devq(ahd, scb);
2138                 }
2139                 /* Return unpausing the sequencer. */
2140                 return (0);
2141         } else if ((ahd_inb(ahd, PERRDIAG) & PARITYERR) != 0) {
2142                 /*
2143                  * Ignore what are really parity errors that
2144                  * occur on the last REQ of a free running
2145                  * clock prior to going busfree.  Some drives
2146                  * do not properly active negate just before
2147                  * going busfree resulting in a parity glitch.
2148                  */
2149                 ahd_outb(ahd, CLRSINT1, CLRSCSIPERR|CLRBUSFREE);
2150 #ifdef AHD_DEBUG
2151                 if ((ahd_debug & AHD_SHOW_MASKED_ERRORS) != 0)
2152                         printf("%s: Parity on last REQ detected "
2153                                "during busfree phase.\n",
2154                                ahd_name(ahd));
2155 #endif
2156                 /* Return unpausing the sequencer. */
2157                 return (0);
2158         }
2159         if (ahd->src_mode != AHD_MODE_SCSI) {
2160                 u_int   scbid;
2161                 struct  scb *scb;
2162
2163                 scbid = ahd_get_scbptr(ahd);
2164                 scb = ahd_lookup_scb(ahd, scbid);
2165                 ahd_print_path(ahd, scb);
2166                 printf("Unexpected PKT busfree condition\n");
2167                 AHD_UNCORRECTABLE_ERROR(ahd);
2168                 ahd_dump_card_state(ahd);
2169                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
2170                                SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2171                                ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
2172
2173                 /* Return restarting the sequencer. */
2174                 return (1);
2175         }
2176         printf("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
2177         AHD_UNCORRECTABLE_ERROR(ahd);
2178         ahd_dump_card_state(ahd);
2179         /* Restart the sequencer. */
2180         return (1);
2181 }
2182
2183 /*
2184  * Non-packetized unexpected or expected busfree.
2185  */
2186 static int
2187 ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
2188 {
2189         struct  ahd_devinfo devinfo;
2190         struct  scb *scb;
2191         u_int   lastphase;
2192         u_int   saved_scsiid;
2193         u_int   saved_lun;
2194         u_int   target;
2195         u_int   initiator_role_id;
2196         u_int   scbid;
2197         u_int   ppr_busfree;
2198         int     printerror;
2199
2200         /*
2201          * Look at what phase we were last in.  If its message out,
2202          * chances are pretty good that the busfree was in response
2203          * to one of our abort requests.
2204          */
2205         lastphase = ahd_inb(ahd, LASTPHASE);
2206         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
2207         saved_lun = ahd_inb(ahd, SAVED_LUN);
2208         target = SCSIID_TARGET(ahd, saved_scsiid);
2209         initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
2210         ahd_compile_devinfo(&devinfo, initiator_role_id,
2211                             target, saved_lun, 'A', ROLE_INITIATOR);
2212         printerror = 1;
2213
2214         scbid = ahd_get_scbptr(ahd);
2215         scb = ahd_lookup_scb(ahd, scbid);
2216         if (scb != NULL
2217          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
2218                 scb = NULL;
2219
2220         ppr_busfree = (ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0;
2221         if (lastphase == P_MESGOUT) {
2222                 u_int tag;
2223
2224                 tag = SCB_LIST_NULL;
2225                 if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT_TAG, TRUE)
2226                  || ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT, TRUE)) {
2227                         int found;
2228                         int sent_msg;
2229
2230                         if (scb == NULL) {
2231                                 ahd_print_devinfo(ahd, &devinfo);
2232                                 printf("Abort for unidentified "
2233                                        "connection completed.\n");
2234                                 /* restart the sequencer. */
2235                                 return (1);
2236                         }
2237                         sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
2238                         ahd_print_path(ahd, scb);
2239                         printf("SCB %d - Abort%s Completed.\n",
2240                                SCB_GET_TAG(scb),
2241                                sent_msg == MSG_ABORT_TAG ? "" : " Tag");
2242
2243                         if (sent_msg == MSG_ABORT_TAG)
2244                                 tag = SCB_GET_TAG(scb);
2245
2246                         if ((scb->flags & SCB_CMDPHASE_ABORT) != 0) {
2247                                 /*
2248                                  * This abort is in response to an
2249                                  * unexpected switch to command phase
2250                                  * for a packetized connection.  Since
2251                                  * the identify message was never sent,
2252                                  * "saved lun" is 0.  We really want to
2253                                  * abort only the SCB that encountered
2254                                  * this error, which could have a different
2255                                  * lun.  The SCB will be retried so the OS
2256                                  * will see the UA after renegotiating to
2257                                  * packetized.
2258                                  */
2259                                 tag = SCB_GET_TAG(scb);
2260                                 saved_lun = scb->hscb->lun;
2261                         }
2262                         found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
2263                                                tag, ROLE_INITIATOR,
2264                                                CAM_REQ_ABORTED);
2265                         printf("found == 0x%x\n", found);
2266                         printerror = 0;
2267                 } else if (ahd_sent_msg(ahd, AHDMSG_1B,
2268                                         MSG_BUS_DEV_RESET, TRUE)) {
2269 #ifdef __FreeBSD__
2270                         /*
2271                          * Don't mark the user's request for this BDR
2272                          * as completing with CAM_BDR_SENT.  CAM3
2273                          * specifies CAM_REQ_CMP.
2274                          */
2275                         if (scb != NULL
2276                          && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
2277                          && ahd_match_scb(ahd, scb, target, 'A',
2278                                           CAM_LUN_WILDCARD, SCB_LIST_NULL,
2279                                           ROLE_INITIATOR))
2280                                 aic_set_transaction_status(scb, CAM_REQ_CMP);
2281 #endif
2282                         ahd_handle_devreset(ahd, &devinfo, CAM_LUN_WILDCARD,
2283                                             CAM_BDR_SENT, "Bus Device Reset",
2284                                             /*verbose_level*/0);
2285                         printerror = 0;
2286                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, FALSE)
2287                         && ppr_busfree == 0) {
2288                         struct ahd_initiator_tinfo *tinfo;
2289                         struct ahd_tmode_tstate *tstate;
2290
2291                         /*
2292                          * PPR Rejected.
2293                          *
2294                          * If the previous negotiation was packetized,
2295                          * this could be because the device has been
2296                          * reset without our knowledge.  Force our
2297                          * current negotiation to async and retry the
2298                          * negotiation.  Otherwise retry the command
2299                          * with non-ppr negotiation.
2300                          */
2301 #ifdef AHD_DEBUG
2302                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2303                                 printf("PPR negotiation rejected busfree.\n");
2304 #endif
2305                         tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
2306                                                     devinfo.our_scsiid,
2307                                                     devinfo.target, &tstate);
2308                         if ((tinfo->curr.ppr_options & MSG_EXT_PPR_IU_REQ)!=0) {
2309                                 ahd_set_width(ahd, &devinfo,
2310                                               MSG_EXT_WDTR_BUS_8_BIT,
2311                                               AHD_TRANS_CUR,
2312                                               /*paused*/TRUE);
2313                                 ahd_set_syncrate(ahd, &devinfo,
2314                                                 /*period*/0, /*offset*/0,
2315                                                 /*ppr_options*/0,
2316                                                 AHD_TRANS_CUR,
2317                                                 /*paused*/TRUE);
2318                                 /*
2319                                  * The expect PPR busfree handler below
2320                                  * will effect the retry and necessary
2321                                  * abort.
2322                                  */
2323                         } else {
2324                                 tinfo->curr.transport_version = 2;
2325                                 tinfo->goal.transport_version = 2;
2326                                 tinfo->goal.ppr_options = 0;
2327                                 /*
2328                                  * Remove any SCBs in the waiting for selection
2329                                  * queue that may also be for this target so
2330                                  * that command ordering is preserved.
2331                                  */
2332                                 ahd_freeze_devq(ahd, scb);
2333                                 ahd_qinfifo_requeue_tail(ahd, scb);
2334                                 printerror = 0;
2335                         }
2336                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, FALSE)
2337                         && ppr_busfree == 0) {
2338                         /*
2339                          * Negotiation Rejected.  Go-narrow and
2340                          * retry command.
2341                          */
2342 #ifdef AHD_DEBUG
2343                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2344                                 printf("WDTR negotiation rejected busfree.\n");
2345 #endif
2346                         ahd_set_width(ahd, &devinfo,
2347                                       MSG_EXT_WDTR_BUS_8_BIT,
2348                                       AHD_TRANS_CUR|AHD_TRANS_GOAL,
2349                                       /*paused*/TRUE);
2350                         /*
2351                          * Remove any SCBs in the waiting for selection
2352                          * queue that may also be for this target so that
2353                          * command ordering is preserved.
2354                          */
2355                         ahd_freeze_devq(ahd, scb);
2356                         ahd_qinfifo_requeue_tail(ahd, scb);
2357                         printerror = 0;
2358                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, FALSE)
2359                         && ppr_busfree == 0) {
2360                         /*
2361                          * Negotiation Rejected.  Go-async and
2362                          * retry command.
2363                          */
2364 #ifdef AHD_DEBUG
2365                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2366                                 printf("SDTR negotiation rejected busfree.\n");
2367 #endif
2368                         ahd_set_syncrate(ahd, &devinfo,
2369                                         /*period*/0, /*offset*/0,
2370                                         /*ppr_options*/0,
2371                                         AHD_TRANS_CUR|AHD_TRANS_GOAL,
2372                                         /*paused*/TRUE);
2373                         /*
2374                          * Remove any SCBs in the waiting for selection
2375                          * queue that may also be for this target so that
2376                          * command ordering is preserved.
2377                          */
2378                         ahd_freeze_devq(ahd, scb);
2379                         ahd_qinfifo_requeue_tail(ahd, scb);
2380                         printerror = 0;
2381                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
2382                         && ahd_sent_msg(ahd, AHDMSG_1B,
2383                                          MSG_INITIATOR_DET_ERR, TRUE)) {
2384
2385 #ifdef AHD_DEBUG
2386                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2387                                 printf("Expected IDE Busfree\n");
2388 #endif
2389                         printerror = 0;
2390                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_QASREJ_BUSFREE)
2391                         && ahd_sent_msg(ahd, AHDMSG_1B,
2392                                         MSG_MESSAGE_REJECT, TRUE)) {
2393
2394 #ifdef AHD_DEBUG
2395                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2396                                 printf("Expected QAS Reject Busfree\n");
2397 #endif
2398                         printerror = 0;
2399                 }
2400         }
2401
2402         /*
2403          * The busfree required flag is honored at the end of
2404          * the message phases.  We check it last in case we
2405          * had to send some other message that caused a busfree.
2406          */
2407         if (printerror != 0
2408          && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
2409          && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
2410
2411                 ahd_freeze_devq(ahd, scb);
2412                 aic_set_transaction_status(scb, CAM_REQUEUE_REQ);
2413                 aic_freeze_scb(scb);
2414                 if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
2415                         ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
2416                                        SCB_GET_CHANNEL(ahd, scb),
2417                                        SCB_GET_LUN(scb), SCB_LIST_NULL,
2418                                        ROLE_INITIATOR, CAM_REQ_ABORTED);
2419                 } else {
2420 #ifdef AHD_DEBUG
2421                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2422                                 printf("PPR Negotiation Busfree.\n");
2423 #endif
2424                         ahd_done(ahd, scb);
2425                 }
2426                 printerror = 0;
2427         }
2428         if (printerror != 0) {
2429                 int aborted;
2430
2431                 aborted = 0;
2432                 if (scb != NULL) {
2433                         u_int tag;
2434
2435                         if ((scb->hscb->control & TAG_ENB) != 0)
2436                                 tag = SCB_GET_TAG(scb);
2437                         else
2438                                 tag = SCB_LIST_NULL;
2439                         ahd_print_path(ahd, scb);
2440                         aborted = ahd_abort_scbs(ahd, target, 'A',
2441                                        SCB_GET_LUN(scb), tag,
2442                                        ROLE_INITIATOR,
2443                                        CAM_UNEXP_BUSFREE);
2444                 } else {
2445                         /*
2446                          * We had not fully identified this connection,
2447                          * so we cannot abort anything.
2448                          */
2449                         printf("%s: ", ahd_name(ahd));
2450                 }
2451                 printf("Unexpected busfree %s, %d SCBs aborted, "
2452                        "PRGMCNT == 0x%x\n",
2453                        ahd_lookup_phase_entry(lastphase)->phasemsg,
2454                        aborted,
2455                        ahd_inw(ahd, PRGMCNT));
2456                 AHD_UNCORRECTABLE_ERROR(ahd);
2457                 ahd_dump_card_state(ahd);
2458                 if (lastphase != P_BUSFREE)
2459                         ahd_force_renegotiation(ahd, &devinfo);
2460         }
2461         /* Always restart the sequencer. */
2462         return (1);
2463 }
2464
2465 static void
2466 ahd_handle_proto_violation(struct ahd_softc *ahd)
2467 {
2468         struct  ahd_devinfo devinfo;
2469         struct  scb *scb;
2470         u_int   scbid;
2471         u_int   seq_flags;
2472         u_int   curphase;
2473         u_int   lastphase;
2474         int     found;
2475
2476         ahd_fetch_devinfo(ahd, &devinfo);
2477         scbid = ahd_get_scbptr(ahd);
2478         scb = ahd_lookup_scb(ahd, scbid);
2479         seq_flags = ahd_inb(ahd, SEQ_FLAGS);
2480         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2481         lastphase = ahd_inb(ahd, LASTPHASE);
2482         if ((seq_flags & NOT_IDENTIFIED) != 0) {
2483
2484                 /*
2485                  * The reconnecting target either did not send an
2486                  * identify message, or did, but we didn't find an SCB
2487                  * to match.
2488                  */
2489                 ahd_print_devinfo(ahd, &devinfo);
2490                 printf("Target did not send an IDENTIFY message. "
2491                        "LASTPHASE = 0x%x.\n", lastphase);
2492                 AHD_UNCORRECTABLE_ERROR(ahd);
2493                 scb = NULL;
2494         } else if (scb == NULL) {
2495                 /*
2496                  * We don't seem to have an SCB active for this
2497                  * transaction.  Print an error and reset the bus.
2498                  */
2499                 ahd_print_devinfo(ahd, &devinfo);
2500                 printf("No SCB found during protocol violation\n");
2501                 AHD_UNCORRECTABLE_ERROR(ahd);
2502                 goto proto_violation_reset;
2503         } else {
2504                 aic_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
2505                 if ((seq_flags & NO_CDB_SENT) != 0) {
2506                         ahd_print_path(ahd, scb);
2507                         printf("No or incomplete CDB sent to device.\n");
2508                         AHD_UNCORRECTABLE_ERROR(ahd);
2509                 } else if ((ahd_inb_scbram(ahd, SCB_CONTROL)
2510                           & STATUS_RCVD) == 0) {
2511                         /*
2512                          * The target never bothered to provide status to
2513                          * us prior to completing the command.  Since we don't
2514                          * know the disposition of this command, we must attempt
2515                          * to abort it.  Assert ATN and prepare to send an abort
2516                          * message.
2517                          */
2518                         ahd_print_path(ahd, scb);
2519                         printf("Completed command without status.\n");
2520                 } else {
2521                         ahd_print_path(ahd, scb);
2522                         printf("Unknown protocol violation.\n");
2523                         AHD_UNCORRECTABLE_ERROR(ahd);
2524                         ahd_dump_card_state(ahd);
2525                 }
2526         }
2527         if ((lastphase & ~P_DATAIN_DT) == 0
2528          || lastphase == P_COMMAND) {
2529 proto_violation_reset:
2530                 /*
2531                  * Target either went directly to data
2532                  * phase or didn't respond to our ATN.
2533                  * The only safe thing to do is to blow
2534                  * it away with a bus reset.
2535                  */
2536                 found = ahd_reset_channel(ahd, 'A', TRUE);
2537                 printf("%s: Issued Channel %c Bus Reset. "
2538                        "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
2539                 AHD_UNCORRECTABLE_ERROR(ahd);
2540         } else {
2541                 /*
2542                  * Leave the selection hardware off in case
2543                  * this abort attempt will affect yet to
2544                  * be sent commands.
2545                  */
2546                 ahd_outb(ahd, SCSISEQ0,
2547                          ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2548                 ahd_assert_atn(ahd);
2549                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
2550                 if (scb == NULL) {
2551                         ahd_print_devinfo(ahd, &devinfo);
2552                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
2553                         ahd->msgout_len = 1;
2554                         ahd->msgout_index = 0;
2555                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2556                 } else {
2557                         ahd_print_path(ahd, scb);
2558                         scb->flags |= SCB_ABORT;
2559                 }
2560                 printf("Protocol violation %s.  Attempting to abort.\n",
2561                        ahd_lookup_phase_entry(curphase)->phasemsg);
2562                 AHD_UNCORRECTABLE_ERROR(ahd);
2563         }
2564 }
2565
2566 /*
2567  * Force renegotiation to occur the next time we initiate
2568  * a command to the current device.
2569  */
2570 static void
2571 ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
2572 {
2573         struct  ahd_initiator_tinfo *targ_info;
2574         struct  ahd_tmode_tstate *tstate;
2575
2576 #ifdef AHD_DEBUG
2577         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
2578                 ahd_print_devinfo(ahd, devinfo);
2579                 printf("Forcing renegotiation\n");
2580         }
2581 #endif
2582         targ_info = ahd_fetch_transinfo(ahd,
2583                                         devinfo->channel,
2584                                         devinfo->our_scsiid,
2585                                         devinfo->target,
2586                                         &tstate);
2587         ahd_update_neg_request(ahd, devinfo, tstate,
2588                                targ_info, AHD_NEG_IF_NON_ASYNC);
2589 }
2590
2591 #define AHD_MAX_STEPS 2000
2592 void
2593 ahd_clear_critical_section(struct ahd_softc *ahd)
2594 {
2595         ahd_mode_state  saved_modes;
2596         int             stepping;
2597         int             steps;
2598         int             first_instr;
2599         u_int           simode0;
2600         u_int           simode1;
2601         u_int           simode3;
2602         u_int           lqimode0;
2603         u_int           lqimode1;
2604         u_int           lqomode0;
2605         u_int           lqomode1;
2606
2607         if (ahd->num_critical_sections == 0)
2608                 return;
2609
2610         stepping = FALSE;
2611         steps = 0;
2612         first_instr = 0;
2613         simode0 = 0;
2614         simode1 = 0;
2615         simode3 = 0;
2616         lqimode0 = 0;
2617         lqimode1 = 0;
2618         lqomode0 = 0;
2619         lqomode1 = 0;
2620         saved_modes = ahd_save_modes(ahd);
2621         for (;;) {
2622                 struct  cs *cs;
2623                 u_int   seqaddr;
2624                 u_int   i;
2625
2626                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2627                 seqaddr = ahd_inw(ahd, CURADDR);
2628
2629                 cs = ahd->critical_sections;
2630                 for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
2631
2632                         if (cs->begin < seqaddr && cs->end >= seqaddr)
2633                                 break;
2634                 }
2635
2636                 if (i == ahd->num_critical_sections)
2637                         break;
2638
2639                 if (steps > AHD_MAX_STEPS) {
2640                         printf("%s: Infinite loop in critical section\n"
2641                                "%s: First Instruction 0x%x now 0x%x\n",
2642                                ahd_name(ahd), ahd_name(ahd), first_instr,
2643                                seqaddr);
2644                         AHD_FATAL_ERROR(ahd);
2645                         ahd_dump_card_state(ahd);
2646                         panic("critical section loop");
2647                 }
2648
2649                 steps++;
2650 #ifdef AHD_DEBUG
2651                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
2652                         printf("%s: Single stepping at 0x%x\n", ahd_name(ahd),
2653                                seqaddr);
2654 #endif
2655                 if (stepping == FALSE) {
2656
2657                         first_instr = seqaddr;
2658                         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2659                         simode0 = ahd_inb(ahd, SIMODE0);
2660                         simode3 = ahd_inb(ahd, SIMODE3);
2661                         lqimode0 = ahd_inb(ahd, LQIMODE0);
2662                         lqimode1 = ahd_inb(ahd, LQIMODE1);
2663                         lqomode0 = ahd_inb(ahd, LQOMODE0);
2664                         lqomode1 = ahd_inb(ahd, LQOMODE1);
2665                         ahd_outb(ahd, SIMODE0, 0);
2666                         ahd_outb(ahd, SIMODE3, 0);
2667                         ahd_outb(ahd, LQIMODE0, 0);
2668                         ahd_outb(ahd, LQIMODE1, 0);
2669                         ahd_outb(ahd, LQOMODE0, 0);
2670                         ahd_outb(ahd, LQOMODE1, 0);
2671                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2672                         simode1 = ahd_inb(ahd, SIMODE1);
2673                         /*
2674                          * We don't clear ENBUSFREE.  Unfortunately
2675                          * we cannot re-enable busfree detection within
2676                          * the current connection, so we must leave it
2677                          * on while single stepping.
2678                          */
2679                         ahd_outb(ahd, SIMODE1, simode1 & ENBUSFREE);
2680                         ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
2681                         stepping = TRUE;
2682                 }
2683                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
2684                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2685                 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
2686                 ahd_outb(ahd, HCNTRL, ahd->unpause);
2687                 while (!ahd_is_paused(ahd))
2688                         aic_delay(200);
2689                 ahd_update_modes(ahd);
2690         }
2691         if (stepping) {
2692                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2693                 ahd_outb(ahd, SIMODE0, simode0);
2694                 ahd_outb(ahd, SIMODE3, simode3);
2695                 ahd_outb(ahd, LQIMODE0, lqimode0);
2696                 ahd_outb(ahd, LQIMODE1, lqimode1);
2697                 ahd_outb(ahd, LQOMODE0, lqomode0);
2698                 ahd_outb(ahd, LQOMODE1, lqomode1);
2699                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2700                 ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
2701                 ahd_outb(ahd, SIMODE1, simode1);
2702                 /*
2703                  * SCSIINT seems to glitch occassionally when
2704                  * the interrupt masks are restored.  Clear SCSIINT
2705                  * one more time so that only persistent errors
2706                  * are seen as a real interrupt.
2707                  */
2708                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2709         }
2710         ahd_restore_modes(ahd, saved_modes);
2711 }
2712
2713 /*
2714  * Clear any pending interrupt status.
2715  */
2716 void
2717 ahd_clear_intstat(struct ahd_softc *ahd)
2718 {
2719         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2720                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2721         /* Clear any interrupt conditions this may have caused */
2722         ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
2723                                  |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
2724         ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
2725                                  |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
2726                                  |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
2727         ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
2728                                  |CLRLQOATNPKT|CLRLQOTCRC);
2729         ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
2730                                  |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
2731         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
2732                 ahd_outb(ahd, CLRLQOINT0, 0);
2733                 ahd_outb(ahd, CLRLQOINT1, 0);
2734         }
2735         ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
2736         ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
2737                                 |CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
2738         ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO
2739                                 |CLRIOERR|CLROVERRUN);
2740         ahd_outb(ahd, CLRINT, CLRSCSIINT);
2741 }
2742
2743 /**************************** Debugging Routines ******************************/
2744 #ifdef AHD_DEBUG
2745 uint32_t ahd_debug = AHD_DEBUG_OPTS;
2746 #endif
2747 void
2748 ahd_print_scb(struct scb *scb)
2749 {
2750         struct hardware_scb *hscb;
2751         int i;
2752
2753         hscb = scb->hscb;
2754         printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
2755                (void *)scb,
2756                hscb->control,
2757                hscb->scsiid,
2758                hscb->lun,
2759                hscb->cdb_len);
2760         printf("Shared Data: ");
2761         for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
2762                 printf("%#02x", hscb->shared_data.idata.cdb[i]);
2763         printf("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
2764                (uint32_t)((aic_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
2765                (uint32_t)(aic_le64toh(hscb->dataptr) & 0xFFFFFFFF),
2766                aic_le32toh(hscb->datacnt),
2767                aic_le32toh(hscb->sgptr),
2768                SCB_GET_TAG(scb));
2769         ahd_dump_sglist(scb);
2770 }
2771
2772 void
2773 ahd_dump_sglist(struct scb *scb)
2774 {
2775         int i;
2776
2777         if (scb->sg_count > 0) {
2778                 if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
2779                         struct ahd_dma64_seg *sg_list;
2780
2781                         sg_list = (struct ahd_dma64_seg*)scb->sg_list;
2782                         for (i = 0; i < scb->sg_count; i++) {
2783                                 uint64_t addr;
2784                                 uint32_t len;
2785
2786                                 addr = aic_le64toh(sg_list[i].addr);
2787                                 len = aic_le32toh(sg_list[i].len);
2788                                 printf("sg[%d] - Addr 0x%x%x : Length %d%s\n",
2789                                        i,
2790                                        (uint32_t)((addr >> 32) & 0xFFFFFFFF),
2791                                        (uint32_t)(addr & 0xFFFFFFFF),
2792                                        sg_list[i].len & AHD_SG_LEN_MASK,
2793                                        (sg_list[i].len & AHD_DMA_LAST_SEG)
2794                                      ? " Last" : "");
2795                         }
2796                 } else {
2797                         struct ahd_dma_seg *sg_list;
2798
2799                         sg_list = (struct ahd_dma_seg*)scb->sg_list;
2800                         for (i = 0; i < scb->sg_count; i++) {
2801                                 uint32_t len;
2802
2803                                 len = aic_le32toh(sg_list[i].len);
2804                                 printf("sg[%d] - Addr 0x%x%x : Length %d%s\n",
2805                                        i,
2806                                        (len & AHD_SG_HIGH_ADDR_MASK) >> 24,
2807                                        aic_le32toh(sg_list[i].addr),
2808                                        len & AHD_SG_LEN_MASK,
2809                                        len & AHD_DMA_LAST_SEG ? " Last" : "");
2810                         }
2811                 }
2812         }
2813 }
2814
2815 /************************* Transfer Negotiation *******************************/
2816 /*
2817  * Allocate per target mode instance (ID we respond to as a target)
2818  * transfer negotiation data structures.
2819  */
2820 static struct ahd_tmode_tstate *
2821 ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
2822 {
2823         struct ahd_tmode_tstate *master_tstate;
2824         struct ahd_tmode_tstate *tstate;
2825         int i;
2826
2827         master_tstate = ahd->enabled_targets[ahd->our_id];
2828         if (ahd->enabled_targets[scsi_id] != NULL
2829          && ahd->enabled_targets[scsi_id] != master_tstate)
2830                 panic("%s: ahd_alloc_tstate - Target already allocated",
2831                       ahd_name(ahd));
2832         tstate = malloc(sizeof(*tstate), M_DEVBUF, M_NOWAIT);
2833         if (tstate == NULL)
2834                 return (NULL);
2835
2836         /*
2837          * If we have allocated a master tstate, copy user settings from
2838          * the master tstate (taken from SRAM or the EEPROM) for this
2839          * channel, but reset our current and goal settings to async/narrow
2840          * until an initiator talks to us.
2841          */
2842         if (master_tstate != NULL) {
2843                 memcpy(tstate, master_tstate, sizeof(*tstate));
2844                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
2845                 for (i = 0; i < 16; i++) {
2846                         memset(&tstate->transinfo[i].curr, 0,
2847                               sizeof(tstate->transinfo[i].curr));
2848                         memset(&tstate->transinfo[i].goal, 0,
2849                               sizeof(tstate->transinfo[i].goal));
2850                 }
2851         } else
2852                 memset(tstate, 0, sizeof(*tstate));
2853         ahd->enabled_targets[scsi_id] = tstate;
2854         return (tstate);
2855 }
2856
2857 #ifdef AHD_TARGET_MODE
2858 /*
2859  * Free per target mode instance (ID we respond to as a target)
2860  * transfer negotiation data structures.
2861  */
2862 static void
2863 ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
2864 {
2865         struct ahd_tmode_tstate *tstate;
2866
2867         /*
2868          * Don't clean up our "master" tstate.
2869          * It has our default user settings.
2870          */
2871         if (scsi_id == ahd->our_id
2872          && force == FALSE)
2873                 return;
2874
2875         tstate = ahd->enabled_targets[scsi_id];
2876         if (tstate != NULL)
2877                 free(tstate, M_DEVBUF);
2878         ahd->enabled_targets[scsi_id] = NULL;
2879 }
2880 #endif
2881
2882 /*
2883  * Called when we have an active connection to a target on the bus,
2884  * this function finds the nearest period to the input period limited
2885  * by the capabilities of the bus connectivity of and sync settings for
2886  * the target.
2887  */
2888 void
2889 ahd_devlimited_syncrate(struct ahd_softc *ahd,
2890                         struct ahd_initiator_tinfo *tinfo,
2891                         u_int *period, u_int *ppr_options, role_t role)
2892 {
2893         struct  ahd_transinfo *transinfo;
2894         u_int   maxsync;
2895
2896         if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
2897          && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
2898                 maxsync = AHD_SYNCRATE_PACED;
2899         } else {
2900                 maxsync = AHD_SYNCRATE_ULTRA;
2901                 /* Can't do DT related options on an SE bus */
2902                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
2903         }
2904         /*
2905          * Never allow a value higher than our current goal
2906          * period otherwise we may allow a target initiated
2907          * negotiation to go above the limit as set by the
2908          * user.  In the case of an initiator initiated
2909          * sync negotiation, we limit based on the user
2910          * setting.  This allows the system to still accept
2911          * incoming negotiations even if target initiated
2912          * negotiation is not performed.
2913          */
2914         if (role == ROLE_TARGET)
2915                 transinfo = &tinfo->user;
2916         else
2917                 transinfo = &tinfo->goal;
2918         *ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
2919         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
2920                 maxsync = MAX(maxsync, AHD_SYNCRATE_ULTRA2);
2921                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2922         }
2923         if (transinfo->period == 0) {
2924                 *period = 0;
2925                 *ppr_options = 0;
2926         } else {
2927                 *period = MAX(*period, transinfo->period);
2928                 ahd_find_syncrate(ahd, period, ppr_options, maxsync);
2929         }
2930 }
2931
2932 /*
2933  * Look up the valid period to SCSIRATE conversion in our table.
2934  * Return the period and offset that should be sent to the target
2935  * if this was the beginning of an SDTR.
2936  */
2937 void
2938 ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
2939                   u_int *ppr_options, u_int maxsync)
2940 {
2941         if (*period < maxsync)
2942                 *period = maxsync;
2943
2944         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
2945          && *period > AHD_SYNCRATE_MIN_DT)
2946                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2947
2948         if (*period > AHD_SYNCRATE_MIN)
2949                 *period = 0;
2950
2951         /* Honor PPR option conformance rules. */
2952         if (*period > AHD_SYNCRATE_PACED)
2953                 *ppr_options &= ~MSG_EXT_PPR_RTI;
2954
2955         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
2956                 *ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
2957
2958         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
2959                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
2960
2961         /* Skip all PACED only entries if IU is not available */
2962         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
2963          && *period < AHD_SYNCRATE_DT)
2964                 *period = AHD_SYNCRATE_DT;
2965
2966         /* Skip all DT only entries if DT is not available */
2967         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
2968          && *period < AHD_SYNCRATE_ULTRA2)
2969                 *period = AHD_SYNCRATE_ULTRA2;
2970 }
2971
2972 /*
2973  * Truncate the given synchronous offset to a value the
2974  * current adapter type and syncrate are capable of.
2975  */
2976 void
2977 ahd_validate_offset(struct ahd_softc *ahd,
2978                     struct ahd_initiator_tinfo *tinfo,
2979                     u_int period, u_int *offset, int wide,
2980                     role_t role)
2981 {
2982         u_int maxoffset;
2983
2984         /* Limit offset to what we can do */
2985         if (period == 0)
2986                 maxoffset = 0;
2987         else if (period <= AHD_SYNCRATE_PACED) {
2988                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0)
2989                         maxoffset = MAX_OFFSET_PACED_BUG;
2990                 else
2991                         maxoffset = MAX_OFFSET_PACED;
2992         } else
2993                 maxoffset = MAX_OFFSET_NON_PACED;
2994         *offset = MIN(*offset, maxoffset);
2995         if (tinfo != NULL) {
2996                 if (role == ROLE_TARGET)
2997                         *offset = MIN(*offset, tinfo->user.offset);
2998                 else
2999                         *offset = MIN(*offset, tinfo->goal.offset);
3000         }
3001 }
3002
3003 /*
3004  * Truncate the given transfer width parameter to a value the
3005  * current adapter type is capable of.
3006  */
3007 void
3008 ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
3009                    u_int *bus_width, role_t role)
3010 {
3011         switch (*bus_width) {
3012         default:
3013                 if (ahd->features & AHD_WIDE) {
3014                         /* Respond Wide */
3015                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
3016                         break;
3017                 }
3018                 /* FALLTHROUGH */
3019         case MSG_EXT_WDTR_BUS_8_BIT:
3020                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
3021                 break;
3022         }
3023         if (tinfo != NULL) {
3024                 if (role == ROLE_TARGET)
3025                         *bus_width = MIN(tinfo->user.width, *bus_width);
3026                 else
3027                         *bus_width = MIN(tinfo->goal.width, *bus_width);
3028         }
3029 }
3030
3031 /*
3032  * Update the bitmask of targets for which the controller should
3033  * negotiate with at the next convenient oportunity.  This currently
3034  * means the next time we send the initial identify messages for
3035  * a new transaction.
3036  */
3037 int
3038 ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3039                        struct ahd_tmode_tstate *tstate,
3040                        struct ahd_initiator_tinfo *tinfo, ahd_neg_type neg_type)
3041 {
3042         u_int auto_negotiate_orig;
3043
3044         auto_negotiate_orig = tstate->auto_negotiate;
3045         if (neg_type == AHD_NEG_ALWAYS) {
3046                 /*
3047                  * Force our "current" settings to be
3048                  * unknown so that unless a bus reset
3049                  * occurs the need to renegotiate is
3050                  * recorded persistently.
3051                  */
3052                 if ((ahd->features & AHD_WIDE) != 0)
3053                         tinfo->curr.width = AHD_WIDTH_UNKNOWN;
3054                 tinfo->curr.period = AHD_PERIOD_UNKNOWN;
3055                 tinfo->curr.offset = AHD_OFFSET_UNKNOWN;
3056         }
3057         if (tinfo->curr.period != tinfo->goal.period
3058          || tinfo->curr.width != tinfo->goal.width
3059          || tinfo->curr.offset != tinfo->goal.offset
3060          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
3061          || (neg_type == AHD_NEG_IF_NON_ASYNC
3062           && (tinfo->goal.offset != 0
3063            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
3064            || tinfo->goal.ppr_options != 0)))
3065                 tstate->auto_negotiate |= devinfo->target_mask;
3066         else
3067                 tstate->auto_negotiate &= ~devinfo->target_mask;
3068
3069         return (auto_negotiate_orig != tstate->auto_negotiate);
3070 }
3071
3072 /*
3073  * Update the user/goal/curr tables of synchronous negotiation
3074  * parameters as well as, in the case of a current or active update,
3075  * any data structures on the host controller.  In the case of an
3076  * active update, the specified target is currently talking to us on
3077  * the bus, so the transfer parameter update must take effect
3078  * immediately.
3079  */
3080 void
3081 ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3082                  u_int period, u_int offset, u_int ppr_options,
3083                  u_int type, int paused)
3084 {
3085         struct  ahd_initiator_tinfo *tinfo;
3086         struct  ahd_tmode_tstate *tstate;
3087         u_int   old_period;
3088         u_int   old_offset;
3089         u_int   old_ppr;
3090         int     active;
3091         int     update_needed;
3092
3093         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3094         update_needed = 0;
3095
3096         if (period == 0 || offset == 0) {
3097                 period = 0;
3098                 offset = 0;
3099         }
3100
3101         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3102                                     devinfo->target, &tstate);
3103
3104         if ((type & AHD_TRANS_USER) != 0) {
3105                 tinfo->user.period = period;
3106                 tinfo->user.offset = offset;
3107                 tinfo->user.ppr_options = ppr_options;
3108         }
3109
3110         if ((type & AHD_TRANS_GOAL) != 0) {
3111                 tinfo->goal.period = period;
3112                 tinfo->goal.offset = offset;
3113                 tinfo->goal.ppr_options = ppr_options;
3114         }
3115
3116         old_period = tinfo->curr.period;
3117         old_offset = tinfo->curr.offset;
3118         old_ppr    = tinfo->curr.ppr_options;
3119
3120         if ((type & AHD_TRANS_CUR) != 0
3121          && (old_period != period
3122           || old_offset != offset
3123           || old_ppr != ppr_options)) {
3124
3125                 update_needed++;
3126
3127                 tinfo->curr.period = period;
3128                 tinfo->curr.offset = offset;
3129                 tinfo->curr.ppr_options = ppr_options;
3130
3131                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
3132                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
3133                 if (bootverbose) {
3134                         if (offset != 0) {
3135                                 int options;
3136
3137                                 printf("%s: target %d synchronous with "
3138                                        "period = 0x%x, offset = 0x%x",
3139                                        ahd_name(ahd), devinfo->target,
3140                                        period, offset);
3141                                 options = 0;
3142                                 if ((ppr_options & MSG_EXT_PPR_RD_STRM) != 0) {
3143                                         printf("(RDSTRM");
3144                                         options++;
3145                                 }
3146                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0) {
3147                                         printf("%s", options ? "|DT" : "(DT");
3148                                         options++;
3149                                 }
3150                                 if ((ppr_options & MSG_EXT_PPR_IU_REQ) != 0) {
3151                                         printf("%s", options ? "|IU" : "(IU");
3152                                         options++;
3153                                 }
3154                                 if ((ppr_options & MSG_EXT_PPR_RTI) != 0) {
3155                                         printf("%s", options ? "|RTI" : "(RTI");
3156                                         options++;
3157                                 }
3158                                 if ((ppr_options & MSG_EXT_PPR_QAS_REQ) != 0) {
3159                                         printf("%s", options ? "|QAS" : "(QAS");
3160                                         options++;
3161                                 }
3162                                 if (options != 0)
3163                                         printf(")\n");
3164                                 else
3165                                         printf("\n");
3166                         } else {
3167                                 printf("%s: target %d using "
3168                                        "asynchronous transfers%s\n",
3169                                        ahd_name(ahd), devinfo->target,
3170                                        (ppr_options & MSG_EXT_PPR_QAS_REQ) != 0
3171                                      ?  "(QAS)" : "");
3172                         }
3173                 }
3174         }
3175         /*
3176          * Always refresh the neg-table to handle the case of the
3177          * sequencer setting the ENATNO bit for a MK_MESSAGE request.
3178          * We will always renegotiate in that case if this is a
3179          * packetized request.  Also manage the busfree expected flag
3180          * from this common routine so that we catch changes due to
3181          * WDTR or SDTR messages.
3182          */
3183         if ((type & AHD_TRANS_CUR) != 0) {
3184                 if (!paused)
3185                         ahd_pause(ahd);
3186                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3187                 if (!paused)
3188                         ahd_unpause(ahd);
3189                 if (ahd->msg_type != MSG_TYPE_NONE) {
3190                         if ((old_ppr & MSG_EXT_PPR_IU_REQ)
3191                          != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
3192 #ifdef AHD_DEBUG
3193                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3194                                         ahd_print_devinfo(ahd, devinfo);
3195                                         printf("Expecting IU Change busfree\n");
3196                                 }
3197 #endif
3198                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
3199                                                |  MSG_FLAG_IU_REQ_CHANGED;
3200                         }
3201                         if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
3202 #ifdef AHD_DEBUG
3203                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3204                                         printf("PPR with IU_REQ outstanding\n");
3205 #endif
3206                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
3207                         }
3208                 }
3209         }
3210
3211         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3212                                                 tinfo, AHD_NEG_TO_GOAL);
3213
3214         if (update_needed && active)
3215                 ahd_update_pending_scbs(ahd);
3216 }
3217
3218 /*
3219  * Update the user/goal/curr tables of wide negotiation
3220  * parameters as well as, in the case of a current or active update,
3221  * any data structures on the host controller.  In the case of an
3222  * active update, the specified target is currently talking to us on
3223  * the bus, so the transfer parameter update must take effect
3224  * immediately.
3225  */
3226 void
3227 ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3228               u_int width, u_int type, int paused)
3229 {
3230         struct  ahd_initiator_tinfo *tinfo;
3231         struct  ahd_tmode_tstate *tstate;
3232         u_int   oldwidth;
3233         int     active;
3234         int     update_needed;
3235
3236         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3237         update_needed = 0;
3238         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3239                                     devinfo->target, &tstate);
3240
3241         if ((type & AHD_TRANS_USER) != 0)
3242                 tinfo->user.width = width;
3243
3244         if ((type & AHD_TRANS_GOAL) != 0)
3245                 tinfo->goal.width = width;
3246
3247         oldwidth = tinfo->curr.width;
3248         if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
3249
3250                 update_needed++;
3251
3252                 tinfo->curr.width = width;
3253                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
3254                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
3255                 if (bootverbose) {
3256                         printf("%s: target %d using %dbit transfers\n",
3257                                ahd_name(ahd), devinfo->target,
3258                                8 * (0x01 << width));
3259                 }
3260         }
3261
3262         if ((type & AHD_TRANS_CUR) != 0) {
3263                 if (!paused)
3264                         ahd_pause(ahd);
3265                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3266                 if (!paused)
3267                         ahd_unpause(ahd);
3268         }
3269
3270         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3271                                                 tinfo, AHD_NEG_TO_GOAL);
3272         if (update_needed && active)
3273                 ahd_update_pending_scbs(ahd);
3274
3275 }
3276
3277 /*
3278  * Update the current state of tagged queuing for a given target.
3279  */
3280 void
3281 ahd_set_tags(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3282              ahd_queue_alg alg)
3283 {
3284         ahd_platform_set_tags(ahd, devinfo, alg);
3285         ahd_send_async(ahd, devinfo->channel, devinfo->target,
3286                        devinfo->lun, AC_TRANSFER_NEG, &alg);
3287 }
3288
3289 static void
3290 ahd_update_neg_table(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3291                      struct ahd_transinfo *tinfo)
3292 {
3293         ahd_mode_state  saved_modes;
3294         u_int           period;
3295         u_int           ppr_opts;
3296         u_int           con_opts;
3297         u_int           offset;
3298         u_int           saved_negoaddr;
3299         uint8_t         iocell_opts[sizeof(ahd->iocell_opts)];
3300
3301         saved_modes = ahd_save_modes(ahd);
3302         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3303
3304         saved_negoaddr = ahd_inb(ahd, NEGOADDR);
3305         ahd_outb(ahd, NEGOADDR, devinfo->target);
3306         period = tinfo->period;
3307         offset = tinfo->offset;
3308         memcpy(iocell_opts, ahd->iocell_opts, sizeof(ahd->iocell_opts));
3309         ppr_opts = tinfo->ppr_options & (MSG_EXT_PPR_QAS_REQ|MSG_EXT_PPR_DT_REQ
3310                                         |MSG_EXT_PPR_IU_REQ|MSG_EXT_PPR_RTI);
3311         con_opts = 0;
3312         if (period == 0)
3313                 period = AHD_SYNCRATE_ASYNC;
3314         if (period == AHD_SYNCRATE_160) {
3315
3316                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
3317                         /*
3318                          * When the SPI4 spec was finalized, PACE transfers
3319                          * was not made a configurable option in the PPR
3320                          * message.  Instead it is assumed to be enabled for
3321                          * any syncrate faster than 80MHz.  Nevertheless,
3322                          * Harpoon2A4 allows this to be configurable.
3323                          *
3324                          * Harpoon2A4 also assumes at most 2 data bytes per
3325                          * negotiated REQ/ACK offset.  Paced transfers take
3326                          * 4, so we must adjust our offset.
3327                          */
3328                         ppr_opts |= PPROPT_PACE;
3329                         offset *= 2;
3330
3331                         /*
3332                          * Harpoon2A assumed that there would be a
3333                          * fallback rate between 160MHz and 80Mhz,
3334                          * so 7 is used as the period factor rather
3335                          * than 8 for 160MHz.
3336                          */
3337                         period = AHD_SYNCRATE_REVA_160;
3338                 }
3339                 if ((tinfo->ppr_options & MSG_EXT_PPR_PCOMP_EN) == 0)
3340                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
3341                             ~AHD_PRECOMP_MASK;
3342         } else {
3343                 /*
3344                  * Precomp should be disabled for non-paced transfers.
3345                  */
3346                 iocell_opts[AHD_PRECOMP_SLEW_INDEX] &= ~AHD_PRECOMP_MASK;
3347
3348                 if ((ahd->features & AHD_NEW_IOCELL_OPTS) != 0
3349                  && (ppr_opts & MSG_EXT_PPR_DT_REQ) != 0
3350                  && (ppr_opts & MSG_EXT_PPR_IU_REQ) == 0) {
3351                         /*
3352                          * Slow down our CRC interval to be
3353                          * compatible with non-packetized
3354                          * U160 devices that can't handle a
3355                          * CRC at full speed.
3356                          */
3357                         con_opts |= ENSLOWCRC;
3358                 }
3359
3360                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
3361                         /*
3362                          * On H2A4, revert to a slower slewrate
3363                          * on non-paced transfers.
3364                          */
3365                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
3366                             ~AHD_SLEWRATE_MASK;
3367                 }
3368         }
3369
3370         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP_SLEW);
3371         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_PRECOMP_SLEW_INDEX]);
3372         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_AMPLITUDE);
3373         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_AMPLITUDE_INDEX]);
3374
3375         ahd_outb(ahd, NEGPERIOD, period);
3376         ahd_outb(ahd, NEGPPROPTS, ppr_opts);
3377         ahd_outb(ahd, NEGOFFSET, offset);
3378
3379         if (tinfo->width == MSG_EXT_WDTR_BUS_16_BIT)
3380                 con_opts |= WIDEXFER;
3381
3382         /*
3383          * During packetized transfers, the target will
3384          * give us the oportunity to send command packets
3385          * without us asserting attention.
3386          */
3387         if ((tinfo->ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
3388                 con_opts |= ENAUTOATNO;
3389         ahd_outb(ahd, NEGCONOPTS, con_opts);
3390         ahd_outb(ahd, NEGOADDR, saved_negoaddr);
3391         ahd_restore_modes(ahd, saved_modes);
3392 }
3393
3394 /*
3395  * When the transfer settings for a connection change, setup for
3396  * negotiation in pending SCBs to effect the change as quickly as
3397  * possible.  We also cancel any negotiations that are scheduled
3398  * for inflight SCBs that have not been started yet.
3399  */
3400 static void
3401 ahd_update_pending_scbs(struct ahd_softc *ahd)
3402 {
3403         struct          scb *pending_scb;
3404         int             pending_scb_count;
3405         int             paused;
3406         u_int           saved_scbptr;
3407         ahd_mode_state  saved_modes;
3408
3409         /*
3410          * Traverse the pending SCB list and ensure that all of the
3411          * SCBs there have the proper settings.  We can only safely
3412          * clear the negotiation required flag (setting requires the
3413          * execution queue to be modified) and this is only possible
3414          * if we are not already attempting to select out for this
3415          * SCB.  For this reason, all callers only call this routine
3416          * if we are changing the negotiation settings for the currently
3417          * active transaction on the bus.
3418          */
3419         pending_scb_count = 0;
3420         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
3421                 struct ahd_devinfo devinfo;
3422                 struct ahd_initiator_tinfo *tinfo;
3423                 struct ahd_tmode_tstate *tstate;
3424
3425                 ahd_scb_devinfo(ahd, &devinfo, pending_scb);
3426                 tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
3427                                             devinfo.our_scsiid,
3428                                             devinfo.target, &tstate);
3429                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
3430                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
3431                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
3432                         pending_scb->hscb->control &= ~MK_MESSAGE;
3433                 }
3434                 ahd_sync_scb(ahd, pending_scb,
3435                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
3436                 pending_scb_count++;
3437         }
3438
3439         if (pending_scb_count == 0)
3440                 return;
3441
3442         if (ahd_is_paused(ahd)) {
3443                 paused = 1;
3444         } else {
3445                 paused = 0;
3446                 ahd_pause(ahd);
3447         }
3448
3449         /*
3450          * Force the sequencer to reinitialize the selection for
3451          * the command at the head of the execution queue if it
3452          * has already been setup.  The negotiation changes may
3453          * effect whether we select-out with ATN.  It is only
3454          * safe to clear ENSELO when the bus is not free and no
3455          * selection is in progres or completed.
3456          */
3457         saved_modes = ahd_save_modes(ahd);
3458         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3459         if ((ahd_inb(ahd, SCSISIGI) & BSYI) != 0
3460          && (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) == 0)
3461                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
3462         saved_scbptr = ahd_get_scbptr(ahd);
3463         /* Ensure that the hscbs down on the card match the new information */
3464         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
3465                 u_int   scb_tag;
3466                 u_int   control;
3467
3468                 scb_tag = SCB_GET_TAG(pending_scb);
3469                 ahd_set_scbptr(ahd, scb_tag);
3470                 control = ahd_inb_scbram(ahd, SCB_CONTROL);
3471                 control &= ~MK_MESSAGE;
3472                 control |= pending_scb->hscb->control & MK_MESSAGE;
3473                 ahd_outb(ahd, SCB_CONTROL, control);
3474         }
3475         ahd_set_scbptr(ahd, saved_scbptr);
3476         ahd_restore_modes(ahd, saved_modes);
3477
3478         if (paused == 0)
3479                 ahd_unpause(ahd);
3480 }
3481
3482 /**************************** Pathing Information *****************************/
3483 static void
3484 ahd_fetch_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3485 {
3486         ahd_mode_state  saved_modes;
3487         u_int           saved_scsiid;
3488         role_t          role;
3489         int             our_id;
3490
3491         saved_modes = ahd_save_modes(ahd);
3492         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3493
3494         if (ahd_inb(ahd, SSTAT0) & TARGET)
3495                 role = ROLE_TARGET;
3496         else
3497                 role = ROLE_INITIATOR;
3498
3499         if (role == ROLE_TARGET
3500          && (ahd_inb(ahd, SEQ_FLAGS) & CMDPHASE_PENDING) != 0) {
3501                 /* We were selected, so pull our id from TARGIDIN */
3502                 our_id = ahd_inb(ahd, TARGIDIN) & OID;
3503         } else if (role == ROLE_TARGET)
3504                 our_id = ahd_inb(ahd, TOWNID);
3505         else
3506                 our_id = ahd_inb(ahd, IOWNID);
3507
3508         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
3509         ahd_compile_devinfo(devinfo,
3510                             our_id,
3511                             SCSIID_TARGET(ahd, saved_scsiid),
3512                             ahd_inb(ahd, SAVED_LUN),
3513                             SCSIID_CHANNEL(ahd, saved_scsiid),
3514                             role);
3515         ahd_restore_modes(ahd, saved_modes);
3516 }
3517
3518 void
3519 ahd_print_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3520 {
3521         printf("%s:%c:%d:%d: ", ahd_name(ahd), 'A',
3522                devinfo->target, devinfo->lun);
3523 }
3524
3525 struct ahd_phase_table_entry*
3526 ahd_lookup_phase_entry(int phase)
3527 {
3528         struct ahd_phase_table_entry *entry;
3529         struct ahd_phase_table_entry *last_entry;
3530
3531         /*
3532          * num_phases doesn't include the default entry which
3533          * will be returned if the phase doesn't match.
3534          */
3535         last_entry = &ahd_phase_table[num_phases];
3536         for (entry = ahd_phase_table; entry < last_entry; entry++) {
3537                 if (phase == entry->phase)
3538                         break;
3539         }
3540         return (entry);
3541 }
3542
3543 void
3544 ahd_compile_devinfo(struct ahd_devinfo *devinfo, u_int our_id, u_int target,
3545                     u_int lun, char channel, role_t role)
3546 {
3547         devinfo->our_scsiid = our_id;
3548         devinfo->target = target;
3549         devinfo->lun = lun;
3550         devinfo->target_offset = target;
3551         devinfo->channel = channel;
3552         devinfo->role = role;
3553         if (channel == 'B')
3554                 devinfo->target_offset += 8;
3555         devinfo->target_mask = (0x01 << devinfo->target_offset);
3556 }
3557
3558 static void
3559 ahd_scb_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3560                 struct scb *scb)
3561 {
3562         role_t  role;
3563         int     our_id;
3564
3565         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
3566         role = ROLE_INITIATOR;
3567         if ((scb->hscb->control & TARGET_SCB) != 0)
3568                 role = ROLE_TARGET;
3569         ahd_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahd, scb),
3570                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahd, scb), role);
3571 }
3572
3573
3574 /************************ Message Phase Processing ****************************/
3575 /*
3576  * When an initiator transaction with the MK_MESSAGE flag either reconnects
3577  * or enters the initial message out phase, we are interrupted.  Fill our
3578  * outgoing message buffer with the appropriate message and beging handing
3579  * the message phase(s) manually.
3580  */
3581 static void
3582 ahd_setup_initiator_msgout(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3583                            struct scb *scb)
3584 {
3585         /*
3586          * To facilitate adding multiple messages together,
3587          * each routine should increment the index and len
3588          * variables instead of setting them explicitly.
3589          */
3590         ahd->msgout_index = 0;
3591         ahd->msgout_len = 0;
3592
3593         if (ahd_currently_packetized(ahd))
3594                 ahd->msg_flags |= MSG_FLAG_PACKETIZED;
3595
3596         if (ahd->send_msg_perror
3597          && ahd_inb(ahd, MSG_OUT) == HOST_MSG) {
3598                 ahd->msgout_buf[ahd->msgout_index++] = ahd->send_msg_perror;
3599                 ahd->msgout_len++;
3600                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3601 #ifdef AHD_DEBUG
3602                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3603                         printf("Setting up for Parity Error delivery\n");
3604 #endif
3605                 return;
3606         } else if (scb == NULL) {
3607                 printf("%s: WARNING. No pending message for "
3608                        "I_T msgin.  Issuing NO-OP\n", ahd_name(ahd));
3609                 AHD_CORRECTABLE_ERROR(ahd);
3610                 ahd->msgout_buf[ahd->msgout_index++] = MSG_NOOP;
3611                 ahd->msgout_len++;
3612                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3613                 return;
3614         }
3615
3616         if ((scb->flags & SCB_DEVICE_RESET) == 0
3617          && (scb->flags & SCB_PACKETIZED) == 0
3618          && ahd_inb(ahd, MSG_OUT) == MSG_IDENTIFYFLAG) {
3619                 u_int identify_msg;
3620
3621                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
3622                 if ((scb->hscb->control & DISCENB) != 0)
3623                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
3624                 ahd->msgout_buf[ahd->msgout_index++] = identify_msg;
3625                 ahd->msgout_len++;
3626
3627                 if ((scb->hscb->control & TAG_ENB) != 0) {
3628                         ahd->msgout_buf[ahd->msgout_index++] =
3629                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
3630                         ahd->msgout_buf[ahd->msgout_index++] = SCB_GET_TAG(scb);
3631                         ahd->msgout_len += 2;
3632                 }
3633         }
3634
3635         if (scb->flags & SCB_DEVICE_RESET) {
3636                 ahd->msgout_buf[ahd->msgout_index++] = MSG_BUS_DEV_RESET;
3637                 ahd->msgout_len++;
3638                 ahd_print_path(ahd, scb);
3639                 printf("Bus Device Reset Message Sent\n");
3640                 AHD_CORRECTABLE_ERROR(ahd);
3641                 /*
3642                  * Clear our selection hardware in advance of
3643                  * the busfree.  We may have an entry in the waiting
3644                  * Q for this target, and we don't want to go about
3645                  * selecting while we handle the busfree and blow it
3646                  * away.
3647                  */
3648                 ahd_outb(ahd, SCSISEQ0, 0);
3649         } else if ((scb->flags & SCB_ABORT) != 0) {
3650
3651                 if ((scb->hscb->control & TAG_ENB) != 0) {
3652                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT_TAG;
3653                 } else {
3654                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT;
3655                 }
3656                 ahd->msgout_len++;
3657                 ahd_print_path(ahd, scb);
3658                 printf("Abort%s Message Sent\n",
3659                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
3660                 AHD_CORRECTABLE_ERROR(ahd);
3661                 /*
3662                  * Clear our selection hardware in advance of
3663                  * the busfree.  We may have an entry in the waiting
3664                  * Q for this target, and we don't want to go about
3665                  * selecting while we handle the busfree and blow it
3666                  * away.
3667                  */
3668                 ahd_outb(ahd, SCSISEQ0, 0);
3669         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
3670                 ahd_build_transfer_msg(ahd, devinfo);
3671                 /*
3672                  * Clear our selection hardware in advance of potential
3673                  * PPR IU status change busfree.  We may have an entry in
3674                  * the waiting Q for this target, and we don't want to go
3675                  * about selecting while we handle the busfree and blow
3676                  * it away.
3677                  */
3678                 ahd_outb(ahd, SCSISEQ0, 0);
3679         } else {
3680                 printf("ahd_intr: AWAITING_MSG for an SCB that "
3681                        "does not have a waiting message\n");
3682                 printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
3683                        devinfo->target_mask);
3684                 AHD_FATAL_ERROR(ahd);
3685                 panic("SCB = %d, SCB Control = %x:%x, MSG_OUT = %x "
3686                       "SCB flags = %x", SCB_GET_TAG(scb), scb->hscb->control,
3687                       ahd_inb_scbram(ahd, SCB_CONTROL), ahd_inb(ahd, MSG_OUT),
3688                       scb->flags);
3689         }
3690
3691         /*
3692          * Clear the MK_MESSAGE flag from the SCB so we aren't
3693          * asked to send this message again.
3694          */
3695         ahd_outb(ahd, SCB_CONTROL,
3696                  ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
3697         scb->hscb->control &= ~MK_MESSAGE;
3698         ahd->msgout_index = 0;
3699         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3700 }
3701
3702 /*
3703  * Build an appropriate transfer negotiation message for the
3704  * currently active target.
3705  */
3706 static void
3707 ahd_build_transfer_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3708 {
3709         /*
3710          * We need to initiate transfer negotiations.
3711          * If our current and goal settings are identical,
3712          * we want to renegotiate due to a check condition.
3713          */
3714         struct  ahd_initiator_tinfo *tinfo;
3715         struct  ahd_tmode_tstate *tstate;
3716         int     dowide;
3717         int     dosync;
3718         int     doppr;
3719         u_int   period;
3720         u_int   ppr_options;
3721         u_int   offset;
3722
3723         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3724                                     devinfo->target, &tstate);
3725         /*
3726          * Filter our period based on the current connection.
3727          * If we can't perform DT transfers on this segment (not in LVD
3728          * mode for instance), then our decision to issue a PPR message
3729          * may change.
3730          */
3731         period = tinfo->goal.period;
3732         offset = tinfo->goal.offset;
3733         ppr_options = tinfo->goal.ppr_options;
3734         /* Target initiated PPR is not allowed in the SCSI spec */
3735         if (devinfo->role == ROLE_TARGET)
3736                 ppr_options = 0;
3737         ahd_devlimited_syncrate(ahd, tinfo, &period,
3738                                 &ppr_options, devinfo->role);
3739         dowide = tinfo->curr.width != tinfo->goal.width;
3740         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
3741         /*
3742          * Only use PPR if we have options that need it, even if the device
3743          * claims to support it.  There might be an expander in the way
3744          * that doesn't.
3745          */
3746         doppr = ppr_options != 0;
3747
3748         if (!dowide && !dosync && !doppr) {
3749                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
3750                 dosync = tinfo->goal.offset != 0;
3751         }
3752
3753         if (!dowide && !dosync && !doppr) {
3754                 /*
3755                  * Force async with a WDTR message if we have a wide bus,
3756                  * or just issue an SDTR with a 0 offset.
3757                  */
3758                 if ((ahd->features & AHD_WIDE) != 0)
3759                         dowide = 1;
3760                 else
3761                         dosync = 1;
3762
3763                 if (bootverbose) {
3764                         ahd_print_devinfo(ahd, devinfo);
3765                         printf("Ensuring async\n");
3766                 }
3767         }
3768         /* Target initiated PPR is not allowed in the SCSI spec */
3769         if (devinfo->role == ROLE_TARGET)
3770                 doppr = 0;
3771
3772         /*
3773          * Both the PPR message and SDTR message require the
3774          * goal syncrate to be limited to what the target device
3775          * is capable of handling (based on whether an LVD->SE
3776          * expander is on the bus), so combine these two cases.
3777          * Regardless, guarantee that if we are using WDTR and SDTR
3778          * messages that WDTR comes first.
3779          */
3780         if (doppr || (dosync && !dowide)) {
3781
3782                 offset = tinfo->goal.offset;
3783                 ahd_validate_offset(ahd, tinfo, period, &offset,
3784                                     doppr ? tinfo->goal.width
3785                                           : tinfo->curr.width,
3786                                     devinfo->role);
3787                 if (doppr) {
3788                         ahd_construct_ppr(ahd, devinfo, period, offset,
3789                                           tinfo->goal.width, ppr_options);
3790                 } else {
3791                         ahd_construct_sdtr(ahd, devinfo, period, offset);
3792                 }
3793         } else {
3794                 ahd_construct_wdtr(ahd, devinfo, tinfo->goal.width);
3795         }
3796 }
3797
3798 /*
3799  * Build a synchronous negotiation message in our message
3800  * buffer based on the input parameters.
3801  */
3802 static void
3803 ahd_construct_sdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3804                    u_int period, u_int offset)
3805 {
3806         if (offset == 0)
3807                 period = AHD_ASYNC_XFER_PERIOD;
3808         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3809         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_SDTR_LEN;
3810         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_SDTR;
3811         ahd->msgout_buf[ahd->msgout_index++] = period;
3812         ahd->msgout_buf[ahd->msgout_index++] = offset;
3813         ahd->msgout_len += 5;
3814         if (bootverbose) {
3815                 printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
3816                        ahd_name(ahd), devinfo->channel, devinfo->target,
3817                        devinfo->lun, period, offset);
3818         }
3819 }
3820
3821 /*
3822  * Build a wide negotiateion message in our message
3823  * buffer based on the input parameters.
3824  */
3825 static void
3826 ahd_construct_wdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3827                    u_int bus_width)
3828 {
3829         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3830         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_WDTR_LEN;
3831         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_WDTR;
3832         ahd->msgout_buf[ahd->msgout_index++] = bus_width;
3833         ahd->msgout_len += 4;
3834         if (bootverbose) {
3835                 printf("(%s:%c:%d:%d): Sending WDTR %x\n",
3836                        ahd_name(ahd), devinfo->channel, devinfo->target,
3837                        devinfo->lun, bus_width);
3838         }
3839 }
3840
3841 /*
3842  * Build a parallel protocol request message in our message
3843  * buffer based on the input parameters.
3844  */
3845 static void
3846 ahd_construct_ppr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3847                   u_int period, u_int offset, u_int bus_width,
3848                   u_int ppr_options)
3849 {
3850         /*
3851          * Always request precompensation from
3852          * the other target if we are running
3853          * at paced syncrates.
3854          */
3855         if (period <= AHD_SYNCRATE_PACED)
3856                 ppr_options |= MSG_EXT_PPR_PCOMP_EN;
3857         if (offset == 0)
3858                 period = AHD_ASYNC_XFER_PERIOD;
3859         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3860         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_PPR_LEN;
3861         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_PPR;
3862         ahd->msgout_buf[ahd->msgout_index++] = period;
3863         ahd->msgout_buf[ahd->msgout_index++] = 0;
3864         ahd->msgout_buf[ahd->msgout_index++] = offset;
3865         ahd->msgout_buf[ahd->msgout_index++] = bus_width;
3866         ahd->msgout_buf[ahd->msgout_index++] = ppr_options;
3867         ahd->msgout_len += 8;
3868         if (bootverbose) {
3869                 printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
3870                        "offset %x, ppr_options %x\n", ahd_name(ahd),
3871                        devinfo->channel, devinfo->target, devinfo->lun,
3872                        bus_width, period, offset, ppr_options);
3873         }
3874 }
3875
3876 /*
3877  * Clear any active message state.
3878  */
3879 static void
3880 ahd_clear_msg_state(struct ahd_softc *ahd)
3881 {
3882         ahd_mode_state saved_modes;
3883
3884         saved_modes = ahd_save_modes(ahd);
3885         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3886         ahd->send_msg_perror = 0;
3887         ahd->msg_flags = MSG_FLAG_NONE;
3888         ahd->msgout_len = 0;
3889         ahd->msgin_index = 0;
3890         ahd->msg_type = MSG_TYPE_NONE;
3891         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
3892                 /*
3893                  * The target didn't care to respond to our
3894                  * message request, so clear ATN.
3895                  */
3896                 ahd_outb(ahd, CLRSINT1, CLRATNO);
3897         }
3898         ahd_outb(ahd, MSG_OUT, MSG_NOOP);
3899         ahd_outb(ahd, SEQ_FLAGS2,
3900                  ahd_inb(ahd, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
3901         ahd_restore_modes(ahd, saved_modes);
3902 }
3903
3904 /*
3905  * Manual message loop handler.
3906  */
3907 static void
3908 ahd_handle_message_phase(struct ahd_softc *ahd)
3909 {
3910         struct  ahd_devinfo devinfo;
3911         u_int   bus_phase;
3912         int     end_session;
3913
3914         ahd_fetch_devinfo(ahd, &devinfo);
3915         end_session = FALSE;
3916         bus_phase = ahd_inb(ahd, LASTPHASE);
3917
3918         if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0) {
3919                 printf("LQIRETRY for LQIPHASE_OUTPKT\n");
3920                 ahd_outb(ahd, LQCTL2, LQIRETRY);
3921         }
3922 reswitch:
3923         switch (ahd->msg_type) {
3924         case MSG_TYPE_INITIATOR_MSGOUT:
3925         {
3926                 int lastbyte;
3927                 int phasemis;
3928                 int msgdone;
3929
3930                 if (ahd->msgout_len == 0 && ahd->send_msg_perror == 0)
3931                         panic("HOST_MSG_LOOP interrupt with no active message");
3932
3933 #ifdef AHD_DEBUG
3934                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3935                         ahd_print_devinfo(ahd, &devinfo);
3936                         printf("INITIATOR_MSG_OUT");
3937                 }
3938 #endif
3939                 phasemis = bus_phase != P_MESGOUT;
3940                 if (phasemis) {
3941 #ifdef AHD_DEBUG
3942                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3943                                 printf(" PHASEMIS %s\n",
3944                                        ahd_lookup_phase_entry(bus_phase)
3945                                                              ->phasemsg);
3946                         }
3947 #endif
3948                         if (bus_phase == P_MESGIN) {
3949                                 /*
3950                                  * Change gears and see if
3951                                  * this messages is of interest to
3952                                  * us or should be passed back to
3953                                  * the sequencer.
3954                                  */
3955                                 ahd_outb(ahd, CLRSINT1, CLRATNO);
3956                                 ahd->send_msg_perror = 0;
3957                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGIN;
3958                                 ahd->msgin_index = 0;
3959                                 goto reswitch;
3960                         }
3961                         end_session = TRUE;
3962                         break;
3963                 }
3964
3965                 if (ahd->send_msg_perror) {
3966                         ahd_outb(ahd, CLRSINT1, CLRATNO);
3967                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
3968 #ifdef AHD_DEBUG
3969                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3970                                 printf(" byte 0x%x\n", ahd->send_msg_perror);
3971 #endif
3972                         /*
3973                          * If we are notifying the target of a CRC error
3974                          * during packetized operations, the target is
3975                          * within its rights to acknowledge our message
3976                          * with a busfree.
3977                          */
3978                         if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0
3979                          && ahd->send_msg_perror == MSG_INITIATOR_DET_ERR)
3980                                 ahd->msg_flags |= MSG_FLAG_EXPECT_IDE_BUSFREE;
3981
3982                         ahd_outb(ahd, RETURN_2, ahd->send_msg_perror);
3983                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
3984                         break;
3985                 }
3986
3987                 msgdone = ahd->msgout_index == ahd->msgout_len;
3988                 if (msgdone) {
3989                         /*
3990                          * The target has requested a retry.
3991                          * Re-assert ATN, reset our message index to
3992                          * 0, and try again.
3993                          */
3994                         ahd->msgout_index = 0;
3995                         ahd_assert_atn(ahd);
3996                 }
3997
3998                 lastbyte = ahd->msgout_index == (ahd->msgout_len - 1);
3999                 if (lastbyte) {
4000                         /* Last byte is signified by dropping ATN */
4001                         ahd_outb(ahd, CLRSINT1, CLRATNO);
4002                 }
4003
4004                 /*
4005                  * Clear our interrupt status and present
4006                  * the next byte on the bus.
4007                  */
4008                 ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4009 #ifdef AHD_DEBUG
4010                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4011                         printf(" byte 0x%x\n",
4012                                ahd->msgout_buf[ahd->msgout_index]);
4013 #endif
4014                 ahd_outb(ahd, RETURN_2, ahd->msgout_buf[ahd->msgout_index++]);
4015                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
4016                 break;
4017         }
4018         case MSG_TYPE_INITIATOR_MSGIN:
4019         {
4020                 int phasemis;
4021                 int message_done;
4022
4023 #ifdef AHD_DEBUG
4024                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4025                         ahd_print_devinfo(ahd, &devinfo);
4026                         printf("INITIATOR_MSG_IN");
4027                 }
4028 #endif
4029                 phasemis = bus_phase != P_MESGIN;
4030                 if (phasemis) {
4031 #ifdef AHD_DEBUG
4032                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4033                                 printf(" PHASEMIS %s\n",
4034                                        ahd_lookup_phase_entry(bus_phase)
4035                                                              ->phasemsg);
4036                         }
4037 #endif
4038                         ahd->msgin_index = 0;
4039                         if (bus_phase == P_MESGOUT
4040                          && (ahd->send_msg_perror != 0
4041                           || (ahd->msgout_len != 0
4042                            && ahd->msgout_index == 0))) {
4043                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4044                                 goto reswitch;
4045                         }
4046                         end_session = TRUE;
4047                         break;
4048                 }
4049
4050                 /* Pull the byte in without acking it */
4051                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIBUS);
4052 #ifdef AHD_DEBUG
4053                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4054                         printf(" byte 0x%x\n",
4055                                ahd->msgin_buf[ahd->msgin_index]);
4056 #endif
4057
4058                 message_done = ahd_parse_msg(ahd, &devinfo);
4059
4060                 if (message_done) {
4061                         /*
4062                          * Clear our incoming message buffer in case there
4063                          * is another message following this one.
4064                          */
4065                         ahd->msgin_index = 0;
4066
4067                         /*
4068                          * If this message illicited a response,
4069                          * assert ATN so the target takes us to the
4070                          * message out phase.
4071                          */
4072                         if (ahd->msgout_len != 0) {
4073 #ifdef AHD_DEBUG
4074                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4075                                         ahd_print_devinfo(ahd, &devinfo);
4076                                         printf("Asserting ATN for response\n");
4077                                 }
4078 #endif
4079                                 ahd_assert_atn(ahd);
4080                         }
4081                 } else
4082                         ahd->msgin_index++;
4083
4084                 if (message_done == MSGLOOP_TERMINATED) {
4085                         end_session = TRUE;
4086                 } else {
4087                         /* Ack the byte */
4088                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4089                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_READ);
4090                 }
4091                 break;
4092         }
4093         case MSG_TYPE_TARGET_MSGIN:
4094         {
4095                 int msgdone;
4096                 int msgout_request;
4097
4098                 /*
4099                  * By default, the message loop will continue.
4100                  */
4101                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4102
4103                 if (ahd->msgout_len == 0)
4104                         panic("Target MSGIN with no active message");
4105
4106                 /*
4107                  * If we interrupted a mesgout session, the initiator
4108                  * will not know this until our first REQ.  So, we
4109                  * only honor mesgout requests after we've sent our
4110                  * first byte.
4111                  */
4112                 if ((ahd_inb(ahd, SCSISIGI) & ATNI) != 0
4113                  && ahd->msgout_index > 0)
4114                         msgout_request = TRUE;
4115                 else
4116                         msgout_request = FALSE;
4117
4118                 if (msgout_request) {
4119
4120                         /*
4121                          * Change gears and see if
4122                          * this messages is of interest to
4123                          * us or should be passed back to
4124                          * the sequencer.
4125                          */
4126                         ahd->msg_type = MSG_TYPE_TARGET_MSGOUT;
4127                         ahd_outb(ahd, SCSISIGO, P_MESGOUT | BSYO);
4128                         ahd->msgin_index = 0;
4129                         /* Dummy read to REQ for first byte */
4130                         ahd_inb(ahd, SCSIDAT);
4131                         ahd_outb(ahd, SXFRCTL0,
4132                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4133                         break;
4134                 }
4135
4136                 msgdone = ahd->msgout_index == ahd->msgout_len;
4137                 if (msgdone) {
4138                         ahd_outb(ahd, SXFRCTL0,
4139                                  ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4140                         end_session = TRUE;
4141                         break;
4142                 }
4143
4144                 /*
4145                  * Present the next byte on the bus.
4146                  */
4147                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4148                 ahd_outb(ahd, SCSIDAT, ahd->msgout_buf[ahd->msgout_index++]);
4149                 break;
4150         }
4151         case MSG_TYPE_TARGET_MSGOUT:
4152         {
4153                 int lastbyte;
4154                 int msgdone;
4155
4156                 /*
4157                  * By default, the message loop will continue.
4158                  */
4159                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4160
4161                 /*
4162                  * The initiator signals that this is
4163                  * the last byte by dropping ATN.
4164                  */
4165                 lastbyte = (ahd_inb(ahd, SCSISIGI) & ATNI) == 0;
4166
4167                 /*
4168                  * Read the latched byte, but turn off SPIOEN first
4169                  * so that we don't inadvertently cause a REQ for the
4170                  * next byte.
4171                  */
4172                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4173                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIDAT);
4174                 msgdone = ahd_parse_msg(ahd, &devinfo);
4175                 if (msgdone == MSGLOOP_TERMINATED) {
4176                         /*
4177                          * The message is *really* done in that it caused
4178                          * us to go to bus free.  The sequencer has already
4179                          * been reset at this point, so pull the ejection
4180                          * handle.
4181                          */
4182                         return;
4183                 }
4184
4185                 ahd->msgin_index++;
4186
4187                 /*
4188                  * XXX Read spec about initiator dropping ATN too soon
4189                  *     and use msgdone to detect it.
4190                  */
4191                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
4192                         ahd->msgin_index = 0;
4193
4194                         /*
4195                          * If this message illicited a response, transition
4196                          * to the Message in phase and send it.
4197                          */
4198                         if (ahd->msgout_len != 0) {
4199                                 ahd_outb(ahd, SCSISIGO, P_MESGIN | BSYO);
4200                                 ahd_outb(ahd, SXFRCTL0,
4201                                          ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4202                                 ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
4203                                 ahd->msgin_index = 0;
4204                                 break;
4205                         }
4206                 }
4207
4208                 if (lastbyte)
4209                         end_session = TRUE;
4210                 else {
4211                         /* Ask for the next byte. */
4212                         ahd_outb(ahd, SXFRCTL0,
4213                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4214                 }
4215
4216                 break;
4217         }
4218         default:
4219                 panic("Unknown REQINIT message type");
4220         }
4221
4222         if (end_session) {
4223                 if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0) {
4224                         printf("%s: Returning to Idle Loop\n",
4225                                ahd_name(ahd));
4226                         ahd_clear_msg_state(ahd);
4227
4228                         /*
4229                          * Perform the equivalent of a clear_target_state.
4230                          */
4231                         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
4232                         ahd_outb(ahd, SEQ_FLAGS, NOT_IDENTIFIED|NO_CDB_SENT);
4233                         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
4234                 } else {
4235                         ahd_clear_msg_state(ahd);
4236                         ahd_outb(ahd, RETURN_1, EXIT_MSG_LOOP);
4237                 }
4238         }
4239 }
4240
4241 /*
4242  * See if we sent a particular extended message to the target.
4243  * If "full" is true, return true only if the target saw the full
4244  * message.  If "full" is false, return true if the target saw at
4245  * least the first byte of the message.
4246  */
4247 static int
4248 ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type, u_int msgval, int full)
4249 {
4250         int found;
4251         u_int index;
4252
4253         found = FALSE;
4254         index = 0;
4255
4256         while (index < ahd->msgout_len) {
4257                 if (ahd->msgout_buf[index] == MSG_EXTENDED) {
4258                         u_int end_index;
4259
4260                         end_index = index + 1 + ahd->msgout_buf[index + 1];
4261                         if (ahd->msgout_buf[index+2] == msgval
4262                          && type == AHDMSG_EXT) {
4263
4264                                 if (full) {
4265                                         if (ahd->msgout_index > end_index)
4266                                                 found = TRUE;
4267                                 } else if (ahd->msgout_index > index)
4268                                         found = TRUE;
4269                         }
4270                         index = end_index;
4271                 } else if (ahd->msgout_buf[index] >= MSG_SIMPLE_TASK
4272                         && ahd->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
4273
4274                         /* Skip tag type and tag id or residue param*/
4275                         index += 2;
4276                 } else {
4277                         /* Single byte message */
4278                         if (type == AHDMSG_1B
4279                          && ahd->msgout_index > index
4280                          && (ahd->msgout_buf[index] == msgval
4281                           || ((ahd->msgout_buf[index] & MSG_IDENTIFYFLAG) != 0
4282                            && msgval == MSG_IDENTIFYFLAG)))
4283                                 found = TRUE;
4284                         index++;
4285                 }
4286
4287                 if (found)
4288                         break;
4289         }
4290         return (found);
4291 }
4292
4293 /*
4294  * Wait for a complete incoming message, parse it, and respond accordingly.
4295  */
4296 static int
4297 ahd_parse_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4298 {
4299         struct  ahd_initiator_tinfo *tinfo;
4300         struct  ahd_tmode_tstate *tstate;
4301         int     reject;
4302         int     done;
4303         int     response;
4304
4305         done = MSGLOOP_IN_PROG;
4306         response = FALSE;
4307         reject = FALSE;
4308         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4309                                     devinfo->target, &tstate);
4310
4311         /*
4312          * Parse as much of the message as is available,
4313          * rejecting it if we don't support it.  When
4314          * the entire message is available and has been
4315          * handled, return MSGLOOP_MSGCOMPLETE, indicating
4316          * that we have parsed an entire message.
4317          *
4318          * In the case of extended messages, we accept the length
4319          * byte outright and perform more checking once we know the
4320          * extended message type.
4321          */
4322         switch (ahd->msgin_buf[0]) {
4323         case MSG_DISCONNECT:
4324         case MSG_SAVEDATAPOINTER:
4325         case MSG_CMDCOMPLETE:
4326         case MSG_RESTOREPOINTERS:
4327         case MSG_IGN_WIDE_RESIDUE:
4328                 /*
4329                  * End our message loop as these are messages
4330                  * the sequencer handles on its own.
4331                  */
4332                 done = MSGLOOP_TERMINATED;
4333                 break;
4334         case MSG_MESSAGE_REJECT:
4335                 response = ahd_handle_msg_reject(ahd, devinfo);
4336                 /* FALLTHROUGH */
4337         case MSG_NOOP:
4338                 done = MSGLOOP_MSGCOMPLETE;
4339                 break;
4340         case MSG_EXTENDED:
4341         {
4342                 /* Wait for enough of the message to begin validation */
4343                 if (ahd->msgin_index < 2)
4344                         break;
4345                 switch (ahd->msgin_buf[2]) {
4346                 case MSG_EXT_SDTR:
4347                 {
4348                         u_int    period;
4349                         u_int    ppr_options;
4350                         u_int    offset;
4351                         u_int    saved_offset;
4352
4353                         if (ahd->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
4354                                 reject = TRUE;
4355                                 break;
4356                         }
4357
4358                         /*
4359                          * Wait until we have both args before validating
4360                          * and acting on this message.
4361                          *
4362                          * Add one to MSG_EXT_SDTR_LEN to account for
4363                          * the extended message preamble.
4364                          */
4365                         if (ahd->msgin_index < (MSG_EXT_SDTR_LEN + 1))
4366                                 break;
4367
4368                         period = ahd->msgin_buf[3];
4369                         ppr_options = 0;
4370                         saved_offset = offset = ahd->msgin_buf[4];
4371                         ahd_devlimited_syncrate(ahd, tinfo, &period,
4372                                                 &ppr_options, devinfo->role);
4373                         ahd_validate_offset(ahd, tinfo, period, &offset,
4374                                             tinfo->curr.width, devinfo->role);
4375                         if (bootverbose) {
4376                                 printf("(%s:%c:%d:%d): Received "
4377                                        "SDTR period %x, offset %x\n\t"
4378                                        "Filtered to period %x, offset %x\n",
4379                                        ahd_name(ahd), devinfo->channel,
4380                                        devinfo->target, devinfo->lun,
4381                                        ahd->msgin_buf[3], saved_offset,
4382                                        period, offset);
4383                         }
4384                         ahd_set_syncrate(ahd, devinfo, period,
4385                                          offset, ppr_options,
4386                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4387                                          /*paused*/TRUE);
4388
4389                         /*
4390                          * See if we initiated Sync Negotiation
4391                          * and didn't have to fall down to async
4392                          * transfers.
4393                          */
4394                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, TRUE)) {
4395                                 /* We started it */
4396                                 if (saved_offset != offset) {
4397                                         /* Went too low - force async */
4398                                         reject = TRUE;
4399                                 }
4400                         } else {
4401                                 /*
4402                                  * Send our own SDTR in reply
4403                                  */
4404                                 if (bootverbose
4405                                  && devinfo->role == ROLE_INITIATOR) {
4406                                         printf("(%s:%c:%d:%d): Target "
4407                                                "Initiated SDTR\n",
4408                                                ahd_name(ahd), devinfo->channel,
4409                                                devinfo->target, devinfo->lun);
4410                                 }
4411                                 ahd->msgout_index = 0;
4412                                 ahd->msgout_len = 0;
4413                                 ahd_construct_sdtr(ahd, devinfo,
4414                                                    period, offset);
4415                                 ahd->msgout_index = 0;
4416                                 response = TRUE;
4417                         }
4418                         done = MSGLOOP_MSGCOMPLETE;
4419                         break;
4420                 }
4421                 case MSG_EXT_WDTR:
4422                 {
4423                         u_int bus_width;
4424                         u_int saved_width;
4425                         u_int sending_reply;
4426
4427                         sending_reply = FALSE;
4428                         if (ahd->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
4429                                 reject = TRUE;
4430                                 break;
4431                         }
4432
4433                         /*
4434                          * Wait until we have our arg before validating
4435                          * and acting on this message.
4436                          *
4437                          * Add one to MSG_EXT_WDTR_LEN to account for
4438                          * the extended message preamble.
4439                          */
4440                         if (ahd->msgin_index < (MSG_EXT_WDTR_LEN + 1))
4441                                 break;
4442
4443                         bus_width = ahd->msgin_buf[3];
4444                         saved_width = bus_width;
4445                         ahd_validate_width(ahd, tinfo, &bus_width,
4446                                            devinfo->role);
4447                         if (bootverbose) {
4448                                 printf("(%s:%c:%d:%d): Received WDTR "
4449                                        "%x filtered to %x\n",
4450                                        ahd_name(ahd), devinfo->channel,
4451                                        devinfo->target, devinfo->lun,
4452                                        saved_width, bus_width);
4453                         }
4454
4455                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, TRUE)) {
4456                                 /*
4457                                  * Don't send a WDTR back to the
4458                                  * target, since we asked first.
4459                                  * If the width went higher than our
4460                                  * request, reject it.
4461                                  */
4462                                 if (saved_width > bus_width) {
4463                                         reject = TRUE;
4464                                         printf("(%s:%c:%d:%d): requested %dBit "
4465                                                "transfers.  Rejecting...\n",
4466                                                ahd_name(ahd), devinfo->channel,
4467                                                devinfo->target, devinfo->lun,
4468                                                8 * (0x01 << bus_width));
4469                                         bus_width = 0;
4470                                 }
4471                         } else {
4472                                 /*
4473                                  * Send our own WDTR in reply
4474                                  */
4475                                 if (bootverbose
4476                                  && devinfo->role == ROLE_INITIATOR) {
4477                                         printf("(%s:%c:%d:%d): Target "
4478                                                "Initiated WDTR\n",
4479                                                ahd_name(ahd), devinfo->channel,
4480                                                devinfo->target, devinfo->lun);
4481                                 }
4482                                 ahd->msgout_index = 0;
4483                                 ahd->msgout_len = 0;
4484                                 ahd_construct_wdtr(ahd, devinfo, bus_width);
4485                                 ahd->msgout_index = 0;
4486                                 response = TRUE;
4487                                 sending_reply = TRUE;
4488                         }
4489                         /*
4490                          * After a wide message, we are async, but
4491                          * some devices don't seem to honor this portion
4492                          * of the spec.  Force a renegotiation of the
4493                          * sync component of our transfer agreement even
4494                          * if our goal is async.  By updating our width
4495                          * after forcing the negotiation, we avoid
4496                          * renegotiating for width.
4497                          */
4498                         ahd_update_neg_request(ahd, devinfo, tstate,
4499                                                tinfo, AHD_NEG_ALWAYS);
4500                         ahd_set_width(ahd, devinfo, bus_width,
4501                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4502                                       /*paused*/TRUE);
4503                         if (sending_reply == FALSE && reject == FALSE) {
4504
4505                                 /*
4506                                  * We will always have an SDTR to send.
4507                                  */
4508                                 ahd->msgout_index = 0;
4509                                 ahd->msgout_len = 0;
4510                                 ahd_build_transfer_msg(ahd, devinfo);
4511                                 ahd->msgout_index = 0;
4512                                 response = TRUE;
4513                         }
4514                         done = MSGLOOP_MSGCOMPLETE;
4515                         break;
4516                 }
4517                 case MSG_EXT_PPR:
4518                 {
4519                         u_int   period;
4520                         u_int   offset;
4521                         u_int   bus_width;
4522                         u_int   ppr_options;
4523                         u_int   saved_width;
4524                         u_int   saved_offset;
4525                         u_int   saved_ppr_options;
4526
4527                         if (ahd->msgin_buf[1] != MSG_EXT_PPR_LEN) {
4528                                 reject = TRUE;
4529                                 break;
4530                         }
4531
4532                         /*
4533                          * Wait until we have all args before validating
4534                          * and acting on this message.
4535                          *
4536                          * Add one to MSG_EXT_PPR_LEN to account for
4537                          * the extended message preamble.
4538                          */
4539                         if (ahd->msgin_index < (MSG_EXT_PPR_LEN + 1))
4540                                 break;
4541
4542                         period = ahd->msgin_buf[3];
4543                         offset = ahd->msgin_buf[5];
4544                         bus_width = ahd->msgin_buf[6];
4545                         saved_width = bus_width;
4546                         ppr_options = ahd->msgin_buf[7];
4547                         /*
4548                          * According to the spec, a DT only
4549                          * period factor with no DT option
4550                          * set implies async.
4551                          */
4552                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
4553                          && period <= 9)
4554                                 offset = 0;
4555                         saved_ppr_options = ppr_options;
4556                         saved_offset = offset;
4557
4558                         /*
4559                          * Transfer options are only available if we
4560                          * are negotiating wide.
4561                          */
4562                         if (bus_width == 0)
4563                                 ppr_options &= MSG_EXT_PPR_QAS_REQ;
4564
4565                         ahd_validate_width(ahd, tinfo, &bus_width,
4566                                            devinfo->role);
4567                         ahd_devlimited_syncrate(ahd, tinfo, &period,
4568                                                 &ppr_options, devinfo->role);
4569                         ahd_validate_offset(ahd, tinfo, period, &offset,
4570                                             bus_width, devinfo->role);
4571
4572                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, TRUE)) {
4573                                 /*
4574                                  * If we are unable to do any of the
4575                                  * requested options (we went too low),
4576                                  * then we'll have to reject the message.
4577                                  */
4578                                 if (saved_width > bus_width
4579                                  || saved_offset != offset
4580                                  || saved_ppr_options != ppr_options) {
4581                                         reject = TRUE;
4582                                         period = 0;
4583                                         offset = 0;
4584                                         bus_width = 0;
4585                                         ppr_options = 0;
4586                                 }
4587                         } else {
4588                                 if (devinfo->role != ROLE_TARGET)
4589                                         printf("(%s:%c:%d:%d): Target "
4590                                                "Initiated PPR\n",
4591                                                ahd_name(ahd), devinfo->channel,
4592                                                devinfo->target, devinfo->lun);
4593                                 else
4594                                         printf("(%s:%c:%d:%d): Initiator "
4595                                                "Initiated PPR\n",
4596                                                ahd_name(ahd), devinfo->channel,
4597                                                devinfo->target, devinfo->lun);
4598                                 ahd->msgout_index = 0;
4599                                 ahd->msgout_len = 0;
4600                                 ahd_construct_ppr(ahd, devinfo, period, offset,
4601                                                   bus_width, ppr_options);
4602                                 ahd->msgout_index = 0;
4603                                 response = TRUE;
4604                         }
4605                         if (bootverbose) {
4606                                 printf("(%s:%c:%d:%d): Received PPR width %x, "
4607                                        "period %x, offset %x,options %x\n"
4608                                        "\tFiltered to width %x, period %x, "
4609                                        "offset %x, options %x\n",
4610                                        ahd_name(ahd), devinfo->channel,
4611                                        devinfo->target, devinfo->lun,
4612                                        saved_width, ahd->msgin_buf[3],
4613                                        saved_offset, saved_ppr_options,
4614                                        bus_width, period, offset, ppr_options);
4615                         }
4616                         ahd_set_width(ahd, devinfo, bus_width,
4617                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4618                                       /*paused*/TRUE);
4619                         ahd_set_syncrate(ahd, devinfo, period,
4620                                          offset, ppr_options,
4621                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4622                                          /*paused*/TRUE);
4623
4624                         done = MSGLOOP_MSGCOMPLETE;
4625                         break;
4626                 }
4627                 default:
4628                         /* Unknown extended message.  Reject it. */
4629                         reject = TRUE;
4630                         break;
4631                 }
4632                 break;
4633         }
4634 #ifdef AHD_TARGET_MODE
4635         case MSG_BUS_DEV_RESET:
4636                 ahd_handle_devreset(ahd, devinfo, CAM_LUN_WILDCARD,
4637                                     CAM_BDR_SENT,
4638                                     "Bus Device Reset Received",
4639                                     /*verbose_level*/0);
4640                 ahd_restart(ahd);
4641                 done = MSGLOOP_TERMINATED;
4642                 break;
4643         case MSG_ABORT_TAG:
4644         case MSG_ABORT:
4645         case MSG_CLEAR_QUEUE:
4646         {
4647                 int tag;
4648
4649                 /* Target mode messages */
4650                 if (devinfo->role != ROLE_TARGET) {
4651                         reject = TRUE;
4652                         break;
4653                 }
4654                 tag = SCB_LIST_NULL;
4655                 if (ahd->msgin_buf[0] == MSG_ABORT_TAG)
4656                         tag = ahd_inb(ahd, INITIATOR_TAG);
4657                 ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
4658                                devinfo->lun, tag, ROLE_TARGET,
4659                                CAM_REQ_ABORTED);
4660
4661                 tstate = ahd->enabled_targets[devinfo->our_scsiid];
4662                 if (tstate != NULL) {
4663                         struct ahd_tmode_lstate* lstate;
4664
4665                         lstate = tstate->enabled_luns[devinfo->lun];
4666                         if (lstate != NULL) {
4667                                 ahd_queue_lstate_event(ahd, lstate,
4668                                                        devinfo->our_scsiid,
4669                                                        ahd->msgin_buf[0],
4670                                                        /*arg*/tag);
4671                                 ahd_send_lstate_events(ahd, lstate);
4672                         }
4673                 }
4674                 ahd_restart(ahd);
4675                 done = MSGLOOP_TERMINATED;
4676                 break;
4677         }
4678 #endif
4679         case MSG_QAS_REQUEST:
4680 #ifdef AHD_DEBUG
4681                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4682                         printf("%s: QAS request.  SCSISIGI == 0x%x\n",
4683                                ahd_name(ahd), ahd_inb(ahd, SCSISIGI));
4684 #endif
4685                 ahd->msg_flags |= MSG_FLAG_EXPECT_QASREJ_BUSFREE;
4686                 /* FALLTHROUGH */
4687         case MSG_TERM_IO_PROC:
4688         default:
4689                 reject = TRUE;
4690                 break;
4691         }
4692
4693         if (reject) {
4694                 /*
4695                  * Setup to reject the message.
4696                  */
4697                 ahd->msgout_index = 0;
4698                 ahd->msgout_len = 1;
4699                 ahd->msgout_buf[0] = MSG_MESSAGE_REJECT;
4700                 done = MSGLOOP_MSGCOMPLETE;
4701                 response = TRUE;
4702         }
4703
4704         if (done != MSGLOOP_IN_PROG && !response)
4705                 /* Clear the outgoing message buffer */
4706                 ahd->msgout_len = 0;
4707
4708         return (done);
4709 }
4710
4711 /*
4712  * Process a message reject message.
4713  */
4714 static int
4715 ahd_handle_msg_reject(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4716 {
4717         /*
4718          * What we care about here is if we had an
4719          * outstanding SDTR or WDTR message for this
4720          * target.  If we did, this is a signal that
4721          * the target is refusing negotiation.
4722          */
4723         struct scb *scb;
4724         struct ahd_initiator_tinfo *tinfo;
4725         struct ahd_tmode_tstate *tstate;
4726         u_int scb_index;
4727         u_int last_msg;
4728         int   response = 0;
4729
4730         scb_index = ahd_get_scbptr(ahd);
4731         scb = ahd_lookup_scb(ahd, scb_index);
4732         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel,
4733                                     devinfo->our_scsiid,
4734                                     devinfo->target, &tstate);
4735         /* Might be necessary */
4736         last_msg = ahd_inb(ahd, LAST_MSG);
4737
4738         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
4739                 if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/TRUE)
4740                  && tinfo->goal.period <= AHD_SYNCRATE_PACED) {
4741                         /*
4742                          * Target may not like our SPI-4 PPR Options.
4743                          * Attempt to negotiate 80MHz which will turn
4744                          * off these options.
4745                          */
4746                         if (bootverbose) {
4747                                 printf("(%s:%c:%d:%d): PPR Rejected. "
4748                                        "Trying simple U160 PPR\n",
4749                                        ahd_name(ahd), devinfo->channel,
4750                                        devinfo->target, devinfo->lun);
4751                         }
4752                         tinfo->goal.period = AHD_SYNCRATE_DT;
4753                         tinfo->goal.ppr_options &= MSG_EXT_PPR_IU_REQ
4754                                                 |  MSG_EXT_PPR_QAS_REQ
4755                                                 |  MSG_EXT_PPR_DT_REQ;
4756                 } else {
4757                         /*
4758                          * Target does not support the PPR message.
4759                          * Attempt to negotiate SPI-2 style.
4760                          */
4761                         if (bootverbose) {
4762                                 printf("(%s:%c:%d:%d): PPR Rejected. "
4763                                        "Trying WDTR/SDTR\n",
4764                                        ahd_name(ahd), devinfo->channel,
4765                                        devinfo->target, devinfo->lun);
4766                         }
4767                         tinfo->goal.ppr_options = 0;
4768                         tinfo->curr.transport_version = 2;
4769                         tinfo->goal.transport_version = 2;
4770                 }
4771                 ahd->msgout_index = 0;
4772                 ahd->msgout_len = 0;
4773                 ahd_build_transfer_msg(ahd, devinfo);
4774                 ahd->msgout_index = 0;
4775                 response = 1;
4776         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
4777
4778                 /* note 8bit xfers */
4779                 printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
4780                        "8bit transfers\n", ahd_name(ahd),
4781                        devinfo->channel, devinfo->target, devinfo->lun);
4782                 ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4783                               AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4784                               /*paused*/TRUE);
4785                 /*
4786                  * No need to clear the sync rate.  If the target
4787                  * did not accept the command, our syncrate is
4788                  * unaffected.  If the target started the negotiation,
4789                  * but rejected our response, we already cleared the
4790                  * sync rate before sending our WDTR.
4791                  */
4792                 if (tinfo->goal.offset != tinfo->curr.offset) {
4793
4794                         /* Start the sync negotiation */
4795                         ahd->msgout_index = 0;
4796                         ahd->msgout_len = 0;
4797                         ahd_build_transfer_msg(ahd, devinfo);
4798                         ahd->msgout_index = 0;
4799                         response = 1;
4800                 }
4801         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
4802                 /* note asynch xfers and clear flag */
4803                 ahd_set_syncrate(ahd, devinfo, /*period*/0,
4804                                  /*offset*/0, /*ppr_options*/0,
4805                                  AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4806                                  /*paused*/TRUE);
4807                 printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
4808                        "Using asynchronous transfers\n",
4809                        ahd_name(ahd), devinfo->channel,
4810                        devinfo->target, devinfo->lun);
4811         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
4812                 int tag_type;
4813                 int mask;
4814
4815                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
4816
4817                 if (tag_type == MSG_SIMPLE_TASK) {
4818                         printf("(%s:%c:%d:%d): refuses tagged commands.  "
4819                                "Performing non-tagged I/O\n", ahd_name(ahd),
4820                                devinfo->channel, devinfo->target, devinfo->lun);
4821                         ahd_set_tags(ahd, devinfo, AHD_QUEUE_NONE);
4822                         mask = ~0x23;
4823                 } else {
4824                         printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
4825                                "Performing simple queue tagged I/O only\n",
4826                                ahd_name(ahd), devinfo->channel, devinfo->target,
4827                                devinfo->lun, tag_type == MSG_ORDERED_TASK
4828                                ? "ordered" : "head of queue");
4829                         ahd_set_tags(ahd, devinfo, AHD_QUEUE_BASIC);
4830                         mask = ~0x03;
4831                 }
4832
4833                 /*
4834                  * Resend the identify for this CCB as the target
4835                  * may believe that the selection is invalid otherwise.
4836                  */
4837                 ahd_outb(ahd, SCB_CONTROL,
4838                          ahd_inb_scbram(ahd, SCB_CONTROL) & mask);
4839                 scb->hscb->control &= mask;
4840                 aic_set_transaction_tag(scb, /*enabled*/FALSE,
4841                                         /*type*/MSG_SIMPLE_TASK);
4842                 ahd_outb(ahd, MSG_OUT, MSG_IDENTIFYFLAG);
4843                 ahd_assert_atn(ahd);
4844                 ahd_busy_tcl(ahd, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
4845                              SCB_GET_TAG(scb));
4846
4847                 /*
4848                  * Requeue all tagged commands for this target
4849                  * currently in our posession so they can be
4850                  * converted to untagged commands.
4851                  */
4852                 ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
4853                                    SCB_GET_CHANNEL(ahd, scb),
4854                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
4855                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
4856                                    SEARCH_COMPLETE);
4857         } else if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_IDENTIFYFLAG, TRUE)) {
4858                 /*
4859                  * Most likely the device believes that we had
4860                  * previously negotiated packetized.
4861                  */
4862                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
4863                                |  MSG_FLAG_IU_REQ_CHANGED;
4864
4865                 ahd_force_renegotiation(ahd, devinfo);
4866                 ahd->msgout_index = 0;
4867                 ahd->msgout_len = 0;
4868                 ahd_build_transfer_msg(ahd, devinfo);
4869                 ahd->msgout_index = 0;
4870                 response = 1;
4871         } else {
4872                 /*
4873                  * Otherwise, we ignore it.
4874                  */
4875                 printf("%s:%c:%d: Message reject for %x -- ignored\n",
4876                        ahd_name(ahd), devinfo->channel, devinfo->target,
4877                        last_msg);
4878         }
4879         return (response);
4880 }
4881
4882 /*
4883  * Process an ingnore wide residue message.
4884  */
4885 static void
4886 ahd_handle_ign_wide_residue(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4887 {
4888         u_int scb_index;
4889         struct scb *scb;
4890
4891         scb_index = ahd_get_scbptr(ahd);
4892         scb = ahd_lookup_scb(ahd, scb_index);
4893         /*
4894          * XXX Actually check data direction in the sequencer?
4895          * Perhaps add datadir to some spare bits in the hscb?
4896          */
4897         if ((ahd_inb(ahd, SEQ_FLAGS) & DPHASE) == 0
4898          || aic_get_transfer_dir(scb) != CAM_DIR_IN) {
4899                 /*
4900                  * Ignore the message if we haven't
4901                  * seen an appropriate data phase yet.
4902                  */
4903         } else {
4904                 /*
4905                  * If the residual occurred on the last
4906                  * transfer and the transfer request was
4907                  * expected to end on an odd count, do
4908                  * nothing.  Otherwise, subtract a byte
4909                  * and update the residual count accordingly.
4910                  */
4911                 uint32_t sgptr;
4912
4913                 sgptr = ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
4914                 if ((sgptr & SG_LIST_NULL) != 0
4915                  && (ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
4916                      & SCB_XFERLEN_ODD) != 0) {
4917                         /*
4918                          * If the residual occurred on the last
4919                          * transfer and the transfer request was
4920                          * expected to end on an odd count, do
4921                          * nothing.
4922                          */
4923                 } else {
4924                         uint32_t data_cnt;
4925                         uint64_t data_addr;
4926                         uint32_t sglen;
4927
4928                         /* Pull in the rest of the sgptr */
4929                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
4930                         data_cnt = ahd_inl_scbram(ahd, SCB_RESIDUAL_DATACNT);
4931                         if ((sgptr & SG_LIST_NULL) != 0) {
4932                                 /*
4933                                  * The residual data count is not updated
4934                                  * for the command run to completion case.
4935                                  * Explicitly zero the count.
4936                                  */
4937                                 data_cnt &= ~AHD_SG_LEN_MASK;
4938                         }
4939                         data_addr = ahd_inq(ahd, SHADDR);
4940                         data_cnt += 1;
4941                         data_addr -= 1;
4942                         sgptr &= SG_PTR_MASK;
4943                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
4944                                 struct ahd_dma64_seg *sg;
4945
4946                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
4947
4948                                 /*
4949                                  * The residual sg ptr points to the next S/G
4950                                  * to load so we must go back one.
4951                                  */
4952                                 sg--;
4953                                 sglen = aic_le32toh(sg->len) & AHD_SG_LEN_MASK;
4954                                 if (sg != scb->sg_list
4955                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
4956
4957                                         sg--;
4958                                         sglen = aic_le32toh(sg->len);
4959                                         /*
4960                                          * Preserve High Address and SG_LIST
4961                                          * bits while setting the count to 1.
4962                                          */
4963                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
4964                                         data_addr = aic_le64toh(sg->addr)
4965                                                   + (sglen & AHD_SG_LEN_MASK)
4966                                                   - 1;
4967
4968                                         /*
4969                                          * Increment sg so it points to the
4970                                          * "next" sg.
4971                                          */
4972                                         sg++;
4973                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
4974                                                                    sg);
4975                                 }
4976                         } else {
4977                                 struct ahd_dma_seg *sg;
4978
4979                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
4980
4981                                 /*
4982                                  * The residual sg ptr points to the next S/G
4983                                  * to load so we must go back one.
4984                                  */
4985                                 sg--;
4986                                 sglen = aic_le32toh(sg->len) & AHD_SG_LEN_MASK;
4987                                 if (sg != scb->sg_list
4988                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
4989
4990                                         sg--;
4991                                         sglen = aic_le32toh(sg->len);
4992                                         /*
4993                                          * Preserve High Address and SG_LIST
4994                                          * bits while setting the count to 1.
4995                                          */
4996                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
4997                                         data_addr = aic_le32toh(sg->addr)
4998                                                   + (sglen & AHD_SG_LEN_MASK)
4999                                                   - 1;
5000
5001                                         /*
5002                                          * Increment sg so it points to the
5003                                          * "next" sg.
5004                                          */
5005                                         sg++;
5006                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
5007                                                                   sg);
5008                                 }
5009                         }
5010                         /*
5011                          * Toggle the "oddness" of the transfer length
5012                          * to handle this mid-transfer ignore wide
5013                          * residue.  This ensures that the oddness is
5014                          * correct for subsequent data transfers.
5015                          */
5016                         ahd_outb(ahd, SCB_TASK_ATTRIBUTE,
5017                             ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
5018                             ^ SCB_XFERLEN_ODD);
5019
5020                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
5021                         ahd_outl(ahd, SCB_RESIDUAL_DATACNT, data_cnt);
5022                         /*
5023                          * The FIFO's pointers will be updated if/when the
5024                          * sequencer re-enters a data phase.
5025                          */
5026                 }
5027         }
5028 }
5029
5030
5031 /*
5032  * Reinitialize the data pointers for the active transfer
5033  * based on its current residual.
5034  */
5035 static void
5036 ahd_reinitialize_dataptrs(struct ahd_softc *ahd)
5037 {
5038         struct           scb *scb;
5039         ahd_mode_state   saved_modes;
5040         u_int            scb_index;
5041         u_int            wait;
5042         uint32_t         sgptr;
5043         uint32_t         resid;
5044         uint64_t         dataptr;
5045
5046         AHD_ASSERT_MODES(ahd, AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK,
5047                          AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK);
5048
5049         scb_index = ahd_get_scbptr(ahd);
5050         scb = ahd_lookup_scb(ahd, scb_index);
5051
5052         /*
5053          * Release and reacquire the FIFO so we
5054          * have a clean slate.
5055          */
5056         ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
5057         wait = 1000;
5058         while (--wait && !(ahd_inb(ahd, MDFFSTAT) & FIFOFREE))
5059                 aic_delay(100);
5060         if (wait == 0) {
5061                 ahd_print_path(ahd, scb);
5062                 printf("ahd_reinitialize_dataptrs: Forcing FIFO free.\n");
5063                 ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
5064         }
5065         saved_modes = ahd_save_modes(ahd);
5066         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5067         ahd_outb(ahd, DFFSTAT,
5068                  ahd_inb(ahd, DFFSTAT)
5069                 | (saved_modes == 0x11 ? CURRFIFO_1 : CURRFIFO_0));
5070
5071         /*
5072          * Determine initial values for data_addr and data_cnt
5073          * for resuming the data phase.
5074          */
5075         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5076         sgptr &= SG_PTR_MASK;
5077
5078         resid = (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 2) << 16)
5079               | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 1) << 8)
5080               | ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT);
5081
5082         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5083                 struct ahd_dma64_seg *sg;
5084
5085                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5086
5087                 /* The residual sg_ptr always points to the next sg */
5088                 sg--;
5089
5090                 dataptr = aic_le64toh(sg->addr)
5091                         + (aic_le32toh(sg->len) & AHD_SG_LEN_MASK)
5092                         - resid;
5093                 ahd_outl(ahd, HADDR + 4, dataptr >> 32);
5094         } else {
5095                 struct   ahd_dma_seg *sg;
5096
5097                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5098
5099                 /* The residual sg_ptr always points to the next sg */
5100                 sg--;
5101
5102                 dataptr = aic_le32toh(sg->addr)
5103                         + (aic_le32toh(sg->len) & AHD_SG_LEN_MASK)
5104                         - resid;
5105                 ahd_outb(ahd, HADDR + 4,
5106                          (aic_le32toh(sg->len) & ~AHD_SG_LEN_MASK) >> 24);
5107         }
5108         ahd_outl(ahd, HADDR, dataptr);
5109         ahd_outb(ahd, HCNT + 2, resid >> 16);
5110         ahd_outb(ahd, HCNT + 1, resid >> 8);
5111         ahd_outb(ahd, HCNT, resid);
5112 }
5113
5114 /*
5115  * Handle the effects of issuing a bus device reset message.
5116  */
5117 static void
5118 ahd_handle_devreset(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5119                     u_int lun, cam_status status, char *message,
5120                     int verbose_level)
5121 {
5122 #ifdef AHD_TARGET_MODE
5123         struct ahd_tmode_tstate* tstate;
5124 #endif
5125         int found;
5126
5127         found = ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5128                                lun, SCB_LIST_NULL, devinfo->role,
5129                                status);
5130
5131 #ifdef AHD_TARGET_MODE
5132         /*
5133          * Send an immediate notify ccb to all target mord peripheral
5134          * drivers affected by this action.
5135          */
5136         tstate = ahd->enabled_targets[devinfo->our_scsiid];
5137         if (tstate != NULL) {
5138                 u_int cur_lun;
5139                 u_int max_lun;
5140
5141                 if (lun != CAM_LUN_WILDCARD) {
5142                         cur_lun = 0;
5143                         max_lun = AHD_NUM_LUNS - 1;
5144                 } else {
5145                         cur_lun = lun;
5146                         max_lun = lun;
5147                 }
5148                 for (cur_lun <= max_lun; cur_lun++) {
5149                         struct ahd_tmode_lstate* lstate;
5150
5151                         lstate = tstate->enabled_luns[cur_lun];
5152                         if (lstate == NULL)
5153                                 continue;
5154
5155                         ahd_queue_lstate_event(ahd, lstate, devinfo->our_scsiid,
5156                                                MSG_BUS_DEV_RESET, /*arg*/0);
5157                         ahd_send_lstate_events(ahd, lstate);
5158                 }
5159         }
5160 #endif
5161
5162         /*
5163          * Go back to async/narrow transfers and renegotiate.
5164          */
5165         ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5166                       AHD_TRANS_CUR, /*paused*/TRUE);
5167         ahd_set_syncrate(ahd, devinfo, /*period*/0, /*offset*/0,
5168                          /*ppr_options*/0, AHD_TRANS_CUR,
5169                          /*paused*/TRUE);
5170
5171         if (status != CAM_SEL_TIMEOUT)
5172                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
5173                                lun, AC_SENT_BDR, NULL);
5174
5175         if (message != NULL
5176          && (verbose_level <= bootverbose)) {
5177                 AHD_CORRECTABLE_ERROR(ahd);
5178                 printf("%s: %s on %c:%d. %d SCBs aborted\n", ahd_name(ahd),
5179                        message, devinfo->channel, devinfo->target, found);
5180         }
5181 }
5182
5183 #ifdef AHD_TARGET_MODE
5184 static void
5185 ahd_setup_target_msgin(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5186                        struct scb *scb)
5187 {
5188
5189         /*
5190          * To facilitate adding multiple messages together,
5191          * each routine should increment the index and len
5192          * variables instead of setting them explicitly.
5193          */
5194         ahd->msgout_index = 0;
5195         ahd->msgout_len = 0;
5196
5197         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
5198                 ahd_build_transfer_msg(ahd, devinfo);
5199         else
5200                 panic("ahd_intr: AWAITING target message with no message");
5201
5202         ahd->msgout_index = 0;
5203         ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
5204 }
5205 #endif
5206 /**************************** Initialization **********************************/
5207 static u_int
5208 ahd_sglist_size(struct ahd_softc *ahd)
5209 {
5210         bus_size_t list_size;
5211
5212         list_size = sizeof(struct ahd_dma_seg) * AHD_NSEG;
5213         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
5214                 list_size = sizeof(struct ahd_dma64_seg) * AHD_NSEG;
5215         return (list_size);
5216 }
5217
5218 /*
5219  * Calculate the optimum S/G List allocation size.  S/G elements used
5220  * for a given transaction must be physically contiguous.  Assume the
5221  * OS will allocate full pages to us, so it doesn't make sense to request
5222  * less than a page.
5223  */
5224 static u_int
5225 ahd_sglist_allocsize(struct ahd_softc *ahd)
5226 {
5227         bus_size_t sg_list_increment;
5228         bus_size_t sg_list_size;
5229         bus_size_t max_list_size;
5230         bus_size_t best_list_size;
5231
5232         /* Start out with the minimum required for AHD_NSEG. */
5233         sg_list_increment = ahd_sglist_size(ahd);
5234         sg_list_size = sg_list_increment;
5235
5236         /* Get us as close as possible to a page in size. */
5237         while ((sg_list_size + sg_list_increment) <= PAGE_SIZE)
5238                 sg_list_size += sg_list_increment;
5239
5240         /*
5241          * Try to reduce the amount of wastage by allocating
5242          * multiple pages.
5243          */
5244         best_list_size = sg_list_size;
5245         max_list_size = roundup(sg_list_increment, PAGE_SIZE);
5246         if (max_list_size < 4 * PAGE_SIZE)
5247                 max_list_size = 4 * PAGE_SIZE;
5248         if (max_list_size > (AHD_SCB_MAX_ALLOC * sg_list_increment))
5249                 max_list_size = (AHD_SCB_MAX_ALLOC * sg_list_increment);
5250         while ((sg_list_size + sg_list_increment) <= max_list_size
5251            &&  (sg_list_size % PAGE_SIZE) != 0) {
5252                 bus_size_t new_mod;
5253                 bus_size_t best_mod;
5254
5255                 sg_list_size += sg_list_increment;
5256                 new_mod = sg_list_size % PAGE_SIZE;
5257                 best_mod = best_list_size % PAGE_SIZE;
5258                 if (new_mod > best_mod || new_mod == 0) {
5259                         best_list_size = sg_list_size;
5260                 }
5261         }
5262         return (best_list_size);
5263 }
5264
5265 /*
5266  * Allocate a controller structure for a new device
5267  * and perform initial initializion.
5268  */
5269 struct ahd_softc *
5270 ahd_alloc(void *platform_arg, char *name)
5271 {
5272         struct  ahd_softc *ahd;
5273
5274 #ifndef __FreeBSD__
5275         ahd = malloc(sizeof(*ahd), M_DEVBUF, M_NOWAIT);
5276         if (!ahd) {
5277                 printf("aic7xxx: cannot malloc softc!\n");
5278                 free(name, M_DEVBUF);
5279                 return NULL;
5280         }
5281 #else
5282         ahd = device_get_softc((device_t)platform_arg);
5283 #endif
5284         memset(ahd, 0, sizeof(*ahd));
5285         ahd->seep_config = malloc(sizeof(*ahd->seep_config),
5286                                   M_DEVBUF, M_NOWAIT);
5287         if (ahd->seep_config == NULL) {
5288 #ifndef __FreeBSD__
5289                 free(ahd, M_DEVBUF);
5290 #endif
5291                 free(name, M_DEVBUF);
5292                 return (NULL);
5293         }
5294         LIST_INIT(&ahd->pending_scbs);
5295         LIST_INIT(&ahd->timedout_scbs);
5296         /* We don't know our unit number until the OSM sets it */
5297         ahd->name = name;
5298         ahd->unit = -1;
5299         ahd->description = NULL;
5300         ahd->bus_description = NULL;
5301         ahd->channel = 'A';
5302         ahd->chip = AHD_NONE;
5303         ahd->features = AHD_FENONE;
5304         ahd->bugs = AHD_BUGNONE;
5305         ahd->flags = AHD_SPCHK_ENB_A|AHD_RESET_BUS_A|AHD_TERM_ENB_A
5306                    | AHD_EXTENDED_TRANS_A|AHD_STPWLEVEL_A;
5307         aic_timer_init(&ahd->reset_timer);
5308         aic_timer_init(&ahd->stat_timer);
5309         ahd->int_coalescing_timer = AHD_INT_COALESCING_TIMER_DEFAULT;
5310         ahd->int_coalescing_maxcmds = AHD_INT_COALESCING_MAXCMDS_DEFAULT;
5311         ahd->int_coalescing_mincmds = AHD_INT_COALESCING_MINCMDS_DEFAULT;
5312         ahd->int_coalescing_threshold = AHD_INT_COALESCING_THRESHOLD_DEFAULT;
5313         ahd->int_coalescing_stop_threshold =
5314             AHD_INT_COALESCING_STOP_THRESHOLD_DEFAULT;
5315
5316         if (ahd_platform_alloc(ahd, platform_arg) != 0) {
5317                 ahd_free(ahd);
5318                 ahd = NULL;
5319         }
5320         ahd_lockinit(ahd);
5321 #ifdef AHD_DEBUG
5322         if ((ahd_debug & AHD_SHOW_MEMORY) != 0) {
5323                 printf("%s: scb size = 0x%x, hscb size = 0x%x\n",
5324                        ahd_name(ahd), (u_int)sizeof(struct scb),
5325                        (u_int)sizeof(struct hardware_scb));
5326         }
5327 #endif
5328         return (ahd);
5329 }
5330
5331 int
5332 ahd_softc_init(struct ahd_softc *ahd)
5333 {
5334
5335         ahd->unpause = 0;
5336         ahd->pause = PAUSE;
5337         return (0);
5338 }
5339
5340 void
5341 ahd_softc_insert(struct ahd_softc *ahd)
5342 {
5343         struct ahd_softc *list_ahd;
5344
5345 #if AIC_PCI_CONFIG > 0
5346         /*
5347          * Second Function PCI devices need to inherit some
5348          * settings from function 0.
5349          */
5350         if ((ahd->features & AHD_MULTI_FUNC) != 0) {
5351                 TAILQ_FOREACH(list_ahd, &ahd_tailq, links) {
5352                         aic_dev_softc_t list_pci;
5353                         aic_dev_softc_t pci;
5354
5355                         list_pci = list_ahd->dev_softc;
5356                         pci = ahd->dev_softc;
5357                         if (aic_get_pci_slot(list_pci) == aic_get_pci_slot(pci)
5358                          && aic_get_pci_bus(list_pci) == aic_get_pci_bus(pci)) {
5359                                 struct ahd_softc *master;
5360                                 struct ahd_softc *slave;
5361
5362                                 if (aic_get_pci_function(list_pci) == 0) {
5363                                         master = list_ahd;
5364                                         slave = ahd;
5365                                 } else {
5366                                         master = ahd;
5367                                         slave = list_ahd;
5368                                 }
5369                                 slave->flags &= ~AHD_BIOS_ENABLED;
5370                                 slave->flags |=
5371                                     master->flags & AHD_BIOS_ENABLED;
5372                                 break;
5373                         }
5374                 }
5375         }
5376 #endif
5377
5378         /*
5379          * Insertion sort into our list of softcs.
5380          */
5381         list_ahd = TAILQ_FIRST(&ahd_tailq);
5382         while (list_ahd != NULL
5383             && ahd_softc_comp(ahd, list_ahd) <= 0)
5384                 list_ahd = TAILQ_NEXT(list_ahd, links);
5385         if (list_ahd != NULL)
5386                 TAILQ_INSERT_BEFORE(list_ahd, ahd, links);
5387         else
5388                 TAILQ_INSERT_TAIL(&ahd_tailq, ahd, links);
5389         ahd->init_level++;
5390 }
5391
5392 void
5393 ahd_set_unit(struct ahd_softc *ahd, int unit)
5394 {
5395         ahd->unit = unit;
5396 }
5397
5398 void
5399 ahd_set_name(struct ahd_softc *ahd, char *name)
5400 {
5401         if (ahd->name != NULL)
5402                 free(ahd->name, M_DEVBUF);
5403         ahd->name = name;
5404 }
5405
5406 void
5407 ahd_free(struct ahd_softc *ahd)
5408 {
5409         int i;
5410
5411         ahd_terminate_recovery_thread(ahd);
5412         switch (ahd->init_level) {
5413         default:
5414         case 5:
5415                 ahd_shutdown(ahd);
5416                 /* FALLTHROUGH */
5417         case 4:
5418                 aic_dmamap_unload(ahd, ahd->shared_data_dmat,
5419                                   ahd->shared_data_map.dmamap);
5420                 /* FALLTHROUGH */
5421         case 3:
5422                 aic_dmamem_free(ahd, ahd->shared_data_dmat, ahd->qoutfifo,
5423                                 ahd->shared_data_map.dmamap);
5424                 aic_dmamap_destroy(ahd, ahd->shared_data_dmat,
5425                                    ahd->shared_data_map.dmamap);
5426                 /* FALLTHROUGH */
5427         case 2:
5428                 aic_dma_tag_destroy(ahd, ahd->shared_data_dmat);
5429         case 1:
5430 #ifndef __linux__
5431                 aic_dma_tag_destroy(ahd, ahd->buffer_dmat);
5432 #endif
5433                 break;
5434         case 0:
5435                 break;
5436         }
5437
5438 #ifndef __linux__
5439         aic_dma_tag_destroy(ahd, ahd->parent_dmat);
5440 #endif
5441         ahd_platform_free(ahd);
5442         ahd_fini_scbdata(ahd);
5443         for (i = 0; i < AHD_NUM_TARGETS; i++) {
5444                 struct ahd_tmode_tstate *tstate;
5445
5446                 tstate = ahd->enabled_targets[i];
5447                 if (tstate != NULL) {
5448 #ifdef AHD_TARGET_MODE
5449                         int j;
5450
5451                         for (j = 0; j < AHD_NUM_LUNS; j++) {
5452                                 struct ahd_tmode_lstate *lstate;
5453
5454                                 lstate = tstate->enabled_luns[j];
5455                                 if (lstate != NULL) {
5456                                         xpt_free_path(lstate->path);
5457                                         free(lstate, M_DEVBUF);
5458                                 }
5459                         }
5460 #endif
5461                         free(tstate, M_DEVBUF);
5462                 }
5463         }
5464 #ifdef AHD_TARGET_MODE
5465         if (ahd->black_hole != NULL) {
5466                 xpt_free_path(ahd->black_hole->path);
5467                 free(ahd->black_hole, M_DEVBUF);
5468         }
5469 #endif
5470         if (ahd->name != NULL)
5471                 free(ahd->name, M_DEVBUF);
5472         if (ahd->seep_config != NULL)
5473                 free(ahd->seep_config, M_DEVBUF);
5474         if (ahd->saved_stack != NULL)
5475                 free(ahd->saved_stack, M_DEVBUF);
5476 #ifndef __FreeBSD__
5477         free(ahd, M_DEVBUF);
5478 #endif
5479         return;
5480 }
5481
5482 void
5483 ahd_shutdown(void *arg)
5484 {
5485         struct  ahd_softc *ahd;
5486
5487         ahd = (struct ahd_softc *)arg;
5488
5489         /*
5490          * Stop periodic timer callbacks.
5491          */
5492         aic_timer_stop(&ahd->reset_timer);
5493         aic_timer_stop(&ahd->stat_timer);
5494
5495         /* This will reset most registers to 0, but not all */
5496         ahd_reset(ahd, /*reinit*/FALSE);
5497 }
5498
5499 /*
5500  * Reset the controller and record some information about it
5501  * that is only available just after a reset.  If "reinit" is
5502  * non-zero, this reset occured after initial configuration
5503  * and the caller requests that the chip be fully reinitialized
5504  * to a runable state.  Chip interrupts are *not* enabled after
5505  * a reinitialization.  The caller must enable interrupts via
5506  * ahd_intr_enable().
5507  */
5508 int
5509 ahd_reset(struct ahd_softc *ahd, int reinit)
5510 {
5511         u_int    sxfrctl1;
5512         int      wait;
5513         uint32_t cmd;
5514
5515         /*
5516          * Preserve the value of the SXFRCTL1 register for all channels.
5517          * It contains settings that affect termination and we don't want
5518          * to disturb the integrity of the bus.
5519          */
5520         ahd_pause(ahd);
5521         ahd_update_modes(ahd);
5522         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5523         sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
5524
5525         cmd = aic_pci_read_config(ahd->dev_softc, PCIR_COMMAND, /*bytes*/2);
5526         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
5527                 uint32_t mod_cmd;
5528
5529                 /*
5530                  * A4 Razor #632
5531                  * During the assertion of CHIPRST, the chip
5532                  * does not disable its parity logic prior to
5533                  * the start of the reset.  This may cause a
5534                  * parity error to be detected and thus a
5535                  * spurious SERR or PERR assertion.  Disble
5536                  * PERR and SERR responses during the CHIPRST.
5537                  */
5538                 mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
5539                 aic_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
5540                                      mod_cmd, /*bytes*/2);
5541         }
5542         ahd_outb(ahd, HCNTRL, CHIPRST | ahd->pause);
5543
5544         /*
5545          * Ensure that the reset has finished.  We delay 1000us
5546          * prior to reading the register to make sure the chip
5547          * has sufficiently completed its reset to handle register
5548          * accesses.
5549          */
5550         wait = 1000;
5551         do {
5552                 aic_delay(1000);
5553         } while (--wait && !(ahd_inb(ahd, HCNTRL) & CHIPRSTACK));
5554
5555         if (wait == 0) {
5556                 printf("%s: WARNING - Failed chip reset!  "
5557                        "Trying to initialize anyway.\n", ahd_name(ahd));
5558                 AHD_FATAL_ERROR(ahd);
5559         }
5560         ahd_outb(ahd, HCNTRL, ahd->pause);
5561
5562         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
5563                 /*
5564                  * Clear any latched PCI error status and restore
5565                  * previous SERR and PERR response enables.
5566                  */
5567                 aic_pci_write_config(ahd->dev_softc, PCIR_STATUS + 1,
5568                                      0xFF, /*bytes*/1);
5569                 aic_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
5570                                      cmd, /*bytes*/2);
5571         }
5572
5573         /*
5574          * Mode should be SCSI after a chip reset, but lets
5575          * set it just to be safe.  We touch the MODE_PTR
5576          * register directly so as to bypass the lazy update
5577          * code in ahd_set_modes().
5578          */
5579         ahd_known_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5580         ahd_outb(ahd, MODE_PTR,
5581                  ahd_build_mode_state(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI));
5582
5583         /*
5584          * Restore SXFRCTL1.
5585          *
5586          * We must always initialize STPWEN to 1 before we
5587          * restore the saved values.  STPWEN is initialized
5588          * to a tri-state condition which can only be cleared
5589          * by turning it on.
5590          */
5591         ahd_outb(ahd, SXFRCTL1, sxfrctl1|STPWEN);
5592         ahd_outb(ahd, SXFRCTL1, sxfrctl1);
5593
5594         /* Determine chip configuration */
5595         ahd->features &= ~AHD_WIDE;
5596         if ((ahd_inb(ahd, SBLKCTL) & SELWIDE) != 0)
5597                 ahd->features |= AHD_WIDE;
5598
5599         /*
5600          * If a recovery action has forced a chip reset,
5601          * re-initialize the chip to our liking.
5602          */
5603         if (reinit != 0)
5604                 ahd_chip_init(ahd);
5605
5606         return (0);
5607 }
5608
5609 /*
5610  * Determine the number of SCBs available on the controller
5611  */
5612 int
5613 ahd_probe_scbs(struct ahd_softc *ahd) {
5614         int i;
5615
5616         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
5617                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
5618         for (i = 0; i < AHD_SCB_MAX; i++) {
5619                 int j;
5620
5621                 ahd_set_scbptr(ahd, i);
5622                 ahd_outw(ahd, SCB_BASE, i);
5623                 for (j = 2; j < 64; j++)
5624                         ahd_outb(ahd, SCB_BASE+j, 0);
5625                 /* Start out life as unallocated (needing an abort) */
5626                 ahd_outb(ahd, SCB_CONTROL, MK_MESSAGE);
5627                 if (ahd_inw_scbram(ahd, SCB_BASE) != i)
5628                         break;
5629                 ahd_set_scbptr(ahd, 0);
5630                 if (ahd_inw_scbram(ahd, SCB_BASE) != 0)
5631                         break;
5632         }
5633         return (i);
5634 }
5635
5636 static void
5637 ahd_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
5638 {
5639         bus_addr_t *baddr;
5640
5641         baddr = (bus_addr_t *)arg;
5642         *baddr = segs->ds_addr;
5643 }
5644
5645 static void
5646 ahd_initialize_hscbs(struct ahd_softc *ahd)
5647 {
5648         int i;
5649
5650         for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
5651                 ahd_set_scbptr(ahd, i);
5652
5653                 /* Clear the control byte. */
5654                 ahd_outb(ahd, SCB_CONTROL, 0);
5655
5656                 /* Set the next pointer */
5657                 ahd_outw(ahd, SCB_NEXT, SCB_LIST_NULL);
5658         }
5659 }
5660
5661 static int
5662 ahd_init_scbdata(struct ahd_softc *ahd)
5663 {
5664         struct  scb_data *scb_data;
5665         int     i;
5666
5667         scb_data = &ahd->scb_data;
5668         TAILQ_INIT(&scb_data->free_scbs);
5669         for (i = 0; i < AHD_NUM_TARGETS * AHD_NUM_LUNS_NONPKT; i++)
5670                 LIST_INIT(&scb_data->free_scb_lists[i]);
5671         LIST_INIT(&scb_data->any_dev_free_scb_list);
5672         SLIST_INIT(&scb_data->hscb_maps);
5673         SLIST_INIT(&scb_data->sg_maps);
5674         SLIST_INIT(&scb_data->sense_maps);
5675
5676         /* Determine the number of hardware SCBs and initialize them */
5677         scb_data->maxhscbs = ahd_probe_scbs(ahd);
5678         if (scb_data->maxhscbs == 0) {
5679                 printf("%s: No SCB space found\n", ahd_name(ahd));
5680                 AHD_FATAL_ERROR(ahd);
5681                 return (ENXIO);
5682         }
5683
5684         ahd_initialize_hscbs(ahd);
5685
5686         /*
5687          * Create our DMA tags.  These tags define the kinds of device
5688          * accessible memory allocations and memory mappings we will
5689          * need to perform during normal operation.
5690          *
5691          * Unless we need to further restrict the allocation, we rely
5692          * on the restrictions of the parent dmat, hence the common
5693          * use of MAXADDR and MAXSIZE.
5694          */
5695
5696         /* DMA tag for our hardware scb structures */
5697         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
5698                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5699                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5700                                /*highaddr*/BUS_SPACE_MAXADDR,
5701                                /*filter*/NULL, /*filterarg*/NULL,
5702                                PAGE_SIZE, /*nsegments*/1,
5703                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5704                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
5705                 goto error_exit;
5706         }
5707
5708         scb_data->init_level++;
5709
5710         /* DMA tag for our S/G structures. */
5711         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/8,
5712                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5713                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5714                                /*highaddr*/BUS_SPACE_MAXADDR,
5715                                /*filter*/NULL, /*filterarg*/NULL,
5716                                ahd_sglist_allocsize(ahd), /*nsegments*/1,
5717                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5718                                /*flags*/0, &scb_data->sg_dmat) != 0) {
5719                 goto error_exit;
5720         }
5721 #ifdef AHD_DEBUG
5722         if ((ahd_debug & AHD_SHOW_MEMORY) != 0)
5723                 printf("%s: ahd_sglist_allocsize = 0x%x\n", ahd_name(ahd),
5724                        ahd_sglist_allocsize(ahd));
5725 #endif
5726
5727         scb_data->init_level++;
5728
5729         /* DMA tag for our sense buffers.  We allocate in page sized chunks */
5730         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
5731                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5732                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5733                                /*highaddr*/BUS_SPACE_MAXADDR,
5734                                /*filter*/NULL, /*filterarg*/NULL,
5735                                PAGE_SIZE, /*nsegments*/1,
5736                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5737                                /*flags*/0, &scb_data->sense_dmat) != 0) {
5738                 goto error_exit;
5739         }
5740
5741         scb_data->init_level++;
5742
5743         /* Perform initial CCB allocation */
5744         while (ahd_alloc_scbs(ahd) != 0)
5745                 ;
5746
5747         if (scb_data->numscbs == 0) {
5748                 printf("%s: ahd_init_scbdata - "
5749                        "Unable to allocate initial scbs\n",
5750                        ahd_name(ahd));
5751                 goto error_exit;
5752         }
5753
5754         /*
5755          * Note that we were successfull
5756          */
5757         return (0);
5758
5759 error_exit:
5760
5761         return (ENOMEM);
5762 }
5763
5764 static struct scb *
5765 ahd_find_scb_by_tag(struct ahd_softc *ahd, u_int tag)
5766 {
5767         struct scb *scb;
5768
5769         /*
5770          * Look on the pending list.
5771          */
5772         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
5773                 if (SCB_GET_TAG(scb) == tag)
5774                         return (scb);
5775         }
5776
5777         /*
5778          * Then on all of the collision free lists.
5779          */
5780         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
5781                 struct scb *list_scb;
5782
5783                 list_scb = scb;
5784                 do {
5785                         if (SCB_GET_TAG(list_scb) == tag)
5786                                 return (list_scb);
5787                         list_scb = LIST_NEXT(list_scb, collision_links);
5788                 } while (list_scb);
5789         }
5790
5791         /*
5792          * And finally on the generic free list.
5793          */
5794         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
5795                 if (SCB_GET_TAG(scb) == tag)
5796                         return (scb);
5797         }
5798
5799         return (NULL);
5800 }
5801
5802 static void
5803 ahd_fini_scbdata(struct ahd_softc *ahd)
5804 {
5805         struct scb_data *scb_data;
5806
5807         scb_data = &ahd->scb_data;
5808         if (scb_data == NULL)
5809                 return;
5810
5811         switch (scb_data->init_level) {
5812         default:
5813         case 7:
5814         {
5815                 struct map_node *sns_map;
5816
5817                 while ((sns_map = SLIST_FIRST(&scb_data->sense_maps)) != NULL) {
5818                         SLIST_REMOVE_HEAD(&scb_data->sense_maps, links);
5819                         aic_dmamap_unload(ahd, scb_data->sense_dmat,
5820                                           sns_map->dmamap);
5821                         aic_dmamem_free(ahd, scb_data->sense_dmat,
5822                                         sns_map->vaddr, sns_map->dmamap);
5823                         free(sns_map, M_DEVBUF);
5824                 }
5825                 aic_dma_tag_destroy(ahd, scb_data->sense_dmat);
5826                 /* FALLTHROUGH */
5827         }
5828         case 6:
5829         {
5830                 struct map_node *sg_map;
5831
5832                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps)) != NULL) {
5833                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
5834                         aic_dmamap_unload(ahd, scb_data->sg_dmat,
5835                                           sg_map->dmamap);
5836                         aic_dmamem_free(ahd, scb_data->sg_dmat,
5837                                         sg_map->vaddr, sg_map->dmamap);
5838                         free(sg_map, M_DEVBUF);
5839                 }
5840                 aic_dma_tag_destroy(ahd, scb_data->sg_dmat);
5841                 /* FALLTHROUGH */
5842         }
5843         case 5:
5844         {
5845                 struct map_node *hscb_map;
5846
5847                 while ((hscb_map = SLIST_FIRST(&scb_data->hscb_maps)) != NULL) {
5848                         SLIST_REMOVE_HEAD(&scb_data->hscb_maps, links);
5849                         aic_dmamap_unload(ahd, scb_data->hscb_dmat,
5850                                           hscb_map->dmamap);
5851                         aic_dmamem_free(ahd, scb_data->hscb_dmat,
5852                                         hscb_map->vaddr, hscb_map->dmamap);
5853                         free(hscb_map, M_DEVBUF);
5854                 }
5855                 aic_dma_tag_destroy(ahd, scb_data->hscb_dmat);
5856                 /* FALLTHROUGH */
5857         }
5858         case 4:
5859         case 3:
5860         case 2:
5861         case 1:
5862         case 0:
5863                 break;
5864         }
5865 }
5866
5867 /*
5868  * DSP filter Bypass must be enabled until the first selection
5869  * after a change in bus mode (Razor #491 and #493).
5870  */
5871 static void
5872 ahd_setup_iocell_workaround(struct ahd_softc *ahd)
5873 {
5874         ahd_mode_state saved_modes;
5875
5876         saved_modes = ahd_save_modes(ahd);
5877         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
5878         ahd_outb(ahd, DSPDATACTL, ahd_inb(ahd, DSPDATACTL)
5879                | BYPASSENAB | RCVROFFSTDIS | XMITOFFSTDIS);
5880         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) | (ENSELDO|ENSELDI));
5881 #ifdef AHD_DEBUG
5882         if ((ahd_debug & AHD_SHOW_MISC) != 0)
5883                 printf("%s: Setting up iocell workaround\n", ahd_name(ahd));
5884 #endif
5885         ahd_restore_modes(ahd, saved_modes);
5886         ahd->flags &= ~AHD_HAD_FIRST_SEL;
5887 }
5888
5889 static void
5890 ahd_iocell_first_selection(struct ahd_softc *ahd)
5891 {
5892         ahd_mode_state  saved_modes;
5893         u_int           sblkctl;
5894
5895         if ((ahd->flags & AHD_HAD_FIRST_SEL) != 0)
5896                 return;
5897         saved_modes = ahd_save_modes(ahd);
5898         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5899         sblkctl = ahd_inb(ahd, SBLKCTL);
5900         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
5901 #ifdef AHD_DEBUG
5902         if ((ahd_debug & AHD_SHOW_MISC) != 0)
5903                 printf("%s: iocell first selection\n", ahd_name(ahd));
5904 #endif
5905         if ((sblkctl & ENAB40) != 0) {
5906                 ahd_outb(ahd, DSPDATACTL,
5907                          ahd_inb(ahd, DSPDATACTL) & ~BYPASSENAB);
5908 #ifdef AHD_DEBUG
5909                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
5910                         printf("%s: BYPASS now disabled\n", ahd_name(ahd));
5911 #endif
5912         }
5913         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) & ~(ENSELDO|ENSELDI));
5914         ahd_outb(ahd, CLRINT, CLRSCSIINT);
5915         ahd_restore_modes(ahd, saved_modes);
5916         ahd->flags |= AHD_HAD_FIRST_SEL;
5917 }
5918
5919 /*************************** SCB Management ***********************************/
5920 static void
5921 ahd_add_col_list(struct ahd_softc *ahd, struct scb *scb, u_int col_idx)
5922 {
5923         struct  scb_list *free_list;
5924         struct  scb_tailq *free_tailq;
5925         struct  scb *first_scb;
5926
5927         scb->flags |= SCB_ON_COL_LIST;
5928         AHD_SET_SCB_COL_IDX(scb, col_idx);
5929         free_list = &ahd->scb_data.free_scb_lists[col_idx];
5930         free_tailq = &ahd->scb_data.free_scbs;
5931         first_scb = LIST_FIRST(free_list);
5932         if (first_scb != NULL) {
5933                 LIST_INSERT_AFTER(first_scb, scb, collision_links);
5934         } else {
5935                 LIST_INSERT_HEAD(free_list, scb, collision_links);
5936                 TAILQ_INSERT_TAIL(free_tailq, scb, links.tqe);
5937         }
5938 }
5939
5940 static void
5941 ahd_rem_col_list(struct ahd_softc *ahd, struct scb *scb)
5942 {
5943         struct  scb_list *free_list;
5944         struct  scb_tailq *free_tailq;
5945         struct  scb *first_scb;
5946         u_int   col_idx;
5947
5948         scb->flags &= ~SCB_ON_COL_LIST;
5949         col_idx = AHD_GET_SCB_COL_IDX(ahd, scb);
5950         free_list = &ahd->scb_data.free_scb_lists[col_idx];
5951         free_tailq = &ahd->scb_data.free_scbs;
5952         first_scb = LIST_FIRST(free_list);
5953         if (first_scb == scb) {
5954                 struct scb *next_scb;
5955
5956                 /*
5957                  * Maintain order in the collision free
5958                  * lists for fairness if this device has
5959                  * other colliding tags active.
5960                  */
5961                 next_scb = LIST_NEXT(scb, collision_links);
5962                 if (next_scb != NULL) {
5963                         TAILQ_INSERT_AFTER(free_tailq, scb,
5964                                            next_scb, links.tqe);
5965                 }
5966                 TAILQ_REMOVE(free_tailq, scb, links.tqe);
5967         }
5968         LIST_REMOVE(scb, collision_links);
5969 }
5970
5971 /*
5972  * Get a free scb. If there are none, see if we can allocate a new SCB.
5973  */
5974 struct scb *
5975 ahd_get_scb(struct ahd_softc *ahd, u_int col_idx)
5976 {
5977         struct scb *scb;
5978         int tries;
5979
5980         tries = 0;
5981 look_again:
5982         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
5983                 if (AHD_GET_SCB_COL_IDX(ahd, scb) != col_idx) {
5984                         ahd_rem_col_list(ahd, scb);
5985                         goto found;
5986                 }
5987         }
5988         if ((scb = LIST_FIRST(&ahd->scb_data.any_dev_free_scb_list)) == NULL) {
5989
5990                 if (tries++ != 0)
5991                         return (NULL);
5992                 if (ahd_alloc_scbs(ahd) == 0)
5993                         return (NULL);
5994                 goto look_again;
5995         }
5996         LIST_REMOVE(scb, links.le);
5997         if (col_idx != AHD_NEVER_COL_IDX
5998          && (scb->col_scb != NULL)
5999          && (scb->col_scb->flags & SCB_ACTIVE) == 0) {
6000                 LIST_REMOVE(scb->col_scb, links.le);
6001                 ahd_add_col_list(ahd, scb->col_scb, col_idx);
6002         }
6003 found:
6004         scb->flags |= SCB_ACTIVE;
6005         return (scb);
6006 }
6007
6008 /*
6009  * Return an SCB resource to the free list.
6010  */
6011 void
6012 ahd_free_scb(struct ahd_softc *ahd, struct scb *scb)
6013 {
6014
6015         /* Clean up for the next user */
6016         scb->flags = SCB_FLAG_NONE;
6017         scb->hscb->control = 0;
6018         ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = NULL;
6019
6020         if (scb->col_scb == NULL) {
6021
6022                 /*
6023                  * No collision possible.  Just free normally.
6024                  */
6025                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6026                                  scb, links.le);
6027         } else if ((scb->col_scb->flags & SCB_ON_COL_LIST) != 0) {
6028
6029                 /*
6030                  * The SCB we might have collided with is on
6031                  * a free collision list.  Put both SCBs on
6032                  * the generic list.
6033                  */
6034                 ahd_rem_col_list(ahd, scb->col_scb);
6035                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6036                                  scb, links.le);
6037                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6038                                  scb->col_scb, links.le);
6039         } else if ((scb->col_scb->flags
6040                   & (SCB_PACKETIZED|SCB_ACTIVE)) == SCB_ACTIVE
6041                 && (scb->col_scb->hscb->control & TAG_ENB) != 0) {
6042
6043                 /*
6044                  * The SCB we might collide with on the next allocation
6045                  * is still active in a non-packetized, tagged, context.
6046                  * Put us on the SCB collision list.
6047                  */
6048                 ahd_add_col_list(ahd, scb,
6049                                  AHD_GET_SCB_COL_IDX(ahd, scb->col_scb));
6050         } else {
6051                 /*
6052                  * The SCB we might collide with on the next allocation
6053                  * is either active in a packetized context, or free.
6054                  * Since we can't collide, put this SCB on the generic
6055                  * free list.
6056                  */
6057                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6058                                  scb, links.le);
6059         }
6060
6061         aic_platform_scb_free(ahd, scb);
6062 }
6063
6064 int
6065 ahd_alloc_scbs(struct ahd_softc *ahd)
6066 {
6067         struct scb_data *scb_data;
6068         struct scb      *next_scb;
6069         struct hardware_scb *hscb;
6070         struct map_node *hscb_map;
6071         struct map_node *sg_map;
6072         struct map_node *sense_map;
6073         uint8_t         *segs;
6074         uint8_t         *sense_data;
6075         bus_addr_t       hscb_busaddr;
6076         bus_addr_t       sg_busaddr;
6077         bus_addr_t       sense_busaddr;
6078         int              newcount;
6079         int              i;
6080
6081         scb_data = &ahd->scb_data;
6082         if (scb_data->numscbs >= AHD_SCB_MAX_ALLOC)
6083                 /* Can't allocate any more */
6084                 return (0);
6085
6086         if (scb_data->scbs_left != 0) {
6087                 int offset;
6088
6089                 offset = (PAGE_SIZE / sizeof(*hscb)) - scb_data->scbs_left;
6090                 hscb_map = SLIST_FIRST(&scb_data->hscb_maps);
6091                 hscb = &((struct hardware_scb *)hscb_map->vaddr)[offset];
6092                 hscb_busaddr = hscb_map->busaddr + (offset * sizeof(*hscb));
6093         } else {
6094                 hscb_map = malloc(sizeof(*hscb_map), M_DEVBUF, M_NOWAIT);
6095
6096                 if (hscb_map == NULL)
6097                         return (0);
6098
6099                 /* Allocate the next batch of hardware SCBs */
6100                 if (aic_dmamem_alloc(ahd, scb_data->hscb_dmat,
6101                                      (void **)&hscb_map->vaddr,
6102                                      BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
6103                                      &hscb_map->dmamap) != 0) {
6104                         free(hscb_map, M_DEVBUF);
6105                         return (0);
6106                 }
6107
6108                 SLIST_INSERT_HEAD(&scb_data->hscb_maps, hscb_map, links);
6109
6110                 aic_dmamap_load(ahd, scb_data->hscb_dmat, hscb_map->dmamap,
6111                                 hscb_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6112                                 &hscb_map->busaddr, /*flags*/0);
6113
6114                 hscb = (struct hardware_scb *)hscb_map->vaddr;
6115                 hscb_busaddr = hscb_map->busaddr;
6116                 scb_data->scbs_left = PAGE_SIZE / sizeof(*hscb);
6117         }
6118
6119         if (scb_data->sgs_left != 0) {
6120                 int offset;
6121
6122                 offset = ((ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd))
6123                        - scb_data->sgs_left) * ahd_sglist_size(ahd);
6124                 sg_map = SLIST_FIRST(&scb_data->sg_maps);
6125                 segs = sg_map->vaddr + offset;
6126                 sg_busaddr = sg_map->busaddr + offset;
6127         } else {
6128                 sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
6129
6130                 if (sg_map == NULL)
6131                         return (0);
6132
6133                 /* Allocate the next batch of S/G lists */
6134                 if (aic_dmamem_alloc(ahd, scb_data->sg_dmat,
6135                                      (void **)&sg_map->vaddr,
6136                                      BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
6137                                      &sg_map->dmamap) != 0) {
6138                         free(sg_map, M_DEVBUF);
6139                         return (0);
6140                 }
6141
6142                 SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
6143
6144                 aic_dmamap_load(ahd, scb_data->sg_dmat, sg_map->dmamap,
6145                                 sg_map->vaddr, ahd_sglist_allocsize(ahd),
6146                                 ahd_dmamap_cb, &sg_map->busaddr, /*flags*/0);
6147
6148                 segs = sg_map->vaddr;
6149                 sg_busaddr = sg_map->busaddr;
6150                 scb_data->sgs_left =
6151                     ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd);
6152 #ifdef AHD_DEBUG
6153                 if (ahd_debug & AHD_SHOW_MEMORY)
6154                         printf("Mapped SG data\n");
6155 #endif
6156         }
6157
6158         if (scb_data->sense_left != 0) {
6159                 int offset;
6160
6161                 offset = PAGE_SIZE - (AHD_SENSE_BUFSIZE * scb_data->sense_left);
6162                 sense_map = SLIST_FIRST(&scb_data->sense_maps);
6163                 sense_data = sense_map->vaddr + offset;
6164                 sense_busaddr = sense_map->busaddr + offset;
6165         } else {
6166                 sense_map = malloc(sizeof(*sense_map), M_DEVBUF, M_NOWAIT);
6167
6168                 if (sense_map == NULL)
6169                         return (0);
6170
6171                 /* Allocate the next batch of sense buffers */
6172                 if (aic_dmamem_alloc(ahd, scb_data->sense_dmat,
6173                                      (void **)&sense_map->vaddr,
6174                                      BUS_DMA_NOWAIT, &sense_map->dmamap) != 0) {
6175                         free(sense_map, M_DEVBUF);
6176                         return (0);
6177                 }
6178
6179                 SLIST_INSERT_HEAD(&scb_data->sense_maps, sense_map, links);
6180
6181                 aic_dmamap_load(ahd, scb_data->sense_dmat, sense_map->dmamap,
6182                                 sense_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6183                                 &sense_map->busaddr, /*flags*/0);
6184
6185                 sense_data = sense_map->vaddr;
6186                 sense_busaddr = sense_map->busaddr;
6187                 scb_data->sense_left = PAGE_SIZE / AHD_SENSE_BUFSIZE;
6188 #ifdef AHD_DEBUG
6189                 if (ahd_debug & AHD_SHOW_MEMORY)
6190                         printf("Mapped sense data\n");
6191 #endif
6192         }
6193
6194         newcount = MIN(scb_data->sense_left, scb_data->scbs_left);
6195         newcount = MIN(newcount, scb_data->sgs_left);
6196         newcount = MIN(newcount, (AHD_SCB_MAX_ALLOC - scb_data->numscbs));
6197         scb_data->sense_left -= newcount;
6198         scb_data->scbs_left -= newcount;
6199         scb_data->sgs_left -= newcount;
6200         for (i = 0; i < newcount; i++) {
6201                 struct scb_platform_data *pdata;
6202                 u_int col_tag;
6203 #ifndef __linux__
6204                 int error;
6205 #endif
6206
6207                 next_scb = (struct scb *)malloc(sizeof(*next_scb),
6208                                                 M_DEVBUF, M_NOWAIT);
6209                 if (next_scb == NULL)
6210                         break;
6211
6212                 pdata = (struct scb_platform_data *)malloc(sizeof(*pdata),
6213                                                            M_DEVBUF, M_NOWAIT);
6214                 if (pdata == NULL) {
6215                         free(next_scb, M_DEVBUF);
6216                         break;
6217                 }
6218                 next_scb->platform_data = pdata;
6219                 next_scb->hscb_map = hscb_map;
6220                 next_scb->sg_map = sg_map;
6221                 next_scb->sense_map = sense_map;
6222                 next_scb->sg_list = segs;
6223                 next_scb->sense_data = sense_data;
6224                 next_scb->sense_busaddr = sense_busaddr;
6225                 memset(hscb, 0, sizeof(*hscb));
6226                 next_scb->hscb = hscb;
6227                 hscb->hscb_busaddr = aic_htole32(hscb_busaddr);
6228
6229                 /*
6230                  * The sequencer always starts with the second entry.
6231                  * The first entry is embedded in the scb.
6232                  */
6233                 next_scb->sg_list_busaddr = sg_busaddr;
6234                 if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
6235                         next_scb->sg_list_busaddr
6236                             += sizeof(struct ahd_dma64_seg);
6237                 else
6238                         next_scb->sg_list_busaddr += sizeof(struct ahd_dma_seg);
6239                 next_scb->ahd_softc = ahd;
6240                 next_scb->flags = SCB_FLAG_NONE;
6241 #ifndef __linux__
6242                 error = aic_dmamap_create(ahd, ahd->buffer_dmat, /*flags*/0,
6243                                           &next_scb->dmamap);
6244                 if (error != 0) {
6245                         free(next_scb, M_DEVBUF);
6246                         free(pdata, M_DEVBUF);
6247                         break;
6248                 }
6249 #endif
6250                 next_scb->hscb->tag = aic_htole16(scb_data->numscbs);
6251                 col_tag = scb_data->numscbs ^ 0x100;
6252                 next_scb->col_scb = ahd_find_scb_by_tag(ahd, col_tag);
6253                 if (next_scb->col_scb != NULL)
6254                         next_scb->col_scb->col_scb = next_scb;
6255                 aic_timer_init(&next_scb->io_timer);
6256                 ahd_free_scb(ahd, next_scb);
6257                 hscb++;
6258                 hscb_busaddr += sizeof(*hscb);
6259                 segs += ahd_sglist_size(ahd);
6260                 sg_busaddr += ahd_sglist_size(ahd);
6261                 sense_data += AHD_SENSE_BUFSIZE;
6262                 sense_busaddr += AHD_SENSE_BUFSIZE;
6263                 scb_data->numscbs++;
6264         }
6265         return (i);
6266 }
6267
6268 void
6269 ahd_controller_info(struct ahd_softc *ahd, char *buf)
6270 {
6271         const char *speed;
6272         const char *type;
6273         int len;
6274
6275         len = sprintf(buf, "%s: ", ahd_chip_names[ahd->chip & AHD_CHIPID_MASK]);
6276         buf += len;
6277
6278         speed = "Ultra320 ";
6279         if ((ahd->features & AHD_WIDE) != 0) {
6280                 type = "Wide ";
6281         } else {
6282                 type = "Single ";
6283         }
6284         len = sprintf(buf, "%s%sChannel %c, SCSI Id=%d, ",
6285                       speed, type, ahd->channel, ahd->our_id);
6286         buf += len;
6287
6288         sprintf(buf, "%s, %d SCBs", ahd->bus_description,
6289                 ahd->scb_data.maxhscbs);
6290 }
6291
6292 static const char *channel_strings[] = {
6293         "Primary Low",
6294         "Primary High",
6295         "Secondary Low",
6296         "Secondary High"
6297 };
6298
6299 static const char *termstat_strings[] = {
6300         "Terminated Correctly",
6301         "Over Terminated",
6302         "Under Terminated",
6303         "Not Configured"
6304 };
6305
6306 /*
6307  * Start the board, ready for normal operation
6308  */
6309 int
6310 ahd_init(struct ahd_softc *ahd)
6311 {
6312         uint8_t         *next_vaddr;
6313         bus_addr_t       next_baddr;
6314         size_t           driver_data_size;
6315         int              i;
6316         int              error;
6317         u_int            warn_user;
6318         uint8_t          current_sensing;
6319         uint8_t          fstat;
6320
6321         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
6322
6323         ahd->stack_size = ahd_probe_stack_size(ahd);
6324         ahd->saved_stack = malloc(ahd->stack_size * sizeof(uint16_t),
6325                                   M_DEVBUF, M_NOWAIT);
6326         if (ahd->saved_stack == NULL)
6327                 return (ENOMEM);
6328
6329         /*
6330          * Verify that the compiler hasn't over-agressively
6331          * padded important structures.
6332          */
6333         if (sizeof(struct hardware_scb) != 64)
6334                 panic("Hardware SCB size is incorrect");
6335
6336 #ifdef AHD_DEBUG
6337         if ((ahd_debug & AHD_DEBUG_SEQUENCER) != 0)
6338                 ahd->flags |= AHD_SEQUENCER_DEBUG;
6339 #endif
6340
6341         /*
6342          * Default to allowing initiator operations.
6343          */
6344         ahd->flags |= AHD_INITIATORROLE;
6345
6346         /*
6347          * Only allow target mode features if this unit has them enabled.
6348          */
6349         if ((AHD_TMODE_ENABLE & (0x1 << ahd->unit)) == 0)
6350                 ahd->features &= ~AHD_TARGETMODE;
6351
6352 #ifndef __linux__
6353         /* DMA tag for mapping buffers into device visible space. */
6354         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6355                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6356                                /*lowaddr*/ahd->flags & AHD_39BIT_ADDRESSING
6357                                         ? (bus_addr_t)0x7FFFFFFFFFULL
6358                                         : BUS_SPACE_MAXADDR_32BIT,
6359                                /*highaddr*/BUS_SPACE_MAXADDR,
6360                                /*filter*/NULL, /*filterarg*/NULL,
6361                                /*maxsize*/(AHD_NSEG - 1) * PAGE_SIZE,
6362                                /*nsegments*/AHD_NSEG,
6363                                /*maxsegsz*/AHD_MAXTRANSFER_SIZE,
6364                                /*flags*/BUS_DMA_ALLOCNOW,
6365                                &ahd->buffer_dmat) != 0) {
6366                 return (ENOMEM);
6367         }
6368 #endif
6369
6370         ahd->init_level++;
6371
6372         /*
6373          * DMA tag for our command fifos and other data in system memory
6374          * the card's sequencer must be able to access.  For initiator
6375          * roles, we need to allocate space for the qoutfifo.  When providing
6376          * for the target mode role, we must additionally provide space for
6377          * the incoming target command fifo.
6378          */
6379         driver_data_size = AHD_SCB_MAX * sizeof(*ahd->qoutfifo)
6380                          + sizeof(struct hardware_scb);
6381         if ((ahd->features & AHD_TARGETMODE) != 0)
6382                 driver_data_size += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6383         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0)
6384                 driver_data_size += PKT_OVERRUN_BUFSIZE;
6385         if (aic_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6386                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6387                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6388                                /*highaddr*/BUS_SPACE_MAXADDR,
6389                                /*filter*/NULL, /*filterarg*/NULL,
6390                                driver_data_size,
6391                                /*nsegments*/1,
6392                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6393                                /*flags*/0, &ahd->shared_data_dmat) != 0) {
6394                 return (ENOMEM);
6395         }
6396
6397         ahd->init_level++;
6398
6399         /* Allocation of driver data */
6400         if (aic_dmamem_alloc(ahd, ahd->shared_data_dmat,
6401                              (void **)&ahd->shared_data_map.vaddr,
6402                              BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
6403                              &ahd->shared_data_map.dmamap) != 0) {
6404                 return (ENOMEM);
6405         }
6406
6407         ahd->init_level++;
6408
6409         /* And permanently map it in */
6410         aic_dmamap_load(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
6411                         ahd->shared_data_map.vaddr, driver_data_size,
6412                         ahd_dmamap_cb, &ahd->shared_data_map.busaddr,
6413                         /*flags*/0);
6414         ahd->qoutfifo = (struct ahd_completion *)ahd->shared_data_map.vaddr;
6415         next_vaddr = (uint8_t *)&ahd->qoutfifo[AHD_QOUT_SIZE];
6416         next_baddr = ahd->shared_data_map.busaddr
6417                    + AHD_QOUT_SIZE*sizeof(struct ahd_completion);
6418         if ((ahd->features & AHD_TARGETMODE) != 0) {
6419                 ahd->targetcmds = (struct target_cmd *)next_vaddr;
6420                 next_vaddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6421                 next_baddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6422         }
6423
6424         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0) {
6425                 ahd->overrun_buf = next_vaddr;
6426                 next_vaddr += PKT_OVERRUN_BUFSIZE;
6427                 next_baddr += PKT_OVERRUN_BUFSIZE;
6428         }
6429
6430         /*
6431          * We need one SCB to serve as the "next SCB".  Since the
6432          * tag identifier in this SCB will never be used, there is
6433          * no point in using a valid HSCB tag from an SCB pulled from
6434          * the standard free pool.  So, we allocate this "sentinel"
6435          * specially from the DMA safe memory chunk used for the QOUTFIFO.
6436          */
6437         ahd->next_queued_hscb = (struct hardware_scb *)next_vaddr;
6438         ahd->next_queued_hscb_map = &ahd->shared_data_map;
6439         ahd->next_queued_hscb->hscb_busaddr = aic_htole32(next_baddr);
6440
6441         ahd->init_level++;
6442
6443         /* Allocate SCB data now that buffer_dmat is initialized */
6444         if (ahd_init_scbdata(ahd) != 0)
6445                 return (ENOMEM);
6446
6447         if ((ahd->flags & AHD_INITIATORROLE) == 0)
6448                 ahd->flags &= ~AHD_RESET_BUS_A;
6449
6450         /*
6451          * Before committing these settings to the chip, give
6452          * the OSM one last chance to modify our configuration.
6453          */
6454         ahd_platform_init(ahd);
6455
6456         /* Bring up the chip. */
6457         ahd_chip_init(ahd);
6458
6459         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
6460
6461         if ((ahd->flags & AHD_CURRENT_SENSING) == 0)
6462                 goto init_done;
6463
6464         /*
6465          * Verify termination based on current draw and
6466          * warn user if the bus is over/under terminated.
6467          */
6468         error = ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL,
6469                                    CURSENSE_ENB);
6470         if (error != 0) {
6471                 printf("%s: current sensing timeout 1\n", ahd_name(ahd));
6472                 goto init_done;
6473         }
6474         for (i = 20, fstat = FLX_FSTAT_BUSY;
6475              (fstat & FLX_FSTAT_BUSY) != 0 && i; i--) {
6476                 error = ahd_read_flexport(ahd, FLXADDR_FLEXSTAT, &fstat);
6477                 if (error != 0) {
6478                         printf("%s: current sensing timeout 2\n",
6479                                ahd_name(ahd));
6480                         goto init_done;
6481                 }
6482         }
6483         if (i == 0) {
6484                 printf("%s: Timedout during current-sensing test\n",
6485                        ahd_name(ahd));
6486                 goto init_done;
6487         }
6488
6489         /* Latch Current Sensing status. */
6490         error = ahd_read_flexport(ahd, FLXADDR_CURRENT_STAT, &current_sensing);
6491         if (error != 0) {
6492                 printf("%s: current sensing timeout 3\n", ahd_name(ahd));
6493                 goto init_done;
6494         }
6495
6496         /* Diable current sensing. */
6497         ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, 0);
6498
6499 #ifdef AHD_DEBUG
6500         if ((ahd_debug & AHD_SHOW_TERMCTL) != 0) {
6501                 printf("%s: current_sensing == 0x%x\n",
6502                        ahd_name(ahd), current_sensing);
6503         }
6504 #endif
6505         warn_user = 0;
6506         for (i = 0; i < 4; i++, current_sensing >>= FLX_CSTAT_SHIFT) {
6507                 u_int term_stat;
6508
6509                 term_stat = (current_sensing & FLX_CSTAT_MASK);
6510                 switch (term_stat) {
6511                 case FLX_CSTAT_OVER:
6512                 case FLX_CSTAT_UNDER:
6513                         warn_user++;
6514                 case FLX_CSTAT_INVALID:
6515                 case FLX_CSTAT_OKAY:
6516                         if (warn_user == 0 && bootverbose == 0)
6517                                 break;
6518                         printf("%s: %s Channel %s\n", ahd_name(ahd),
6519                                channel_strings[i], termstat_strings[term_stat]);
6520                         break;
6521                 }
6522         }
6523         if (warn_user) {
6524                 printf("%s: WARNING. Termination is not configured correctly.\n"
6525                        "%s: WARNING. SCSI bus operations may FAIL.\n",
6526                        ahd_name(ahd), ahd_name(ahd));
6527                 AHD_CORRECTABLE_ERROR(ahd);
6528         }
6529 init_done:
6530         ahd_restart(ahd);
6531         aic_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_MS,
6532                         ahd_stat_timer, ahd);
6533         return (0);
6534 }
6535
6536 /*
6537  * (Re)initialize chip state after a chip reset.
6538  */
6539 static void
6540 ahd_chip_init(struct ahd_softc *ahd)
6541 {
6542         uint32_t busaddr;
6543         u_int    sxfrctl1;
6544         u_int    scsiseq_template;
6545         u_int    wait;
6546         u_int    i;
6547         u_int    target;
6548
6549         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6550         /*
6551          * Take the LED out of diagnostic mode
6552          */
6553         ahd_outb(ahd, SBLKCTL, ahd_inb(ahd, SBLKCTL) & ~(DIAGLEDEN|DIAGLEDON));
6554
6555         /*
6556          * Return HS_MAILBOX to its default value.
6557          */
6558         ahd->hs_mailbox = 0;
6559         ahd_outb(ahd, HS_MAILBOX, 0);
6560
6561         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1. */
6562         ahd_outb(ahd, IOWNID, ahd->our_id);
6563         ahd_outb(ahd, TOWNID, ahd->our_id);
6564         sxfrctl1 = (ahd->flags & AHD_TERM_ENB_A) != 0 ? STPWEN : 0;
6565         sxfrctl1 |= (ahd->flags & AHD_SPCHK_ENB_A) != 0 ? ENSPCHK : 0;
6566         if ((ahd->bugs & AHD_LONG_SETIMO_BUG)
6567          && (ahd->seltime != STIMESEL_MIN)) {
6568                 /*
6569                  * The selection timer duration is twice as long
6570                  * as it should be.  Halve it by adding "1" to
6571                  * the user specified setting.
6572                  */
6573                 sxfrctl1 |= ahd->seltime + STIMESEL_BUG_ADJ;
6574         } else {
6575                 sxfrctl1 |= ahd->seltime;
6576         }
6577
6578         ahd_outb(ahd, SXFRCTL0, DFON);
6579         ahd_outb(ahd, SXFRCTL1, sxfrctl1|ahd->seltime|ENSTIMER|ACTNEGEN);
6580         ahd_outb(ahd, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
6581
6582         /*
6583          * Now that termination is set, wait for up
6584          * to 500ms for our transceivers to settle.  If
6585          * the adapter does not have a cable attached,
6586          * the transceivers may never settle, so don't
6587          * complain if we fail here.
6588          */
6589         for (wait = 10000;
6590              (ahd_inb(ahd, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
6591              wait--)
6592                 aic_delay(100);
6593
6594         /* Clear any false bus resets due to the transceivers settling */
6595         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
6596         ahd_outb(ahd, CLRINT, CLRSCSIINT);
6597
6598         /* Initialize mode specific S/G state. */
6599         for (i = 0; i < 2; i++) {
6600                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
6601                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
6602                 ahd_outb(ahd, SG_STATE, 0);
6603                 ahd_outb(ahd, CLRSEQINTSRC, 0xFF);
6604                 ahd_outb(ahd, SEQIMODE,
6605                          ENSAVEPTRS|ENCFG4DATA|ENCFG4ISTAT
6606                         |ENCFG4TSTAT|ENCFG4ICMD|ENCFG4TCMD);
6607         }
6608
6609         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6610         ahd_outb(ahd, DSCOMMAND0, ahd_inb(ahd, DSCOMMAND0)|MPARCKEN|CACHETHEN);
6611         ahd_outb(ahd, DFF_THRSH, RD_DFTHRSH_75|WR_DFTHRSH_75);
6612         ahd_outb(ahd, SIMODE0, ENIOERR|ENOVERRUN);
6613         ahd_outb(ahd, SIMODE3, ENNTRAMPERR|ENOSRAMPERR);
6614         if ((ahd->bugs & AHD_BUSFREEREV_BUG) != 0) {
6615                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|AUTO_MSGOUT_DE);
6616         } else {
6617                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|BUSFREEREV|AUTO_MSGOUT_DE);
6618         }
6619         ahd_outb(ahd, SCSCHKN, CURRFIFODEF|WIDERESEN|SHVALIDSTDIS);
6620         if ((ahd->chip & AHD_BUS_MASK) == AHD_PCIX)
6621                 /*
6622                  * Do not issue a target abort when a split completion
6623                  * error occurs.  Let our PCIX interrupt handler deal
6624                  * with it instead. H2A4 Razor #625
6625                  */
6626                 ahd_outb(ahd, PCIXCTL, ahd_inb(ahd, PCIXCTL) | SPLTSTADIS);
6627
6628         if ((ahd->bugs & AHD_LQOOVERRUN_BUG) != 0)
6629                 ahd_outb(ahd, LQOSCSCTL, LQONOCHKOVER);
6630
6631         /*
6632          * Tweak IOCELL settings.
6633          */
6634         if ((ahd->flags & AHD_HP_BOARD) != 0) {
6635                 for (i = 0; i < NUMDSPS; i++) {
6636                         ahd_outb(ahd, DSPSELECT, i);
6637                         ahd_outb(ahd, WRTBIASCTL, WRTBIASCTL_HP_DEFAULT);
6638                 }
6639 #ifdef AHD_DEBUG
6640                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
6641                         printf("%s: WRTBIASCTL now 0x%x\n", ahd_name(ahd),
6642                                WRTBIASCTL_HP_DEFAULT);
6643 #endif
6644         }
6645         ahd_setup_iocell_workaround(ahd);
6646
6647         /*
6648          * Enable LQI Manager interrupts.
6649          */
6650         ahd_outb(ahd, LQIMODE1, ENLQIPHASE_LQ|ENLQIPHASE_NLQ|ENLIQABORT
6651                               | ENLQICRCI_LQ|ENLQICRCI_NLQ|ENLQIBADLQI
6652                               | ENLQIOVERI_LQ|ENLQIOVERI_NLQ);
6653         ahd_outb(ahd, LQOMODE0, ENLQOATNLQ|ENLQOATNPKT|ENLQOTCRC);
6654         /*
6655          * We choose to have the sequencer catch LQOPHCHGINPKT errors
6656          * manually for the command phase at the start of a packetized
6657          * selection case.  ENLQOBUSFREE should be made redundant by
6658          * the BUSFREE interrupt, but it seems that some LQOBUSFREE
6659          * events fail to assert the BUSFREE interrupt so we must
6660          * also enable LQOBUSFREE interrupts.
6661          */
6662         ahd_outb(ahd, LQOMODE1, ENLQOBUSFREE);
6663
6664         /*
6665          * Setup sequencer interrupt handlers.
6666          */
6667         ahd_outw(ahd, INTVEC1_ADDR, ahd_resolve_seqaddr(ahd, LABEL_seq_isr));
6668         ahd_outw(ahd, INTVEC2_ADDR, ahd_resolve_seqaddr(ahd, LABEL_timer_isr));
6669
6670         /*
6671          * Setup SCB Offset registers.
6672          */
6673         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
6674                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb,
6675                          pkt_long_lun));
6676         } else {
6677                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb, lun));
6678         }
6679         ahd_outb(ahd, CMDLENPTR, offsetof(struct hardware_scb, cdb_len));
6680         ahd_outb(ahd, ATTRPTR, offsetof(struct hardware_scb, task_attribute));
6681         ahd_outb(ahd, FLAGPTR, offsetof(struct hardware_scb, task_management));
6682         ahd_outb(ahd, CMDPTR, offsetof(struct hardware_scb,
6683                                        shared_data.idata.cdb));
6684         ahd_outb(ahd, QNEXTPTR,
6685                  offsetof(struct hardware_scb, next_hscb_busaddr));
6686         ahd_outb(ahd, ABRTBITPTR, MK_MESSAGE_BIT_OFFSET);
6687         ahd_outb(ahd, ABRTBYTEPTR, offsetof(struct hardware_scb, control));
6688         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
6689                 ahd_outb(ahd, LUNLEN,
6690                          sizeof(ahd->next_queued_hscb->pkt_long_lun) - 1);
6691         } else {
6692                 ahd_outb(ahd, LUNLEN, LUNLEN_SINGLE_LEVEL_LUN);
6693         }
6694         ahd_outb(ahd, CDBLIMIT, SCB_CDB_LEN_PTR - 1);
6695         ahd_outb(ahd, MAXCMD, 0xFF);
6696         ahd_outb(ahd, SCBAUTOPTR,
6697                  AUSCBPTR_EN | offsetof(struct hardware_scb, tag));
6698
6699         /* We haven't been enabled for target mode yet. */
6700         ahd_outb(ahd, MULTARGID, 0);
6701         ahd_outb(ahd, MULTARGID + 1, 0);
6702
6703         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6704         /* Initialize the negotiation table. */
6705         if ((ahd->features & AHD_NEW_IOCELL_OPTS) == 0) {
6706                 /*
6707                  * Clear the spare bytes in the neg table to avoid
6708                  * spurious parity errors.
6709                  */
6710                 for (target = 0; target < AHD_NUM_TARGETS; target++) {
6711                         ahd_outb(ahd, NEGOADDR, target);
6712                         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PER_DEV0);
6713                         for (i = 0; i < AHD_NUM_PER_DEV_ANNEXCOLS; i++)
6714                                 ahd_outb(ahd, ANNEXDAT, 0);
6715                 }
6716         }
6717         for (target = 0; target < AHD_NUM_TARGETS; target++) {
6718                 struct   ahd_devinfo devinfo;
6719                 struct   ahd_initiator_tinfo *tinfo;
6720                 struct   ahd_tmode_tstate *tstate;
6721
6722                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6723                                             target, &tstate);
6724                 ahd_compile_devinfo(&devinfo, ahd->our_id,
6725                                     target, CAM_LUN_WILDCARD,
6726                                     'A', ROLE_INITIATOR);
6727                 ahd_update_neg_table(ahd, &devinfo, &tinfo->curr);
6728         }
6729
6730         ahd_outb(ahd, CLRSINT3, NTRAMPERR|OSRAMPERR);
6731         ahd_outb(ahd, CLRINT, CLRSCSIINT);
6732
6733 #ifdef NEEDS_MORE_TESTING
6734         /*
6735          * Always enable abort on incoming L_Qs if this feature is
6736          * supported.  We use this to catch invalid SCB references.
6737          */
6738         if ((ahd->bugs & AHD_ABORT_LQI_BUG) == 0)
6739                 ahd_outb(ahd, LQCTL1, ABORTPENDING);
6740         else
6741 #endif
6742                 ahd_outb(ahd, LQCTL1, 0);
6743
6744         /* All of our queues are empty */
6745         ahd->qoutfifonext = 0;
6746         ahd->qoutfifonext_valid_tag = QOUTFIFO_ENTRY_VALID;
6747         ahd_outb(ahd, QOUTFIFO_ENTRY_VALID_TAG, QOUTFIFO_ENTRY_VALID);
6748         for (i = 0; i < AHD_QOUT_SIZE; i++)
6749                 ahd->qoutfifo[i].valid_tag = 0;
6750         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_PREREAD);
6751
6752         ahd->qinfifonext = 0;
6753         for (i = 0; i < AHD_QIN_SIZE; i++)
6754                 ahd->qinfifo[i] = SCB_LIST_NULL;
6755
6756         if ((ahd->features & AHD_TARGETMODE) != 0) {
6757                 /* All target command blocks start out invalid. */
6758                 for (i = 0; i < AHD_TMODE_CMDS; i++)
6759                         ahd->targetcmds[i].cmd_valid = 0;
6760                 ahd_sync_tqinfifo(ahd, BUS_DMASYNC_PREREAD);
6761                 ahd->tqinfifonext = 1;
6762                 ahd_outb(ahd, KERNEL_TQINPOS, ahd->tqinfifonext - 1);
6763                 ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
6764         }
6765
6766         /* Initialize Scratch Ram. */
6767         ahd_outb(ahd, SEQ_FLAGS, 0);
6768         ahd_outb(ahd, SEQ_FLAGS2, 0);
6769
6770         /* We don't have any waiting selections */
6771         ahd_outw(ahd, WAITING_TID_HEAD, SCB_LIST_NULL);
6772         ahd_outw(ahd, WAITING_TID_TAIL, SCB_LIST_NULL);
6773         ahd_outw(ahd, MK_MESSAGE_SCB, SCB_LIST_NULL);
6774         ahd_outw(ahd, MK_MESSAGE_SCSIID, 0xFF);
6775         for (i = 0; i < AHD_NUM_TARGETS; i++)
6776                 ahd_outw(ahd, WAITING_SCB_TAILS + (2 * i), SCB_LIST_NULL);
6777
6778         /*
6779          * Nobody is waiting to be DMAed into the QOUTFIFO.
6780          */
6781         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
6782         ahd_outw(ahd, COMPLETE_SCB_DMAINPROG_HEAD, SCB_LIST_NULL);
6783         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
6784         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
6785         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
6786
6787         /*
6788          * The Freeze Count is 0.
6789          */
6790         ahd->qfreeze_cnt = 0;
6791         ahd_outw(ahd, QFREEZE_COUNT, 0);
6792         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, 0);
6793
6794         /*
6795          * Tell the sequencer where it can find our arrays in memory.
6796          */
6797         busaddr = ahd->shared_data_map.busaddr;
6798         ahd_outl(ahd, SHARED_DATA_ADDR, busaddr);
6799         ahd_outl(ahd, QOUTFIFO_NEXT_ADDR, busaddr);
6800
6801         /*
6802          * Setup the allowed SCSI Sequences based on operational mode.
6803          * If we are a target, we'll enable select in operations once
6804          * we've had a lun enabled.
6805          */
6806         scsiseq_template = ENAUTOATNP;
6807         if ((ahd->flags & AHD_INITIATORROLE) != 0)
6808                 scsiseq_template |= ENRSELI;
6809         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq_template);
6810
6811         /* There are no busy SCBs yet. */
6812         for (target = 0; target < AHD_NUM_TARGETS; target++) {
6813                 int lun;
6814
6815                 for (lun = 0; lun < AHD_NUM_LUNS_NONPKT; lun++)
6816                         ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(target, 'A', lun));
6817         }
6818
6819         /*
6820          * Initialize the group code to command length table.
6821          * Vendor Unique codes are set to 0 so we only capture
6822          * the first byte of the cdb.  These can be overridden
6823          * when target mode is enabled.
6824          */
6825         ahd_outb(ahd, CMDSIZE_TABLE, 5);
6826         ahd_outb(ahd, CMDSIZE_TABLE + 1, 9);
6827         ahd_outb(ahd, CMDSIZE_TABLE + 2, 9);
6828         ahd_outb(ahd, CMDSIZE_TABLE + 3, 0);
6829         ahd_outb(ahd, CMDSIZE_TABLE + 4, 15);
6830         ahd_outb(ahd, CMDSIZE_TABLE + 5, 11);
6831         ahd_outb(ahd, CMDSIZE_TABLE + 6, 0);
6832         ahd_outb(ahd, CMDSIZE_TABLE + 7, 0);
6833
6834         /* Tell the sequencer of our initial queue positions */
6835         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
6836         ahd_outb(ahd, QOFF_CTLSTA, SCB_QSIZE_512);
6837         ahd->qinfifonext = 0;
6838         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
6839         ahd_set_hescb_qoff(ahd, 0);
6840         ahd_set_snscb_qoff(ahd, 0);
6841         ahd_set_sescb_qoff(ahd, 0);
6842         ahd_set_sdscb_qoff(ahd, 0);
6843
6844         /*
6845          * Tell the sequencer which SCB will be the next one it receives.
6846          */
6847         busaddr = aic_le32toh(ahd->next_queued_hscb->hscb_busaddr);
6848         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
6849
6850         /*
6851          * Default to coalescing disabled.
6852          */
6853         ahd_outw(ahd, INT_COALESCING_CMDCOUNT, 0);
6854         ahd_outw(ahd, CMDS_PENDING, 0);
6855         ahd_update_coalescing_values(ahd, ahd->int_coalescing_timer,
6856                                      ahd->int_coalescing_maxcmds,
6857                                      ahd->int_coalescing_mincmds);
6858         ahd_enable_coalescing(ahd, FALSE);
6859
6860         ahd_loadseq(ahd);
6861         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6862 }
6863
6864 /*
6865  * Setup default device and controller settings.
6866  * This should only be called if our probe has
6867  * determined that no configuration data is available.
6868  */
6869 int
6870 ahd_default_config(struct ahd_softc *ahd)
6871 {
6872         int     targ;
6873
6874         ahd->our_id = 7;
6875
6876         /*
6877          * Allocate a tstate to house information for our
6878          * initiator presence on the bus as well as the user
6879          * data for any target mode initiator.
6880          */
6881         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
6882                 printf("%s: unable to allocate ahd_tmode_tstate.  "
6883                        "Failing attach\n", ahd_name(ahd));
6884                 AHD_FATAL_ERROR(ahd);
6885                 return (ENOMEM);
6886         }
6887
6888         for (targ = 0; targ < AHD_NUM_TARGETS; targ++) {
6889                 struct   ahd_devinfo devinfo;
6890                 struct   ahd_initiator_tinfo *tinfo;
6891                 struct   ahd_tmode_tstate *tstate;
6892                 uint16_t target_mask;
6893
6894                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6895                                             targ, &tstate);
6896                 /*
6897                  * We support SPC2 and SPI4.
6898                  */
6899                 tinfo->user.protocol_version = 4;
6900                 tinfo->user.transport_version = 4;
6901
6902                 target_mask = 0x01 << targ;
6903                 ahd->user_discenable |= target_mask;
6904                 tstate->discenable |= target_mask;
6905                 ahd->user_tagenable |= target_mask;
6906 #ifdef AHD_FORCE_160
6907                 tinfo->user.period = AHD_SYNCRATE_DT;
6908 #else
6909                 tinfo->user.period = AHD_SYNCRATE_160;
6910 #endif
6911                 tinfo->user.offset = MAX_OFFSET;
6912                 tinfo->user.ppr_options = MSG_EXT_PPR_RD_STRM
6913                                         | MSG_EXT_PPR_WR_FLOW
6914                                         | MSG_EXT_PPR_HOLD_MCS
6915                                         | MSG_EXT_PPR_IU_REQ
6916                                         | MSG_EXT_PPR_QAS_REQ
6917                                         | MSG_EXT_PPR_DT_REQ;
6918                 if ((ahd->features & AHD_RTI) != 0)
6919                         tinfo->user.ppr_options |= MSG_EXT_PPR_RTI;
6920
6921                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
6922
6923                 /*
6924                  * Start out Async/Narrow/Untagged and with
6925                  * conservative protocol support.
6926                  */
6927                 tinfo->goal.protocol_version = 2;
6928                 tinfo->goal.transport_version = 2;
6929                 tinfo->curr.protocol_version = 2;
6930                 tinfo->curr.transport_version = 2;
6931                 ahd_compile_devinfo(&devinfo, ahd->our_id,
6932                                     targ, CAM_LUN_WILDCARD,
6933                                     'A', ROLE_INITIATOR);
6934                 tstate->tagenable &= ~target_mask;
6935                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6936                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
6937                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
6938                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
6939                                  /*paused*/TRUE);
6940         }
6941         return (0);
6942 }
6943
6944 /*
6945  * Parse device configuration information.
6946  */
6947 int
6948 ahd_parse_cfgdata(struct ahd_softc *ahd, struct seeprom_config *sc)
6949 {
6950         int targ;
6951         int max_targ;
6952
6953         max_targ = sc->max_targets & CFMAXTARG;
6954         ahd->our_id = sc->brtime_id & CFSCSIID;
6955
6956         /*
6957          * Allocate a tstate to house information for our
6958          * initiator presence on the bus as well as the user
6959          * data for any target mode initiator.
6960          */
6961         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
6962                 printf("%s: unable to allocate ahd_tmode_tstate.  "
6963                        "Failing attach\n", ahd_name(ahd));
6964                 AHD_FATAL_ERROR(ahd);
6965                 return (ENOMEM);
6966         }
6967
6968         for (targ = 0; targ < max_targ; targ++) {
6969                 struct   ahd_devinfo devinfo;
6970                 struct   ahd_initiator_tinfo *tinfo;
6971                 struct   ahd_transinfo *user_tinfo;
6972                 struct   ahd_tmode_tstate *tstate;
6973                 uint16_t target_mask;
6974
6975                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6976                                             targ, &tstate);
6977                 user_tinfo = &tinfo->user;
6978
6979                 /*
6980                  * We support SPC2 and SPI4.
6981                  */
6982                 tinfo->user.protocol_version = 4;
6983                 tinfo->user.transport_version = 4;
6984
6985                 target_mask = 0x01 << targ;
6986                 ahd->user_discenable &= ~target_mask;
6987                 tstate->discenable &= ~target_mask;
6988                 ahd->user_tagenable &= ~target_mask;
6989                 if (sc->device_flags[targ] & CFDISC) {
6990                         tstate->discenable |= target_mask;
6991                         ahd->user_discenable |= target_mask;
6992                         ahd->user_tagenable |= target_mask;
6993                 } else {
6994                         /*
6995                          * Cannot be packetized without disconnection.
6996                          */
6997                         sc->device_flags[targ] &= ~CFPACKETIZED;
6998                 }
6999
7000                 user_tinfo->ppr_options = 0;
7001                 user_tinfo->period = (sc->device_flags[targ] & CFXFER);
7002                 if (user_tinfo->period < CFXFER_ASYNC) {
7003                         if (user_tinfo->period <= AHD_PERIOD_10MHz)
7004                                 user_tinfo->ppr_options |= MSG_EXT_PPR_DT_REQ;
7005                         user_tinfo->offset = MAX_OFFSET;
7006                 } else  {
7007                         user_tinfo->offset = 0;
7008                         user_tinfo->period = AHD_ASYNC_XFER_PERIOD;
7009                 }
7010 #ifdef AHD_FORCE_160
7011                 if (user_tinfo->period <= AHD_SYNCRATE_160)
7012                         user_tinfo->period = AHD_SYNCRATE_DT;
7013 #endif
7014
7015                 if ((sc->device_flags[targ] & CFPACKETIZED) != 0) {
7016                         user_tinfo->ppr_options |= MSG_EXT_PPR_RD_STRM
7017                                                 |  MSG_EXT_PPR_WR_FLOW
7018                                                 |  MSG_EXT_PPR_HOLD_MCS
7019                                                 |  MSG_EXT_PPR_IU_REQ;
7020                         if ((ahd->features & AHD_RTI) != 0)
7021                                 user_tinfo->ppr_options |= MSG_EXT_PPR_RTI;
7022                 }
7023
7024                 if ((sc->device_flags[targ] & CFQAS) != 0)
7025                         user_tinfo->ppr_options |= MSG_EXT_PPR_QAS_REQ;
7026
7027                 if ((sc->device_flags[targ] & CFWIDEB) != 0)
7028                         user_tinfo->width = MSG_EXT_WDTR_BUS_16_BIT;
7029                 else
7030                         user_tinfo->width = MSG_EXT_WDTR_BUS_8_BIT;
7031 #ifdef AHD_DEBUG
7032                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
7033                         printf("(%d): %x:%x:%x:%x\n", targ, user_tinfo->width,
7034                                user_tinfo->period, user_tinfo->offset,
7035                                user_tinfo->ppr_options);
7036 #endif
7037                 /*
7038                  * Start out Async/Narrow/Untagged and with
7039                  * conservative protocol support.
7040                  */
7041                 tstate->tagenable &= ~target_mask;
7042                 tinfo->goal.protocol_version = 2;
7043                 tinfo->goal.transport_version = 2;
7044                 tinfo->curr.protocol_version = 2;
7045                 tinfo->curr.transport_version = 2;
7046                 ahd_compile_devinfo(&devinfo, ahd->our_id,
7047                                     targ, CAM_LUN_WILDCARD,
7048                                     'A', ROLE_INITIATOR);
7049                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7050                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
7051                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
7052                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
7053                                  /*paused*/TRUE);
7054         }
7055
7056         ahd->flags &= ~AHD_SPCHK_ENB_A;
7057         if (sc->bios_control & CFSPARITY)
7058                 ahd->flags |= AHD_SPCHK_ENB_A;
7059
7060         ahd->flags &= ~AHD_RESET_BUS_A;
7061         if (sc->bios_control & CFRESETB)
7062                 ahd->flags |= AHD_RESET_BUS_A;
7063
7064         ahd->flags &= ~AHD_EXTENDED_TRANS_A;
7065         if (sc->bios_control & CFEXTEND)
7066                 ahd->flags |= AHD_EXTENDED_TRANS_A;
7067
7068         ahd->flags &= ~AHD_BIOS_ENABLED;
7069         if ((sc->bios_control & CFBIOSSTATE) == CFBS_ENABLED)
7070                 ahd->flags |= AHD_BIOS_ENABLED;
7071
7072         ahd->flags &= ~AHD_STPWLEVEL_A;
7073         if ((sc->adapter_control & CFSTPWLEVEL) != 0)
7074                 ahd->flags |= AHD_STPWLEVEL_A;
7075
7076         return (0);
7077 }
7078
7079 /*
7080  * Parse device configuration information.
7081  */
7082 int
7083 ahd_parse_vpddata(struct ahd_softc *ahd, struct vpd_config *vpd)
7084 {
7085         int error;
7086
7087         error = ahd_verify_vpd_cksum(vpd);
7088         if (error == 0)
7089                 return (EINVAL);
7090         if ((vpd->bios_flags & VPDBOOTHOST) != 0)
7091                 ahd->flags |= AHD_BOOT_CHANNEL;
7092         return (0);
7093 }
7094
7095 void
7096 ahd_intr_enable(struct ahd_softc *ahd, int enable)
7097 {
7098         u_int hcntrl;
7099
7100         hcntrl = ahd_inb(ahd, HCNTRL);
7101         hcntrl &= ~INTEN;
7102         ahd->pause &= ~INTEN;
7103         ahd->unpause &= ~INTEN;
7104         if (enable) {
7105                 hcntrl |= INTEN;
7106                 ahd->pause |= INTEN;
7107                 ahd->unpause |= INTEN;
7108         }
7109         ahd_outb(ahd, HCNTRL, hcntrl);
7110 }
7111
7112 void
7113 ahd_update_coalescing_values(struct ahd_softc *ahd, u_int timer, u_int maxcmds,
7114                              u_int mincmds)
7115 {
7116         if (timer > AHD_TIMER_MAX_US)
7117                 timer = AHD_TIMER_MAX_US;
7118         ahd->int_coalescing_timer = timer;
7119
7120         if (maxcmds > AHD_INT_COALESCING_MAXCMDS_MAX)
7121                 maxcmds = AHD_INT_COALESCING_MAXCMDS_MAX;
7122         if (mincmds > AHD_INT_COALESCING_MINCMDS_MAX)
7123                 mincmds = AHD_INT_COALESCING_MINCMDS_MAX;
7124         ahd->int_coalescing_maxcmds = maxcmds;
7125         ahd_outw(ahd, INT_COALESCING_TIMER, timer / AHD_TIMER_US_PER_TICK);
7126         ahd_outb(ahd, INT_COALESCING_MAXCMDS, -maxcmds);
7127         ahd_outb(ahd, INT_COALESCING_MINCMDS, -mincmds);
7128 }
7129
7130 void
7131 ahd_enable_coalescing(struct ahd_softc *ahd, int enable)
7132 {
7133
7134         ahd->hs_mailbox &= ~ENINT_COALESCE;
7135         if (enable)
7136                 ahd->hs_mailbox |= ENINT_COALESCE;
7137         ahd_outb(ahd, HS_MAILBOX, ahd->hs_mailbox);
7138         ahd_flush_device_writes(ahd);
7139         ahd_run_qoutfifo(ahd);
7140 }
7141
7142 /*
7143  * Ensure that the card is paused in a location
7144  * outside of all critical sections and that all
7145  * pending work is completed prior to returning.
7146  * This routine should only be called from outside
7147  * an interrupt context.
7148  */
7149 void
7150 ahd_pause_and_flushwork(struct ahd_softc *ahd)
7151 {
7152         u_int intstat;
7153         u_int maxloops;
7154
7155         maxloops = 1000;
7156         ahd->flags |= AHD_ALL_INTERRUPTS;
7157         ahd_pause(ahd);
7158         /*
7159          * Freeze the outgoing selections.  We do this only
7160          * until we are safely paused without further selections
7161          * pending.
7162          */
7163         ahd->qfreeze_cnt--;
7164         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7165         ahd_outb(ahd, SEQ_FLAGS2, ahd_inb(ahd, SEQ_FLAGS2) | SELECTOUT_QFROZEN);
7166         do {
7167
7168                 ahd_unpause(ahd);
7169                 /*
7170                  * Give the sequencer some time to service
7171                  * any active selections.
7172                  */
7173                 aic_delay(500);
7174
7175                 ahd_intr(ahd);
7176                 ahd_pause(ahd);
7177                 intstat = ahd_inb(ahd, INTSTAT);
7178                 if ((intstat & INT_PEND) == 0) {
7179                         ahd_clear_critical_section(ahd);
7180                         intstat = ahd_inb(ahd, INTSTAT);
7181                 }
7182         } while (--maxloops
7183               && (intstat != 0xFF || (ahd->features & AHD_REMOVABLE) == 0)
7184               && ((intstat & INT_PEND) != 0
7185                || (ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
7186                || (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) != 0));
7187
7188         if (maxloops == 0) {
7189                 printf("Infinite interrupt loop, INTSTAT = %x",
7190                       ahd_inb(ahd, INTSTAT));
7191                 AHD_FATAL_ERROR(ahd);
7192         }
7193         ahd->qfreeze_cnt++;
7194         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7195
7196         ahd_flush_qoutfifo(ahd);
7197
7198         ahd_platform_flushwork(ahd);
7199         ahd->flags &= ~AHD_ALL_INTERRUPTS;
7200 }
7201
7202 int
7203 ahd_suspend(struct ahd_softc *ahd)
7204 {
7205
7206         ahd_pause_and_flushwork(ahd);
7207
7208         if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
7209                 ahd_unpause(ahd);
7210                 return (EBUSY);
7211         }
7212         ahd_shutdown(ahd);
7213         return (0);
7214 }
7215
7216 int
7217 ahd_resume(struct ahd_softc *ahd)
7218 {
7219
7220         ahd_reset(ahd, /*reinit*/TRUE);
7221         ahd_intr_enable(ahd, TRUE);
7222         ahd_restart(ahd);
7223         return (0);
7224 }
7225
7226 /************************** Busy Target Table *********************************/
7227 /*
7228  * Set SCBPTR to the SCB that contains the busy
7229  * table entry for TCL.  Return the offset into
7230  * the SCB that contains the entry for TCL.
7231  * saved_scbid is dereferenced and set to the
7232  * scbid that should be restored once manipualtion
7233  * of the TCL entry is complete.
7234  */
7235 static __inline u_int
7236 ahd_index_busy_tcl(struct ahd_softc *ahd, u_int *saved_scbid, u_int tcl)
7237 {
7238         /*
7239          * Index to the SCB that contains the busy entry.
7240          */
7241         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7242         *saved_scbid = ahd_get_scbptr(ahd);
7243         ahd_set_scbptr(ahd, TCL_LUN(tcl)
7244                      | ((TCL_TARGET_OFFSET(tcl) & 0xC) << 4));
7245
7246         /*
7247          * And now calculate the SCB offset to the entry.
7248          * Each entry is 2 bytes wide, hence the
7249          * multiplication by 2.
7250          */
7251         return (((TCL_TARGET_OFFSET(tcl) & 0x3) << 1) + SCB_DISCONNECTED_LISTS);
7252 }
7253
7254 /*
7255  * Return the untagged transaction id for a given target/channel lun.
7256  */
7257 u_int
7258 ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl)
7259 {
7260         u_int scbid;
7261         u_int scb_offset;
7262         u_int saved_scbptr;
7263
7264         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7265         scbid = ahd_inw_scbram(ahd, scb_offset);
7266         ahd_set_scbptr(ahd, saved_scbptr);
7267         return (scbid);
7268 }
7269
7270 void
7271 ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl, u_int scbid)
7272 {
7273         u_int scb_offset;
7274         u_int saved_scbptr;
7275
7276         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7277         ahd_outw(ahd, scb_offset, scbid);
7278         ahd_set_scbptr(ahd, saved_scbptr);
7279 }
7280
7281 /************************** SCB and SCB queue management **********************/
7282 int
7283 ahd_match_scb(struct ahd_softc *ahd, struct scb *scb, int target,
7284               char channel, int lun, u_int tag, role_t role)
7285 {
7286         int targ = SCB_GET_TARGET(ahd, scb);
7287         char chan = SCB_GET_CHANNEL(ahd, scb);
7288         int slun = SCB_GET_LUN(scb);
7289         int match;
7290
7291         match = ((chan == channel) || (channel == ALL_CHANNELS));
7292         if (match != 0)
7293                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
7294         if (match != 0)
7295                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
7296         if (match != 0) {
7297 #ifdef AHD_TARGET_MODE
7298                 int group;
7299
7300                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
7301                 if (role == ROLE_INITIATOR) {
7302                         match = (group != XPT_FC_GROUP_TMODE)
7303                               && ((tag == SCB_GET_TAG(scb))
7304                                || (tag == SCB_LIST_NULL));
7305                 } else if (role == ROLE_TARGET) {
7306                         match = (group == XPT_FC_GROUP_TMODE)
7307                               && ((tag == scb->io_ctx->csio.tag_id)
7308                                || (tag == SCB_LIST_NULL));
7309                 }
7310 #else /* !AHD_TARGET_MODE */
7311                 match = ((tag == SCB_GET_TAG(scb)) || (tag == SCB_LIST_NULL));
7312 #endif /* AHD_TARGET_MODE */
7313         }
7314
7315         return match;
7316 }
7317
7318 void
7319 ahd_freeze_devq(struct ahd_softc *ahd, struct scb *scb)
7320 {
7321         int     target;
7322         char    channel;
7323         int     lun;
7324
7325         target = SCB_GET_TARGET(ahd, scb);
7326         lun = SCB_GET_LUN(scb);
7327         channel = SCB_GET_CHANNEL(ahd, scb);
7328
7329         ahd_search_qinfifo(ahd, target, channel, lun,
7330                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
7331                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
7332
7333         ahd_platform_freeze_devq(ahd, scb);
7334 }
7335
7336 void
7337 ahd_qinfifo_requeue_tail(struct ahd_softc *ahd, struct scb *scb)
7338 {
7339         struct scb      *prev_scb;
7340         ahd_mode_state   saved_modes;
7341
7342         saved_modes = ahd_save_modes(ahd);
7343         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7344         prev_scb = NULL;
7345         if (ahd_qinfifo_count(ahd) != 0) {
7346                 u_int prev_tag;
7347                 u_int prev_pos;
7348
7349                 prev_pos = AHD_QIN_WRAP(ahd->qinfifonext - 1);
7350                 prev_tag = ahd->qinfifo[prev_pos];
7351                 prev_scb = ahd_lookup_scb(ahd, prev_tag);
7352         }
7353         ahd_qinfifo_requeue(ahd, prev_scb, scb);
7354         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7355         ahd_restore_modes(ahd, saved_modes);
7356 }
7357
7358 static void
7359 ahd_qinfifo_requeue(struct ahd_softc *ahd, struct scb *prev_scb,
7360                     struct scb *scb)
7361 {
7362         if (prev_scb == NULL) {
7363                 uint32_t busaddr;
7364
7365                 busaddr = aic_le32toh(scb->hscb->hscb_busaddr);
7366                 ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7367         } else {
7368                 prev_scb->hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
7369                 ahd_sync_scb(ahd, prev_scb,
7370                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
7371         }
7372         ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
7373         ahd->qinfifonext++;
7374         scb->hscb->next_hscb_busaddr = ahd->next_queued_hscb->hscb_busaddr;
7375         ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
7376 }
7377
7378 static int
7379 ahd_qinfifo_count(struct ahd_softc *ahd)
7380 {
7381         u_int qinpos;
7382         u_int wrap_qinpos;
7383         u_int wrap_qinfifonext;
7384
7385         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
7386         qinpos = ahd_get_snscb_qoff(ahd);
7387         wrap_qinpos = AHD_QIN_WRAP(qinpos);
7388         wrap_qinfifonext = AHD_QIN_WRAP(ahd->qinfifonext);
7389         if (wrap_qinfifonext >= wrap_qinpos)
7390                 return (wrap_qinfifonext - wrap_qinpos);
7391         else
7392                 return (wrap_qinfifonext
7393                       + NUM_ELEMENTS(ahd->qinfifo) - wrap_qinpos);
7394 }
7395
7396 void
7397 ahd_reset_cmds_pending(struct ahd_softc *ahd)
7398 {
7399         struct          scb *scb;
7400         ahd_mode_state  saved_modes;
7401         u_int           pending_cmds;
7402
7403         saved_modes = ahd_save_modes(ahd);
7404         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7405
7406         /*
7407          * Don't count any commands as outstanding that the
7408          * sequencer has already marked for completion.
7409          */
7410         ahd_flush_qoutfifo(ahd);
7411
7412         pending_cmds = 0;
7413         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
7414                 pending_cmds++;
7415         }
7416         ahd_outw(ahd, CMDS_PENDING, pending_cmds - ahd_qinfifo_count(ahd));
7417         ahd_restore_modes(ahd, saved_modes);
7418         ahd->flags &= ~AHD_UPDATE_PEND_CMDS;
7419 }
7420
7421 void
7422 ahd_done_with_status(struct ahd_softc *ahd, struct scb *scb, uint32_t status)
7423 {
7424         cam_status ostat;
7425         cam_status cstat;
7426
7427         ostat = aic_get_transaction_status(scb);
7428         if (ostat == CAM_REQ_INPROG)
7429                 aic_set_transaction_status(scb, status);
7430         cstat = aic_get_transaction_status(scb);
7431         if (cstat != CAM_REQ_CMP)
7432                 aic_freeze_scb(scb);
7433         ahd_done(ahd, scb);
7434 }
7435
7436 int
7437 ahd_search_qinfifo(struct ahd_softc *ahd, int target, char channel,
7438                    int lun, u_int tag, role_t role, uint32_t status,
7439                    ahd_search_action action)
7440 {
7441         struct scb      *scb;
7442         struct scb      *mk_msg_scb;
7443         struct scb      *prev_scb;
7444         ahd_mode_state   saved_modes;
7445         u_int            qinstart;
7446         u_int            qinpos;
7447         u_int            qintail;
7448         u_int            tid_next;
7449         u_int            tid_prev;
7450         u_int            scbid;
7451         u_int            seq_flags2;
7452         u_int            savedscbptr;
7453         uint32_t         busaddr;
7454         int              found;
7455         int              targets;
7456
7457         /* Must be in CCHAN mode */
7458         saved_modes = ahd_save_modes(ahd);
7459         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7460
7461         /*
7462          * Halt any pending SCB DMA.  The sequencer will reinitiate
7463          * this dma if the qinfifo is not empty once we unpause.
7464          */
7465         if ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN|CCSCBDIR))
7466          == (CCARREN|CCSCBEN|CCSCBDIR)) {
7467                 ahd_outb(ahd, CCSCBCTL,
7468                          ahd_inb(ahd, CCSCBCTL) & ~(CCARREN|CCSCBEN));
7469                 while ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN)) != 0)
7470                         ;
7471         }
7472         /* Determine sequencer's position in the qinfifo. */
7473         qintail = AHD_QIN_WRAP(ahd->qinfifonext);
7474         qinstart = ahd_get_snscb_qoff(ahd);
7475         qinpos = AHD_QIN_WRAP(qinstart);
7476         found = 0;
7477         prev_scb = NULL;
7478
7479         if (action == SEARCH_PRINT) {
7480                 printf("qinstart = %d qinfifonext = %d\nQINFIFO:",
7481                        qinstart, ahd->qinfifonext);
7482         }
7483
7484         /*
7485          * Start with an empty queue.  Entries that are not chosen
7486          * for removal will be re-added to the queue as we go.
7487          */
7488         ahd->qinfifonext = qinstart;
7489         busaddr = aic_le32toh(ahd->next_queued_hscb->hscb_busaddr);
7490         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7491
7492         while (qinpos != qintail) {
7493                 scb = ahd_lookup_scb(ahd, ahd->qinfifo[qinpos]);
7494                 if (scb == NULL) {
7495                         printf("qinpos = %d, SCB index = %d\n",
7496                                 qinpos, ahd->qinfifo[qinpos]);
7497                         AHD_FATAL_ERROR(ahd);
7498                         panic("Loop 1\n");
7499                 }
7500
7501                 if (ahd_match_scb(ahd, scb, target, channel, lun, tag, role)) {
7502                         /*
7503                          * We found an scb that needs to be acted on.
7504                          */
7505                         found++;
7506                         switch (action) {
7507                         case SEARCH_COMPLETE:
7508                                 if ((scb->flags & SCB_ACTIVE) == 0)
7509                                         printf("Inactive SCB in qinfifo\n");
7510                                 ahd_done_with_status(ahd, scb, status);
7511                                 /* FALLTHROUGH */
7512                         case SEARCH_REMOVE:
7513                                 break;
7514                         case SEARCH_PRINT:
7515                                 printf(" 0x%x", ahd->qinfifo[qinpos]);
7516                                 /* FALLTHROUGH */
7517                         case SEARCH_COUNT:
7518                                 ahd_qinfifo_requeue(ahd, prev_scb, scb);
7519                                 prev_scb = scb;
7520                                 break;
7521                         }
7522                 } else {
7523                         ahd_qinfifo_requeue(ahd, prev_scb, scb);
7524                         prev_scb = scb;
7525                 }
7526                 qinpos = AHD_QIN_WRAP(qinpos+1);
7527         }
7528
7529         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7530
7531         if (action == SEARCH_PRINT)
7532                 printf("\nWAITING_TID_QUEUES:\n");
7533
7534         /*
7535          * Search waiting for selection lists.  We traverse the
7536          * list of "their ids" waiting for selection and, if
7537          * appropriate, traverse the SCBs of each "their id"
7538          * looking for matches.
7539          */
7540         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7541         seq_flags2 = ahd_inb(ahd, SEQ_FLAGS2);
7542         if ((seq_flags2 & PENDING_MK_MESSAGE) != 0) {
7543                 scbid = ahd_inw(ahd, MK_MESSAGE_SCB);
7544                 mk_msg_scb = ahd_lookup_scb(ahd, scbid);
7545         } else
7546                 mk_msg_scb = NULL;
7547         savedscbptr = ahd_get_scbptr(ahd);
7548         tid_next = ahd_inw(ahd, WAITING_TID_HEAD);
7549         tid_prev = SCB_LIST_NULL;
7550         targets = 0;
7551         for (scbid = tid_next; !SCBID_IS_NULL(scbid); scbid = tid_next) {
7552                 u_int tid_head;
7553                 u_int tid_tail;
7554
7555                 targets++;
7556                 if (targets > AHD_NUM_TARGETS)
7557                         panic("TID LIST LOOP");
7558
7559                 if (scbid >= ahd->scb_data.numscbs) {
7560                         printf("%s: Waiting TID List inconsistency. "
7561                                "SCB index == 0x%x, yet numscbs == 0x%x.",
7562                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
7563                         ahd_dump_card_state(ahd);
7564                         panic("for safety");
7565                 }
7566                 scb = ahd_lookup_scb(ahd, scbid);
7567                 if (scb == NULL) {
7568                         printf("%s: SCB = 0x%x Not Active!\n",
7569                                ahd_name(ahd), scbid);
7570                         panic("Waiting TID List traversal\n");
7571                 }
7572                 ahd_set_scbptr(ahd, scbid);
7573                 tid_next = ahd_inw_scbram(ahd, SCB_NEXT2);
7574                 if (ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
7575                                   SCB_LIST_NULL, ROLE_UNKNOWN) == 0) {
7576                         tid_prev = scbid;
7577                         continue;
7578                 }
7579
7580                 /*
7581                  * We found a list of scbs that needs to be searched.
7582                  */
7583                 if (action == SEARCH_PRINT)
7584                         printf("       %d ( ", SCB_GET_TARGET(ahd, scb));
7585                 tid_head = scbid;
7586                 found += ahd_search_scb_list(ahd, target, channel,
7587                                              lun, tag, role, status,
7588                                              action, &tid_head, &tid_tail,
7589                                              SCB_GET_TARGET(ahd, scb));
7590                 /*
7591                  * Check any MK_MESSAGE SCB that is still waiting to
7592                  * enter this target's waiting for selection queue.
7593                  */
7594                 if (mk_msg_scb != NULL
7595                  && ahd_match_scb(ahd, mk_msg_scb, target, channel,
7596                                   lun, tag, role)) {
7597
7598                         /*
7599                          * We found an scb that needs to be acted on.
7600                          */
7601                         found++;
7602                         switch (action) {
7603                         case SEARCH_COMPLETE:
7604                                 if ((mk_msg_scb->flags & SCB_ACTIVE) == 0)
7605                                         printf("Inactive SCB pending MK_MSG\n");
7606                                 ahd_done_with_status(ahd, mk_msg_scb, status);
7607                                 /* FALLTHROUGH */
7608                         case SEARCH_REMOVE:
7609                         {
7610                                 u_int tail_offset;
7611
7612                                 printf("Removing MK_MSG scb\n");
7613
7614                                 /*
7615                                  * Reset our tail to the tail of the
7616                                  * main per-target list.
7617                                  */
7618                                 tail_offset = WAITING_SCB_TAILS
7619                                     + (2 * SCB_GET_TARGET(ahd, mk_msg_scb));
7620                                 ahd_outw(ahd, tail_offset, tid_tail);
7621
7622                                 seq_flags2 &= ~PENDING_MK_MESSAGE;
7623                                 ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
7624                                 ahd_outw(ahd, CMDS_PENDING,
7625                                          ahd_inw(ahd, CMDS_PENDING)-1);
7626                                 mk_msg_scb = NULL;
7627                                 break;
7628                         }
7629                         case SEARCH_PRINT:
7630                                 printf(" 0x%x", SCB_GET_TAG(scb));
7631                                 /* FALLTHROUGH */
7632                         case SEARCH_COUNT:
7633                                 break;
7634                         }
7635                 }
7636
7637                 if (mk_msg_scb != NULL
7638                  && SCBID_IS_NULL(tid_head)
7639                  && ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
7640                                   SCB_LIST_NULL, ROLE_UNKNOWN)) {
7641
7642                         /*
7643                          * When removing the last SCB for a target
7644                          * queue with a pending MK_MESSAGE scb, we
7645                          * must queue the MK_MESSAGE scb.
7646                          */
7647                         printf("Queueing mk_msg_scb\n");
7648                         tid_head = ahd_inw(ahd, MK_MESSAGE_SCB);
7649                         seq_flags2 &= ~PENDING_MK_MESSAGE;
7650                         ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
7651                         mk_msg_scb = NULL;
7652                 }
7653                 if (tid_head != scbid)
7654                         ahd_stitch_tid_list(ahd, tid_prev, tid_head, tid_next);
7655                 if (!SCBID_IS_NULL(tid_head))
7656                         tid_prev = tid_head;
7657                 if (action == SEARCH_PRINT)
7658                         printf(")\n");
7659         }
7660
7661         /* Restore saved state. */
7662         ahd_set_scbptr(ahd, savedscbptr);
7663         ahd_restore_modes(ahd, saved_modes);
7664         return (found);
7665 }
7666
7667 static int
7668 ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel,
7669                     int lun, u_int tag, role_t role, uint32_t status,
7670                     ahd_search_action action, u_int *list_head,
7671                     u_int *list_tail, u_int tid)
7672 {
7673         struct  scb *scb;
7674         u_int   scbid;
7675         u_int   next;
7676         u_int   prev;
7677         int     found;
7678
7679         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7680         found = 0;
7681         prev = SCB_LIST_NULL;
7682         next = *list_head;
7683         *list_tail = SCB_LIST_NULL;
7684         for (scbid = next; !SCBID_IS_NULL(scbid); scbid = next) {
7685                 if (scbid >= ahd->scb_data.numscbs) {
7686                         printf("%s:SCB List inconsistency. "
7687                                "SCB == 0x%x, yet numscbs == 0x%x.",
7688                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
7689                         ahd_dump_card_state(ahd);
7690                         panic("for safety");
7691                 }
7692                 scb = ahd_lookup_scb(ahd, scbid);
7693                 if (scb == NULL) {
7694                         printf("%s: SCB = %d Not Active!\n",
7695                                ahd_name(ahd), scbid);
7696                         panic("Waiting List traversal\n");
7697                 }
7698                 ahd_set_scbptr(ahd, scbid);
7699                 *list_tail = scbid;
7700                 next = ahd_inw_scbram(ahd, SCB_NEXT);
7701                 if (ahd_match_scb(ahd, scb, target, channel,
7702                                   lun, SCB_LIST_NULL, role) == 0) {
7703                         prev = scbid;
7704                         continue;
7705                 }
7706                 found++;
7707                 switch (action) {
7708                 case SEARCH_COMPLETE:
7709                         if ((scb->flags & SCB_ACTIVE) == 0)
7710                                 printf("Inactive SCB in Waiting List\n");
7711                         ahd_done_with_status(ahd, scb, status);
7712                         /* FALLTHROUGH */
7713                 case SEARCH_REMOVE:
7714                         ahd_rem_wscb(ahd, scbid, prev, next, tid);
7715                         *list_tail = prev;
7716                         if (SCBID_IS_NULL(prev))
7717                                 *list_head = next;
7718                         break;
7719                 case SEARCH_PRINT:
7720                         printf("0x%x ", scbid);
7721                 case SEARCH_COUNT:
7722                         prev = scbid;
7723                         break;
7724                 }
7725                 if (found > AHD_SCB_MAX)
7726                         panic("SCB LIST LOOP");
7727         }
7728         if (action == SEARCH_COMPLETE
7729          || action == SEARCH_REMOVE)
7730                 ahd_outw(ahd, CMDS_PENDING, ahd_inw(ahd, CMDS_PENDING) - found);
7731         return (found);
7732 }
7733
7734 static void
7735 ahd_stitch_tid_list(struct ahd_softc *ahd, u_int tid_prev,
7736                     u_int tid_cur, u_int tid_next)
7737 {
7738         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7739
7740         if (SCBID_IS_NULL(tid_cur)) {
7741
7742                 /* Bypass current TID list */
7743                 if (SCBID_IS_NULL(tid_prev)) {
7744                         ahd_outw(ahd, WAITING_TID_HEAD, tid_next);
7745                 } else {
7746                         ahd_set_scbptr(ahd, tid_prev);
7747                         ahd_outw(ahd, SCB_NEXT2, tid_next);
7748                 }
7749                 if (SCBID_IS_NULL(tid_next))
7750                         ahd_outw(ahd, WAITING_TID_TAIL, tid_prev);
7751         } else {
7752
7753                 /* Stitch through tid_cur */
7754                 if (SCBID_IS_NULL(tid_prev)) {
7755                         ahd_outw(ahd, WAITING_TID_HEAD, tid_cur);
7756                 } else {
7757                         ahd_set_scbptr(ahd, tid_prev);
7758                         ahd_outw(ahd, SCB_NEXT2, tid_cur);
7759                 }
7760                 ahd_set_scbptr(ahd, tid_cur);
7761                 ahd_outw(ahd, SCB_NEXT2, tid_next);
7762
7763                 if (SCBID_IS_NULL(tid_next))
7764                         ahd_outw(ahd, WAITING_TID_TAIL, tid_cur);
7765         }
7766 }
7767
7768 /*
7769  * Manipulate the waiting for selection list and return the
7770  * scb that follows the one that we remove.
7771  */
7772 static u_int
7773 ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
7774              u_int prev, u_int next, u_int tid)
7775 {
7776         u_int tail_offset;
7777
7778         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7779         if (!SCBID_IS_NULL(prev)) {
7780                 ahd_set_scbptr(ahd, prev);
7781                 ahd_outw(ahd, SCB_NEXT, next);
7782         }
7783
7784         /*
7785          * SCBs that have MK_MESSAGE set in them may
7786          * cause the tail pointer to be updated without
7787          * setting the next pointer of the previous tail.
7788          * Only clear the tail if the removed SCB was
7789          * the tail.
7790          */
7791         tail_offset = WAITING_SCB_TAILS + (2 * tid);
7792         if (SCBID_IS_NULL(next)
7793          && ahd_inw(ahd, tail_offset) == scbid)
7794                 ahd_outw(ahd, tail_offset, prev);
7795
7796         ahd_add_scb_to_free_list(ahd, scbid);
7797         return (next);
7798 }
7799
7800 /*
7801  * Add the SCB as selected by SCBPTR onto the on chip list of
7802  * free hardware SCBs.  This list is empty/unused if we are not
7803  * performing SCB paging.
7804  */
7805 static void
7806 ahd_add_scb_to_free_list(struct ahd_softc *ahd, u_int scbid)
7807 {
7808 /* XXX Need some other mechanism to designate "free". */
7809         /*
7810          * Invalidate the tag so that our abort
7811          * routines don't think it's active.
7812         ahd_outb(ahd, SCB_TAG, SCB_LIST_NULL);
7813          */
7814 }
7815
7816 /******************************** Error Handling ******************************/
7817 /*
7818  * Abort all SCBs that match the given description (target/channel/lun/tag),
7819  * setting their status to the passed in status if the status has not already
7820  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
7821  * is paused before it is called.
7822  */
7823 int
7824 ahd_abort_scbs(struct ahd_softc *ahd, int target, char channel,
7825                int lun, u_int tag, role_t role, uint32_t status)
7826 {
7827         struct          scb *scbp;
7828         struct          scb *scbp_next;
7829         u_int           i, j;
7830         u_int           maxtarget;
7831         u_int           minlun;
7832         u_int           maxlun;
7833         int             found;
7834         ahd_mode_state  saved_modes;
7835
7836         /* restore this when we're done */
7837         saved_modes = ahd_save_modes(ahd);
7838         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7839
7840         found = ahd_search_qinfifo(ahd, target, channel, lun, SCB_LIST_NULL,
7841                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
7842
7843         /*
7844          * Clean out the busy target table for any untagged commands.
7845          */
7846         i = 0;
7847         maxtarget = 16;
7848         if (target != CAM_TARGET_WILDCARD) {
7849                 i = target;
7850                 if (channel == 'B')
7851                         i += 8;
7852                 maxtarget = i + 1;
7853         }
7854
7855         if (lun == CAM_LUN_WILDCARD) {
7856                 minlun = 0;
7857                 maxlun = AHD_NUM_LUNS_NONPKT;
7858         } else if (lun >= AHD_NUM_LUNS_NONPKT) {
7859                 minlun = maxlun = 0;
7860         } else {
7861                 minlun = lun;
7862                 maxlun = lun + 1;
7863         }
7864
7865         if (role != ROLE_TARGET) {
7866                 for (;i < maxtarget; i++) {
7867                         for (j = minlun;j < maxlun; j++) {
7868                                 u_int scbid;
7869                                 u_int tcl;
7870
7871                                 tcl = BUILD_TCL_RAW(i, 'A', j);
7872                                 scbid = ahd_find_busy_tcl(ahd, tcl);
7873                                 scbp = ahd_lookup_scb(ahd, scbid);
7874                                 if (scbp == NULL
7875                                  || ahd_match_scb(ahd, scbp, target, channel,
7876                                                   lun, tag, role) == 0)
7877                                         continue;
7878                                 ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(i, 'A', j));
7879                         }
7880                 }
7881         }
7882
7883         /*
7884          * Don't abort commands that have already completed,
7885          * but haven't quite made it up to the host yet.
7886          */
7887         ahd_flush_qoutfifo(ahd);
7888
7889         /*
7890          * Go through the pending CCB list and look for
7891          * commands for this target that are still active.
7892          * These are other tagged commands that were
7893          * disconnected when the reset occurred.
7894          */
7895         scbp_next = LIST_FIRST(&ahd->pending_scbs);
7896         while (scbp_next != NULL) {
7897                 scbp = scbp_next;
7898                 scbp_next = LIST_NEXT(scbp, pending_links);
7899                 if (ahd_match_scb(ahd, scbp, target, channel, lun, tag, role)) {
7900                         cam_status ostat;
7901
7902                         ostat = aic_get_transaction_status(scbp);
7903                         if (ostat == CAM_REQ_INPROG)
7904                                 aic_set_transaction_status(scbp, status);
7905                         if (aic_get_transaction_status(scbp) != CAM_REQ_CMP)
7906                                 aic_freeze_scb(scbp);
7907                         if ((scbp->flags & SCB_ACTIVE) == 0)
7908                                 printf("Inactive SCB on pending list\n");
7909                         ahd_done(ahd, scbp);
7910                         found++;
7911                 }
7912         }
7913         ahd_restore_modes(ahd, saved_modes);
7914         ahd_platform_abort_scbs(ahd, target, channel, lun, tag, role, status);
7915         ahd->flags |= AHD_UPDATE_PEND_CMDS;
7916         return found;
7917 }
7918
7919 static void
7920 ahd_reset_current_bus(struct ahd_softc *ahd)
7921 {
7922         uint8_t scsiseq;
7923
7924         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7925         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) & ~ENSCSIRST);
7926         scsiseq = ahd_inb(ahd, SCSISEQ0) & ~(ENSELO|ENARBO|SCSIRSTO);
7927         ahd_outb(ahd, SCSISEQ0, scsiseq | SCSIRSTO);
7928         ahd_flush_device_writes(ahd);
7929         aic_delay(AHD_BUSRESET_DELAY);
7930         /* Turn off the bus reset */
7931         ahd_outb(ahd, SCSISEQ0, scsiseq);
7932         ahd_flush_device_writes(ahd);
7933         aic_delay(AHD_BUSRESET_DELAY);
7934         if ((ahd->bugs & AHD_SCSIRST_BUG) != 0) {
7935                 /*
7936                  * 2A Razor #474
7937                  * Certain chip state is not cleared for
7938                  * SCSI bus resets that we initiate, so
7939                  * we must reset the chip.
7940                  */
7941                 ahd_reset(ahd, /*reinit*/TRUE);
7942                 ahd_intr_enable(ahd, /*enable*/TRUE);
7943                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7944         }
7945
7946         ahd_clear_intstat(ahd);
7947 }
7948
7949 int
7950 ahd_reset_channel(struct ahd_softc *ahd, char channel, int initiate_reset)
7951 {
7952         struct  ahd_devinfo devinfo;
7953         u_int   initiator;
7954         u_int   target;
7955         u_int   max_scsiid;
7956         int     found;
7957         u_int   fifo;
7958         u_int   next_fifo;
7959
7960         ahd->pending_device = NULL;
7961
7962         ahd_compile_devinfo(&devinfo,
7963                             CAM_TARGET_WILDCARD,
7964                             CAM_TARGET_WILDCARD,
7965                             CAM_LUN_WILDCARD,
7966                             channel, ROLE_UNKNOWN);
7967         ahd_pause(ahd);
7968
7969         /* Make sure the sequencer is in a safe location. */
7970         ahd_clear_critical_section(ahd);
7971
7972 #ifdef AHD_TARGET_MODE
7973         if ((ahd->flags & AHD_TARGETROLE) != 0) {
7974                 ahd_run_tqinfifo(ahd, /*paused*/TRUE);
7975         }
7976 #endif
7977         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7978
7979         /*
7980          * Disable selections so no automatic hardware
7981          * functions will modify chip state.
7982          */
7983         ahd_outb(ahd, SCSISEQ0, 0);
7984         ahd_outb(ahd, SCSISEQ1, 0);
7985
7986         /*
7987          * Safely shut down our DMA engines.  Always start with
7988          * the FIFO that is not currently active (if any are
7989          * actively connected).
7990          */
7991         next_fifo = fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
7992         if (next_fifo > CURRFIFO_1)
7993                 /* If disconneced, arbitrarily start with FIFO1. */
7994                 next_fifo = fifo = 0;
7995         do {
7996                 next_fifo ^= CURRFIFO_1;
7997                 ahd_set_modes(ahd, next_fifo, next_fifo);
7998                 ahd_outb(ahd, DFCNTRL,
7999                          ahd_inb(ahd, DFCNTRL) & ~(SCSIEN|HDMAEN));
8000                 while ((ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0)
8001                         aic_delay(10);
8002                 /*
8003                  * Set CURRFIFO to the now inactive channel.
8004                  */
8005                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8006                 ahd_outb(ahd, DFFSTAT, next_fifo);
8007         } while (next_fifo != fifo);
8008
8009         /*
8010          * Reset the bus if we are initiating this reset
8011          */
8012         ahd_clear_msg_state(ahd);
8013         ahd_outb(ahd, SIMODE1,
8014                  ahd_inb(ahd, SIMODE1) & ~(ENBUSFREE|ENSCSIRST));
8015
8016         if (initiate_reset)
8017                 ahd_reset_current_bus(ahd);
8018
8019         ahd_clear_intstat(ahd);
8020
8021         /*
8022          * Clean up all the state information for the
8023          * pending transactions on this bus.
8024          */
8025         found = ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, channel,
8026                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
8027                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
8028
8029         /*
8030          * Cleanup anything left in the FIFOs.
8031          */
8032         ahd_clear_fifo(ahd, 0);
8033         ahd_clear_fifo(ahd, 1);
8034
8035         /*
8036          * Revert to async/narrow transfers until we renegotiate.
8037          */
8038         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
8039         for (target = 0; target <= max_scsiid; target++) {
8040
8041                 if (ahd->enabled_targets[target] == NULL)
8042                         continue;
8043                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
8044                         struct ahd_devinfo devinfo;
8045
8046                         ahd_compile_devinfo(&devinfo, target, initiator,
8047                                             CAM_LUN_WILDCARD,
8048                                             'A', ROLE_UNKNOWN);
8049                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
8050                                       AHD_TRANS_CUR, /*paused*/TRUE);
8051                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
8052                                          /*offset*/0, /*ppr_options*/0,
8053                                          AHD_TRANS_CUR, /*paused*/TRUE);
8054                 }
8055         }
8056
8057 #ifdef AHD_TARGET_MODE
8058         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
8059
8060         /*
8061          * Send an immediate notify ccb to all target more peripheral
8062          * drivers affected by this action.
8063          */
8064         for (target = 0; target <= max_scsiid; target++) {
8065                 struct ahd_tmode_tstate* tstate;
8066                 u_int lun;
8067
8068                 tstate = ahd->enabled_targets[target];
8069                 if (tstate == NULL)
8070                         continue;
8071                 for (lun = 0; lun < AHD_NUM_LUNS; lun++) {
8072                         struct ahd_tmode_lstate* lstate;
8073
8074                         lstate = tstate->enabled_luns[lun];
8075                         if (lstate == NULL)
8076                                 continue;
8077
8078                         ahd_queue_lstate_event(ahd, lstate, CAM_TARGET_WILDCARD,
8079                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
8080                         ahd_send_lstate_events(ahd, lstate);
8081                 }
8082         }
8083 #endif
8084         /* Notify the XPT that a bus reset occurred */
8085         ahd_send_async(ahd, devinfo.channel, CAM_TARGET_WILDCARD,
8086                        CAM_LUN_WILDCARD, AC_BUS_RESET, NULL);
8087         ahd_restart(ahd);
8088         /*
8089          * Freeze the SIMQ until our poller can determine that
8090          * the bus reset has really gone away.  We set the initial
8091          * timer to 0 to have the check performed as soon as possible
8092          * from the timer context.
8093          */
8094         if ((ahd->flags & AHD_RESET_POLL_ACTIVE) == 0) {
8095                 ahd->flags |= AHD_RESET_POLL_ACTIVE;
8096                 aic_freeze_simq(ahd);
8097                 aic_timer_reset(&ahd->reset_timer, 0, ahd_reset_poll, ahd);
8098         }
8099         return (found);
8100 }
8101
8102
8103 #define AHD_RESET_POLL_MS 1
8104 static void
8105 ahd_reset_poll(void *arg)
8106 {
8107         struct  ahd_softc *ahd = (struct ahd_softc *)arg;
8108         u_int   scsiseq1;
8109
8110         ahd_lock(ahd);
8111         ahd_pause(ahd);
8112         ahd_update_modes(ahd);
8113         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8114         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
8115         if ((ahd_inb(ahd, SSTAT1) & SCSIRSTI) != 0) {
8116                 aic_timer_reset(&ahd->reset_timer, AHD_RESET_POLL_MS,
8117                                 ahd_reset_poll, ahd);
8118                 ahd_unpause(ahd);
8119                 ahd_unlock(ahd);
8120                 return;
8121         }
8122
8123         /* Reset is now low.  Complete chip reinitialization. */
8124         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) | ENSCSIRST);
8125         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
8126         ahd_outb(ahd, SCSISEQ1, scsiseq1 & (ENSELI|ENRSELI|ENAUTOATNP));
8127         ahd_unpause(ahd);
8128         ahd->flags &= ~AHD_RESET_POLL_ACTIVE;
8129         aic_release_simq(ahd);
8130         ahd_unlock(ahd);
8131 }
8132
8133 /**************************** Statistics Processing ***************************/
8134 static void
8135 ahd_stat_timer(void *arg)
8136 {
8137         struct  ahd_softc *ahd = (struct ahd_softc *)arg;
8138         int     enint_coal;
8139
8140         ahd_lock(ahd);
8141         enint_coal = ahd->hs_mailbox & ENINT_COALESCE;
8142         if (ahd->cmdcmplt_total > ahd->int_coalescing_threshold)
8143                 enint_coal |= ENINT_COALESCE;
8144         else if (ahd->cmdcmplt_total < ahd->int_coalescing_stop_threshold)
8145                 enint_coal &= ~ENINT_COALESCE;
8146
8147         if (enint_coal != (ahd->hs_mailbox & ENINT_COALESCE)) {
8148                 ahd_enable_coalescing(ahd, enint_coal);
8149 #ifdef AHD_DEBUG
8150                 if ((ahd_debug & AHD_SHOW_INT_COALESCING) != 0)
8151                         printf("%s: Interrupt coalescing "
8152                                "now %sabled. Cmds %d\n",
8153                                ahd_name(ahd),
8154                                (enint_coal & ENINT_COALESCE) ? "en" : "dis",
8155                                ahd->cmdcmplt_total);
8156 #endif
8157         }
8158
8159         ahd->cmdcmplt_bucket = (ahd->cmdcmplt_bucket+1) & (AHD_STAT_BUCKETS-1);
8160         ahd->cmdcmplt_total -= ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket];
8161         ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket] = 0;
8162         aic_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_MS,
8163                         ahd_stat_timer, ahd);
8164         ahd_unlock(ahd);
8165 }
8166
8167 /****************************** Status Processing *****************************/
8168 void
8169 ahd_handle_scb_status(struct ahd_softc *ahd, struct scb *scb)
8170 {
8171         if (scb->hscb->shared_data.istatus.scsi_status != 0) {
8172                 ahd_handle_scsi_status(ahd, scb);
8173         } else {
8174                 ahd_calc_residual(ahd, scb);
8175                 ahd_done(ahd, scb);
8176         }
8177 }
8178
8179 void
8180 ahd_handle_scsi_status(struct ahd_softc *ahd, struct scb *scb)
8181 {
8182         struct  hardware_scb *hscb;
8183         int     paused;
8184
8185         /*
8186          * The sequencer freezes its select-out queue
8187          * anytime a SCSI status error occurs.  We must
8188          * handle the error and increment our qfreeze count
8189          * to allow the sequencer to continue.  We don't
8190          * bother clearing critical sections here since all
8191          * operations are on data structures that the sequencer
8192          * is not touching once the queue is frozen.
8193          */
8194         hscb = scb->hscb;
8195
8196         if (ahd_is_paused(ahd)) {
8197                 paused = 1;
8198         } else {
8199                 paused = 0;
8200                 ahd_pause(ahd);
8201         }
8202
8203         /* Freeze the queue until the client sees the error. */
8204         ahd_freeze_devq(ahd, scb);
8205         aic_freeze_scb(scb);
8206         ahd->qfreeze_cnt++;
8207         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
8208
8209         if (paused == 0)
8210                 ahd_unpause(ahd);
8211
8212         /* Don't want to clobber the original sense code */
8213         if ((scb->flags & SCB_SENSE) != 0) {
8214                 /*
8215                  * Clear the SCB_SENSE Flag and perform
8216                  * a normal command completion.
8217                  */
8218                 scb->flags &= ~SCB_SENSE;
8219                 aic_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
8220                 ahd_done(ahd, scb);
8221                 return;
8222         }
8223         aic_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
8224         aic_set_scsi_status(scb, hscb->shared_data.istatus.scsi_status);
8225         switch (hscb->shared_data.istatus.scsi_status) {
8226         case STATUS_PKT_SENSE:
8227         {
8228                 struct scsi_status_iu_header *siu;
8229
8230                 ahd_sync_sense(ahd, scb, BUS_DMASYNC_POSTREAD);
8231                 siu = (struct scsi_status_iu_header *)scb->sense_data;
8232                 aic_set_scsi_status(scb, siu->status);
8233 #ifdef AHD_DEBUG
8234                 if ((ahd_debug & AHD_SHOW_SENSE) != 0) {
8235                         ahd_print_path(ahd, scb);
8236                         printf("SCB 0x%x Received PKT Status of 0x%x\n",
8237                                SCB_GET_TAG(scb), siu->status);
8238                         printf("\tflags = 0x%x, sense len = 0x%x, "
8239                                "pktfail = 0x%x\n",
8240                                siu->flags, scsi_4btoul(siu->sense_length),
8241                                scsi_4btoul(siu->pkt_failures_length));
8242                 }
8243 #endif
8244                 if ((siu->flags & SIU_RSPVALID) != 0) {
8245                         ahd_print_path(ahd, scb);
8246                         if (scsi_4btoul(siu->pkt_failures_length) < 4) {
8247                                 printf("Unable to parse pkt_failures\n");
8248                         } else {
8249
8250                                 switch (SIU_PKTFAIL_CODE(siu)) {
8251                                 case SIU_PFC_NONE:
8252                                         printf("No packet failure found\n");
8253                                         AHD_UNCORRECTABLE_ERROR(ahd);
8254                                         break;
8255                                 case SIU_PFC_CIU_FIELDS_INVALID:
8256                                         printf("Invalid Command IU Field\n");
8257                                         AHD_UNCORRECTABLE_ERROR(ahd);
8258                                         break;
8259                                 case SIU_PFC_TMF_NOT_SUPPORTED:
8260                                         printf("TMF not supportd\n");
8261                                         AHD_UNCORRECTABLE_ERROR(ahd);
8262                                         break;
8263                                 case SIU_PFC_TMF_FAILED:
8264                                         printf("TMF failed\n");
8265                                         AHD_UNCORRECTABLE_ERROR(ahd);
8266                                         break;
8267                                 case SIU_PFC_INVALID_TYPE_CODE:
8268                                         printf("Invalid L_Q Type code\n");
8269                                         AHD_UNCORRECTABLE_ERROR(ahd);
8270                                         break;
8271                                 case SIU_PFC_ILLEGAL_REQUEST:
8272                                         AHD_UNCORRECTABLE_ERROR(ahd);
8273                                         printf("Illegal request\n");
8274                                 default:
8275                                         break;
8276                                 }
8277                         }
8278                         if (siu->status == SCSI_STATUS_OK)
8279                                 aic_set_transaction_status(scb,
8280                                                            CAM_REQ_CMP_ERR);
8281                 }
8282                 if ((siu->flags & SIU_SNSVALID) != 0) {
8283                         scb->flags |= SCB_PKT_SENSE;
8284 #ifdef AHD_DEBUG
8285                         if ((ahd_debug & AHD_SHOW_SENSE) != 0)
8286                                 printf("Sense data available\n");
8287 #endif
8288                 }
8289                 ahd_done(ahd, scb);
8290                 break;
8291         }
8292         case SCSI_STATUS_CMD_TERMINATED:
8293         case SCSI_STATUS_CHECK_COND:
8294         {
8295                 struct ahd_devinfo devinfo;
8296                 struct ahd_dma_seg *sg;
8297                 struct scsi_sense *sc;
8298                 struct ahd_initiator_tinfo *targ_info;
8299                 struct ahd_tmode_tstate *tstate;
8300                 struct ahd_transinfo *tinfo;
8301 #ifdef AHD_DEBUG
8302                 if (ahd_debug & AHD_SHOW_SENSE) {
8303                         ahd_print_path(ahd, scb);
8304                         printf("SCB %d: requests Check Status\n",
8305                                SCB_GET_TAG(scb));
8306                 }
8307 #endif
8308
8309                 if (aic_perform_autosense(scb) == 0)
8310                         break;
8311
8312                 ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
8313                                     SCB_GET_TARGET(ahd, scb),
8314                                     SCB_GET_LUN(scb),
8315                                     SCB_GET_CHANNEL(ahd, scb),
8316                                     ROLE_INITIATOR);
8317                 targ_info = ahd_fetch_transinfo(ahd,
8318                                                 devinfo.channel,
8319                                                 devinfo.our_scsiid,
8320                                                 devinfo.target,
8321                                                 &tstate);
8322                 tinfo = &targ_info->curr;
8323                 sg = scb->sg_list;
8324                 sc = (struct scsi_sense *)hscb->shared_data.idata.cdb;
8325                 /*
8326                  * Save off the residual if there is one.
8327                  */
8328                 ahd_update_residual(ahd, scb);
8329 #ifdef AHD_DEBUG
8330                 if (ahd_debug & AHD_SHOW_SENSE) {
8331                         ahd_print_path(ahd, scb);
8332                         printf("Sending Sense\n");
8333                 }
8334 #endif
8335                 scb->sg_count = 0;
8336                 sg = ahd_sg_setup(ahd, scb, sg, ahd_get_sense_bufaddr(ahd, scb),
8337                                   aic_get_sense_bufsize(ahd, scb),
8338                                   /*last*/TRUE);
8339                 sc->opcode = REQUEST_SENSE;
8340                 sc->byte2 = 0;
8341                 if (tinfo->protocol_version <= SCSI_REV_2
8342                  && SCB_GET_LUN(scb) < 8)
8343                         sc->byte2 = SCB_GET_LUN(scb) << 5;
8344                 sc->unused[0] = 0;
8345                 sc->unused[1] = 0;
8346                 sc->length = aic_get_sense_bufsize(ahd, scb);
8347                 sc->control = 0;
8348
8349                 /*
8350                  * We can't allow the target to disconnect.
8351                  * This will be an untagged transaction and
8352                  * having the target disconnect will make this
8353                  * transaction indestinguishable from outstanding
8354                  * tagged transactions.
8355                  */
8356                 hscb->control = 0;
8357
8358                 /*
8359                  * This request sense could be because the
8360                  * the device lost power or in some other
8361                  * way has lost our transfer negotiations.
8362                  * Renegotiate if appropriate.  Unit attention
8363                  * errors will be reported before any data
8364                  * phases occur.
8365                  */
8366                 if (aic_get_residual(scb) == aic_get_transfer_length(scb)) {
8367                         ahd_update_neg_request(ahd, &devinfo,
8368                                                tstate, targ_info,
8369                                                AHD_NEG_IF_NON_ASYNC);
8370                 }
8371                 if (tstate->auto_negotiate & devinfo.target_mask) {
8372                         hscb->control |= MK_MESSAGE;
8373                         scb->flags &=
8374                             ~(SCB_NEGOTIATE|SCB_ABORT|SCB_DEVICE_RESET);
8375                         scb->flags |= SCB_AUTO_NEGOTIATE;
8376                 }
8377                 hscb->cdb_len = sizeof(*sc);
8378                 ahd_setup_data_scb(ahd, scb);
8379                 scb->flags |= SCB_SENSE;
8380                 ahd_queue_scb(ahd, scb);
8381                 /*
8382                  * Ensure we have enough time to actually
8383                  * retrieve the sense, but only schedule
8384                  * the timer if we are not in recovery or
8385                  * this is a recovery SCB that is allowed
8386                  * to have an active timer.
8387                  */
8388                 if (ahd->scb_data.recovery_scbs == 0
8389                  || (scb->flags & SCB_RECOVERY_SCB) != 0)
8390                         aic_scb_timer_reset(scb, 5 * 1000);
8391                 break;
8392         }
8393         case SCSI_STATUS_OK:
8394                 printf("%s: Interrupted for staus of 0???\n",
8395                        ahd_name(ahd));
8396                 /* FALLTHROUGH */
8397         default:
8398                 ahd_done(ahd, scb);
8399                 break;
8400         }
8401 }
8402
8403 /*
8404  * Calculate the residual for a just completed SCB.
8405  */
8406 void
8407 ahd_calc_residual(struct ahd_softc *ahd, struct scb *scb)
8408 {
8409         struct hardware_scb *hscb;
8410         struct initiator_status *spkt;
8411         uint32_t sgptr;
8412         uint32_t resid_sgptr;
8413         uint32_t resid;
8414
8415         /*
8416          * 5 cases.
8417          * 1) No residual.
8418          *    SG_STATUS_VALID clear in sgptr.
8419          * 2) Transferless command
8420          * 3) Never performed any transfers.
8421          *    sgptr has SG_FULL_RESID set.
8422          * 4) No residual but target did not
8423          *    save data pointers after the
8424          *    last transfer, so sgptr was
8425          *    never updated.
8426          * 5) We have a partial residual.
8427          *    Use residual_sgptr to determine
8428          *    where we are.
8429          */
8430
8431         hscb = scb->hscb;
8432         sgptr = aic_le32toh(hscb->sgptr);
8433         if ((sgptr & SG_STATUS_VALID) == 0)
8434                 /* Case 1 */
8435                 return;
8436         sgptr &= ~SG_STATUS_VALID;
8437
8438         if ((sgptr & SG_LIST_NULL) != 0)
8439                 /* Case 2 */
8440                 return;
8441
8442         /*
8443          * Residual fields are the same in both
8444          * target and initiator status packets,
8445          * so we can always use the initiator fields
8446          * regardless of the role for this SCB.
8447          */
8448         spkt = &hscb->shared_data.istatus;
8449         resid_sgptr = aic_le32toh(spkt->residual_sgptr);
8450         if ((sgptr & SG_FULL_RESID) != 0) {
8451                 /* Case 3 */
8452                 resid = aic_get_transfer_length(scb);
8453         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
8454                 /* Case 4 */
8455                 return;
8456         } else if ((resid_sgptr & SG_OVERRUN_RESID) != 0) {
8457                 ahd_print_path(ahd, scb);
8458                 printf("data overrun detected Tag == 0x%x.\n",
8459                        SCB_GET_TAG(scb));
8460                 ahd_freeze_devq(ahd, scb);
8461                 aic_set_transaction_status(scb, CAM_DATA_RUN_ERR);
8462                 aic_freeze_scb(scb);
8463                 return;
8464         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
8465                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
8466                 /* NOTREACHED */
8467         } else {
8468                 struct ahd_dma_seg *sg;
8469
8470                 /*
8471                  * Remainder of the SG where the transfer
8472                  * stopped.
8473                  */
8474                 resid = aic_le32toh(spkt->residual_datacnt) & AHD_SG_LEN_MASK;
8475                 sg = ahd_sg_bus_to_virt(ahd, scb, resid_sgptr & SG_PTR_MASK);
8476
8477                 /* The residual sg_ptr always points to the next sg */
8478                 sg--;
8479
8480                 /*
8481                  * Add up the contents of all residual
8482                  * SG segments that are after the SG where
8483                  * the transfer stopped.
8484                  */
8485                 while ((aic_le32toh(sg->len) & AHD_DMA_LAST_SEG) == 0) {
8486                         sg++;
8487                         resid += aic_le32toh(sg->len) & AHD_SG_LEN_MASK;
8488                 }
8489         }
8490         if ((scb->flags & SCB_SENSE) == 0)
8491                 aic_set_residual(scb, resid);
8492         else
8493                 aic_set_sense_residual(scb, resid);
8494
8495 #ifdef AHD_DEBUG
8496         if ((ahd_debug & AHD_SHOW_MISC) != 0) {
8497                 ahd_print_path(ahd, scb);
8498                 printf("Handled %sResidual of %d bytes\n",
8499                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
8500         }
8501 #endif
8502 }
8503
8504 /******************************* Target Mode **********************************/
8505 #ifdef AHD_TARGET_MODE
8506 /*
8507  * Add a target mode event to this lun's queue
8508  */
8509 static void
8510 ahd_queue_lstate_event(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate,
8511                        u_int initiator_id, u_int event_type, u_int event_arg)
8512 {
8513         struct ahd_tmode_event *event;
8514         int pending;
8515
8516         xpt_freeze_devq(lstate->path, /*count*/1);
8517         if (lstate->event_w_idx >= lstate->event_r_idx)
8518                 pending = lstate->event_w_idx - lstate->event_r_idx;
8519         else
8520                 pending = AHD_TMODE_EVENT_BUFFER_SIZE + 1
8521                         - (lstate->event_r_idx - lstate->event_w_idx);
8522
8523         if (event_type == EVENT_TYPE_BUS_RESET
8524          || event_type == MSG_BUS_DEV_RESET) {
8525                 /*
8526                  * Any earlier events are irrelevant, so reset our buffer.
8527                  * This has the effect of allowing us to deal with reset
8528                  * floods (an external device holding down the reset line)
8529                  * without losing the event that is really interesting.
8530                  */
8531                 lstate->event_r_idx = 0;
8532                 lstate->event_w_idx = 0;
8533                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
8534         }
8535
8536         if (pending == AHD_TMODE_EVENT_BUFFER_SIZE) {
8537                 xpt_print_path(lstate->path);
8538                 printf("immediate event %x:%x lost\n",
8539                        lstate->event_buffer[lstate->event_r_idx].event_type,
8540                        lstate->event_buffer[lstate->event_r_idx].event_arg);
8541                 lstate->event_r_idx++;
8542                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8543                         lstate->event_r_idx = 0;
8544                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
8545         }
8546
8547         event = &lstate->event_buffer[lstate->event_w_idx];
8548         event->initiator_id = initiator_id;
8549         event->event_type = event_type;
8550         event->event_arg = event_arg;
8551         lstate->event_w_idx++;
8552         if (lstate->event_w_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8553                 lstate->event_w_idx = 0;
8554 }
8555
8556 /*
8557  * Send any target mode events queued up waiting
8558  * for immediate notify resources.
8559  */
8560 void
8561 ahd_send_lstate_events(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate)
8562 {
8563         struct ccb_hdr *ccbh;
8564         struct ccb_immediate_notify *inot;
8565
8566         while (lstate->event_r_idx != lstate->event_w_idx
8567             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
8568                 struct ahd_tmode_event *event;
8569
8570                 event = &lstate->event_buffer[lstate->event_r_idx];
8571                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
8572                 inot = (struct ccb_immediate_notify *)ccbh;
8573                 switch (event->event_type) {
8574                 case EVENT_TYPE_BUS_RESET:
8575                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
8576                         break;
8577                 default:
8578                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
8579                         inot->arg = event->event_type;
8580                         inot->seq_id = event->event_arg;
8581                         break;
8582                 }
8583                 inot->initiator_id = event->initiator_id;
8584                 xpt_done((union ccb *)inot);
8585                 lstate->event_r_idx++;
8586                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8587                         lstate->event_r_idx = 0;
8588         }
8589 }
8590 #endif
8591
8592 /******************** Sequencer Program Patching/Download *********************/
8593
8594 #ifdef AHD_DUMP_SEQ
8595 void
8596 ahd_dumpseq(struct ahd_softc* ahd)
8597 {
8598         int i;
8599         int max_prog;
8600
8601         max_prog = 2048;
8602
8603         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
8604         ahd_outw(ahd, PRGMCNT, 0);
8605         for (i = 0; i < max_prog; i++) {
8606                 uint8_t ins_bytes[4];
8607
8608                 ahd_insb(ahd, SEQRAM, ins_bytes, 4);
8609                 printf("0x%08x\n", ins_bytes[0] << 24
8610                                  | ins_bytes[1] << 16
8611                                  | ins_bytes[2] << 8
8612                                  | ins_bytes[3]);
8613         }
8614 }
8615 #endif
8616
8617 static void
8618 ahd_loadseq(struct ahd_softc *ahd)
8619 {
8620         struct  cs cs_table[num_critical_sections];
8621         u_int   begin_set[num_critical_sections];
8622         u_int   end_set[num_critical_sections];
8623         struct  patch *cur_patch;
8624         u_int   cs_count;
8625         u_int   cur_cs;
8626         u_int   i;
8627         int     downloaded;
8628         u_int   skip_addr;
8629         u_int   sg_prefetch_cnt;
8630         u_int   sg_prefetch_cnt_limit;
8631         u_int   sg_prefetch_align;
8632         u_int   sg_size;
8633         u_int   cacheline_mask;
8634         uint8_t download_consts[DOWNLOAD_CONST_COUNT];
8635
8636         if (bootverbose)
8637                 printf("%s: Downloading Sequencer Program...",
8638                        ahd_name(ahd));
8639
8640 #if DOWNLOAD_CONST_COUNT != 8
8641 #error "Download Const Mismatch"
8642 #endif
8643         /*
8644          * Start out with 0 critical sections
8645          * that apply to this firmware load.
8646          */
8647         cs_count = 0;
8648         cur_cs = 0;
8649         memset(begin_set, 0, sizeof(begin_set));
8650         memset(end_set, 0, sizeof(end_set));
8651
8652         /*
8653          * Setup downloadable constant table.
8654          *
8655          * The computation for the S/G prefetch variables is
8656          * a bit complicated.  We would like to always fetch
8657          * in terms of cachelined sized increments.  However,
8658          * if the cacheline is not an even multiple of the
8659          * SG element size or is larger than our SG RAM, using
8660          * just the cache size might leave us with only a portion
8661          * of an SG element at the tail of a prefetch.  If the
8662          * cacheline is larger than our S/G prefetch buffer less
8663          * the size of an SG element, we may round down to a cacheline
8664          * that doesn't contain any or all of the S/G of interest
8665          * within the bounds of our S/G ram.  Provide variables to
8666          * the sequencer that will allow it to handle these edge
8667          * cases.
8668          */
8669         /* Start by aligning to the nearest cacheline. */
8670         sg_prefetch_align = ahd->pci_cachesize;
8671         if (sg_prefetch_align == 0)
8672                 sg_prefetch_align = 8;
8673         /* Round down to the nearest power of 2. */
8674         while (powerof2(sg_prefetch_align) == 0)
8675                 sg_prefetch_align--;
8676
8677         cacheline_mask = sg_prefetch_align - 1;
8678
8679         /*
8680          * If the cacheline boundary is greater than half our prefetch RAM
8681          * we risk not being able to fetch even a single complete S/G
8682          * segment if we align to that boundary.
8683          */
8684         if (sg_prefetch_align > CCSGADDR_MAX/2)
8685                 sg_prefetch_align = CCSGADDR_MAX/2;
8686         /* Start by fetching a single cacheline. */
8687         sg_prefetch_cnt = sg_prefetch_align;
8688         /*
8689          * Increment the prefetch count by cachelines until
8690          * at least one S/G element will fit.
8691          */
8692         sg_size = sizeof(struct ahd_dma_seg);
8693         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
8694                 sg_size = sizeof(struct ahd_dma64_seg);
8695         while (sg_prefetch_cnt < sg_size)
8696                 sg_prefetch_cnt += sg_prefetch_align;
8697         /*
8698          * If the cacheline is not an even multiple of
8699          * the S/G size, we may only get a partial S/G when
8700          * we align. Add a cacheline if this is the case.
8701          */
8702         if ((sg_prefetch_align % sg_size) != 0
8703          && (sg_prefetch_cnt < CCSGADDR_MAX))
8704                 sg_prefetch_cnt += sg_prefetch_align;
8705         /*
8706          * Lastly, compute a value that the sequencer can use
8707          * to determine if the remainder of the CCSGRAM buffer
8708          * has a full S/G element in it.
8709          */
8710         sg_prefetch_cnt_limit = -(sg_prefetch_cnt - sg_size + 1);
8711         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
8712         download_consts[SG_PREFETCH_CNT_LIMIT] = sg_prefetch_cnt_limit;
8713         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_align - 1);
8714         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_align - 1);
8715         download_consts[SG_SIZEOF] = sg_size;
8716         download_consts[PKT_OVERRUN_BUFOFFSET] =
8717                 (ahd->overrun_buf - (uint8_t *)ahd->qoutfifo) / 256;
8718         download_consts[SCB_TRANSFER_SIZE] = SCB_TRANSFER_SIZE_1BYTE_LUN;
8719         download_consts[CACHELINE_MASK] = cacheline_mask;
8720         cur_patch = patches;
8721         downloaded = 0;
8722         skip_addr = 0;
8723         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
8724         ahd_outw(ahd, PRGMCNT, 0);
8725
8726         for (i = 0; i < sizeof(seqprog)/4; i++) {
8727                 if (ahd_check_patch(ahd, &cur_patch, i, &skip_addr) == 0) {
8728                         /*
8729                          * Don't download this instruction as it
8730                          * is in a patch that was removed.
8731                          */
8732                         continue;
8733                 }
8734                 /*
8735                  * Move through the CS table until we find a CS
8736                  * that might apply to this instruction.
8737                  */
8738                 for (; cur_cs < num_critical_sections; cur_cs++) {
8739                         if (critical_sections[cur_cs].end <= i) {
8740                                 if (begin_set[cs_count] == TRUE
8741                                  && end_set[cs_count] == FALSE) {
8742                                         cs_table[cs_count].end = downloaded;
8743                                         end_set[cs_count] = TRUE;
8744                                         cs_count++;
8745                                 }
8746                                 continue;
8747                         }
8748                         if (critical_sections[cur_cs].begin <= i
8749                          && begin_set[cs_count] == FALSE) {
8750                                 cs_table[cs_count].begin = downloaded;
8751                                 begin_set[cs_count] = TRUE;
8752                         }
8753                         break;
8754                 }
8755                 ahd_download_instr(ahd, i, download_consts);
8756                 downloaded++;
8757         }
8758
8759         ahd->num_critical_sections = cs_count;
8760         if (cs_count != 0) {
8761
8762                 cs_count *= sizeof(struct cs);
8763                 ahd->critical_sections = malloc(cs_count, M_DEVBUF, M_NOWAIT);
8764                 if (ahd->critical_sections == NULL)
8765                         panic("ahd_loadseq: Could not malloc");
8766                 memcpy(ahd->critical_sections, cs_table, cs_count);
8767         }
8768         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE);
8769
8770         if (bootverbose) {
8771                 printf(" %d instructions downloaded\n", downloaded);
8772                 printf("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
8773                        ahd_name(ahd), ahd->features, ahd->bugs, ahd->flags);
8774         }
8775 }
8776
8777 static int
8778 ahd_check_patch(struct ahd_softc *ahd, struct patch **start_patch,
8779                 u_int start_instr, u_int *skip_addr)
8780 {
8781         struct  patch *cur_patch;
8782         struct  patch *last_patch;
8783         u_int   num_patches;
8784
8785         num_patches = sizeof(patches)/sizeof(struct patch);
8786         last_patch = &patches[num_patches];
8787         cur_patch = *start_patch;
8788
8789         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
8790
8791                 if (cur_patch->patch_func(ahd) == 0) {
8792
8793                         /* Start rejecting code */
8794                         *skip_addr = start_instr + cur_patch->skip_instr;
8795                         cur_patch += cur_patch->skip_patch;
8796                 } else {
8797                         /* Accepted this patch.  Advance to the next
8798                          * one and wait for our intruction pointer to
8799                          * hit this point.
8800                          */
8801                         cur_patch++;
8802                 }
8803         }
8804
8805         *start_patch = cur_patch;
8806         if (start_instr < *skip_addr)
8807                 /* Still skipping */
8808                 return (0);
8809
8810         return (1);
8811 }
8812
8813 static u_int
8814 ahd_resolve_seqaddr(struct ahd_softc *ahd, u_int address)
8815 {
8816         struct patch *cur_patch;
8817         int address_offset;
8818         u_int skip_addr;
8819         u_int i;
8820
8821         address_offset = 0;
8822         cur_patch = patches;
8823         skip_addr = 0;
8824
8825         for (i = 0; i < address;) {
8826
8827                 ahd_check_patch(ahd, &cur_patch, i, &skip_addr);
8828
8829                 if (skip_addr > i) {
8830                         int end_addr;
8831
8832                         end_addr = MIN(address, skip_addr);
8833                         address_offset += end_addr - i;
8834                         i = skip_addr;
8835                 } else {
8836                         i++;
8837                 }
8838         }
8839         return (address - address_offset);
8840 }
8841
8842 static void
8843 ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts)
8844 {
8845         union   ins_formats instr;
8846         struct  ins_format1 *fmt1_ins;
8847         struct  ins_format3 *fmt3_ins;
8848         u_int   opcode;
8849
8850         /*
8851          * The firmware is always compiled into a little endian format.
8852          */
8853         instr.integer = aic_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
8854
8855         fmt1_ins = &instr.format1;
8856         fmt3_ins = NULL;
8857
8858         /* Pull the opcode */
8859         opcode = instr.format1.opcode;
8860         switch (opcode) {
8861         case AIC_OP_JMP:
8862         case AIC_OP_JC:
8863         case AIC_OP_JNC:
8864         case AIC_OP_CALL:
8865         case AIC_OP_JNE:
8866         case AIC_OP_JNZ:
8867         case AIC_OP_JE:
8868         case AIC_OP_JZ:
8869         {
8870                 fmt3_ins = &instr.format3;
8871                 fmt3_ins->address = ahd_resolve_seqaddr(ahd, fmt3_ins->address);
8872                 /* FALLTHROUGH */
8873         }
8874         case AIC_OP_OR:
8875         case AIC_OP_AND:
8876         case AIC_OP_XOR:
8877         case AIC_OP_ADD:
8878         case AIC_OP_ADC:
8879         case AIC_OP_BMOV:
8880                 if (fmt1_ins->parity != 0) {
8881                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
8882                 }
8883                 fmt1_ins->parity = 0;
8884                 /* FALLTHROUGH */
8885         case AIC_OP_ROL:
8886         {
8887                 int i, count;
8888
8889                 /* Calculate odd parity for the instruction */
8890                 for (i = 0, count = 0; i < 31; i++) {
8891                         uint32_t mask;
8892
8893                         mask = 0x01 << i;
8894                         if ((instr.integer & mask) != 0)
8895                                 count++;
8896                 }
8897                 if ((count & 0x01) == 0)
8898                         instr.format1.parity = 1;
8899
8900                 /* The sequencer is a little endian cpu */
8901                 instr.integer = aic_htole32(instr.integer);
8902                 ahd_outsb(ahd, SEQRAM, instr.bytes, 4);
8903                 break;
8904         }
8905         default:
8906                 panic("Unknown opcode encountered in seq program");
8907                 break;
8908         }
8909 }
8910
8911 static int
8912 ahd_probe_stack_size(struct ahd_softc *ahd)
8913 {
8914         int last_probe;
8915
8916         last_probe = 0;
8917         while (1) {
8918                 int i;
8919
8920                 /*
8921                  * We avoid using 0 as a pattern to avoid
8922                  * confusion if the stack implementation
8923                  * "back-fills" with zeros when "poping'
8924                  * entries.
8925                  */
8926                 for (i = 1; i <= last_probe+1; i++) {
8927                        ahd_outb(ahd, STACK, i & 0xFF);
8928                        ahd_outb(ahd, STACK, (i >> 8) & 0xFF);
8929                 }
8930
8931                 /* Verify */
8932                 for (i = last_probe+1; i > 0; i--) {
8933                         u_int stack_entry;
8934
8935                         stack_entry = ahd_inb(ahd, STACK)
8936                                     |(ahd_inb(ahd, STACK) << 8);
8937                         if (stack_entry != i)
8938                                 goto sized;
8939                 }
8940                 last_probe++;
8941         }
8942 sized:
8943         return (last_probe);
8944 }
8945
8946 void
8947 ahd_dump_all_cards_state(void)
8948 {
8949         struct ahd_softc *list_ahd;
8950
8951         TAILQ_FOREACH(list_ahd, &ahd_tailq, links) {
8952                 ahd_dump_card_state(list_ahd);
8953         }
8954 }
8955
8956 int
8957 ahd_print_register(ahd_reg_parse_entry_t *table, u_int num_entries,
8958                    const char *name, u_int address, u_int value,
8959                    u_int *cur_column, u_int wrap_point)
8960 {
8961         int     printed;
8962         u_int   printed_mask;
8963         u_int   dummy_column;
8964
8965         if (cur_column == NULL) {
8966                 dummy_column = 0;
8967                 cur_column = &dummy_column;
8968         }
8969
8970         if (cur_column != NULL && *cur_column >= wrap_point) {
8971                 printf("\n");
8972                 *cur_column = 0;
8973         }
8974         printed = printf("%s[0x%x]", name, value);
8975         if (table == NULL) {
8976                 printed += printf(" ");
8977                 *cur_column += printed;
8978                 return (printed);
8979         }
8980         printed_mask = 0;
8981         while (printed_mask != 0xFF) {
8982                 int entry;
8983
8984                 for (entry = 0; entry < num_entries; entry++) {
8985                         if (((value & table[entry].mask)
8986                           != table[entry].value)
8987                          || ((printed_mask & table[entry].mask)
8988                           == table[entry].mask))
8989                                 continue;
8990
8991                         printed += printf("%s%s",
8992                                           printed_mask == 0 ? ":(" : "|",
8993                                           table[entry].name);
8994                         printed_mask |= table[entry].mask;
8995
8996                         break;
8997                 }
8998                 if (entry >= num_entries)
8999                         break;
9000         }
9001         if (printed_mask != 0)
9002                 printed += printf(") ");
9003         else
9004                 printed += printf(" ");
9005         *cur_column += printed;
9006         return (printed);
9007 }
9008
9009 void
9010 ahd_dump_card_state(struct ahd_softc *ahd)
9011 {
9012         struct scb      *scb;
9013         ahd_mode_state   saved_modes;
9014         u_int            dffstat;
9015         int              paused;
9016         u_int            scb_index;
9017         u_int            saved_scb_index;
9018         u_int            cur_col;
9019         int              i;
9020
9021         if (ahd_is_paused(ahd)) {
9022                 paused = 1;
9023         } else {
9024                 paused = 0;
9025                 ahd_pause(ahd);
9026         }
9027         saved_modes = ahd_save_modes(ahd);
9028         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9029         printf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
9030                "%s: Dumping Card State at program address 0x%x Mode 0x%x\n",
9031                ahd_name(ahd),
9032                ahd_inw(ahd, CURADDR),
9033                ahd_build_mode_state(ahd, ahd->saved_src_mode,
9034                                     ahd->saved_dst_mode));
9035         if (paused)
9036                 printf("Card was paused\n");
9037
9038         if (ahd_check_cmdcmpltqueues(ahd))
9039                 printf("Completions are pending\n");
9040
9041         /*
9042          * Mode independent registers.
9043          */
9044         cur_col = 0;
9045         ahd_intstat_print(ahd_inb(ahd, INTSTAT), &cur_col, 50);
9046         ahd_seloid_print(ahd_inb(ahd, SELOID), &cur_col, 50);
9047         ahd_selid_print(ahd_inb(ahd, SELID), &cur_col, 50);
9048         ahd_hs_mailbox_print(ahd_inb(ahd, LOCAL_HS_MAILBOX), &cur_col, 50);
9049         ahd_intctl_print(ahd_inb(ahd, INTCTL), &cur_col, 50);
9050         ahd_seqintstat_print(ahd_inb(ahd, SEQINTSTAT), &cur_col, 50);
9051         ahd_saved_mode_print(ahd_inb(ahd, SAVED_MODE), &cur_col, 50);
9052         ahd_dffstat_print(ahd_inb(ahd, DFFSTAT), &cur_col, 50);
9053         ahd_scsisigi_print(ahd_inb(ahd, SCSISIGI), &cur_col, 50);
9054         ahd_scsiphase_print(ahd_inb(ahd, SCSIPHASE), &cur_col, 50);
9055         ahd_scsibus_print(ahd_inb(ahd, SCSIBUS), &cur_col, 50);
9056         ahd_lastphase_print(ahd_inb(ahd, LASTPHASE), &cur_col, 50);
9057         ahd_scsiseq0_print(ahd_inb(ahd, SCSISEQ0), &cur_col, 50);
9058         ahd_scsiseq1_print(ahd_inb(ahd, SCSISEQ1), &cur_col, 50);
9059         ahd_seqctl0_print(ahd_inb(ahd, SEQCTL0), &cur_col, 50);
9060         ahd_seqintctl_print(ahd_inb(ahd, SEQINTCTL), &cur_col, 50);
9061         ahd_seq_flags_print(ahd_inb(ahd, SEQ_FLAGS), &cur_col, 50);
9062         ahd_seq_flags2_print(ahd_inb(ahd, SEQ_FLAGS2), &cur_col, 50);
9063         ahd_qfreeze_count_print(ahd_inw(ahd, QFREEZE_COUNT), &cur_col, 50);
9064         ahd_kernel_qfreeze_count_print(ahd_inw(ahd, KERNEL_QFREEZE_COUNT),
9065                                        &cur_col, 50);
9066         ahd_mk_message_scb_print(ahd_inw(ahd, MK_MESSAGE_SCB), &cur_col, 50);
9067         ahd_mk_message_scsiid_print(ahd_inb(ahd, MK_MESSAGE_SCSIID),
9068                                     &cur_col, 50);
9069         ahd_sstat0_print(ahd_inb(ahd, SSTAT0), &cur_col, 50);
9070         ahd_sstat1_print(ahd_inb(ahd, SSTAT1), &cur_col, 50);
9071         ahd_sstat2_print(ahd_inb(ahd, SSTAT2), &cur_col, 50);
9072         ahd_sstat3_print(ahd_inb(ahd, SSTAT3), &cur_col, 50);
9073         ahd_perrdiag_print(ahd_inb(ahd, PERRDIAG), &cur_col, 50);
9074         ahd_simode1_print(ahd_inb(ahd, SIMODE1), &cur_col, 50);
9075         ahd_lqistat0_print(ahd_inb(ahd, LQISTAT0), &cur_col, 50);
9076         ahd_lqistat1_print(ahd_inb(ahd, LQISTAT1), &cur_col, 50);
9077         ahd_lqistat2_print(ahd_inb(ahd, LQISTAT2), &cur_col, 50);
9078         ahd_lqostat0_print(ahd_inb(ahd, LQOSTAT0), &cur_col, 50);
9079         ahd_lqostat1_print(ahd_inb(ahd, LQOSTAT1), &cur_col, 50);
9080         ahd_lqostat2_print(ahd_inb(ahd, LQOSTAT2), &cur_col, 50);
9081         printf("\n");
9082         printf("\nSCB Count = %d CMDS_PENDING = %d LASTSCB 0x%x "
9083                "CURRSCB 0x%x NEXTSCB 0x%x\n",
9084                ahd->scb_data.numscbs, ahd_inw(ahd, CMDS_PENDING),
9085                ahd_inw(ahd, LASTSCB), ahd_inw(ahd, CURRSCB),
9086                ahd_inw(ahd, NEXTSCB));
9087         cur_col = 0;
9088         /* QINFIFO */
9089         ahd_search_qinfifo(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
9090                            CAM_LUN_WILDCARD, SCB_LIST_NULL,
9091                            ROLE_UNKNOWN, /*status*/0, SEARCH_PRINT);
9092         saved_scb_index = ahd_get_scbptr(ahd);
9093         printf("Pending list:");
9094         i = 0;
9095         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
9096                 if (i++ > AHD_SCB_MAX)
9097                         break;
9098                 cur_col = printf("\n%3d FIFO_USE[0x%x] ", SCB_GET_TAG(scb),
9099                                  ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT));
9100                 ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
9101                 ahd_scb_control_print(ahd_inb_scbram(ahd, SCB_CONTROL),
9102                                       &cur_col, 60);
9103                 ahd_scb_scsiid_print(ahd_inb_scbram(ahd, SCB_SCSIID),
9104                                      &cur_col, 60);
9105         }
9106         printf("\nTotal %d\n", i);
9107
9108         printf("Kernel Free SCB lists: ");
9109         i = 0;
9110         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
9111                 struct scb *list_scb;
9112
9113                 printf("\n  COLIDX[%d]: ", AHD_GET_SCB_COL_IDX(ahd, scb));
9114                 list_scb = scb;
9115                 do {
9116                         printf("%d ", SCB_GET_TAG(list_scb));
9117                         list_scb = LIST_NEXT(list_scb, collision_links);
9118                 } while (list_scb && i++ < AHD_SCB_MAX);
9119         }
9120
9121         printf("\n  Any Device: ");
9122         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
9123                 if (i++ > AHD_SCB_MAX)
9124                         break;
9125                 printf("%d ", SCB_GET_TAG(scb));
9126         }
9127         printf("\n");
9128
9129         printf("Sequencer Complete DMA-inprog list: ");
9130         scb_index = ahd_inw(ahd, COMPLETE_SCB_DMAINPROG_HEAD);
9131         i = 0;
9132         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9133                 ahd_set_scbptr(ahd, scb_index);
9134                 printf("%d ", scb_index);
9135                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9136         }
9137         printf("\n");
9138
9139         printf("Sequencer Complete list: ");
9140         scb_index = ahd_inw(ahd, COMPLETE_SCB_HEAD);
9141         i = 0;
9142         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9143                 ahd_set_scbptr(ahd, scb_index);
9144                 printf("%d ", scb_index);
9145                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9146         }
9147         printf("\n");
9148
9149
9150         printf("Sequencer DMA-Up and Complete list: ");
9151         scb_index = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
9152         i = 0;
9153         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9154                 ahd_set_scbptr(ahd, scb_index);
9155                 printf("%d ", scb_index);
9156                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9157         }
9158         printf("\n");
9159         printf("Sequencer On QFreeze and Complete list: ");
9160         scb_index = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
9161         i = 0;
9162         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9163                 ahd_set_scbptr(ahd, scb_index);
9164                 printf("%d ", scb_index);
9165                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9166         }
9167         printf("\n");
9168         ahd_set_scbptr(ahd, saved_scb_index);
9169         dffstat = ahd_inb(ahd, DFFSTAT);
9170         for (i = 0; i < 2; i++) {
9171 #ifdef AHD_DEBUG
9172                 struct scb *fifo_scb;
9173 #endif
9174                 u_int       fifo_scbptr;
9175
9176                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
9177                 fifo_scbptr = ahd_get_scbptr(ahd);
9178                 printf("\n\n%s: FIFO%d %s, LONGJMP == 0x%x, SCB 0x%x\n",
9179                        ahd_name(ahd), i,
9180                        (dffstat & (FIFO0FREE << i)) ? "Free" : "Active",
9181                        ahd_inw(ahd, LONGJMP_ADDR), fifo_scbptr);
9182                 cur_col = 0;
9183                 ahd_seqimode_print(ahd_inb(ahd, SEQIMODE), &cur_col, 50);
9184                 ahd_seqintsrc_print(ahd_inb(ahd, SEQINTSRC), &cur_col, 50);
9185                 ahd_dfcntrl_print(ahd_inb(ahd, DFCNTRL), &cur_col, 50);
9186                 ahd_dfstatus_print(ahd_inb(ahd, DFSTATUS), &cur_col, 50);
9187                 ahd_sg_cache_shadow_print(ahd_inb(ahd, SG_CACHE_SHADOW),
9188                                           &cur_col, 50);
9189                 ahd_sg_state_print(ahd_inb(ahd, SG_STATE), &cur_col, 50);
9190                 ahd_dffsxfrctl_print(ahd_inb(ahd, DFFSXFRCTL), &cur_col, 50);
9191                 ahd_soffcnt_print(ahd_inb(ahd, SOFFCNT), &cur_col, 50);
9192                 ahd_mdffstat_print(ahd_inb(ahd, MDFFSTAT), &cur_col, 50);
9193                 if (cur_col > 50) {
9194                         printf("\n");
9195                         cur_col = 0;
9196                 }
9197                 cur_col += printf("SHADDR = 0x%x%x, SHCNT = 0x%x ",
9198                                   ahd_inl(ahd, SHADDR+4),
9199                                   ahd_inl(ahd, SHADDR),
9200                                   (ahd_inb(ahd, SHCNT)
9201                                 | (ahd_inb(ahd, SHCNT + 1) << 8)
9202                                 | (ahd_inb(ahd, SHCNT + 2) << 16)));
9203                 if (cur_col > 50) {
9204                         printf("\n");
9205                         cur_col = 0;
9206                 }
9207                 cur_col += printf("HADDR = 0x%x%x, HCNT = 0x%x ",
9208                                   ahd_inl(ahd, HADDR+4),
9209                                   ahd_inl(ahd, HADDR),
9210                                   (ahd_inb(ahd, HCNT)
9211                                 | (ahd_inb(ahd, HCNT + 1) << 8)
9212                                 | (ahd_inb(ahd, HCNT + 2) << 16)));
9213                 ahd_ccsgctl_print(ahd_inb(ahd, CCSGCTL), &cur_col, 50);
9214 #ifdef AHD_DEBUG
9215                 if ((ahd_debug & AHD_SHOW_SG) != 0) {
9216                         fifo_scb = ahd_lookup_scb(ahd, fifo_scbptr);
9217                         if (fifo_scb != NULL)
9218                                 ahd_dump_sglist(fifo_scb);
9219                 }
9220 #endif
9221         }
9222         printf("\nLQIN: ");
9223         for (i = 0; i < 20; i++)
9224                 printf("0x%x ", ahd_inb(ahd, LQIN + i));
9225         printf("\n");
9226         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
9227         printf("%s: LQISTATE = 0x%x, LQOSTATE = 0x%x, OPTIONMODE = 0x%x\n",
9228                ahd_name(ahd), ahd_inb(ahd, LQISTATE), ahd_inb(ahd, LQOSTATE),
9229                ahd_inb(ahd, OPTIONMODE));
9230         printf("%s: OS_SPACE_CNT = 0x%x MAXCMDCNT = 0x%x\n",
9231                ahd_name(ahd), ahd_inb(ahd, OS_SPACE_CNT),
9232                ahd_inb(ahd, MAXCMDCNT));
9233         printf("%s: SAVED_SCSIID = 0x%x SAVED_LUN = 0x%x\n",
9234                ahd_name(ahd), ahd_inb(ahd, SAVED_SCSIID),
9235                ahd_inb(ahd, SAVED_LUN));
9236         ahd_simode0_print(ahd_inb(ahd, SIMODE0), &cur_col, 50);
9237         printf("\n");
9238         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
9239         cur_col = 0;
9240         ahd_ccscbctl_print(ahd_inb(ahd, CCSCBCTL), &cur_col, 50);
9241         printf("\n");
9242         ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
9243         printf("%s: REG0 == 0x%x, SINDEX = 0x%x, DINDEX = 0x%x\n",
9244                ahd_name(ahd), ahd_inw(ahd, REG0), ahd_inw(ahd, SINDEX),
9245                ahd_inw(ahd, DINDEX));
9246         printf("%s: SCBPTR == 0x%x, SCB_NEXT == 0x%x, SCB_NEXT2 == 0x%x\n",
9247                ahd_name(ahd), ahd_get_scbptr(ahd),
9248                ahd_inw_scbram(ahd, SCB_NEXT),
9249                ahd_inw_scbram(ahd, SCB_NEXT2));
9250         printf("CDB %x %x %x %x %x %x\n",
9251                ahd_inb_scbram(ahd, SCB_CDB_STORE),
9252                ahd_inb_scbram(ahd, SCB_CDB_STORE+1),
9253                ahd_inb_scbram(ahd, SCB_CDB_STORE+2),
9254                ahd_inb_scbram(ahd, SCB_CDB_STORE+3),
9255                ahd_inb_scbram(ahd, SCB_CDB_STORE+4),
9256                ahd_inb_scbram(ahd, SCB_CDB_STORE+5));
9257         printf("STACK:");
9258         for (i = 0; i < ahd->stack_size; i++) {
9259                 ahd->saved_stack[i] =
9260                     ahd_inb(ahd, STACK)|(ahd_inb(ahd, STACK) << 8);
9261                 printf(" 0x%x", ahd->saved_stack[i]);
9262         }
9263         for (i = ahd->stack_size-1; i >= 0; i--) {
9264                 ahd_outb(ahd, STACK, ahd->saved_stack[i] & 0xFF);
9265                 ahd_outb(ahd, STACK, (ahd->saved_stack[i] >> 8) & 0xFF);
9266         }
9267         printf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
9268         ahd_platform_dump_card_state(ahd);
9269         ahd_restore_modes(ahd, saved_modes);
9270         if (paused == 0)
9271                 ahd_unpause(ahd);
9272 }
9273
9274 void
9275 ahd_dump_scbs(struct ahd_softc *ahd)
9276 {
9277         ahd_mode_state saved_modes;
9278         u_int          saved_scb_index;
9279         int            i;
9280
9281         saved_modes = ahd_save_modes(ahd);
9282         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9283         saved_scb_index = ahd_get_scbptr(ahd);
9284         for (i = 0; i < AHD_SCB_MAX; i++) {
9285                 ahd_set_scbptr(ahd, i);
9286                 printf("%3d", i);
9287                 printf("(CTRL 0x%x ID 0x%x N 0x%x N2 0x%x SG 0x%x, RSG 0x%x)\n",
9288                        ahd_inb_scbram(ahd, SCB_CONTROL),
9289                        ahd_inb_scbram(ahd, SCB_SCSIID),
9290                        ahd_inw_scbram(ahd, SCB_NEXT),
9291                        ahd_inw_scbram(ahd, SCB_NEXT2),
9292                        ahd_inl_scbram(ahd, SCB_SGPTR),
9293                        ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR));
9294         }
9295         printf("\n");
9296         ahd_set_scbptr(ahd, saved_scb_index);
9297         ahd_restore_modes(ahd, saved_modes);
9298 }
9299
9300
9301 /*************************** Timeout Handling *********************************/
9302 void
9303 ahd_timeout(struct scb *scb)
9304 {
9305         struct ahd_softc *ahd;
9306
9307         ahd = scb->ahd_softc;
9308         if ((scb->flags & SCB_ACTIVE) != 0) {
9309                 if ((scb->flags & SCB_TIMEDOUT) == 0) {
9310                         LIST_INSERT_HEAD(&ahd->timedout_scbs, scb,
9311                                          timedout_links);
9312                         scb->flags |= SCB_TIMEDOUT;
9313                 }
9314                 ahd_wakeup_recovery_thread(ahd);
9315         }
9316 }
9317
9318 /*
9319  * ahd_recover_commands determines if any of the commands that have currently
9320  * timedout are the root cause for this timeout.  Innocent commands are given
9321  * a new timeout while we wait for the command executing on the bus to timeout.
9322  * This routine is invoked from a thread context so we are allowed to sleep.
9323  * Our lock is not held on entry.
9324  */
9325 void
9326 ahd_recover_commands(struct ahd_softc *ahd)
9327 {
9328         struct  scb *scb;
9329         struct  scb *active_scb;
9330         int     found;
9331         int     was_paused;
9332         u_int   active_scbptr;
9333         u_int   last_phase;
9334
9335         /*
9336          * Pause the controller and manually flush any
9337          * commands that have just completed but that our
9338          * interrupt handler has yet to see.
9339          */
9340         was_paused = ahd_is_paused(ahd);
9341
9342         printf("%s: Recovery Initiated - Card was %spaused\n", ahd_name(ahd),
9343                was_paused ? "" : "not ");
9344         AHD_CORRECTABLE_ERROR(ahd);
9345         ahd_dump_card_state(ahd);
9346
9347         ahd_pause_and_flushwork(ahd);
9348
9349         if (LIST_EMPTY(&ahd->timedout_scbs) != 0) {
9350                 /*
9351                  * The timedout commands have already
9352                  * completed.  This typically means
9353                  * that either the timeout value was on
9354                  * the hairy edge of what the device
9355                  * requires or - more likely - interrupts
9356                  * are not happening.
9357                  */
9358                 printf("%s: Timedout SCBs already complete. "
9359                        "Interrupts may not be functioning.\n", ahd_name(ahd));
9360                 ahd_unpause(ahd);
9361                 return;
9362         }
9363
9364         /*
9365          * Determine identity of SCB acting on the bus.
9366          * This test only catches non-packetized transactions.
9367          * Due to the fleeting nature of packetized operations,
9368          * we can't easily determine that a packetized operation
9369          * is on the bus.
9370          */
9371         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9372         last_phase = ahd_inb(ahd, LASTPHASE);
9373         active_scbptr = ahd_get_scbptr(ahd);
9374         active_scb = NULL;
9375         if (last_phase != P_BUSFREE
9376          || (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) == 0)
9377                 active_scb = ahd_lookup_scb(ahd, active_scbptr);
9378
9379         while ((scb = LIST_FIRST(&ahd->timedout_scbs)) != NULL) {
9380                 int     target;
9381                 int     lun;
9382                 char    channel;
9383
9384                 target = SCB_GET_TARGET(ahd, scb);
9385                 channel = SCB_GET_CHANNEL(ahd, scb);
9386                 lun = SCB_GET_LUN(scb);
9387
9388                 ahd_print_path(ahd, scb);
9389                 printf("SCB %d - timed out\n", SCB_GET_TAG(scb));
9390
9391                 if (scb->flags & (SCB_DEVICE_RESET|SCB_ABORT)) {
9392                         /*
9393                          * Been down this road before.
9394                          * Do a full bus reset.
9395                          */
9396                         aic_set_transaction_status(scb, CAM_CMD_TIMEOUT);
9397 bus_reset:
9398                         found = ahd_reset_channel(ahd, channel,
9399                                                   /*Initiate Reset*/TRUE);
9400                         printf("%s: Issued Channel %c Bus Reset. "
9401                                "%d SCBs aborted\n", ahd_name(ahd), channel,
9402                                found);
9403                         continue;
9404                 }
9405
9406                 /*
9407                  * Remove the command from the timedout list in
9408                  * preparation for requeing it.
9409                  */
9410                 LIST_REMOVE(scb, timedout_links);
9411                 scb->flags &= ~SCB_TIMEDOUT;
9412
9413                 if (active_scb != NULL) {
9414
9415                         if (active_scb != scb) {
9416
9417                                 /*
9418                                  * If the active SCB is not us, assume that
9419                                  * the active SCB has a longer timeout than
9420                                  * the timedout SCB, and wait for the active
9421                                  * SCB to timeout.  As a safeguard, only
9422                                  * allow this deferral to continue if some
9423                                  * untimed-out command is outstanding.
9424                                  */
9425                                 if (ahd_other_scb_timeout(ahd, scb,
9426                                                           active_scb) == 0)
9427                                         goto bus_reset;
9428                                 continue;
9429                         }
9430
9431                         /*
9432                          * We're active on the bus, so assert ATN
9433                          * and hope that the target responds.
9434                          */
9435                         ahd_set_recoveryscb(ahd, active_scb);
9436                         active_scb->flags |= SCB_RECOVERY_SCB|SCB_DEVICE_RESET;
9437                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
9438                         ahd_outb(ahd, SCSISIGO, last_phase|ATNO);
9439                         ahd_print_path(ahd, active_scb);
9440                         printf("BDR message in message buffer\n");
9441                         aic_scb_timer_reset(scb, 2 * 1000);
9442                         break;
9443                 } else if (last_phase != P_BUSFREE
9444                         && ahd_inb(ahd, SCSIPHASE) == 0) {
9445                         /*
9446                          * SCB is not identified, there
9447                          * is no pending REQ, and the sequencer
9448                          * has not seen a busfree.  Looks like
9449                          * a stuck connection waiting to
9450                          * go busfree.  Reset the bus.
9451                          */
9452                         printf("%s: Connection stuck awaiting busfree or "
9453                                "Identify Msg.\n", ahd_name(ahd));
9454                         goto bus_reset;
9455                 } else if (ahd_search_qinfifo(ahd, target, channel, lun,
9456                                               SCB_GET_TAG(scb),
9457                                               ROLE_INITIATOR, /*status*/0,
9458                                               SEARCH_COUNT) > 0) {
9459
9460                         /*
9461                          * We haven't even gone out on the bus
9462                          * yet, so the timeout must be due to
9463                          * some other command.  Reset the timer
9464                          * and go on.
9465                          */
9466                         if (ahd_other_scb_timeout(ahd, scb, NULL) == 0)
9467                                 goto bus_reset;
9468                 } else {
9469                         /*
9470                          * This SCB is for a disconnected transaction
9471                          * and we haven't found a better candidate on
9472                          * the bus to explain this timeout.
9473                          */
9474                         ahd_set_recoveryscb(ahd, scb);
9475
9476                         /*
9477                          * Actually re-queue this SCB in an attempt
9478                          * to select the device before it reconnects.
9479                          * In either case (selection or reselection),
9480                          * we will now issue a target reset to the
9481                          * timed-out device.
9482                          */
9483                         scb->flags |= SCB_DEVICE_RESET;
9484                         scb->hscb->cdb_len = 0;
9485                         scb->hscb->task_attribute = 0;
9486                         scb->hscb->task_management = SIU_TASKMGMT_ABORT_TASK;
9487
9488                         ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
9489                         if ((scb->flags & SCB_PACKETIZED) != 0) {
9490                                 /*
9491                                  * Mark the SCB has having an outstanding
9492                                  * task management function.  Should the command
9493                                  * complete normally before the task management
9494                                  * function can be sent, the host will be
9495                                  * notified to abort our requeued SCB.
9496                                  */
9497                                 ahd_outb(ahd, SCB_TASK_MANAGEMENT,
9498                                          scb->hscb->task_management);
9499                         } else {
9500                                 /*
9501                                  * If non-packetized, set the MK_MESSAGE control
9502                                  * bit indicating that we desire to send a
9503                                  * message.  We also set the disconnected flag
9504                                  * since there is no guarantee that our SCB
9505                                  * control byte matches the version on the
9506                                  * card.  We don't want the sequencer to abort
9507                                  * the command thinking an unsolicited
9508                                  * reselection occurred.
9509                                  */
9510                                 scb->hscb->control |= MK_MESSAGE|DISCONNECTED;
9511
9512                                 /*
9513                                  * The sequencer will never re-reference the
9514                                  * in-core SCB.  To make sure we are notified
9515                                  * during reslection, set the MK_MESSAGE flag in
9516                                  * the card's copy of the SCB.
9517                                  */
9518                                 ahd_outb(ahd, SCB_CONTROL,
9519                                          ahd_inb(ahd, SCB_CONTROL)|MK_MESSAGE);
9520                         }
9521
9522                         /*
9523                          * Clear out any entries in the QINFIFO first
9524                          * so we are the next SCB for this target
9525                          * to run.
9526                          */
9527                         ahd_search_qinfifo(ahd, target, channel, lun,
9528                                            SCB_LIST_NULL, ROLE_INITIATOR,
9529                                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
9530                         ahd_qinfifo_requeue_tail(ahd, scb);
9531                         ahd_set_scbptr(ahd, active_scbptr);
9532                         ahd_print_path(ahd, scb);
9533                         printf("Queuing a BDR SCB\n");
9534                         aic_scb_timer_reset(scb, 2 * 1000);
9535                         break;
9536                 }
9537         }
9538
9539         /*
9540          * Any remaining SCBs were not the "culprit", so remove
9541          * them from the timeout list.  The timer for these commands
9542          * will be reset once the recovery SCB completes.
9543          */
9544         while ((scb = LIST_FIRST(&ahd->timedout_scbs)) != NULL) {
9545
9546                 LIST_REMOVE(scb, timedout_links);
9547                 scb->flags &= ~SCB_TIMEDOUT;
9548         }
9549
9550         ahd_unpause(ahd);
9551 }
9552
9553 /*
9554  * Re-schedule a timeout for the passed in SCB if we determine that some
9555  * other SCB is in the process of recovery or an SCB with a longer
9556  * timeout is still pending.  Limit our search to just "other_scb"
9557  * if it is non-NULL.
9558  */
9559 static int
9560 ahd_other_scb_timeout(struct ahd_softc *ahd, struct scb *scb,
9561                       struct scb *other_scb)
9562 {
9563         u_int   newtimeout;
9564         int     found;
9565
9566         ahd_print_path(ahd, scb);
9567         printf("Other SCB Timeout%s",
9568                (scb->flags & SCB_OTHERTCL_TIMEOUT) != 0
9569                ? " again\n" : "\n");
9570
9571         AHD_UNCORRECTABLE_ERROR(ahd);
9572         newtimeout = aic_get_timeout(scb);
9573         scb->flags |= SCB_OTHERTCL_TIMEOUT;
9574         found = 0;
9575         if (other_scb != NULL) {
9576                 if ((other_scb->flags
9577                    & (SCB_OTHERTCL_TIMEOUT|SCB_TIMEDOUT)) == 0
9578                  || (other_scb->flags & SCB_RECOVERY_SCB) != 0) {
9579                         found++;
9580                         newtimeout = MAX(aic_get_timeout(other_scb),
9581                                          newtimeout);
9582                 }
9583         } else {
9584                 LIST_FOREACH(other_scb, &ahd->pending_scbs, pending_links) {
9585                         if ((other_scb->flags
9586                            & (SCB_OTHERTCL_TIMEOUT|SCB_TIMEDOUT)) == 0
9587                          || (other_scb->flags & SCB_RECOVERY_SCB) != 0) {
9588                                 found++;
9589                                 newtimeout = MAX(aic_get_timeout(other_scb),
9590                                                  newtimeout);
9591                         }
9592                 }
9593         }
9594
9595         if (found != 0)
9596                 aic_scb_timer_reset(scb, newtimeout);
9597         else {
9598                 ahd_print_path(ahd, scb);
9599                 printf("No other SCB worth waiting for...\n");
9600         }
9601
9602         return (found != 0);
9603 }
9604
9605 /**************************** Flexport Logic **********************************/
9606 /*
9607  * Read count 16bit words from 16bit word address start_addr from the
9608  * SEEPROM attached to the controller, into buf, using the controller's
9609  * SEEPROM reading state machine.  Optionally treat the data as a byte
9610  * stream in terms of byte order.
9611  */
9612 int
9613 ahd_read_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9614                  u_int start_addr, u_int count, int bytestream)
9615 {
9616         u_int cur_addr;
9617         u_int end_addr;
9618         int   error;
9619
9620         /*
9621          * If we never make it through the loop even once,
9622          * we were passed invalid arguments.
9623          */
9624         error = EINVAL;
9625         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9626         end_addr = start_addr + count;
9627         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
9628
9629                 ahd_outb(ahd, SEEADR, cur_addr);
9630                 ahd_outb(ahd, SEECTL, SEEOP_READ | SEESTART);
9631
9632                 error = ahd_wait_seeprom(ahd);
9633                 if (error)
9634                         break;
9635                 if (bytestream != 0) {
9636                         uint8_t *bytestream_ptr;
9637
9638                         bytestream_ptr = (uint8_t *)buf;
9639                         *bytestream_ptr++ = ahd_inb(ahd, SEEDAT);
9640                         *bytestream_ptr = ahd_inb(ahd, SEEDAT+1);
9641                 } else {
9642                         /*
9643                          * ahd_inw() already handles machine byte order.
9644                          */
9645                         *buf = ahd_inw(ahd, SEEDAT);
9646                 }
9647                 buf++;
9648         }
9649         return (error);
9650 }
9651
9652 /*
9653  * Write count 16bit words from buf, into SEEPROM attache to the
9654  * controller starting at 16bit word address start_addr, using the
9655  * controller's SEEPROM writing state machine.
9656  */
9657 int
9658 ahd_write_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9659                   u_int start_addr, u_int count)
9660 {
9661         u_int cur_addr;
9662         u_int end_addr;
9663         int   error;
9664         int   retval;
9665
9666         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9667         error = ENOENT;
9668
9669         /* Place the chip into write-enable mode */
9670         ahd_outb(ahd, SEEADR, SEEOP_EWEN_ADDR);
9671         ahd_outb(ahd, SEECTL, SEEOP_EWEN | SEESTART);
9672         error = ahd_wait_seeprom(ahd);
9673         if (error)
9674                 return (error);
9675
9676         /*
9677          * Write the data.  If we don't get throught the loop at
9678          * least once, the arguments were invalid.
9679          */
9680         retval = EINVAL;
9681         end_addr = start_addr + count;
9682         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
9683                 ahd_outw(ahd, SEEDAT, *buf++);
9684                 ahd_outb(ahd, SEEADR, cur_addr);
9685                 ahd_outb(ahd, SEECTL, SEEOP_WRITE | SEESTART);
9686
9687                 retval = ahd_wait_seeprom(ahd);
9688                 if (retval)
9689                         break;
9690         }
9691
9692         /*
9693          * Disable writes.
9694          */
9695         ahd_outb(ahd, SEEADR, SEEOP_EWDS_ADDR);
9696         ahd_outb(ahd, SEECTL, SEEOP_EWDS | SEESTART);
9697         error = ahd_wait_seeprom(ahd);
9698         if (error)
9699                 return (error);
9700         return (retval);
9701 }
9702
9703 /*
9704  * Wait ~100us for the serial eeprom to satisfy our request.
9705  */
9706 int
9707 ahd_wait_seeprom(struct ahd_softc *ahd)
9708 {
9709         int cnt;
9710
9711         cnt = 5000;
9712         while ((ahd_inb(ahd, SEESTAT) & (SEEARBACK|SEEBUSY)) != 0 && --cnt)
9713                 aic_delay(5);
9714
9715         if (cnt == 0)
9716                 return (ETIMEDOUT);
9717         return (0);
9718 }
9719
9720 /*
9721  * Validate the two checksums in the per_channel
9722  * vital product data struct.
9723  */
9724 int
9725 ahd_verify_vpd_cksum(struct vpd_config *vpd)
9726 {
9727         int i;
9728         int maxaddr;
9729         uint32_t checksum;
9730         uint8_t *vpdarray;
9731
9732         vpdarray = (uint8_t *)vpd;
9733         maxaddr = offsetof(struct vpd_config, vpd_checksum);
9734         checksum = 0;
9735         for (i = offsetof(struct vpd_config, resource_type); i < maxaddr; i++)
9736                 checksum = checksum + vpdarray[i];
9737         if (checksum == 0
9738          || (-checksum & 0xFF) != vpd->vpd_checksum)
9739                 return (0);
9740
9741         checksum = 0;
9742         maxaddr = offsetof(struct vpd_config, checksum);
9743         for (i = offsetof(struct vpd_config, default_target_flags);
9744              i < maxaddr; i++)
9745                 checksum = checksum + vpdarray[i];
9746         if (checksum == 0
9747          || (-checksum & 0xFF) != vpd->checksum)
9748                 return (0);
9749         return (1);
9750 }
9751
9752 int
9753 ahd_verify_cksum(struct seeprom_config *sc)
9754 {
9755         int i;
9756         int maxaddr;
9757         uint32_t checksum;
9758         uint16_t *scarray;
9759
9760         maxaddr = (sizeof(*sc)/2) - 1;
9761         checksum = 0;
9762         scarray = (uint16_t *)sc;
9763
9764         for (i = 0; i < maxaddr; i++)
9765                 checksum = checksum + scarray[i];
9766         if (checksum == 0
9767          || (checksum & 0xFFFF) != sc->checksum) {
9768                 return (0);
9769         } else {
9770                 return (1);
9771         }
9772 }
9773
9774 int
9775 ahd_acquire_seeprom(struct ahd_softc *ahd)
9776 {
9777         /*
9778          * We should be able to determine the SEEPROM type
9779          * from the flexport logic, but unfortunately not
9780          * all implementations have this logic and there is
9781          * no programatic method for determining if the logic
9782          * is present.
9783          */
9784         return (1);
9785 #if 0
9786         uint8_t seetype;
9787         int     error;
9788
9789         error = ahd_read_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, &seetype);
9790         if (error != 0
9791          || ((seetype & FLX_ROMSTAT_SEECFG) == FLX_ROMSTAT_SEE_NONE))
9792                 return (0);
9793         return (1);
9794 #endif
9795 }
9796
9797 void
9798 ahd_release_seeprom(struct ahd_softc *ahd)
9799 {
9800         /* Currently a no-op */
9801 }
9802
9803 int
9804 ahd_write_flexport(struct ahd_softc *ahd, u_int addr, u_int value)
9805 {
9806         int error;
9807
9808         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9809         if (addr > 7)
9810                 panic("ahd_write_flexport: address out of range");
9811         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
9812         error = ahd_wait_flexport(ahd);
9813         if (error != 0)
9814                 return (error);
9815         ahd_outb(ahd, BRDDAT, value);
9816         ahd_flush_device_writes(ahd);
9817         ahd_outb(ahd, BRDCTL, BRDSTB|BRDEN|(addr << 3));
9818         ahd_flush_device_writes(ahd);
9819         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
9820         ahd_flush_device_writes(ahd);
9821         ahd_outb(ahd, BRDCTL, 0);
9822         ahd_flush_device_writes(ahd);
9823         return (0);
9824 }
9825
9826 int
9827 ahd_read_flexport(struct ahd_softc *ahd, u_int addr, uint8_t *value)
9828 {
9829         int     error;
9830
9831         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9832         if (addr > 7)
9833                 panic("ahd_read_flexport: address out of range");
9834         ahd_outb(ahd, BRDCTL, BRDRW|BRDEN|(addr << 3));
9835         error = ahd_wait_flexport(ahd);
9836         if (error != 0)
9837                 return (error);
9838         *value = ahd_inb(ahd, BRDDAT);
9839         ahd_outb(ahd, BRDCTL, 0);
9840         ahd_flush_device_writes(ahd);
9841         return (0);
9842 }
9843
9844 /*
9845  * Wait at most 2 seconds for flexport arbitration to succeed.
9846  */
9847 int
9848 ahd_wait_flexport(struct ahd_softc *ahd)
9849 {
9850         int cnt;
9851
9852         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9853         cnt = 1000000 * 2 / 5;
9854         while ((ahd_inb(ahd, BRDCTL) & FLXARBACK) == 0 && --cnt)
9855                 aic_delay(5);
9856
9857         if (cnt == 0)
9858                 return (ETIMEDOUT);
9859         return (0);
9860 }
9861
9862 /************************* Target Mode ****************************************/
9863 #ifdef AHD_TARGET_MODE
9864 cam_status
9865 ahd_find_tmode_devs(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb,
9866                     struct ahd_tmode_tstate **tstate,
9867                     struct ahd_tmode_lstate **lstate,
9868                     int notfound_failure)
9869 {
9870
9871         if ((ahd->features & AHD_TARGETMODE) == 0)
9872                 return (CAM_REQ_INVALID);
9873
9874         /*
9875          * Handle the 'black hole' device that sucks up
9876          * requests to unattached luns on enabled targets.
9877          */
9878         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
9879          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
9880                 *tstate = NULL;
9881                 *lstate = ahd->black_hole;
9882         } else {
9883                 u_int max_id;
9884
9885                 max_id = (ahd->features & AHD_WIDE) ? 15 : 7;
9886                 if (ccb->ccb_h.target_id > max_id)
9887                         return (CAM_TID_INVALID);
9888
9889                 if (ccb->ccb_h.target_lun >= AHD_NUM_LUNS)
9890                         return (CAM_LUN_INVALID);
9891
9892                 *tstate = ahd->enabled_targets[ccb->ccb_h.target_id];
9893                 *lstate = NULL;
9894                 if (*tstate != NULL)
9895                         *lstate =
9896                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
9897         }
9898
9899         if (notfound_failure != 0 && *lstate == NULL)
9900                 return (CAM_PATH_INVALID);
9901
9902         return (CAM_REQ_CMP);
9903 }
9904
9905 void
9906 ahd_handle_en_lun(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb)
9907 {
9908 #if NOT_YET
9909         struct     ahd_tmode_tstate *tstate;
9910         struct     ahd_tmode_lstate *lstate;
9911         struct     ccb_en_lun *cel;
9912         cam_status status;
9913         u_int      target;
9914         u_int      lun;
9915         u_int      target_mask;
9916         u_long     s;
9917         char       channel;
9918
9919         status = ahd_find_tmode_devs(ahd, sim, ccb, &tstate, &lstate,
9920                                      /*notfound_failure*/FALSE);
9921
9922         if (status != CAM_REQ_CMP) {
9923                 ccb->ccb_h.status = status;
9924                 return;
9925         }
9926
9927         if ((ahd->features & AHD_MULTIROLE) != 0) {
9928                 u_int      our_id;
9929
9930                 our_id = ahd->our_id;
9931                 if (ccb->ccb_h.target_id != our_id) {
9932                         if ((ahd->features & AHD_MULTI_TID) != 0
9933                          && (ahd->flags & AHD_INITIATORROLE) != 0) {
9934                                 /*
9935                                  * Only allow additional targets if
9936                                  * the initiator role is disabled.
9937                                  * The hardware cannot handle a re-select-in
9938                                  * on the initiator id during a re-select-out
9939                                  * on a different target id.
9940                                  */
9941                                 status = CAM_TID_INVALID;
9942                         } else if ((ahd->flags & AHD_INITIATORROLE) != 0
9943                                 || ahd->enabled_luns > 0) {
9944                                 /*
9945                                  * Only allow our target id to change
9946                                  * if the initiator role is not configured
9947                                  * and there are no enabled luns which
9948                                  * are attached to the currently registered
9949                                  * scsi id.
9950                                  */
9951                                 status = CAM_TID_INVALID;
9952                         }
9953                 }
9954         }
9955
9956         if (status != CAM_REQ_CMP) {
9957                 ccb->ccb_h.status = status;
9958                 return;
9959         }
9960
9961         /*
9962          * We now have an id that is valid.
9963          * If we aren't in target mode, switch modes.
9964          */
9965         if ((ahd->flags & AHD_TARGETROLE) == 0
9966          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
9967                 printf("Configuring Target Mode\n");
9968                 if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
9969                         ccb->ccb_h.status = CAM_BUSY;
9970                         return;
9971                 }
9972                 ahd->flags |= AHD_TARGETROLE;
9973                 if ((ahd->features & AHD_MULTIROLE) == 0)
9974                         ahd->flags &= ~AHD_INITIATORROLE;
9975                 ahd_pause(ahd);
9976                 ahd_loadseq(ahd);
9977                 ahd_restart(ahd);
9978         }
9979         cel = &ccb->cel;
9980         target = ccb->ccb_h.target_id;
9981         lun = ccb->ccb_h.target_lun;
9982         channel = SIM_CHANNEL(ahd, sim);
9983         target_mask = 0x01 << target;
9984         if (channel == 'B')
9985                 target_mask <<= 8;
9986
9987         if (cel->enable != 0) {
9988                 u_int scsiseq1;
9989
9990                 /* Are we already enabled?? */
9991                 if (lstate != NULL) {
9992                         xpt_print_path(ccb->ccb_h.path);
9993                         printf("Lun already enabled\n");
9994                         AHD_CORRECTABLE_ERROR(ahd);
9995                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
9996                         return;
9997                 }
9998
9999                 if (cel->grp6_len != 0
10000                  || cel->grp7_len != 0) {
10001                         /*
10002                          * Don't (yet?) support vendor
10003                          * specific commands.
10004                          */
10005                         ccb->ccb_h.status = CAM_REQ_INVALID;
10006                         printf("Non-zero Group Codes\n");
10007                         return;
10008                 }
10009
10010                 /*
10011                  * Seems to be okay.
10012                  * Setup our data structures.
10013                  */
10014                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
10015                         tstate = ahd_alloc_tstate(ahd, target, channel);
10016                         if (tstate == NULL) {
10017                                 xpt_print_path(ccb->ccb_h.path);
10018                                 printf("Couldn't allocate tstate\n");
10019                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10020                                 return;
10021                         }
10022                 }
10023                 lstate = malloc(sizeof(*lstate), M_DEVBUF, M_NOWAIT);
10024                 if (lstate == NULL) {
10025                         xpt_print_path(ccb->ccb_h.path);
10026                         printf("Couldn't allocate lstate\n");
10027                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10028                         return;
10029                 }
10030                 memset(lstate, 0, sizeof(*lstate));
10031                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
10032                                          xpt_path_path_id(ccb->ccb_h.path),
10033                                          xpt_path_target_id(ccb->ccb_h.path),
10034                                          xpt_path_lun_id(ccb->ccb_h.path));
10035                 if (status != CAM_REQ_CMP) {
10036                         free(lstate, M_DEVBUF);
10037                         xpt_print_path(ccb->ccb_h.path);
10038                         printf("Couldn't allocate path\n");
10039                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10040                         return;
10041                 }
10042                 SLIST_INIT(&lstate->accept_tios);
10043                 SLIST_INIT(&lstate->immed_notifies);
10044                 ahd_pause(ahd);
10045                 if (target != CAM_TARGET_WILDCARD) {
10046                         tstate->enabled_luns[lun] = lstate;
10047                         ahd->enabled_luns++;
10048
10049                         if ((ahd->features & AHD_MULTI_TID) != 0) {
10050                                 u_int targid_mask;
10051
10052                                 targid_mask = ahd_inw(ahd, TARGID);
10053                                 targid_mask |= target_mask;
10054                                 ahd_outw(ahd, TARGID, targid_mask);
10055                                 ahd_update_scsiid(ahd, targid_mask);
10056                         } else {
10057                                 u_int our_id;
10058                                 char  channel;
10059
10060                                 channel = SIM_CHANNEL(ahd, sim);
10061                                 our_id = SIM_SCSI_ID(ahd, sim);
10062
10063                                 /*
10064                                  * This can only happen if selections
10065                                  * are not enabled
10066                                  */
10067                                 if (target != our_id) {
10068                                         u_int sblkctl;
10069                                         char  cur_channel;
10070                                         int   swap;
10071
10072                                         sblkctl = ahd_inb(ahd, SBLKCTL);
10073                                         cur_channel = (sblkctl & SELBUSB)
10074                                                     ? 'B' : 'A';
10075                                         if ((ahd->features & AHD_TWIN) == 0)
10076                                                 cur_channel = 'A';
10077                                         swap = cur_channel != channel;
10078                                         ahd->our_id = target;
10079
10080                                         if (swap)
10081                                                 ahd_outb(ahd, SBLKCTL,
10082                                                          sblkctl ^ SELBUSB);
10083
10084                                         ahd_outb(ahd, SCSIID, target);
10085
10086                                         if (swap)
10087                                                 ahd_outb(ahd, SBLKCTL, sblkctl);
10088                                 }
10089                         }
10090                 } else
10091                         ahd->black_hole = lstate;
10092                 /* Allow select-in operations */
10093                 if (ahd->black_hole != NULL && ahd->enabled_luns > 0) {
10094                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10095                         scsiseq1 |= ENSELI;
10096                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10097                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10098                         scsiseq1 |= ENSELI;
10099                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
10100                 }
10101                 ahd_unpause(ahd);
10102                 ccb->ccb_h.status = CAM_REQ_CMP;
10103                 xpt_print_path(ccb->ccb_h.path);
10104                 printf("Lun now enabled for target mode\n");
10105         } else {
10106                 struct scb *scb;
10107                 int i, empty;
10108
10109                 if (lstate == NULL) {
10110                         ccb->ccb_h.status = CAM_LUN_INVALID;
10111                         return;
10112                 }
10113
10114                 ccb->ccb_h.status = CAM_REQ_CMP;
10115                 LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
10116                         struct ccb_hdr *ccbh;
10117
10118                         ccbh = &scb->io_ctx->ccb_h;
10119                         if (ccbh->func_code == XPT_CONT_TARGET_IO
10120                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
10121                                 printf("CTIO pending\n");
10122                                 ccb->ccb_h.status = CAM_REQ_INVALID;
10123                                 return;
10124                         }
10125                 }
10126
10127                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
10128                         printf("ATIOs pending\n");
10129                         ccb->ccb_h.status = CAM_REQ_INVALID;
10130                 }
10131
10132                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
10133                         printf("INOTs pending\n");
10134                         ccb->ccb_h.status = CAM_REQ_INVALID;
10135                 }
10136
10137                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
10138                         return;
10139                 }
10140
10141                 xpt_print_path(ccb->ccb_h.path);
10142                 printf("Target mode disabled\n");
10143                 xpt_free_path(lstate->path);
10144                 free(lstate, M_DEVBUF);
10145
10146                 ahd_pause(ahd);
10147                 /* Can we clean up the target too? */
10148                 if (target != CAM_TARGET_WILDCARD) {
10149                         tstate->enabled_luns[lun] = NULL;
10150                         ahd->enabled_luns--;
10151                         for (empty = 1, i = 0; i < 8; i++)
10152                                 if (tstate->enabled_luns[i] != NULL) {
10153                                         empty = 0;
10154                                         break;
10155                                 }
10156
10157                         if (empty) {
10158                                 ahd_free_tstate(ahd, target, channel,
10159                                                 /*force*/FALSE);
10160                                 if (ahd->features & AHD_MULTI_TID) {
10161                                         u_int targid_mask;
10162
10163                                         targid_mask = ahd_inw(ahd, TARGID);
10164                                         targid_mask &= ~target_mask;
10165                                         ahd_outw(ahd, TARGID, targid_mask);
10166                                         ahd_update_scsiid(ahd, targid_mask);
10167                                 }
10168                         }
10169                 } else {
10170
10171                         ahd->black_hole = NULL;
10172
10173                         /*
10174                          * We can't allow selections without
10175                          * our black hole device.
10176                          */
10177                         empty = TRUE;
10178                 }
10179                 if (ahd->enabled_luns == 0) {
10180                         /* Disallow select-in */
10181                         u_int scsiseq1;
10182
10183                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10184                         scsiseq1 &= ~ENSELI;
10185                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10186                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10187                         scsiseq1 &= ~ENSELI;
10188                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
10189
10190                         if ((ahd->features & AHD_MULTIROLE) == 0) {
10191                                 printf("Configuring Initiator Mode\n");
10192                                 ahd->flags &= ~AHD_TARGETROLE;
10193                                 ahd->flags |= AHD_INITIATORROLE;
10194                                 ahd_pause(ahd);
10195                                 ahd_loadseq(ahd);
10196                                 ahd_restart(ahd);
10197                                 /*
10198                                  * Unpaused.  The extra unpause
10199                                  * that follows is harmless.
10200                                  */
10201                         }
10202                 }
10203                 ahd_unpause(ahd);
10204         }
10205 #endif
10206 }
10207
10208 static void
10209 ahd_update_scsiid(struct ahd_softc *ahd, u_int targid_mask)
10210 {
10211 #if NOT_YET
10212         u_int scsiid_mask;
10213         u_int scsiid;
10214
10215         if ((ahd->features & AHD_MULTI_TID) == 0)
10216                 panic("ahd_update_scsiid called on non-multitid unit\n");
10217
10218         /*
10219          * Since we will rely on the TARGID mask
10220          * for selection enables, ensure that OID
10221          * in SCSIID is not set to some other ID
10222          * that we don't want to allow selections on.
10223          */
10224         if ((ahd->features & AHD_ULTRA2) != 0)
10225                 scsiid = ahd_inb(ahd, SCSIID_ULTRA2);
10226         else
10227                 scsiid = ahd_inb(ahd, SCSIID);
10228         scsiid_mask = 0x1 << (scsiid & OID);
10229         if ((targid_mask & scsiid_mask) == 0) {
10230                 u_int our_id;
10231
10232                 /* ffs counts from 1 */
10233                 our_id = ffs(targid_mask);
10234                 if (our_id == 0)
10235                         our_id = ahd->our_id;
10236                 else
10237                         our_id--;
10238                 scsiid &= TID;
10239                 scsiid |= our_id;
10240         }
10241         if ((ahd->features & AHD_ULTRA2) != 0)
10242                 ahd_outb(ahd, SCSIID_ULTRA2, scsiid);
10243         else
10244                 ahd_outb(ahd, SCSIID, scsiid);
10245 #endif
10246 }
10247
10248 void
10249 ahd_run_tqinfifo(struct ahd_softc *ahd, int paused)
10250 {
10251         struct target_cmd *cmd;
10252
10253         ahd_sync_tqinfifo(ahd, BUS_DMASYNC_POSTREAD);
10254         while ((cmd = &ahd->targetcmds[ahd->tqinfifonext])->cmd_valid != 0) {
10255
10256                 /*
10257                  * Only advance through the queue if we
10258                  * have the resources to process the command.
10259                  */
10260                 if (ahd_handle_target_cmd(ahd, cmd) != 0)
10261                         break;
10262
10263                 cmd->cmd_valid = 0;
10264                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
10265                                 ahd->shared_data_dmamap,
10266                                 ahd_targetcmd_offset(ahd, ahd->tqinfifonext),
10267                                 sizeof(struct target_cmd),
10268                                 BUS_DMASYNC_PREREAD);
10269                 ahd->tqinfifonext++;
10270
10271                 /*
10272                  * Lazily update our position in the target mode incoming
10273                  * command queue as seen by the sequencer.
10274                  */
10275                 if ((ahd->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
10276                         u_int hs_mailbox;
10277
10278                         hs_mailbox = ahd_inb(ahd, HS_MAILBOX);
10279                         hs_mailbox &= ~HOST_TQINPOS;
10280                         hs_mailbox |= ahd->tqinfifonext & HOST_TQINPOS;
10281                         ahd_outb(ahd, HS_MAILBOX, hs_mailbox);
10282                 }
10283         }
10284 }
10285
10286 static int
10287 ahd_handle_target_cmd(struct ahd_softc *ahd, struct target_cmd *cmd)
10288 {
10289         struct    ahd_tmode_tstate *tstate;
10290         struct    ahd_tmode_lstate *lstate;
10291         struct    ccb_accept_tio *atio;
10292         uint8_t *byte;
10293         int       initiator;
10294         int       target;
10295         int       lun;
10296
10297         initiator = SCSIID_TARGET(ahd, cmd->scsiid);
10298         target = SCSIID_OUR_ID(cmd->scsiid);
10299         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
10300
10301         byte = cmd->bytes;
10302         tstate = ahd->enabled_targets[target];
10303         lstate = NULL;
10304         if (tstate != NULL)
10305                 lstate = tstate->enabled_luns[lun];
10306
10307         /*
10308          * Commands for disabled luns go to the black hole driver.
10309          */
10310         if (lstate == NULL)
10311                 lstate = ahd->black_hole;
10312
10313         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
10314         if (atio == NULL) {
10315                 ahd->flags |= AHD_TQINFIFO_BLOCKED;
10316                 /*
10317                  * Wait for more ATIOs from the peripheral driver for this lun.
10318                  */
10319                 return (1);
10320         } else
10321                 ahd->flags &= ~AHD_TQINFIFO_BLOCKED;
10322 #ifdef AHD_DEBUG
10323         if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10324                 printf("Incoming command from %d for %d:%d%s\n",
10325                        initiator, target, lun,
10326                        lstate == ahd->black_hole ? "(Black Holed)" : "");
10327 #endif
10328         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
10329
10330         if (lstate == ahd->black_hole) {
10331                 /* Fill in the wildcards */
10332                 atio->ccb_h.target_id = target;
10333                 atio->ccb_h.target_lun = lun;
10334         }
10335
10336         /*
10337          * Package it up and send it off to
10338          * whomever has this lun enabled.
10339          */
10340         atio->sense_len = 0;
10341         atio->init_id = initiator;
10342         if (byte[0] != 0xFF) {
10343                 /* Tag was included */
10344                 atio->tag_action = *byte++;
10345                 atio->tag_id = *byte++;
10346                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
10347         } else {
10348                 atio->ccb_h.flags = 0;
10349         }
10350         byte++;
10351
10352         /* Okay.  Now determine the cdb size based on the command code */
10353         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
10354         case 0:
10355                 atio->cdb_len = 6;
10356                 break;
10357         case 1:
10358         case 2:
10359                 atio->cdb_len = 10;
10360                 break;
10361         case 4:
10362                 atio->cdb_len = 16;
10363                 break;
10364         case 5:
10365                 atio->cdb_len = 12;
10366                 break;
10367         case 3:
10368         default:
10369                 /* Only copy the opcode. */
10370                 atio->cdb_len = 1;
10371                 printf("Reserved or VU command code type encountered\n");
10372                 break;
10373         }
10374
10375         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
10376
10377         atio->ccb_h.status |= CAM_CDB_RECVD;
10378
10379         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
10380                 /*
10381                  * We weren't allowed to disconnect.
10382                  * We're hanging on the bus until a
10383                  * continue target I/O comes in response
10384                  * to this accept tio.
10385                  */
10386 #ifdef AHD_DEBUG
10387                 if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10388                         printf("Received Immediate Command %d:%d:%d - %p\n",
10389                                initiator, target, lun, ahd->pending_device);
10390 #endif
10391                 ahd->pending_device = lstate;
10392                 ahd_freeze_ccb((union ccb *)atio);
10393                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
10394         }
10395         xpt_done((union ccb*)atio);
10396         return (0);
10397 }
10398
10399 #endif