sys/dev/sound/pci/hda/hdac.c

   1 /*-
   2  * Copyright (c) 2006 Stephane E. Potvin <sepotvin@videotron.ca>
   3  * Copyright (c) 2006 Ariff Abdullah <ariff@FreeBSD.org>
   4  * Copyright (c) 2008-2012 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * Intel High Definition Audio (Controller) driver for FreeBSD.
  31  */
  32
  33 #ifdef HAVE_KERNEL_OPTION_HEADERS
  34 #include "opt_snd.h"
  35 #endif
  36
  37 #include <dev/sound/pcm/sound.h>
  38 #include <dev/pci/pcireg.h>
  39 #include <dev/pci/pcivar.h>
  40
  41 #include <sys/ctype.h>
  42 #include <sys/taskqueue.h>
  43
  44 #include <dev/sound/pci/hda/hdac_private.h>
  45 #include <dev/sound/pci/hda/hdac_reg.h>
  46 #include <dev/sound/pci/hda/hda_reg.h>
  47 #include <dev/sound/pci/hda/hdac.h>
  48
  49 #define HDA_DRV_TEST_REV        "20120126_0002"
  50
  51 SND_DECLARE_FILE("$FreeBSD$");
  52
  53 #define hdac_lock(sc)           snd_mtxlock((sc)->lock)
  54 #define hdac_unlock(sc)         snd_mtxunlock((sc)->lock)
  55 #define hdac_lockassert(sc)     snd_mtxassert((sc)->lock)
  56 #define hdac_lockowned(sc)      mtx_owned((sc)->lock)
  57
  58 #define HDAC_QUIRK_64BIT        (1 << 0)
  59 #define HDAC_QUIRK_DMAPOS       (1 << 1)
  60 #define HDAC_QUIRK_MSI          (1 << 2)
  61
  62 static const struct {
  63         const char *key;
  64         uint32_t value;
  65 } hdac_quirks_tab[] = {
  66         { "64bit", HDAC_QUIRK_DMAPOS },
  67         { "dmapos", HDAC_QUIRK_DMAPOS },
  68         { "msi", HDAC_QUIRK_MSI },
  69 };
  70
  71 MALLOC_DEFINE(M_HDAC, "hdac", "HDA Controller");
  72
  73 static const struct {
  74         uint32_t        model;
  75         const char      *desc;
  76         char            quirks_on;
  77         char            quirks_off;
  78 } hdac_devices[] = {
  79         { HDA_INTEL_OAK,     "Intel Oaktrail",  0, 0 },
  80         { HDA_INTEL_BAY,     "Intel BayTrail",  0, 0 },
  81         { HDA_INTEL_HSW1,    "Intel Haswell",   0, 0 },
  82         { HDA_INTEL_HSW2,    "Intel Haswell",   0, 0 },
  83         { HDA_INTEL_HSW3,    "Intel Haswell",   0, 0 },
  84         { HDA_INTEL_CPT,     "Intel Cougar Point",      0, 0 },
  85         { HDA_INTEL_PATSBURG,"Intel Patsburg",  0, 0 },
  86         { HDA_INTEL_PPT1,    "Intel Panther Point",     0, 0 },
  87         { HDA_INTEL_LPT1,    "Intel Lynx Point",        0, 0 },
  88         { HDA_INTEL_LPT2,    "Intel Lynx Point",        0, 0 },
  89         { HDA_INTEL_WELLS1,  "Intel Wellsburg", 0, 0 },
  90         { HDA_INTEL_WELLS2,  "Intel Wellsburg", 0, 0 },
  91         { HDA_INTEL_LPTLP1,  "Intel Lynx Point-LP",     0, 0 },
  92         { HDA_INTEL_LPTLP2,  "Intel Lynx Point-LP",     0, 0 },
  93         { HDA_INTEL_82801F,  "Intel 82801F",    0, 0 },
  94         { HDA_INTEL_63XXESB, "Intel 631x/632xESB",      0, 0 },
  95         { HDA_INTEL_82801G,  "Intel 82801G",    0, 0 },
  96         { HDA_INTEL_82801H,  "Intel 82801H",    0, 0 },
  97         { HDA_INTEL_82801I,  "Intel 82801I",    0, 0 },
  98         { HDA_INTEL_82801JI, "Intel 82801JI",   0, 0 },
  99         { HDA_INTEL_82801JD, "Intel 82801JD",   0, 0 },
 100         { HDA_INTEL_PCH,     "Intel 5 Series/3400 Series",      0, 0 },
 101         { HDA_INTEL_PCH2,    "Intel 5 Series/3400 Series",      0, 0 },
 102         { HDA_INTEL_SCH,     "Intel SCH",       0, 0 },
 103         { HDA_NVIDIA_MCP51,  "NVIDIA MCP51",    0, HDAC_QUIRK_MSI },
 104         { HDA_NVIDIA_MCP55,  "NVIDIA MCP55",    0, HDAC_QUIRK_MSI },
 105         { HDA_NVIDIA_MCP61_1, "NVIDIA MCP61",   0, 0 },
 106         { HDA_NVIDIA_MCP61_2, "NVIDIA MCP61",   0, 0 },
 107         { HDA_NVIDIA_MCP65_1, "NVIDIA MCP65",   0, 0 },
 108         { HDA_NVIDIA_MCP65_2, "NVIDIA MCP65",   0, 0 },
 109         { HDA_NVIDIA_MCP67_1, "NVIDIA MCP67",   0, 0 },
 110         { HDA_NVIDIA_MCP67_2, "NVIDIA MCP67",   0, 0 },
 111         { HDA_NVIDIA_MCP73_1, "NVIDIA MCP73",   0, 0 },
 112         { HDA_NVIDIA_MCP73_2, "NVIDIA MCP73",   0, 0 },
 113         { HDA_NVIDIA_MCP78_1, "NVIDIA MCP78",   0, HDAC_QUIRK_64BIT },
 114         { HDA_NVIDIA_MCP78_2, "NVIDIA MCP78",   0, HDAC_QUIRK_64BIT },
 115         { HDA_NVIDIA_MCP78_3, "NVIDIA MCP78",   0, HDAC_QUIRK_64BIT },
 116         { HDA_NVIDIA_MCP78_4, "NVIDIA MCP78",   0, HDAC_QUIRK_64BIT },
 117         { HDA_NVIDIA_MCP79_1, "NVIDIA MCP79",   0, 0 },
 118         { HDA_NVIDIA_MCP79_2, "NVIDIA MCP79",   0, 0 },
 119         { HDA_NVIDIA_MCP79_3, "NVIDIA MCP79",   0, 0 },
 120         { HDA_NVIDIA_MCP79_4, "NVIDIA MCP79",   0, 0 },
 121         { HDA_NVIDIA_MCP89_1, "NVIDIA MCP89",   0, 0 },
 122         { HDA_NVIDIA_MCP89_2, "NVIDIA MCP89",   0, 0 },
 123         { HDA_NVIDIA_MCP89_3, "NVIDIA MCP89",   0, 0 },
 124         { HDA_NVIDIA_MCP89_4, "NVIDIA MCP89",   0, 0 },
 125         { HDA_NVIDIA_0BE2,   "NVIDIA (0x0be2)", 0, HDAC_QUIRK_MSI },
 126         { HDA_NVIDIA_0BE3,   "NVIDIA (0x0be3)", 0, HDAC_QUIRK_MSI },
 127         { HDA_NVIDIA_0BE4,   "NVIDIA (0x0be4)", 0, HDAC_QUIRK_MSI },
 128         { HDA_NVIDIA_GT100,  "NVIDIA GT100",    0, HDAC_QUIRK_MSI },
 129         { HDA_NVIDIA_GT104,  "NVIDIA GT104",    0, HDAC_QUIRK_MSI },
 130         { HDA_NVIDIA_GT106,  "NVIDIA GT106",    0, HDAC_QUIRK_MSI },
 131         { HDA_NVIDIA_GT108,  "NVIDIA GT108",    0, HDAC_QUIRK_MSI },
 132         { HDA_NVIDIA_GT116,  "NVIDIA GT116",    0, HDAC_QUIRK_MSI },
 133         { HDA_NVIDIA_GF119,  "NVIDIA GF119",    0, 0 },
 134         { HDA_NVIDIA_GF110_1, "NVIDIA GF110",   0, HDAC_QUIRK_MSI },
 135         { HDA_NVIDIA_GF110_2, "NVIDIA GF110",   0, HDAC_QUIRK_MSI },
 136         { HDA_ATI_SB450,     "ATI SB450",       0, 0 },
 137         { HDA_ATI_SB600,     "ATI SB600",       0, 0 },
 138         { HDA_ATI_RS600,     "ATI RS600",       0, 0 },
 139         { HDA_ATI_RS690,     "ATI RS690",       0, 0 },
 140         { HDA_ATI_RS780,     "ATI RS780",       0, 0 },
 141         { HDA_ATI_R600,      "ATI R600",        0, 0 },
 142         { HDA_ATI_RV610,     "ATI RV610",       0, 0 },
 143         { HDA_ATI_RV620,     "ATI RV620",       0, 0 },
 144         { HDA_ATI_RV630,     "ATI RV630",       0, 0 },
 145         { HDA_ATI_RV635,     "ATI RV635",       0, 0 },
 146         { HDA_ATI_RV710,     "ATI RV710",       0, 0 },
 147         { HDA_ATI_RV730,     "ATI RV730",       0, 0 },
 148         { HDA_ATI_RV740,     "ATI RV740",       0, 0 },
 149         { HDA_ATI_RV770,     "ATI RV770",       0, 0 },
 150         { HDA_ATI_RV810,     "ATI RV810",       0, 0 },
 151         { HDA_ATI_RV830,     "ATI RV830",       0, 0 },
 152         { HDA_ATI_RV840,     "ATI RV840",       0, 0 },
 153         { HDA_ATI_RV870,     "ATI RV870",       0, 0 },
 154         { HDA_ATI_RV910,     "ATI RV910",       0, 0 },
 155         { HDA_ATI_RV930,     "ATI RV930",       0, 0 },
 156         { HDA_ATI_RV940,     "ATI RV940",       0, 0 },
 157         { HDA_ATI_RV970,     "ATI RV970",       0, 0 },
 158         { HDA_ATI_R1000,     "ATI R1000",       0, 0 },
 159         { HDA_RDC_M3010,     "RDC M3010",       0, 0 },
 160         { HDA_VIA_VT82XX,    "VIA VT8251/8237A",0, 0 },
 161         { HDA_SIS_966,       "SiS 966",         0, 0 },
 162         { HDA_ULI_M5461,     "ULI M5461",       0, 0 },
 163         /* Unknown */
 164         { HDA_INTEL_ALL,  "Intel",              0, 0 },
 165         { HDA_NVIDIA_ALL, "NVIDIA",             0, 0 },
 166         { HDA_ATI_ALL,    "ATI",                0, 0 },
 167         { HDA_VIA_ALL,    "VIA",                0, 0 },
 168         { HDA_SIS_ALL,    "SiS",                0, 0 },
 169         { HDA_ULI_ALL,    "ULI",                0, 0 },
 170 };
 171
 172 static const struct {
 173         uint16_t vendor;
 174         uint8_t reg;
 175         uint8_t mask;
 176         uint8_t enable;
 177 } hdac_pcie_snoop[] = {
 178         {  INTEL_VENDORID, 0x00, 0x00, 0x00 },
 179         {    ATI_VENDORID, 0x42, 0xf8, 0x02 },
 180         { NVIDIA_VENDORID, 0x4e, 0xf0, 0x0f },
 181 };
 182
 183 /****************************************************************************
 184  * Function prototypes
 185  ****************************************************************************/
 186 static void     hdac_intr_handler(void *);
 187 static int      hdac_reset(struct hdac_softc *, int);
 188 static int      hdac_get_capabilities(struct hdac_softc *);
 189 static void     hdac_dma_cb(void *, bus_dma_segment_t *, int, int);
 190 static int      hdac_dma_alloc(struct hdac_softc *,
 191                                         struct hdac_dma *, bus_size_t);
 192 static void     hdac_dma_free(struct hdac_softc *, struct hdac_dma *);
 193 static int      hdac_mem_alloc(struct hdac_softc *);
 194 static void     hdac_mem_free(struct hdac_softc *);
 195 static int      hdac_irq_alloc(struct hdac_softc *);
 196 static void     hdac_irq_free(struct hdac_softc *);
 197 static void     hdac_corb_init(struct hdac_softc *);
 198 static void     hdac_rirb_init(struct hdac_softc *);
 199 static void     hdac_corb_start(struct hdac_softc *);
 200 static void     hdac_rirb_start(struct hdac_softc *);
 201
 202 static void     hdac_attach2(void *);
 203
 204 static uint32_t hdac_send_command(struct hdac_softc *, nid_t, uint32_t);
 205
 206 static int      hdac_probe(device_t);
 207 static int      hdac_attach(device_t);
 208 static int      hdac_detach(device_t);
 209 static int      hdac_suspend(device_t);
 210 static int      hdac_resume(device_t);
 211
 212 static int      hdac_rirb_flush(struct hdac_softc *sc);
 213 static int      hdac_unsolq_flush(struct hdac_softc *sc);
 214
 215 #define hdac_command(a1, a2, a3)        \
 216                 hdac_send_command(a1, a3, a2)
 217
 218 /* This function surely going to make its way into upper level someday. */
 219 static void
 220 hdac_config_fetch(struct hdac_softc *sc, uint32_t *on, uint32_t *off)
 221 {
 222         const char *res = NULL;
 223         int i = 0, j, k, len, inv;
 224
 225         if (resource_string_value(device_get_name(sc->dev),
 226             device_get_unit(sc->dev), "config", &res) != 0)
 227                 return;
 228         if (!(res != NULL && strlen(res) > 0))
 229                 return;
 230         HDA_BOOTVERBOSE(
 231                 device_printf(sc->dev, "Config options:");
 232         );
 233         for (;;) {
 234                 while (res[i] != '\0' &&
 235                     (res[i] == ',' || isspace(res[i]) != 0))
 236                         i++;
 237                 if (res[i] == '\0') {
 238                         HDA_BOOTVERBOSE(
 239                                 printf("\n");
 240                         );
 241                         return;
 242                 }
 243                 j = i;
 244                 while (res[j] != '\0' &&
 245                     !(res[j] == ',' || isspace(res[j]) != 0))
 246                         j++;
 247                 len = j - i;
 248                 if (len > 2 && strncmp(res + i, "no", 2) == 0)
 249                         inv = 2;
 250                 else
 251                         inv = 0;
 252                 for (k = 0; len > inv && k < nitems(hdac_quirks_tab); k++) {
 253                         if (strncmp(res + i + inv,
 254                             hdac_quirks_tab[k].key, len - inv) != 0)
 255                                 continue;
 256                         if (len - inv != strlen(hdac_quirks_tab[k].key))
 257                                 continue;
 258                         HDA_BOOTVERBOSE(
 259                                 printf(" %s%s", (inv != 0) ? "no" : "",
 260                                     hdac_quirks_tab[k].key);
 261                         );
 262                         if (inv == 0) {
 263                                 *on |= hdac_quirks_tab[k].value;
 264                                 *on &= ~hdac_quirks_tab[k].value;
 265                         } else if (inv != 0) {
 266                                 *off |= hdac_quirks_tab[k].value;
 267                                 *off &= ~hdac_quirks_tab[k].value;
 268                         }
 269                         break;
 270                 }
 271                 i = j;
 272         }
 273 }
 274
 275 /****************************************************************************
 276  * void hdac_intr_handler(void *)
 277  *
 278  * Interrupt handler. Processes interrupts received from the hdac.
 279  ****************************************************************************/
 280 static void
 281 hdac_intr_handler(void *context)
 282 {
 283         struct hdac_softc *sc;
 284         device_t dev;
 285         uint32_t intsts;
 286         uint8_t rirbsts;
 287         int i;
 288
 289         sc = (struct hdac_softc *)context;
 290         hdac_lock(sc);
 291
 292         /* Do we have anything to do? */
 293         intsts = HDAC_READ_4(&sc->mem, HDAC_INTSTS);
 294         if ((intsts & HDAC_INTSTS_GIS) == 0) {
 295                 hdac_unlock(sc);
 296                 return;
 297         }
 298
 299         /* Was this a controller interrupt? */
 300         if (intsts & HDAC_INTSTS_CIS) {
 301                 rirbsts = HDAC_READ_1(&sc->mem, HDAC_RIRBSTS);
 302                 /* Get as many responses that we can */
 303                 while (rirbsts & HDAC_RIRBSTS_RINTFL) {
 304                         HDAC_WRITE_1(&sc->mem,
 305                             HDAC_RIRBSTS, HDAC_RIRBSTS_RINTFL);
 306                         hdac_rirb_flush(sc);
 307                         rirbsts = HDAC_READ_1(&sc->mem, HDAC_RIRBSTS);
 308                 }
 309                 if (sc->unsolq_rp != sc->unsolq_wp)
 310                         taskqueue_enqueue(taskqueue_thread, &sc->unsolq_task);
 311         }
 312
 313         if (intsts & HDAC_INTSTS_SIS_MASK) {
 314                 for (i = 0; i < sc->num_ss; i++) {
 315                         if ((intsts & (1 << i)) == 0)
 316                                 continue;
 317                         HDAC_WRITE_1(&sc->mem, (i << 5) + HDAC_SDSTS,
 318                             HDAC_SDSTS_DESE | HDAC_SDSTS_FIFOE | HDAC_SDSTS_BCIS );
 319                         if ((dev = sc->streams[i].dev) != NULL) {
 320                                 HDAC_STREAM_INTR(dev,
 321                                     sc->streams[i].dir, sc->streams[i].stream);
 322                         }
 323                 }
 324         }
 325
 326         HDAC_WRITE_4(&sc->mem, HDAC_INTSTS, intsts);
 327         hdac_unlock(sc);
 328 }
 329
 330 static void
 331 hdac_poll_callback(void *arg)
 332 {
 333         struct hdac_softc *sc = arg;
 334
 335         if (sc == NULL)
 336                 return;
 337
 338         hdac_lock(sc);
 339         if (sc->polling == 0) {
 340                 hdac_unlock(sc);
 341                 return;
 342         }
 343         callout_reset(&sc->poll_callout, sc->poll_ival,
 344             hdac_poll_callback, sc);
 345         hdac_unlock(sc);
 346
 347         hdac_intr_handler(sc);
 348 }
 349
 350 /****************************************************************************
 351  * int hdac_reset(hdac_softc *, int)
 352  *
 353  * Reset the hdac to a quiescent and known state.
 354  ****************************************************************************/
 355 static int
 356 hdac_reset(struct hdac_softc *sc, int wakeup)
 357 {
 358         uint32_t gctl;
 359         int count, i;
 360
 361         /*
 362          * Stop all Streams DMA engine
 363          */
 364         for (i = 0; i < sc->num_iss; i++)
 365                 HDAC_WRITE_4(&sc->mem, HDAC_ISDCTL(sc, i), 0x0);
 366         for (i = 0; i < sc->num_oss; i++)
 367                 HDAC_WRITE_4(&sc->mem, HDAC_OSDCTL(sc, i), 0x0);
 368         for (i = 0; i < sc->num_bss; i++)
 369                 HDAC_WRITE_4(&sc->mem, HDAC_BSDCTL(sc, i), 0x0);
 370
 371         /*
 372          * Stop Control DMA engines.
 373          */
 374         HDAC_WRITE_1(&sc->mem, HDAC_CORBCTL, 0x0);
 375         HDAC_WRITE_1(&sc->mem, HDAC_RIRBCTL, 0x0);
 376
 377         /*
 378          * Reset DMA position buffer.
 379          */
 380         HDAC_WRITE_4(&sc->mem, HDAC_DPIBLBASE, 0x0);
 381         HDAC_WRITE_4(&sc->mem, HDAC_DPIBUBASE, 0x0);
 382
 383         /*
 384          * Reset the controller. The reset must remain asserted for
 385          * a minimum of 100us.
 386          */
 387         gctl = HDAC_READ_4(&sc->mem, HDAC_GCTL);
 388         HDAC_WRITE_4(&sc->mem, HDAC_GCTL, gctl & ~HDAC_GCTL_CRST);
 389         count = 10000;
 390         do {
 391                 gctl = HDAC_READ_4(&sc->mem, HDAC_GCTL);
 392                 if (!(gctl & HDAC_GCTL_CRST))
 393                         break;
 394                 DELAY(10);
 395         } while (--count);
 396         if (gctl & HDAC_GCTL_CRST) {
 397                 device_printf(sc->dev, "Unable to put hdac in reset\n");
 398                 return (ENXIO);
 399         }
 400
 401         /* If wakeup is not requested - leave the controller in reset state. */
 402         if (!wakeup)
 403                 return (0);
 404
 405         DELAY(100);
 406         gctl = HDAC_READ_4(&sc->mem, HDAC_GCTL);
 407         HDAC_WRITE_4(&sc->mem, HDAC_GCTL, gctl | HDAC_GCTL_CRST);
 408         count = 10000;
 409         do {
 410                 gctl = HDAC_READ_4(&sc->mem, HDAC_GCTL);
 411                 if (gctl & HDAC_GCTL_CRST)
 412                         break;
 413                 DELAY(10);
 414         } while (--count);
 415         if (!(gctl & HDAC_GCTL_CRST)) {
 416                 device_printf(sc->dev, "Device stuck in reset\n");
 417                 return (ENXIO);
 418         }
 419
 420         /*
 421          * Wait for codecs to finish their own reset sequence. The delay here
 422          * should be of 250us but for some reasons, on it's not enough on my
 423          * computer. Let's use twice as much as necessary to make sure that
 424          * it's reset properly.
 425          */
 426         DELAY(1000);
 427
 428         return (0);
 429 }
 430
 431
 432 /****************************************************************************
 433  * int hdac_get_capabilities(struct hdac_softc *);
 434  *
 435  * Retreive the general capabilities of the hdac;
 436  *      Number of Input Streams
 437  *      Number of Output Streams
 438  *      Number of bidirectional Streams
 439  *      64bit ready
 440  *      CORB and RIRB sizes
 441  ****************************************************************************/
 442 static int
 443 hdac_get_capabilities(struct hdac_softc *sc)
 444 {
 445         uint16_t gcap;
 446         uint8_t corbsize, rirbsize;
 447
 448         gcap = HDAC_READ_2(&sc->mem, HDAC_GCAP);
 449         sc->num_iss = HDAC_GCAP_ISS(gcap);
 450         sc->num_oss = HDAC_GCAP_OSS(gcap);
 451         sc->num_bss = HDAC_GCAP_BSS(gcap);
 452         sc->num_ss = sc->num_iss + sc->num_oss + sc->num_bss;
 453         sc->num_sdo = HDAC_GCAP_NSDO(gcap);
 454         sc->support_64bit = (gcap & HDAC_GCAP_64OK) != 0;
 455         if (sc->quirks_on & HDAC_QUIRK_64BIT)
 456                 sc->support_64bit = 1;
 457         else if (sc->quirks_off & HDAC_QUIRK_64BIT)
 458                 sc->support_64bit = 0;
 459
 460         corbsize = HDAC_READ_1(&sc->mem, HDAC_CORBSIZE);
 461         if ((corbsize & HDAC_CORBSIZE_CORBSZCAP_256) ==
 462             HDAC_CORBSIZE_CORBSZCAP_256)
 463                 sc->corb_size = 256;
 464         else if ((corbsize & HDAC_CORBSIZE_CORBSZCAP_16) ==
 465             HDAC_CORBSIZE_CORBSZCAP_16)
 466                 sc->corb_size = 16;
 467         else if ((corbsize & HDAC_CORBSIZE_CORBSZCAP_2) ==
 468             HDAC_CORBSIZE_CORBSZCAP_2)
 469                 sc->corb_size = 2;
 470         else {
 471                 device_printf(sc->dev, "%s: Invalid corb size (%x)\n",
 472                     __func__, corbsize);
 473                 return (ENXIO);
 474         }
 475
 476         rirbsize = HDAC_READ_1(&sc->mem, HDAC_RIRBSIZE);
 477         if ((rirbsize & HDAC_RIRBSIZE_RIRBSZCAP_256) ==
 478             HDAC_RIRBSIZE_RIRBSZCAP_256)
 479                 sc->rirb_size = 256;
 480         else if ((rirbsize & HDAC_RIRBSIZE_RIRBSZCAP_16) ==
 481             HDAC_RIRBSIZE_RIRBSZCAP_16)
 482                 sc->rirb_size = 16;
 483         else if ((rirbsize & HDAC_RIRBSIZE_RIRBSZCAP_2) ==
 484             HDAC_RIRBSIZE_RIRBSZCAP_2)
 485                 sc->rirb_size = 2;
 486         else {
 487                 device_printf(sc->dev, "%s: Invalid rirb size (%x)\n",
 488                     __func__, rirbsize);
 489                 return (ENXIO);
 490         }
 491
 492         HDA_BOOTVERBOSE(
 493                 device_printf(sc->dev, "Caps: OSS %d, ISS %d, BSS %d, "
 494                     "NSDO %d%s, CORB %d, RIRB %d\n",
 495                     sc->num_oss, sc->num_iss, sc->num_bss, 1 << sc->num_sdo,
 496                     sc->support_64bit ? ", 64bit" : "",
 497                     sc->corb_size, sc->rirb_size);
 498         );
 499
 500         return (0);
 501 }
 502
 503
 504 /****************************************************************************
 505  * void hdac_dma_cb
 506  *
 507  * This function is called by bus_dmamap_load when the mapping has been
 508  * established. We just record the physical address of the mapping into
 509  * the struct hdac_dma passed in.
 510  ****************************************************************************/
 511 static void
 512 hdac_dma_cb(void *callback_arg, bus_dma_segment_t *segs, int nseg, int error)
 513 {
 514         struct hdac_dma *dma;
 515
 516         if (error == 0) {
 517                 dma = (struct hdac_dma *)callback_arg;
 518                 dma->dma_paddr = segs[0].ds_addr;
 519         }
 520 }
 521
 522
 523 /****************************************************************************
 524  * int hdac_dma_alloc
 525  *
 526  * This function allocate and setup a dma region (struct hdac_dma).
 527  * It must be freed by a corresponding hdac_dma_free.
 528  ****************************************************************************/
 529 static int
 530 hdac_dma_alloc(struct hdac_softc *sc, struct hdac_dma *dma, bus_size_t size)
 531 {
 532         bus_size_t roundsz;
 533         int result;
 534
 535         roundsz = roundup2(size, HDA_DMA_ALIGNMENT);
 536         bzero(dma, sizeof(*dma));
 537
 538         /*
 539          * Create a DMA tag
 540          */
 541         result = bus_dma_tag_create(
 542             bus_get_dma_tag(sc->dev),           /* parent */
 543             HDA_DMA_ALIGNMENT,                  /* alignment */
 544             0,                                  /* boundary */
 545             (sc->support_64bit) ? BUS_SPACE_MAXADDR :
 546                 BUS_SPACE_MAXADDR_32BIT,        /* lowaddr */
 547             BUS_SPACE_MAXADDR,                  /* highaddr */
 548             NULL,                               /* filtfunc */
 549             NULL,                               /* fistfuncarg */
 550             roundsz,                            /* maxsize */
 551             1,                                  /* nsegments */
 552             roundsz,                            /* maxsegsz */
 553             0,                                  /* flags */
 554             NULL,                               /* lockfunc */
 555             NULL,                               /* lockfuncarg */
 556             &dma->dma_tag);                     /* dmat */
 557         if (result != 0) {
 558                 device_printf(sc->dev, "%s: bus_dma_tag_create failed (%x)\n",
 559                     __func__, result);
 560                 goto hdac_dma_alloc_fail;
 561         }
 562
 563         /*
 564          * Allocate DMA memory
 565          */
 566         result = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
 567             BUS_DMA_NOWAIT | BUS_DMA_ZERO |
 568             ((sc->flags & HDAC_F_DMA_NOCACHE) ? BUS_DMA_NOCACHE : 0),
 569             &dma->dma_map);
 570         if (result != 0) {
 571                 device_printf(sc->dev, "%s: bus_dmamem_alloc failed (%x)\n",
 572                     __func__, result);
 573                 goto hdac_dma_alloc_fail;
 574         }
 575
 576         dma->dma_size = roundsz;
 577
 578         /*
 579          * Map the memory
 580          */
 581         result = bus_dmamap_load(dma->dma_tag, dma->dma_map,
 582             (void *)dma->dma_vaddr, roundsz, hdac_dma_cb, (void *)dma, 0);
 583         if (result != 0 || dma->dma_paddr == 0) {
 584                 if (result == 0)
 585                         result = ENOMEM;
 586                 device_printf(sc->dev, "%s: bus_dmamem_load failed (%x)\n",
 587                     __func__, result);
 588                 goto hdac_dma_alloc_fail;
 589         }
 590
 591         HDA_BOOTHVERBOSE(
 592                 device_printf(sc->dev, "%s: size=%ju -> roundsz=%ju\n",
 593                     __func__, (uintmax_t)size, (uintmax_t)roundsz);
 594         );
 595
 596         return (0);
 597
 598 hdac_dma_alloc_fail:
 599         hdac_dma_free(sc, dma);
 600
 601         return (result);
 602 }
 603
 604
 605 /****************************************************************************
 606  * void hdac_dma_free(struct hdac_softc *, struct hdac_dma *)
 607  *
 608  * Free a struct dhac_dma that has been previously allocated via the
 609  * hdac_dma_alloc function.
 610  ****************************************************************************/
 611 static void
 612 hdac_dma_free(struct hdac_softc *sc, struct hdac_dma *dma)
 613 {
 614         if (dma->dma_map != NULL) {
 615 #if 0
 616                 /* Flush caches */
 617                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
 618                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 619 #endif
 620                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 621         }
 622         if (dma->dma_vaddr != NULL) {
 623                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 624                 dma->dma_vaddr = NULL;
 625         }
 626         dma->dma_map = NULL;
 627         if (dma->dma_tag != NULL) {
 628                 bus_dma_tag_destroy(dma->dma_tag);
 629                 dma->dma_tag = NULL;
 630         }
 631         dma->dma_size = 0;
 632 }
 633
 634 /****************************************************************************
 635  * int hdac_mem_alloc(struct hdac_softc *)
 636  *
 637  * Allocate all the bus resources necessary to speak with the physical
 638  * controller.
 639  ****************************************************************************/
 640 static int
 641 hdac_mem_alloc(struct hdac_softc *sc)
 642 {
 643         struct hdac_mem *mem;
 644
 645         mem = &sc->mem;
 646         mem->mem_rid = PCIR_BAR(0);
 647         mem->mem_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
 648             &mem->mem_rid, RF_ACTIVE);
 649         if (mem->mem_res == NULL) {
 650                 device_printf(sc->dev,
 651                     "%s: Unable to allocate memory resource\n", __func__);
 652                 return (ENOMEM);
 653         }
 654         mem->mem_tag = rman_get_bustag(mem->mem_res);
 655         mem->mem_handle = rman_get_bushandle(mem->mem_res);
 656
 657         return (0);
 658 }
 659
 660 /****************************************************************************
 661  * void hdac_mem_free(struct hdac_softc *)
 662  *
 663  * Free up resources previously allocated by hdac_mem_alloc.
 664  ****************************************************************************/
 665 static void
 666 hdac_mem_free(struct hdac_softc *sc)
 667 {
 668         struct hdac_mem *mem;
 669
 670         mem = &sc->mem;
 671         if (mem->mem_res != NULL)
 672                 bus_release_resource(sc->dev, SYS_RES_MEMORY, mem->mem_rid,
 673                     mem->mem_res);
 674         mem->mem_res = NULL;
 675 }
 676
 677 /****************************************************************************
 678  * int hdac_irq_alloc(struct hdac_softc *)
 679  *
 680  * Allocate and setup the resources necessary for interrupt handling.
 681  ****************************************************************************/
 682 static int
 683 hdac_irq_alloc(struct hdac_softc *sc)
 684 {
 685         struct hdac_irq *irq;
 686         int result;
 687
 688         irq = &sc->irq;
 689         irq->irq_rid = 0x0;
 690
 691         if ((sc->quirks_off & HDAC_QUIRK_MSI) == 0 &&
 692             (result = pci_msi_count(sc->dev)) == 1 &&
 693             pci_alloc_msi(sc->dev, &result) == 0)
 694                 irq->irq_rid = 0x1;
 695
 696         irq->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
 697             &irq->irq_rid, RF_SHAREABLE | RF_ACTIVE);
 698         if (irq->irq_res == NULL) {
 699                 device_printf(sc->dev, "%s: Unable to allocate irq\n",
 700                     __func__);
 701                 goto hdac_irq_alloc_fail;
 702         }
 703         result = bus_setup_intr(sc->dev, irq->irq_res, INTR_MPSAFE | INTR_TYPE_AV,
 704             NULL, hdac_intr_handler, sc, &irq->irq_handle);
 705         if (result != 0) {
 706                 device_printf(sc->dev,
 707                     "%s: Unable to setup interrupt handler (%x)\n",
 708                     __func__, result);
 709                 goto hdac_irq_alloc_fail;
 710         }
 711
 712         return (0);
 713
 714 hdac_irq_alloc_fail:
 715         hdac_irq_free(sc);
 716
 717         return (ENXIO);
 718 }
 719
 720 /****************************************************************************
 721  * void hdac_irq_free(struct hdac_softc *)
 722  *
 723  * Free up resources previously allocated by hdac_irq_alloc.
 724  ****************************************************************************/
 725 static void
 726 hdac_irq_free(struct hdac_softc *sc)
 727 {
 728         struct hdac_irq *irq;
 729
 730         irq = &sc->irq;
 731         if (irq->irq_res != NULL && irq->irq_handle != NULL)
 732                 bus_teardown_intr(sc->dev, irq->irq_res, irq->irq_handle);
 733         if (irq->irq_res != NULL)
 734                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->irq_rid,
 735                     irq->irq_res);
 736         if (irq->irq_rid == 0x1)
 737                 pci_release_msi(sc->dev);
 738         irq->irq_handle = NULL;
 739         irq->irq_res = NULL;
 740         irq->irq_rid = 0x0;
 741 }
 742
 743 /****************************************************************************
 744  * void hdac_corb_init(struct hdac_softc *)
 745  *
 746  * Initialize the corb registers for operations but do not start it up yet.
 747  * The CORB engine must not be running when this function is called.
 748  ****************************************************************************/
 749 static void
 750 hdac_corb_init(struct hdac_softc *sc)
 751 {
 752         uint8_t corbsize;
 753         uint64_t corbpaddr;
 754
 755         /* Setup the CORB size. */
 756         switch (sc->corb_size) {
 757         case 256:
 758                 corbsize = HDAC_CORBSIZE_CORBSIZE(HDAC_CORBSIZE_CORBSIZE_256);
 759                 break;
 760         case 16:
 761                 corbsize = HDAC_CORBSIZE_CORBSIZE(HDAC_CORBSIZE_CORBSIZE_16);
 762                 break;
 763         case 2:
 764                 corbsize = HDAC_CORBSIZE_CORBSIZE(HDAC_CORBSIZE_CORBSIZE_2);
 765                 break;
 766         default:
 767                 panic("%s: Invalid CORB size (%x)\n", __func__, sc->corb_size);
 768         }
 769         HDAC_WRITE_1(&sc->mem, HDAC_CORBSIZE, corbsize);
 770
 771         /* Setup the CORB Address in the hdac */
 772         corbpaddr = (uint64_t)sc->corb_dma.dma_paddr;
 773         HDAC_WRITE_4(&sc->mem, HDAC_CORBLBASE, (uint32_t)corbpaddr);
 774         HDAC_WRITE_4(&sc->mem, HDAC_CORBUBASE, (uint32_t)(corbpaddr >> 32));
 775
 776         /* Set the WP and RP */
 777         sc->corb_wp = 0;
 778         HDAC_WRITE_2(&sc->mem, HDAC_CORBWP, sc->corb_wp);
 779         HDAC_WRITE_2(&sc->mem, HDAC_CORBRP, HDAC_CORBRP_CORBRPRST);
 780         /*
 781          * The HDA specification indicates that the CORBRPRST bit will always
 782          * read as zero. Unfortunately, it seems that at least the 82801G
 783          * doesn't reset the bit to zero, which stalls the corb engine.
 784          * manually reset the bit to zero before continuing.
 785          */
 786         HDAC_WRITE_2(&sc->mem, HDAC_CORBRP, 0x0);
 787
 788         /* Enable CORB error reporting */
 789 #if 0
 790         HDAC_WRITE_1(&sc->mem, HDAC_CORBCTL, HDAC_CORBCTL_CMEIE);
 791 #endif
 792 }
 793
 794 /****************************************************************************
 795  * void hdac_rirb_init(struct hdac_softc *)
 796  *
 797  * Initialize the rirb registers for operations but do not start it up yet.
 798  * The RIRB engine must not be running when this function is called.
 799  ****************************************************************************/
 800 static void
 801 hdac_rirb_init(struct hdac_softc *sc)
 802 {
 803         uint8_t rirbsize;
 804         uint64_t rirbpaddr;
 805
 806         /* Setup the RIRB size. */
 807         switch (sc->rirb_size) {
 808         case 256:
 809                 rirbsize = HDAC_RIRBSIZE_RIRBSIZE(HDAC_RIRBSIZE_RIRBSIZE_256);
 810                 break;
 811         case 16:
 812                 rirbsize = HDAC_RIRBSIZE_RIRBSIZE(HDAC_RIRBSIZE_RIRBSIZE_16);
 813                 break;
 814         case 2:
 815                 rirbsize = HDAC_RIRBSIZE_RIRBSIZE(HDAC_RIRBSIZE_RIRBSIZE_2);
 816                 break;
 817         default:
 818                 panic("%s: Invalid RIRB size (%x)\n", __func__, sc->rirb_size);
 819         }
 820         HDAC_WRITE_1(&sc->mem, HDAC_RIRBSIZE, rirbsize);
 821
 822         /* Setup the RIRB Address in the hdac */
 823         rirbpaddr = (uint64_t)sc->rirb_dma.dma_paddr;
 824         HDAC_WRITE_4(&sc->mem, HDAC_RIRBLBASE, (uint32_t)rirbpaddr);
 825         HDAC_WRITE_4(&sc->mem, HDAC_RIRBUBASE, (uint32_t)(rirbpaddr >> 32));
 826
 827         /* Setup the WP and RP */
 828         sc->rirb_rp = 0;
 829         HDAC_WRITE_2(&sc->mem, HDAC_RIRBWP, HDAC_RIRBWP_RIRBWPRST);
 830
 831         /* Setup the interrupt threshold */
 832         HDAC_WRITE_2(&sc->mem, HDAC_RINTCNT, sc->rirb_size / 2);
 833
 834         /* Enable Overrun and response received reporting */
 835 #if 0
 836         HDAC_WRITE_1(&sc->mem, HDAC_RIRBCTL,
 837             HDAC_RIRBCTL_RIRBOIC | HDAC_RIRBCTL_RINTCTL);
 838 #else
 839         HDAC_WRITE_1(&sc->mem, HDAC_RIRBCTL, HDAC_RIRBCTL_RINTCTL);
 840 #endif
 841
 842 #if 0
 843         /*
 844          * Make sure that the Host CPU cache doesn't contain any dirty
 845          * cache lines that falls in the rirb. If I understood correctly, it
 846          * should be sufficient to do this only once as the rirb is purely
 847          * read-only from now on.
 848          */
 849         bus_dmamap_sync(sc->rirb_dma.dma_tag, sc->rirb_dma.dma_map,
 850             BUS_DMASYNC_PREREAD);
 851 #endif
 852 }
 853
 854 /****************************************************************************
 855  * void hdac_corb_start(hdac_softc *)
 856  *
 857  * Startup the corb DMA engine
 858  ****************************************************************************/
 859 static void
 860 hdac_corb_start(struct hdac_softc *sc)
 861 {
 862         uint32_t corbctl;
 863
 864         corbctl = HDAC_READ_1(&sc->mem, HDAC_CORBCTL);
 865         corbctl |= HDAC_CORBCTL_CORBRUN;
 866         HDAC_WRITE_1(&sc->mem, HDAC_CORBCTL, corbctl);
 867 }
 868
 869 /****************************************************************************
 870  * void hdac_rirb_start(hdac_softc *)
 871  *
 872  * Startup the rirb DMA engine
 873  ****************************************************************************/
 874 static void
 875 hdac_rirb_start(struct hdac_softc *sc)
 876 {
 877         uint32_t rirbctl;
 878
 879         rirbctl = HDAC_READ_1(&sc->mem, HDAC_RIRBCTL);
 880         rirbctl |= HDAC_RIRBCTL_RIRBDMAEN;
 881         HDAC_WRITE_1(&sc->mem, HDAC_RIRBCTL, rirbctl);
 882 }
 883
 884 static int
 885 hdac_rirb_flush(struct hdac_softc *sc)
 886 {
 887         struct hdac_rirb *rirb_base, *rirb;
 888         nid_t cad;
 889         uint32_t resp;
 890         uint8_t rirbwp;
 891         int ret;
 892
 893         rirb_base = (struct hdac_rirb *)sc->rirb_dma.dma_vaddr;
 894         rirbwp = HDAC_READ_1(&sc->mem, HDAC_RIRBWP);
 895 #if 0
 896         bus_dmamap_sync(sc->rirb_dma.dma_tag, sc->rirb_dma.dma_map,
 897             BUS_DMASYNC_POSTREAD);
 898 #endif
 899
 900         ret = 0;
 901         while (sc->rirb_rp != rirbwp) {
 902                 sc->rirb_rp++;
 903                 sc->rirb_rp %= sc->rirb_size;
 904                 rirb = &rirb_base[sc->rirb_rp];
 905                 cad = HDAC_RIRB_RESPONSE_EX_SDATA_IN(rirb->response_ex);
 906                 resp = rirb->response;
 907                 if (rirb->response_ex & HDAC_RIRB_RESPONSE_EX_UNSOLICITED) {
 908                         sc->unsolq[sc->unsolq_wp++] = resp;
 909                         sc->unsolq_wp %= HDAC_UNSOLQ_MAX;
 910                         sc->unsolq[sc->unsolq_wp++] = cad;
 911                         sc->unsolq_wp %= HDAC_UNSOLQ_MAX;
 912                 } else if (sc->codecs[cad].pending <= 0) {
 913                         device_printf(sc->dev, "Unexpected unsolicited "
 914                             "response from address %d: %08x\n", cad, resp);
 915                 } else {
 916                         sc->codecs[cad].response = resp;
 917                         sc->codecs[cad].pending--;
 918                 }
 919                 ret++;
 920         }
 921         return (ret);
 922 }
 923
 924 static int
 925 hdac_unsolq_flush(struct hdac_softc *sc)
 926 {
 927         device_t child;
 928         nid_t cad;
 929         uint32_t resp;
 930         int ret = 0;
 931
 932         if (sc->unsolq_st == HDAC_UNSOLQ_READY) {
 933                 sc->unsolq_st = HDAC_UNSOLQ_BUSY;
 934                 while (sc->unsolq_rp != sc->unsolq_wp) {
 935                         resp = sc->unsolq[sc->unsolq_rp++];
 936                         sc->unsolq_rp %= HDAC_UNSOLQ_MAX;
 937                         cad = sc->unsolq[sc->unsolq_rp++];
 938                         sc->unsolq_rp %= HDAC_UNSOLQ_MAX;
 939                         if ((child = sc->codecs[cad].dev) != NULL)
 940                                 HDAC_UNSOL_INTR(child, resp);
 941                         ret++;
 942                 }
 943                 sc->unsolq_st = HDAC_UNSOLQ_READY;
 944         }
 945
 946         return (ret);
 947 }
 948
 949 /****************************************************************************
 950  * uint32_t hdac_command_sendone_internal
 951  *
 952  * Wrapper function that sends only one command to a given codec
 953  ****************************************************************************/
 954 static uint32_t
 955 hdac_send_command(struct hdac_softc *sc, nid_t cad, uint32_t verb)
 956 {
 957         int timeout;
 958         uint32_t *corb;
 959
 960         if (!hdac_lockowned(sc))
 961                 device_printf(sc->dev, "WARNING!!!! mtx not owned!!!!\n");
 962         verb &= ~HDA_CMD_CAD_MASK;
 963         verb |= ((uint32_t)cad) << HDA_CMD_CAD_SHIFT;
 964         sc->codecs[cad].response = HDA_INVALID;
 965
 966         sc->codecs[cad].pending++;
 967         sc->corb_wp++;
 968         sc->corb_wp %= sc->corb_size;
 969         corb = (uint32_t *)sc->corb_dma.dma_vaddr;
 970 #if 0
 971         bus_dmamap_sync(sc->corb_dma.dma_tag,
 972             sc->corb_dma.dma_map, BUS_DMASYNC_PREWRITE);
 973 #endif
 974         corb[sc->corb_wp] = verb;
 975 #if 0
 976         bus_dmamap_sync(sc->corb_dma.dma_tag,
 977             sc->corb_dma.dma_map, BUS_DMASYNC_POSTWRITE);
 978 #endif
 979         HDAC_WRITE_2(&sc->mem, HDAC_CORBWP, sc->corb_wp);
 980
 981         timeout = 10000;
 982         do {
 983                 if (hdac_rirb_flush(sc) == 0)
 984                         DELAY(10);
 985         } while (sc->codecs[cad].pending != 0 && --timeout);
 986
 987         if (sc->codecs[cad].pending != 0) {
 988                 device_printf(sc->dev, "Command timeout on address %d\n", cad);
 989                 sc->codecs[cad].pending = 0;
 990         }
 991
 992         if (sc->unsolq_rp != sc->unsolq_wp)
 993                 taskqueue_enqueue(taskqueue_thread, &sc->unsolq_task);
 994         return (sc->codecs[cad].response);
 995 }
 996
 997 /****************************************************************************
 998  * Device Methods
 999  ****************************************************************************/
1000
1001 /****************************************************************************
1002  * int hdac_probe(device_t)
1003  *
1004  * Probe for the presence of an hdac. If none is found, check for a generic
1005  * match using the subclass of the device.
1006  ****************************************************************************/
1007 static int
1008 hdac_probe(device_t dev)
1009 {
1010         int i, result;
1011         uint32_t model;
1012         uint16_t class, subclass;
1013         char desc[64];
1014
1015         model = (uint32_t)pci_get_device(dev) << 16;
1016         model |= (uint32_t)pci_get_vendor(dev) & 0x0000ffff;
1017         class = pci_get_class(dev);
1018         subclass = pci_get_subclass(dev);
1019
1020         bzero(desc, sizeof(desc));
1021         result = ENXIO;
1022         for (i = 0; i < nitems(hdac_devices); i++) {
1023                 if (hdac_devices[i].model == model) {
1024                         strlcpy(desc, hdac_devices[i].desc, sizeof(desc));
1025                         result = BUS_PROBE_DEFAULT;
1026                         break;
1027                 }
1028                 if (HDA_DEV_MATCH(hdac_devices[i].model, model) &&
1029                     class == PCIC_MULTIMEDIA &&
1030                     subclass == PCIS_MULTIMEDIA_HDA) {
1031                         snprintf(desc, sizeof(desc),
1032                             "%s (0x%04x)",
1033                             hdac_devices[i].desc, pci_get_device(dev));
1034                         result = BUS_PROBE_GENERIC;
1035                         break;
1036                 }
1037         }
1038         if (result == ENXIO && class == PCIC_MULTIMEDIA &&
1039             subclass == PCIS_MULTIMEDIA_HDA) {
1040                 snprintf(desc, sizeof(desc), "Generic (0x%08x)", model);
1041                 result = BUS_PROBE_GENERIC;
1042         }
1043         if (result != ENXIO) {
1044                 strlcat(desc, " HDA Controller", sizeof(desc));
1045                 device_set_desc_copy(dev, desc);
1046         }
1047
1048         return (result);
1049 }
1050
1051 static void
1052 hdac_unsolq_task(void *context, int pending)
1053 {
1054         struct hdac_softc *sc;
1055
1056         sc = (struct hdac_softc *)context;
1057
1058         hdac_lock(sc);
1059         hdac_unsolq_flush(sc);
1060         hdac_unlock(sc);
1061 }
1062
1063 /****************************************************************************
1064  * int hdac_attach(device_t)
1065  *
1066  * Attach the device into the kernel. Interrupts usually won't be enabled
1067  * when this function is called. Setup everything that doesn't require
1068  * interrupts and defer probing of codecs until interrupts are enabled.
1069  ****************************************************************************/
1070 static int
1071 hdac_attach(device_t dev)
1072 {
1073         struct hdac_softc *sc;
1074         int result;
1075         int i, devid = -1;
1076         uint32_t model;
1077         uint16_t class, subclass;
1078         uint16_t vendor;
1079         uint8_t v;
1080
1081         sc = device_get_softc(dev);
1082         HDA_BOOTVERBOSE(
1083                 device_printf(dev, "PCI card vendor: 0x%04x, device: 0x%04x\n",
1084                     pci_get_subvendor(dev), pci_get_subdevice(dev));
1085                 device_printf(dev, "HDA Driver Revision: %s\n",
1086                     HDA_DRV_TEST_REV);
1087         );
1088
1089         model = (uint32_t)pci_get_device(dev) << 16;
1090         model |= (uint32_t)pci_get_vendor(dev) & 0x0000ffff;
1091         class = pci_get_class(dev);
1092         subclass = pci_get_subclass(dev);
1093
1094         for (i = 0; i < nitems(hdac_devices); i++) {
1095                 if (hdac_devices[i].model == model) {
1096                         devid = i;
1097                         break;
1098                 }
1099                 if (HDA_DEV_MATCH(hdac_devices[i].model, model) &&
1100                     class == PCIC_MULTIMEDIA &&
1101                     subclass == PCIS_MULTIMEDIA_HDA) {
1102                         devid = i;
1103                         break;
1104                 }
1105         }
1106
1107         sc->lock = snd_mtxcreate(device_get_nameunit(dev), "HDA driver mutex");
1108         sc->dev = dev;
1109         TASK_INIT(&sc->unsolq_task, 0, hdac_unsolq_task, sc);
1110         callout_init(&sc->poll_callout, CALLOUT_MPSAFE);
1111         for (i = 0; i < HDAC_CODEC_MAX; i++)
1112                 sc->codecs[i].dev = NULL;
1113         if (devid >= 0) {
1114                 sc->quirks_on = hdac_devices[devid].quirks_on;
1115                 sc->quirks_off = hdac_devices[devid].quirks_off;
1116         } else {
1117                 sc->quirks_on = 0;
1118                 sc->quirks_off = 0;
1119         }
1120         if (resource_int_value(device_get_name(dev),
1121             device_get_unit(dev), "msi", &i) == 0) {
1122                 if (i == 0)
1123                         sc->quirks_off |= HDAC_QUIRK_MSI;
1124                 else {
1125                         sc->quirks_on |= HDAC_QUIRK_MSI;
1126                         sc->quirks_off |= ~HDAC_QUIRK_MSI;
1127                 }
1128         }
1129         hdac_config_fetch(sc, &sc->quirks_on, &sc->quirks_off);
1130         HDA_BOOTVERBOSE(
1131                 device_printf(sc->dev,
1132                     "Config options: on=0x%08x off=0x%08x\n",
1133                     sc->quirks_on, sc->quirks_off);
1134         );
1135         sc->poll_ival = hz;
1136         if (resource_int_value(device_get_name(dev),
1137             device_get_unit(dev), "polling", &i) == 0 && i != 0)
1138                 sc->polling = 1;
1139         else
1140                 sc->polling = 0;
1141
1142         pci_enable_busmaster(dev);
1143
1144         vendor = pci_get_vendor(dev);
1145         if (vendor == INTEL_VENDORID) {
1146                 /* TCSEL -> TC0 */
1147                 v = pci_read_config(dev, 0x44, 1);
1148                 pci_write_config(dev, 0x44, v & 0xf8, 1);
1149                 HDA_BOOTHVERBOSE(
1150                         device_printf(dev, "TCSEL: 0x%02d -> 0x%02d\n", v,
1151                             pci_read_config(dev, 0x44, 1));
1152                 );
1153         }
1154
1155 #if defined(__i386__) || defined(__amd64__)
1156         sc->flags |= HDAC_F_DMA_NOCACHE;
1157
1158         if (resource_int_value(device_get_name(dev),
1159             device_get_unit(dev), "snoop", &i) == 0 && i != 0) {
1160 #else
1161         sc->flags &= ~HDAC_F_DMA_NOCACHE;
1162 #endif
1163                 /*
1164                  * Try to enable PCIe snoop to avoid messing around with
1165                  * uncacheable DMA attribute. Since PCIe snoop register
1166                  * config is pretty much vendor specific, there are no
1167                  * general solutions on how to enable it, forcing us (even
1168                  * Microsoft) to enable uncacheable or write combined DMA
1169                  * by default.
1170                  *
1171                  * http://msdn2.microsoft.com/en-us/library/ms790324.aspx
1172                  */
1173                 for (i = 0; i < nitems(hdac_pcie_snoop); i++) {
1174                         if (hdac_pcie_snoop[i].vendor != vendor)
1175                                 continue;
1176                         sc->flags &= ~HDAC_F_DMA_NOCACHE;
1177                         if (hdac_pcie_snoop[i].reg == 0x00)
1178                                 break;
1179                         v = pci_read_config(dev, hdac_pcie_snoop[i].reg, 1);
1180                         if ((v & hdac_pcie_snoop[i].enable) ==
1181                             hdac_pcie_snoop[i].enable)
1182                                 break;
1183                         v &= hdac_pcie_snoop[i].mask;
1184                         v |= hdac_pcie_snoop[i].enable;
1185                         pci_write_config(dev, hdac_pcie_snoop[i].reg, v, 1);
1186                         v = pci_read_config(dev, hdac_pcie_snoop[i].reg, 1);
1187                         if ((v & hdac_pcie_snoop[i].enable) !=
1188                             hdac_pcie_snoop[i].enable) {
1189                                 HDA_BOOTVERBOSE(
1190                                         device_printf(dev,
1191                                             "WARNING: Failed to enable PCIe "
1192                                             "snoop!\n");
1193                                 );
1194 #if defined(__i386__) || defined(__amd64__)
1195                                 sc->flags |= HDAC_F_DMA_NOCACHE;
1196 #endif
1197                         }
1198                         break;
1199                 }
1200 #if defined(__i386__) || defined(__amd64__)
1201         }
1202 #endif
1203
1204         HDA_BOOTHVERBOSE(
1205                 device_printf(dev, "DMA Coherency: %s / vendor=0x%04x\n",
1206                     (sc->flags & HDAC_F_DMA_NOCACHE) ?
1207                     "Uncacheable" : "PCIe snoop", vendor);
1208         );
1209
1210         /* Allocate resources */
1211         result = hdac_mem_alloc(sc);
1212         if (result != 0)
1213                 goto hdac_attach_fail;
1214         result = hdac_irq_alloc(sc);
1215         if (result != 0)
1216                 goto hdac_attach_fail;
1217
1218         /* Get Capabilities */
1219         result = hdac_get_capabilities(sc);
1220         if (result != 0)
1221                 goto hdac_attach_fail;
1222
1223         /* Allocate CORB, RIRB, POS and BDLs dma memory */
1224         result = hdac_dma_alloc(sc, &sc->corb_dma,
1225             sc->corb_size * sizeof(uint32_t));
1226         if (result != 0)
1227                 goto hdac_attach_fail;
1228         result = hdac_dma_alloc(sc, &sc->rirb_dma,
1229             sc->rirb_size * sizeof(struct hdac_rirb));
1230         if (result != 0)
1231                 goto hdac_attach_fail;
1232         sc->streams = malloc(sizeof(struct hdac_stream) * sc->num_ss,
1233             M_HDAC, M_ZERO | M_WAITOK);
1234         for (i = 0; i < sc->num_ss; i++) {
1235                 result = hdac_dma_alloc(sc, &sc->streams[i].bdl,
1236                     sizeof(struct hdac_bdle) * HDA_BDL_MAX);
1237                 if (result != 0)
1238                         goto hdac_attach_fail;
1239         }
1240         if (sc->quirks_on & HDAC_QUIRK_DMAPOS) {
1241                 if (hdac_dma_alloc(sc, &sc->pos_dma, (sc->num_ss) * 8) != 0) {
1242                         HDA_BOOTVERBOSE(
1243                                 device_printf(dev, "Failed to "
1244                                     "allocate DMA pos buffer "
1245                                     "(non-fatal)\n");
1246                         );
1247                 } else {
1248                         uint64_t addr = sc->pos_dma.dma_paddr;
1249
1250                         HDAC_WRITE_4(&sc->mem, HDAC_DPIBUBASE, addr >> 32);
1251                         HDAC_WRITE_4(&sc->mem, HDAC_DPIBLBASE,
1252                             (addr & HDAC_DPLBASE_DPLBASE_MASK) |
1253                             HDAC_DPLBASE_DPLBASE_DMAPBE);
1254                 }
1255         }
1256
1257         result = bus_dma_tag_create(
1258             bus_get_dma_tag(sc->dev),           /* parent */
1259             HDA_DMA_ALIGNMENT,                  /* alignment */
1260             0,                                  /* boundary */
1261             (sc->support_64bit) ? BUS_SPACE_MAXADDR :
1262                 BUS_SPACE_MAXADDR_32BIT,        /* lowaddr */
1263             BUS_SPACE_MAXADDR,                  /* highaddr */
1264             NULL,                               /* filtfunc */
1265             NULL,                               /* fistfuncarg */
1266             HDA_BUFSZ_MAX,                      /* maxsize */
1267             1,                                  /* nsegments */
1268             HDA_BUFSZ_MAX,                      /* maxsegsz */
1269             0,                                  /* flags */
1270             NULL,                               /* lockfunc */
1271             NULL,                               /* lockfuncarg */
1272             &sc->chan_dmat);                    /* dmat */
1273         if (result != 0) {
1274                 device_printf(dev, "%s: bus_dma_tag_create failed (%x)\n",
1275                      __func__, result);
1276                 goto hdac_attach_fail;
1277         }
1278
1279         /* Quiesce everything */
1280         HDA_BOOTHVERBOSE(
1281                 device_printf(dev, "Reset controller...\n");
1282         );
1283         hdac_reset(sc, 1);
1284
1285         /* Initialize the CORB and RIRB */
1286         hdac_corb_init(sc);
1287         hdac_rirb_init(sc);
1288
1289         /* Defer remaining of initialization until interrupts are enabled */
1290         sc->intrhook.ich_func = hdac_attach2;
1291         sc->intrhook.ich_arg = (void *)sc;
1292         if (cold == 0 || config_intrhook_establish(&sc->intrhook) != 0) {
1293                 sc->intrhook.ich_func = NULL;
1294                 hdac_attach2((void *)sc);
1295         }
1296
1297         return (0);
1298
1299 hdac_attach_fail:
1300         hdac_irq_free(sc);
1301         for (i = 0; i < sc->num_ss; i++)
1302                 hdac_dma_free(sc, &sc->streams[i].bdl);
1303         free(sc->streams, M_HDAC);
1304         hdac_dma_free(sc, &sc->rirb_dma);
1305         hdac_dma_free(sc, &sc->corb_dma);
1306         hdac_mem_free(sc);
1307         snd_mtxfree(sc->lock);
1308
1309         return (ENXIO);
1310 }
1311
1312 static int
1313 sysctl_hdac_pindump(SYSCTL_HANDLER_ARGS)
1314 {
1315         struct hdac_softc *sc;
1316         device_t *devlist;
1317         device_t dev;
1318         int devcount, i, err, val;
1319
1320         dev = oidp->oid_arg1;
1321         sc = device_get_softc(dev);
1322         if (sc == NULL)
1323                 return (EINVAL);
1324         val = 0;
1325         err = sysctl_handle_int(oidp, &val, 0, req);
1326         if (err != 0 || req->newptr == NULL || val == 0)
1327                 return (err);
1328
1329         /* XXX: Temporary. For debugging. */
1330         if (val == 100) {
1331                 hdac_suspend(dev);
1332                 return (0);
1333         } else if (val == 101) {
1334                 hdac_resume(dev);
1335                 return (0);
1336         }
1337
1338         if ((err = device_get_children(dev, &devlist, &devcount)) != 0)
1339                 return (err);
1340         hdac_lock(sc);
1341         for (i = 0; i < devcount; i++)
1342                 HDAC_PINDUMP(devlist[i]);
1343         hdac_unlock(sc);
1344         free(devlist, M_TEMP);
1345         return (0);
1346 }
1347
1348 static int
1349 hdac_mdata_rate(uint16_t fmt)
1350 {
1351         static const int mbits[8] = { 8, 16, 32, 32, 32, 32, 32, 32 };
1352         int rate, bits;
1353
1354         if (fmt & (1 << 14))
1355                 rate = 44100;
1356         else
1357                 rate = 48000;
1358         rate *= ((fmt >> 11) & 0x07) + 1;
1359         rate /= ((fmt >> 8) & 0x07) + 1;
1360         bits = mbits[(fmt >> 4) & 0x03];
1361         bits *= (fmt & 0x0f) + 1;
1362         return (rate * bits);
1363 }
1364
1365 static int
1366 hdac_bdata_rate(uint16_t fmt, int output)
1367 {
1368         static const int bbits[8] = { 8, 16, 20, 24, 32, 32, 32, 32 };
1369         int rate, bits;
1370
1371         rate = 48000;
1372         rate *= ((fmt >> 11) & 0x07) + 1;
1373         bits = bbits[(fmt >> 4) & 0x03];
1374         bits *= (fmt & 0x0f) + 1;
1375         if (!output)
1376                 bits = ((bits + 7) & ~0x07) + 10;
1377         return (rate * bits);
1378 }
1379
1380 static void
1381 hdac_poll_reinit(struct hdac_softc *sc)
1382 {
1383         int i, pollticks, min = 1000000;
1384         struct hdac_stream *s;
1385
1386         if (sc->polling == 0)
1387                 return;
1388         if (sc->unsol_registered > 0)
1389                 min = hz / 2;
1390         for (i = 0; i < sc->num_ss; i++) {
1391                 s = &sc->streams[i];
1392                 if (s->running == 0)
1393                         continue;
1394                 pollticks = ((uint64_t)hz * s->blksz) /
1395                     (hdac_mdata_rate(s->format) / 8);
1396                 pollticks >>= 1;
1397                 if (pollticks > hz)
1398                         pollticks = hz;
1399                 if (pollticks < 1) {
1400                         HDA_BOOTVERBOSE(
1401                                 device_printf(sc->dev,
1402                                     "poll interval < 1 tick !\n");
1403                         );
1404                         pollticks = 1;
1405                 }
1406                 if (min > pollticks)
1407                         min = pollticks;
1408         }
1409         HDA_BOOTVERBOSE(
1410                 device_printf(sc->dev,
1411                     "poll interval %d -> %d ticks\n",
1412                     sc->poll_ival, min);
1413         );
1414         sc->poll_ival = min;
1415         if (min == 1000000)
1416                 callout_stop(&sc->poll_callout);
1417         else
1418                 callout_reset(&sc->poll_callout, 1, hdac_poll_callback, sc);
1419 }
1420
1421 static int
1422 sysctl_hdac_polling(SYSCTL_HANDLER_ARGS)
1423 {
1424         struct hdac_softc *sc;
1425         device_t dev;
1426         uint32_t ctl;
1427         int err, val;
1428
1429         dev = oidp->oid_arg1;
1430         sc = device_get_softc(dev);
1431         if (sc == NULL)
1432                 return (EINVAL);
1433         hdac_lock(sc);
1434         val = sc->polling;
1435         hdac_unlock(sc);
1436         err = sysctl_handle_int(oidp, &val, 0, req);
1437
1438         if (err != 0 || req->newptr == NULL)
1439                 return (err);
1440         if (val < 0 || val > 1)
1441                 return (EINVAL);
1442
1443         hdac_lock(sc);
1444         if (val != sc->polling) {
1445                 if (val == 0) {
1446                         callout_stop(&sc->poll_callout);
1447                         hdac_unlock(sc);
1448                         callout_drain(&sc->poll_callout);
1449                         hdac_lock(sc);
1450                         sc->polling = 0;
1451                         ctl = HDAC_READ_4(&sc->mem, HDAC_INTCTL);
1452                         ctl |= HDAC_INTCTL_GIE;
1453                         HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, ctl);
1454                 } else {
1455                         ctl = HDAC_READ_4(&sc->mem, HDAC_INTCTL);
1456                         ctl &= ~HDAC_INTCTL_GIE;
1457                         HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, ctl);
1458                         sc->polling = 1;
1459                         hdac_poll_reinit(sc);
1460                 }
1461         }
1462         hdac_unlock(sc);
1463
1464         return (err);
1465 }
1466
1467 static void
1468 hdac_attach2(void *arg)
1469 {
1470         struct hdac_softc *sc;
1471         device_t child;
1472         uint32_t vendorid, revisionid;
1473         int i;
1474         uint16_t statests;
1475
1476         sc = (struct hdac_softc *)arg;
1477
1478         hdac_lock(sc);
1479
1480         /* Remove ourselves from the config hooks */
1481         if (sc->intrhook.ich_func != NULL) {
1482                 config_intrhook_disestablish(&sc->intrhook);
1483                 sc->intrhook.ich_func = NULL;
1484         }
1485
1486         HDA_BOOTHVERBOSE(
1487                 device_printf(sc->dev, "Starting CORB Engine...\n");
1488         );
1489         hdac_corb_start(sc);
1490         HDA_BOOTHVERBOSE(
1491                 device_printf(sc->dev, "Starting RIRB Engine...\n");
1492         );
1493         hdac_rirb_start(sc);
1494         HDA_BOOTHVERBOSE(
1495                 device_printf(sc->dev,
1496                     "Enabling controller interrupt...\n");
1497         );
1498         HDAC_WRITE_4(&sc->mem, HDAC_GCTL, HDAC_READ_4(&sc->mem, HDAC_GCTL) |
1499             HDAC_GCTL_UNSOL);
1500         if (sc->polling == 0) {
1501                 HDAC_WRITE_4(&sc->mem, HDAC_INTCTL,
1502                     HDAC_INTCTL_CIE | HDAC_INTCTL_GIE);
1503         }
1504         DELAY(1000);
1505
1506         HDA_BOOTHVERBOSE(
1507                 device_printf(sc->dev, "Scanning HDA codecs ...\n");
1508         );
1509         statests = HDAC_READ_2(&sc->mem, HDAC_STATESTS);
1510         hdac_unlock(sc);
1511         for (i = 0; i < HDAC_CODEC_MAX; i++) {
1512                 if (HDAC_STATESTS_SDIWAKE(statests, i)) {
1513                         HDA_BOOTHVERBOSE(
1514                                 device_printf(sc->dev,
1515                                     "Found CODEC at address %d\n", i);
1516                         );
1517                         hdac_lock(sc);
1518                         vendorid = hdac_send_command(sc, i,
1519                             HDA_CMD_GET_PARAMETER(0, 0x0, HDA_PARAM_VENDOR_ID));
1520                         revisionid = hdac_send_command(sc, i,
1521                             HDA_CMD_GET_PARAMETER(0, 0x0, HDA_PARAM_REVISION_ID));
1522                         hdac_unlock(sc);
1523                         if (vendorid == HDA_INVALID &&
1524                             revisionid == HDA_INVALID) {
1525                                 device_printf(sc->dev,
1526                                     "CODEC is not responding!\n");
1527                                 continue;
1528                         }
1529                         sc->codecs[i].vendor_id =
1530                             HDA_PARAM_VENDOR_ID_VENDOR_ID(vendorid);
1531                         sc->codecs[i].device_id =
1532                             HDA_PARAM_VENDOR_ID_DEVICE_ID(vendorid);
1533                         sc->codecs[i].revision_id =
1534                             HDA_PARAM_REVISION_ID_REVISION_ID(revisionid);
1535                         sc->codecs[i].stepping_id =
1536                             HDA_PARAM_REVISION_ID_STEPPING_ID(revisionid);
1537                         child = device_add_child(sc->dev, "hdacc", -1);
1538                         if (child == NULL) {
1539                                 device_printf(sc->dev,
1540                                     "Failed to add CODEC device\n");
1541                                 continue;
1542                         }
1543                         device_set_ivars(child, (void *)(intptr_t)i);
1544                         sc->codecs[i].dev = child;
1545                 }
1546         }
1547         bus_generic_attach(sc->dev);
1548
1549         SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev),
1550             SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO,
1551             "pindump", CTLTYPE_INT | CTLFLAG_RW, sc->dev, sizeof(sc->dev),
1552             sysctl_hdac_pindump, "I", "Dump pin states/data");
1553         SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev),
1554             SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO,
1555             "polling", CTLTYPE_INT | CTLFLAG_RW, sc->dev, sizeof(sc->dev),
1556             sysctl_hdac_polling, "I", "Enable polling mode");
1557 }
1558
1559 /****************************************************************************
1560  * int hdac_suspend(device_t)
1561  *
1562  * Suspend and power down HDA bus and codecs.
1563  ****************************************************************************/
1564 static int
1565 hdac_suspend(device_t dev)
1566 {
1567         struct hdac_softc *sc = device_get_softc(dev);
1568
1569         HDA_BOOTHVERBOSE(
1570                 device_printf(dev, "Suspend...\n");
1571         );
1572         bus_generic_suspend(dev);
1573
1574         hdac_lock(sc);
1575         HDA_BOOTHVERBOSE(
1576                 device_printf(dev, "Reset controller...\n");
1577         );
1578         callout_stop(&sc->poll_callout);
1579         hdac_reset(sc, 0);
1580         hdac_unlock(sc);
1581         callout_drain(&sc->poll_callout);
1582         taskqueue_drain(taskqueue_thread, &sc->unsolq_task);
1583         HDA_BOOTHVERBOSE(
1584                 device_printf(dev, "Suspend done\n");
1585         );
1586         return (0);
1587 }
1588
1589 /****************************************************************************
1590  * int hdac_resume(device_t)
1591  *
1592  * Powerup and restore HDA bus and codecs state.
1593  ****************************************************************************/
1594 static int
1595 hdac_resume(device_t dev)
1596 {
1597         struct hdac_softc *sc = device_get_softc(dev);
1598         int error;
1599
1600         HDA_BOOTHVERBOSE(
1601                 device_printf(dev, "Resume...\n");
1602         );
1603         hdac_lock(sc);
1604
1605         /* Quiesce everything */
1606         HDA_BOOTHVERBOSE(
1607                 device_printf(dev, "Reset controller...\n");
1608         );
1609         hdac_reset(sc, 1);
1610
1611         /* Initialize the CORB and RIRB */
1612         hdac_corb_init(sc);
1613         hdac_rirb_init(sc);
1614
1615         HDA_BOOTHVERBOSE(
1616                 device_printf(dev, "Starting CORB Engine...\n");
1617         );
1618         hdac_corb_start(sc);
1619         HDA_BOOTHVERBOSE(
1620                 device_printf(dev, "Starting RIRB Engine...\n");
1621         );
1622         hdac_rirb_start(sc);
1623         HDA_BOOTHVERBOSE(
1624                 device_printf(dev, "Enabling controller interrupt...\n");
1625         );
1626         HDAC_WRITE_4(&sc->mem, HDAC_GCTL, HDAC_READ_4(&sc->mem, HDAC_GCTL) |
1627             HDAC_GCTL_UNSOL);
1628         HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, HDAC_INTCTL_CIE | HDAC_INTCTL_GIE);
1629         DELAY(1000);
1630         hdac_poll_reinit(sc);
1631         hdac_unlock(sc);
1632
1633         error = bus_generic_resume(dev);
1634         HDA_BOOTHVERBOSE(
1635                 device_printf(dev, "Resume done\n");
1636         );
1637         return (error);
1638 }
1639
1640 /****************************************************************************
1641  * int hdac_detach(device_t)
1642  *
1643  * Detach and free up resources utilized by the hdac device.
1644  ****************************************************************************/
1645 static int
1646 hdac_detach(device_t dev)
1647 {
1648         struct hdac_softc *sc = device_get_softc(dev);
1649         device_t *devlist;
1650         int cad, i, devcount, error;
1651
1652         if ((error = device_get_children(dev, &devlist, &devcount)) != 0)
1653                 return (error);
1654         for (i = 0; i < devcount; i++) {
1655                 cad = (intptr_t)device_get_ivars(devlist[i]);
1656                 if ((error = device_delete_child(dev, devlist[i])) != 0) {
1657                         free(devlist, M_TEMP);
1658                         return (error);
1659                 }
1660                 sc->codecs[cad].dev = NULL;
1661         }
1662         free(devlist, M_TEMP);
1663
1664         hdac_lock(sc);
1665         hdac_reset(sc, 0);
1666         hdac_unlock(sc);
1667         taskqueue_drain(taskqueue_thread, &sc->unsolq_task);
1668         hdac_irq_free(sc);
1669
1670         for (i = 0; i < sc->num_ss; i++)
1671                 hdac_dma_free(sc, &sc->streams[i].bdl);
1672         free(sc->streams, M_HDAC);
1673         hdac_dma_free(sc, &sc->pos_dma);
1674         hdac_dma_free(sc, &sc->rirb_dma);
1675         hdac_dma_free(sc, &sc->corb_dma);
1676         if (sc->chan_dmat != NULL) {
1677                 bus_dma_tag_destroy(sc->chan_dmat);
1678                 sc->chan_dmat = NULL;
1679         }
1680         hdac_mem_free(sc);
1681         snd_mtxfree(sc->lock);
1682         return (0);
1683 }
1684
1685 static bus_dma_tag_t
1686 hdac_get_dma_tag(device_t dev, device_t child)
1687 {
1688         struct hdac_softc *sc = device_get_softc(dev);
1689
1690         return (sc->chan_dmat);
1691 }
1692
1693 static int
1694 hdac_print_child(device_t dev, device_t child)
1695 {
1696         int retval;
1697
1698         retval = bus_print_child_header(dev, child);
1699         retval += printf(" at cad %d",
1700             (int)(intptr_t)device_get_ivars(child));
1701         retval += bus_print_child_footer(dev, child);
1702
1703         return (retval);
1704 }
1705
1706 static int
1707 hdac_child_location_str(device_t dev, device_t child, char *buf,
1708     size_t buflen)
1709 {
1710
1711         snprintf(buf, buflen, "cad=%d",
1712             (int)(intptr_t)device_get_ivars(child));
1713         return (0);
1714 }
1715
1716 static int
1717 hdac_child_pnpinfo_str_method(device_t dev, device_t child, char *buf,
1718     size_t buflen)
1719 {
1720         struct hdac_softc *sc = device_get_softc(dev);
1721         nid_t cad = (uintptr_t)device_get_ivars(child);
1722
1723         snprintf(buf, buflen, "vendor=0x%04x device=0x%04x revision=0x%02x "
1724             "stepping=0x%02x",
1725             sc->codecs[cad].vendor_id, sc->codecs[cad].device_id,
1726             sc->codecs[cad].revision_id, sc->codecs[cad].stepping_id);
1727         return (0);
1728 }
1729
1730 static int
1731 hdac_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
1732 {
1733         struct hdac_softc *sc = device_get_softc(dev);
1734         nid_t cad = (uintptr_t)device_get_ivars(child);
1735
1736         switch (which) {
1737         case HDA_IVAR_CODEC_ID:
1738                 *result = cad;
1739                 break;
1740         case HDA_IVAR_VENDOR_ID:
1741                 *result = sc->codecs[cad].vendor_id;
1742                 break;
1743         case HDA_IVAR_DEVICE_ID:
1744                 *result = sc->codecs[cad].device_id;
1745                 break;
1746         case HDA_IVAR_REVISION_ID:
1747                 *result = sc->codecs[cad].revision_id;
1748                 break;
1749         case HDA_IVAR_STEPPING_ID:
1750                 *result = sc->codecs[cad].stepping_id;
1751                 break;
1752         case HDA_IVAR_SUBVENDOR_ID:
1753                 *result = pci_get_subvendor(dev);
1754                 break;
1755         case HDA_IVAR_SUBDEVICE_ID:
1756                 *result = pci_get_subdevice(dev);
1757                 break;
1758         case HDA_IVAR_DMA_NOCACHE:
1759                 *result = (sc->flags & HDAC_F_DMA_NOCACHE) != 0;
1760                 break;
1761         default:
1762                 return (ENOENT);
1763         }
1764         return (0);
1765 }
1766
1767 static struct mtx *
1768 hdac_get_mtx(device_t dev, device_t child)
1769 {
1770         struct hdac_softc *sc = device_get_softc(dev);
1771
1772         return (sc->lock);
1773 }
1774
1775 static uint32_t
1776 hdac_codec_command(device_t dev, device_t child, uint32_t verb)
1777 {
1778
1779         return (hdac_send_command(device_get_softc(dev),
1780             (intptr_t)device_get_ivars(child), verb));
1781 }
1782
1783 static int
1784 hdac_find_stream(struct hdac_softc *sc, int dir, int stream)
1785 {
1786         int i, ss;
1787
1788         ss = -1;
1789         /* Allocate ISS/BSS first. */
1790         if (dir == 0) {
1791                 for (i = 0; i < sc->num_iss; i++) {
1792                         if (sc->streams[i].stream == stream) {
1793                                 ss = i;
1794                                 break;
1795                         }
1796                 }
1797         } else {
1798                 for (i = 0; i < sc->num_oss; i++) {
1799                         if (sc->streams[i + sc->num_iss].stream == stream) {
1800                                 ss = i + sc->num_iss;
1801                                 break;
1802                         }
1803                 }
1804         }
1805         /* Fallback to BSS. */
1806         if (ss == -1) {
1807                 for (i = 0; i < sc->num_bss; i++) {
1808                         if (sc->streams[i + sc->num_iss + sc->num_oss].stream
1809                             == stream) {
1810                                 ss = i + sc->num_iss + sc->num_oss;
1811                                 break;
1812                         }
1813                 }
1814         }
1815         return (ss);
1816 }
1817
1818 static int
1819 hdac_stream_alloc(device_t dev, device_t child, int dir, int format, int stripe,
1820     uint32_t **dmapos)
1821 {
1822         struct hdac_softc *sc = device_get_softc(dev);
1823         nid_t cad = (uintptr_t)device_get_ivars(child);
1824         int stream, ss, bw, maxbw, prevbw;
1825
1826         /* Look for empty stream. */
1827         ss = hdac_find_stream(sc, dir, 0);
1828
1829         /* Return if found nothing. */
1830         if (ss < 0)
1831                 return (0);
1832
1833         /* Check bus bandwidth. */
1834         bw = hdac_bdata_rate(format, dir);
1835         if (dir == 1) {
1836                 bw *= 1 << (sc->num_sdo - stripe);
1837                 prevbw = sc->sdo_bw_used;
1838                 maxbw = 48000 * 960 * (1 << sc->num_sdo);
1839         } else {
1840                 prevbw = sc->codecs[cad].sdi_bw_used;
1841                 maxbw = 48000 * 464;
1842         }
1843         HDA_BOOTHVERBOSE(
1844                 device_printf(dev, "%dKbps of %dKbps bandwidth used%s\n",
1845                     (bw + prevbw) / 1000, maxbw / 1000,
1846                     bw + prevbw > maxbw ? " -- OVERFLOW!" : "");
1847         );
1848         if (bw + prevbw > maxbw)
1849                 return (0);
1850         if (dir == 1)
1851                 sc->sdo_bw_used += bw;
1852         else
1853                 sc->codecs[cad].sdi_bw_used += bw;
1854
1855         /* Allocate stream number */
1856         if (ss >= sc->num_iss + sc->num_oss)
1857                 stream = 15 - (ss - sc->num_iss + sc->num_oss);
1858         else if (ss >= sc->num_iss)
1859                 stream = ss - sc->num_iss + 1;
1860         else
1861                 stream = ss + 1;
1862
1863         sc->streams[ss].dev = child;
1864         sc->streams[ss].dir = dir;
1865         sc->streams[ss].stream = stream;
1866         sc->streams[ss].bw = bw;
1867         sc->streams[ss].format = format;
1868         sc->streams[ss].stripe = stripe;
1869         if (dmapos != NULL) {
1870                 if (sc->pos_dma.dma_vaddr != NULL)
1871                         *dmapos = (uint32_t *)(sc->pos_dma.dma_vaddr + ss * 8);
1872                 else
1873                         *dmapos = NULL;
1874         }
1875         return (stream);
1876 }
1877
1878 static void
1879 hdac_stream_free(device_t dev, device_t child, int dir, int stream)
1880 {
1881         struct hdac_softc *sc = device_get_softc(dev);
1882         nid_t cad = (uintptr_t)device_get_ivars(child);
1883         int ss;
1884
1885         ss = hdac_find_stream(sc, dir, stream);
1886         KASSERT(ss >= 0,
1887             ("Free for not allocated stream (%d/%d)\n", dir, stream));
1888         if (dir == 1)
1889                 sc->sdo_bw_used -= sc->streams[ss].bw;
1890         else
1891                 sc->codecs[cad].sdi_bw_used -= sc->streams[ss].bw;
1892         sc->streams[ss].stream = 0;
1893         sc->streams[ss].dev = NULL;
1894 }
1895
1896 static int
1897 hdac_stream_start(device_t dev, device_t child,
1898     int dir, int stream, bus_addr_t buf, int blksz, int blkcnt)
1899 {
1900         struct hdac_softc *sc = device_get_softc(dev);
1901         struct hdac_bdle *bdle;
1902         uint64_t addr;
1903         int i, ss, off;
1904         uint32_t ctl;
1905
1906         ss = hdac_find_stream(sc, dir, stream);
1907         KASSERT(ss >= 0,
1908             ("Start for not allocated stream (%d/%d)\n", dir, stream));
1909
1910         addr = (uint64_t)buf;
1911         bdle = (struct hdac_bdle *)sc->streams[ss].bdl.dma_vaddr;
1912         for (i = 0; i < blkcnt; i++, bdle++) {
1913                 bdle->addrl = (uint32_t)addr;
1914                 bdle->addrh = (uint32_t)(addr >> 32);
1915                 bdle->len = blksz;
1916                 bdle->ioc = 1;
1917                 addr += blksz;
1918         }
1919
1920         off = ss << 5;
1921         HDAC_WRITE_4(&sc->mem, off + HDAC_SDCBL, blksz * blkcnt);
1922         HDAC_WRITE_2(&sc->mem, off + HDAC_SDLVI, blkcnt - 1);
1923         addr = sc->streams[ss].bdl.dma_paddr;
1924         HDAC_WRITE_4(&sc->mem, off + HDAC_SDBDPL, (uint32_t)addr);
1925         HDAC_WRITE_4(&sc->mem, off + HDAC_SDBDPU, (uint32_t)(addr >> 32));
1926
1927         ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL2);
1928         if (dir)
1929                 ctl |= HDAC_SDCTL2_DIR;
1930         else
1931                 ctl &= ~HDAC_SDCTL2_DIR;
1932         ctl &= ~HDAC_SDCTL2_STRM_MASK;
1933         ctl |= stream << HDAC_SDCTL2_STRM_SHIFT;
1934         ctl &= ~HDAC_SDCTL2_STRIPE_MASK;
1935         ctl |= sc->streams[ss].stripe << HDAC_SDCTL2_STRIPE_SHIFT;
1936         HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL2, ctl);
1937
1938         HDAC_WRITE_2(&sc->mem, off + HDAC_SDFMT, sc->streams[ss].format);
1939
1940         ctl = HDAC_READ_4(&sc->mem, HDAC_INTCTL);
1941         ctl |= 1 << ss;
1942         HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, ctl);
1943
1944         HDAC_WRITE_1(&sc->mem, off + HDAC_SDSTS,
1945             HDAC_SDSTS_DESE | HDAC_SDSTS_FIFOE | HDAC_SDSTS_BCIS);
1946         ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
1947         ctl |= HDAC_SDCTL_IOCE | HDAC_SDCTL_FEIE | HDAC_SDCTL_DEIE |
1948             HDAC_SDCTL_RUN;
1949         HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL0, ctl);
1950
1951         sc->streams[ss].blksz = blksz;
1952         sc->streams[ss].running = 1;
1953         hdac_poll_reinit(sc);
1954         return (0);
1955 }
1956
1957 static void
1958 hdac_stream_stop(device_t dev, device_t child, int dir, int stream)
1959 {
1960         struct hdac_softc *sc = device_get_softc(dev);
1961         int ss, off;
1962         uint32_t ctl;
1963
1964         ss = hdac_find_stream(sc, dir, stream);
1965         KASSERT(ss >= 0,
1966             ("Stop for not allocated stream (%d/%d)\n", dir, stream));
1967
1968         off = ss << 5;
1969         ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
1970         ctl &= ~(HDAC_SDCTL_IOCE | HDAC_SDCTL_FEIE | HDAC_SDCTL_DEIE |
1971             HDAC_SDCTL_RUN);
1972         HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL0, ctl);
1973
1974         ctl = HDAC_READ_4(&sc->mem, HDAC_INTCTL);
1975         ctl &= ~(1 << ss);
1976         HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, ctl);
1977
1978         sc->streams[ss].running = 0;
1979         hdac_poll_reinit(sc);
1980 }
1981
1982 static void
1983 hdac_stream_reset(device_t dev, device_t child, int dir, int stream)
1984 {
1985         struct hdac_softc *sc = device_get_softc(dev);
1986         int timeout = 1000;
1987         int to = timeout;
1988         int ss, off;
1989         uint32_t ctl;
1990
1991         ss = hdac_find_stream(sc, dir, stream);
1992         KASSERT(ss >= 0,
1993             ("Reset for not allocated stream (%d/%d)\n", dir, stream));
1994
1995         off = ss << 5;
1996         ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
1997         ctl |= HDAC_SDCTL_SRST;
1998         HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL0, ctl);
1999         do {
2000                 ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
2001                 if (ctl & HDAC_SDCTL_SRST)
2002                         break;
2003                 DELAY(10);
2004         } while (--to);
2005         if (!(ctl & HDAC_SDCTL_SRST))
2006                 device_printf(dev, "Reset setting timeout\n");
2007         ctl &= ~HDAC_SDCTL_SRST;
2008         HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL0, ctl);
2009         to = timeout;
2010         do {
2011                 ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
2012                 if (!(ctl & HDAC_SDCTL_SRST))
2013                         break;
2014                 DELAY(10);
2015         } while (--to);
2016         if (ctl & HDAC_SDCTL_SRST)
2017                 device_printf(dev, "Reset timeout!\n");
2018 }
2019
2020 static uint32_t
2021 hdac_stream_getptr(device_t dev, device_t child, int dir, int stream)
2022 {
2023         struct hdac_softc *sc = device_get_softc(dev);
2024         int ss, off;
2025
2026         ss = hdac_find_stream(sc, dir, stream);
2027         KASSERT(ss >= 0,
2028             ("Reset for not allocated stream (%d/%d)\n", dir, stream));
2029
2030         off = ss << 5;
2031         return (HDAC_READ_4(&sc->mem, off + HDAC_SDLPIB));
2032 }
2033
2034 static int
2035 hdac_unsol_alloc(device_t dev, device_t child, int tag)
2036 {
2037         struct hdac_softc *sc = device_get_softc(dev);
2038
2039         sc->unsol_registered++;
2040         hdac_poll_reinit(sc);
2041         return (tag);
2042 }
2043
2044 static void
2045 hdac_unsol_free(device_t dev, device_t child, int tag)
2046 {
2047         struct hdac_softc *sc = device_get_softc(dev);
2048
2049         sc->unsol_registered--;
2050         hdac_poll_reinit(sc);
2051 }
2052
2053 static device_method_t hdac_methods[] = {
2054         /* device interface */
2055         DEVMETHOD(device_probe,         hdac_probe),
2056         DEVMETHOD(device_attach,        hdac_attach),
2057         DEVMETHOD(device_detach,        hdac_detach),
2058         DEVMETHOD(device_suspend,       hdac_suspend),
2059         DEVMETHOD(device_resume,        hdac_resume),
2060         /* Bus interface */
2061         DEVMETHOD(bus_get_dma_tag,      hdac_get_dma_tag),
2062         DEVMETHOD(bus_print_child,      hdac_print_child),
2063         DEVMETHOD(bus_child_location_str, hdac_child_location_str),
2064         DEVMETHOD(bus_child_pnpinfo_str, hdac_child_pnpinfo_str_method),
2065         DEVMETHOD(bus_read_ivar,        hdac_read_ivar),
2066         DEVMETHOD(hdac_get_mtx,         hdac_get_mtx),
2067         DEVMETHOD(hdac_codec_command,   hdac_codec_command),
2068         DEVMETHOD(hdac_stream_alloc,    hdac_stream_alloc),
2069         DEVMETHOD(hdac_stream_free,     hdac_stream_free),
2070         DEVMETHOD(hdac_stream_start,    hdac_stream_start),
2071         DEVMETHOD(hdac_stream_stop,     hdac_stream_stop),
2072         DEVMETHOD(hdac_stream_reset,    hdac_stream_reset),
2073         DEVMETHOD(hdac_stream_getptr,   hdac_stream_getptr),
2074         DEVMETHOD(hdac_unsol_alloc,     hdac_unsol_alloc),
2075         DEVMETHOD(hdac_unsol_free,      hdac_unsol_free),
2076         DEVMETHOD_END
2077 };
2078
2079 static driver_t hdac_driver = {
2080         "hdac",
2081         hdac_methods,
2082         sizeof(struct hdac_softc),
2083 };
2084
2085 static devclass_t hdac_devclass;
2086
2087 DRIVER_MODULE(snd_hda, pci, hdac_driver, hdac_devclass, NULL, NULL);