sys/dev/rtsx/rtsx.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2006 Uwe Stuehler <uwe@openbsd.org>
   5  * Copyright (c) 2012 Stefan Sperling <stsp@openbsd.org>
   6  * Copyright (c) 2020 Henri Hennebert <hlh@restart.be>
   7  * Copyright (c) 2020 Gary Jennejohn <gj@freebsd.org>
   8  * Copyright (c) 2020 Jesper Schmitz Mouridsen <jsm@FreeBSD.org>
   9  * All rights reserved.
  10  *
  11  * Patch from:
  12  * - Lutz Bichler <Lutz.Bichler@gmail.com>
  13  *
  14  * Base on OpenBSD /sys/dev/pci/rtsx_pci.c & /dev/ic/rtsx.c
  15  *      on Linux   /drivers/mmc/host/rtsx_pci_sdmmc.c,
  16  *                 /include/linux/rtsx_pci.h &
  17  *                 /drivers/misc/cardreader/rtsx_pcr.c
  18  *      on NetBSD  /sys/dev/ic/rtsx.c
  19  *
  20  * Permission to use, copy, modify, and distribute this software for any
  21  * purpose with or without fee is hereby granted, provided that the above
  22  * copyright notice and this permission notice appear in all copies.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  34  * SUCH DAMAGE.
  35  */
  36
  37 #include <sys/cdefs.h>
  38 __FBSDID("$FreeBSD$");
  39
  40 #include <sys/param.h>
  41 #include <sys/module.h>
  42 #include <sys/systm.h> /* For FreeBSD 11 */
  43 #include <sys/types.h> /* For FreeBSD 11 */
  44 #include <sys/errno.h>
  45 #include <sys/kernel.h>
  46 #include <sys/bus.h>
  47 #include <sys/endian.h>
  48 #include <machine/bus.h>
  49 #include <sys/mutex.h>
  50 #include <sys/rman.h>
  51 #include <sys/queue.h>
  52 #include <sys/taskqueue.h>
  53 #include <sys/sysctl.h>
  54 #include <dev/pci/pcivar.h>
  55 #include <dev/pci/pcireg.h>
  56 #include <dev/mmc/bridge.h>
  57 #include <dev/mmc/mmcreg.h>
  58 #include <dev/mmc/mmcbrvar.h>
  59 #include <machine/_inttypes.h>
  60
  61 #include "opt_mmccam.h"
  62
  63 #ifdef MMCCAM
  64 #include <cam/cam.h>
  65 #include <cam/cam_ccb.h>
  66 #include <cam/cam_debug.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_xpt_sim.h>
  69 #endif /* MMCCAM */
  70
  71 #include "rtsxreg.h"
  72
  73 /* The softc holds our per-instance data. */
  74 struct rtsx_softc {
  75         struct mtx      rtsx_mtx;               /* device mutex */
  76         device_t        rtsx_dev;               /* device */
  77         uint16_t        rtsx_flags;             /* device flags */
  78         uint16_t        rtsx_device_id;         /* device ID */
  79         device_t        rtsx_mmc_dev;           /* device of mmc bus */
  80         uint32_t        rtsx_intr_enabled;      /* enabled interrupts */
  81         uint32_t        rtsx_intr_status;       /* soft interrupt status */
  82         int             rtsx_irq_res_id;        /* bus IRQ resource id */
  83         struct resource *rtsx_irq_res;          /* bus IRQ resource */
  84         void            *rtsx_irq_cookie;       /* bus IRQ resource cookie */
  85         struct callout  rtsx_timeout_callout;   /* callout for timeout */
  86         int             rtsx_timeout;           /* interrupt timeout value */
  87         void            (*rtsx_intr_trans_ok)(struct rtsx_softc *sc);
  88                                                 /* function to call if transfer succeed */
  89         void            (*rtsx_intr_trans_ko)(struct rtsx_softc *sc);
  90                                                 /* function to call if transfer fail */
  91         struct timeout_task
  92                         rtsx_card_insert_task;  /* card insert delayed task */
  93         struct task     rtsx_card_remove_task;  /* card remove task */
  94
  95         int             rtsx_res_id;            /* bus memory resource id */
  96         struct resource *rtsx_res;              /* bus memory resource */
  97         int             rtsx_res_type;          /* bus memory resource type */
  98         bus_space_tag_t rtsx_btag;              /* host register set tag */
  99         bus_space_handle_t rtsx_bhandle;        /* host register set handle */
 100
 101         bus_dma_tag_t   rtsx_cmd_dma_tag;       /* DMA tag for command transfer */
 102         bus_dmamap_t    rtsx_cmd_dmamap;        /* DMA map for command transfer */
 103         void            *rtsx_cmd_dmamem;       /* DMA mem for command transfer */
 104         bus_addr_t      rtsx_cmd_buffer;        /* device visible address of the DMA segment */
 105         int             rtsx_cmd_index;         /* index in rtsx_cmd_buffer */
 106
 107         bus_dma_tag_t   rtsx_data_dma_tag;      /* DMA tag for data transfer */
 108         bus_dmamap_t    rtsx_data_dmamap;       /* DMA map for data transfer */
 109         void            *rtsx_data_dmamem;      /* DMA mem for data transfer */
 110         bus_addr_t      rtsx_data_buffer;       /* device visible address of the DMA segment */
 111
 112 #ifdef MMCCAM
 113         struct cam_devq         *rtsx_devq;     /* CAM queue of requests */
 114         struct cam_sim          *rtsx_sim;      /* descriptor of our SCSI Interface Modules (SIM) */
 115         struct mtx              rtsx_sim_mtx;   /* SIM mutex */
 116         union ccb               *rtsx_ccb;      /* CAM control block */
 117         struct mmc_request      rtsx_cam_req;   /* CAM MMC request */
 118 #endif /* MMCCAM */
 119
 120         struct mmc_request *rtsx_req;           /* MMC request */
 121         struct mmc_host rtsx_host;              /* host parameters */
 122         int             rtsx_pcie_cap;          /* PCIe capability offset */
 123         int8_t          rtsx_bus_busy;          /* bus busy status */
 124         int8_t          rtsx_ios_bus_width;     /* current host.ios.bus_width */
 125         int32_t         rtsx_ios_clock;         /* current host.ios.clock */
 126         int8_t          rtsx_ios_power_mode;    /* current host.ios.power mode */
 127         int8_t          rtsx_ios_timing;        /* current host.ios.timing */
 128         int8_t          rtsx_ios_vccq;          /* current host.ios.vccq */
 129         uint8_t         rtsx_read_only;         /* card read only status */
 130         uint8_t         rtsx_inversion;         /* inversion of card detection and read only status */
 131         uint8_t         rtsx_force_timing;      /* force bus_timing_uhs_sdr50 */
 132         uint8_t         rtsx_debug;             /* print debugging */
 133 #ifdef MMCCAM
 134         uint8_t         rtsx_cam_status;        /* CAM status - 1 if card in use */
 135 #endif /* MMCCAM */
 136         uint64_t        rtsx_read_count;        /* count of read operations */
 137         uint64_t        rtsx_write_count;       /* count of write operations */
 138         bool            rtsx_discovery_mode;    /* are we in discovery mode? */
 139         bool            rtsx_tuning_mode;       /* are we tuning */
 140         bool            rtsx_double_clk;        /* double clock freqency */
 141         bool            rtsx_vpclk;             /* voltage at Pulse-width Modulation(PWM) clock? */
 142         uint8_t         rtsx_ssc_depth;         /* Spread spectrum clocking depth */
 143         uint8_t         rtsx_card_drive_sel;    /* value for RTSX_CARD_DRIVE_SEL */
 144         uint8_t         rtsx_sd30_drive_sel_3v3;/* value for RTSX_SD30_DRIVE_SEL */
 145 };
 146
 147 /* rtsx_flags values */
 148 #define RTSX_F_DEFAULT          0x0000
 149 #define RTSX_F_CARD_PRESENT     0x0001
 150 #define RTSX_F_SDIO_SUPPORT     0x0002
 151 #define RTSX_F_VERSION_A        0x0004
 152 #define RTSX_F_VERSION_B        0x0008
 153 #define RTSX_F_VERSION_C        0x0010
 154 #define RTSX_F_VERSION_D        0x0020
 155 #define RTSX_F_8411B_QFN48      0x0040
 156 #define RTSX_F_REVERSE_SOCKET   0x0080
 157
 158 #define RTSX_REALTEK            0x10ec
 159 #define RTSX_RTS5209            0x5209
 160 #define RTSX_RTS5227            0x5227
 161 #define RTSX_RTS5229            0x5229
 162 #define RTSX_RTS522A            0x522a
 163 #define RTSX_RTS525A            0x525a
 164 #define RTSX_RTS5249            0x5249
 165 #define RTSX_RTL8402            0x5286
 166 #define RTSX_RTL8411            0x5289
 167 #define RTSX_RTL8411B           0x5287
 168
 169 #define RTSX_VERSION            "2.0c"
 170
 171 static const struct rtsx_device {
 172         uint16_t        vendor_id;
 173         uint16_t        device_id;
 174         const char      *desc;
 175 } rtsx_devices[] = {
 176         { RTSX_REALTEK, RTSX_RTS5209,   RTSX_VERSION " Realtek RTS5209 PCI MMC/SD Card Reader"},
 177         { RTSX_REALTEK, RTSX_RTS5227,   RTSX_VERSION " Realtek RTS5227 PCI MMC/SD Card Reader"},
 178         { RTSX_REALTEK, RTSX_RTS5229,   RTSX_VERSION " Realtek RTS5229 PCI MMC/SD Card Reader"},
 179         { RTSX_REALTEK, RTSX_RTS522A,   RTSX_VERSION " Realtek RTS522A PCI MMC/SD Card Reader"},
 180         { RTSX_REALTEK, RTSX_RTS525A,   RTSX_VERSION " Realtek RTS525A PCI MMC/SD Card Reader"},
 181         { RTSX_REALTEK, RTSX_RTS5249,   RTSX_VERSION " Realtek RTS5249 PCI MMC/SD Card Reader"},
 182         { RTSX_REALTEK, RTSX_RTL8402,   RTSX_VERSION " Realtek RTL8402 PCI MMC/SD Card Reader"},
 183         { RTSX_REALTEK, RTSX_RTL8411,   RTSX_VERSION " Realtek RTL8411 PCI MMC/SD Card Reader"},
 184         { RTSX_REALTEK, RTSX_RTL8411B,  RTSX_VERSION " Realtek RTL8411B PCI MMC/SD Card Reader"},
 185         { 0,            0,              NULL}
 186 };
 187
 188 static int      rtsx_dma_alloc(struct rtsx_softc *sc);
 189 static void     rtsx_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
 190 static void     rtsx_dma_free(struct rtsx_softc *sc);
 191 static void     rtsx_intr(void *arg);
 192 static void     rtsx_handle_card_present(struct rtsx_softc *sc);
 193 static void     rtsx_card_task(void *arg, int pending __unused);
 194 static bool     rtsx_is_card_present(struct rtsx_softc *sc);
 195 static int      rtsx_init(struct rtsx_softc *sc);
 196 static int      rtsx_map_sd_drive(int index);
 197 static int      rtsx_rts5227_fill_driving(struct rtsx_softc *sc);
 198 static int      rtsx_rts5249_fill_driving(struct rtsx_softc *sc);
 199 static int      rtsx_read(struct rtsx_softc *, uint16_t, uint8_t *);
 200 static int      rtsx_read_cfg(struct rtsx_softc *sc, uint8_t func, uint16_t addr, uint32_t *val);
 201 static int      rtsx_write(struct rtsx_softc *sc, uint16_t addr, uint8_t mask, uint8_t val);
 202 static int      rtsx_read_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t *val);
 203 static int      rtsx_write_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t val);
 204 static int      rtsx_bus_power_off(struct rtsx_softc *sc);
 205 static int      rtsx_bus_power_on(struct rtsx_softc *sc);
 206 static int      rtsx_set_bus_width(struct rtsx_softc *sc, enum mmc_bus_width width);
 207 static int      rtsx_set_sd_timing(struct rtsx_softc *sc, enum mmc_bus_timing timing);
 208 static int      rtsx_set_sd_clock(struct rtsx_softc *sc, uint32_t freq);
 209 static int      rtsx_stop_sd_clock(struct rtsx_softc *sc);
 210 static int      rtsx_switch_sd_clock(struct rtsx_softc *sc, uint8_t clk, uint8_t n, uint8_t div, uint8_t mcu);
 211 static void     rtsx_sd_change_tx_phase(struct rtsx_softc *sc, uint8_t sample_point);
 212 static void     rtsx_sd_change_rx_phase(struct rtsx_softc *sc, uint8_t sample_point);
 213 static void     rtsx_sd_tuning_rx_phase(struct rtsx_softc *sc, uint32_t *phase_map);
 214 static int      rtsx_sd_tuning_rx_cmd(struct rtsx_softc *sc, uint8_t sample_point);
 215 static int      rtsx_sd_tuning_rx_cmd_wait(struct rtsx_softc *sc, struct mmc_command *cmd);
 216 static void     rtsx_sd_tuning_rx_cmd_wakeup(struct rtsx_softc *sc);
 217 static void     rtsx_sd_wait_data_idle(struct rtsx_softc *sc);
 218 static uint8_t  rtsx_sd_search_final_rx_phase(struct rtsx_softc *sc, uint32_t phase_map);
 219 static int      rtsx_sd_get_rx_phase_len(uint32_t phase_map, int start_bit);
 220 #if 0   /* For led */
 221 static int      rtsx_led_enable(struct rtsx_softc *sc);
 222 static int      rtsx_led_disable(struct rtsx_softc *sc);
 223 #endif  /* For led */
 224 static uint8_t  rtsx_response_type(uint16_t mmc_rsp);
 225 static void     rtsx_init_cmd(struct rtsx_softc *sc, struct mmc_command *cmd);
 226 static void     rtsx_push_cmd(struct rtsx_softc *sc, uint8_t cmd, uint16_t reg,
 227                               uint8_t mask, uint8_t data);
 228 static void     rtsx_set_cmd_data_len(struct rtsx_softc *sc, uint16_t block_cnt, uint16_t byte_cnt);
 229 static void     rtsx_send_cmd(struct rtsx_softc *sc);
 230 static void     rtsx_ret_resp(struct rtsx_softc *sc);
 231 static void     rtsx_set_resp(struct rtsx_softc *sc, struct mmc_command *cmd);
 232 static void     rtsx_stop_cmd(struct rtsx_softc *sc);
 233 static void     rtsx_clear_error(struct rtsx_softc *sc);
 234 static void     rtsx_req_done(struct rtsx_softc *sc);
 235 static int      rtsx_send_req(struct rtsx_softc *sc, struct mmc_command *cmd);
 236 static int      rtsx_xfer_short(struct rtsx_softc *sc, struct mmc_command *cmd);
 237 static void     rtsx_ask_ppbuf_part1(struct rtsx_softc *sc);
 238 static void     rtsx_get_ppbuf_part1(struct rtsx_softc *sc);
 239 static void     rtsx_get_ppbuf_part2(struct rtsx_softc *sc);
 240 static void     rtsx_put_ppbuf_part1(struct rtsx_softc *sc);
 241 static void     rtsx_put_ppbuf_part2(struct rtsx_softc *sc);
 242 static void     rtsx_write_ppbuf(struct rtsx_softc *sc);
 243 static int      rtsx_xfer(struct rtsx_softc *sc, struct mmc_command *cmd);
 244 static void     rtsx_xfer_begin(struct rtsx_softc *sc);
 245 static void     rtsx_xfer_start(struct rtsx_softc *sc);
 246 static void     rtsx_xfer_finish(struct rtsx_softc *sc);
 247 static void     rtsx_timeout(void *arg);
 248
 249 #ifdef MMCCAM
 250 static void     rtsx_cam_action(struct cam_sim *sim, union ccb *ccb);
 251 static void     rtsx_cam_poll(struct cam_sim *sim);
 252 static void     rtsx_cam_set_tran_settings(struct rtsx_softc *sc, union ccb *ccb);
 253 static void     rtsx_cam_request(struct rtsx_softc *sc, union ccb *ccb);
 254 #endif /* MMCCAM */
 255
 256 static int      rtsx_read_ivar(device_t bus, device_t child, int which, uintptr_t *result);
 257 static int      rtsx_write_ivar(device_t bus, device_t child, int which, uintptr_t value);
 258
 259 static int      rtsx_mmcbr_update_ios(device_t bus, device_t child __unused);
 260 static int      rtsx_mmcbr_switch_vccq(device_t bus, device_t child __unused);
 261 static int      rtsx_mmcbr_tune(device_t bus, device_t child __unused, bool hs400 __unused);
 262 static int      rtsx_mmcbr_retune(device_t bus, device_t child __unused, bool reset __unused);
 263 static int      rtsx_mmcbr_request(device_t bus, device_t child __unused, struct mmc_request *req);
 264 static int      rtsx_mmcbr_get_ro(device_t bus, device_t child __unused);
 265 static int      rtsx_mmcbr_acquire_host(device_t bus, device_t child __unused);
 266 static int      rtsx_mmcbr_release_host(device_t bus, device_t child __unused);
 267
 268 static int      rtsx_probe(device_t dev);
 269 static int      rtsx_attach(device_t dev);
 270 static int      rtsx_detach(device_t dev);
 271 static int      rtsx_shutdown(device_t dev);
 272 static int      rtsx_suspend(device_t dev);
 273 static int      rtsx_resume(device_t dev);
 274
 275 #define RTSX_LOCK_INIT(_sc)     mtx_init(&(_sc)->rtsx_mtx,      \
 276                                          device_get_nameunit(sc->rtsx_dev), "rtsx", MTX_DEF)
 277 #define RTSX_LOCK(_sc)          mtx_lock(&(_sc)->rtsx_mtx)
 278 #define RTSX_UNLOCK(_sc)        mtx_unlock(&(_sc)->rtsx_mtx)
 279 #define RTSX_LOCK_DESTROY(_sc)  mtx_destroy(&(_sc)->rtsx_mtx)
 280
 281 #define RTSX_SDCLK_OFF                  0
 282 #define RTSX_SDCLK_250KHZ          250000
 283 #define RTSX_SDCLK_400KHZ          400000
 284 #define RTSX_SDCLK_25MHZ         25000000
 285 #define RTSX_SDCLK_50MHZ         50000000
 286 #define RTSX_SDCLK_100MHZ       100000000
 287 #define RTSX_SDCLK_208MHZ       208000000
 288
 289 #define RTSX_MIN_DIV_N          80
 290 #define RTSX_MAX_DIV_N          208
 291
 292 #define RTSX_MAX_DATA_BLKLEN    512
 293
 294 #define RTSX_DMA_ALIGN          4
 295 #define RTSX_HOSTCMD_MAX        256
 296 #define RTSX_DMA_CMD_BIFSIZE    (sizeof(uint32_t) * RTSX_HOSTCMD_MAX)
 297 #define RTSX_DMA_DATA_BUFSIZE   MAXPHYS
 298
 299 #define ISSET(t, f) ((t) & (f))
 300
 301 #define READ4(sc, reg)                                          \
 302         (bus_space_read_4((sc)->rtsx_btag, (sc)->rtsx_bhandle, (reg)))
 303 #define WRITE4(sc, reg, val)                                    \
 304         (bus_space_write_4((sc)->rtsx_btag, (sc)->rtsx_bhandle, (reg), (val)))
 305
 306 #define RTSX_READ(sc, reg, val)                                 \
 307         do {                                                    \
 308                 int err = rtsx_read((sc), (reg), (val));        \
 309                 if (err)                                        \
 310                         return (err);                           \
 311         } while (0)
 312
 313 #define RTSX_WRITE(sc, reg, val)                                \
 314         do {                                                    \
 315                 int err = rtsx_write((sc), (reg), 0xff, (val)); \
 316                 if (err)                                        \
 317                         return (err);                           \
 318         } while (0)
 319 #define RTSX_CLR(sc, reg, bits)                                 \
 320         do {                                                    \
 321                 int err = rtsx_write((sc), (reg), (bits), 0);   \
 322                 if (err)                                        \
 323                         return (err);                           \
 324         } while (0)
 325
 326 #define RTSX_SET(sc, reg, bits)                                 \
 327         do {                                                    \
 328                 int err = rtsx_write((sc), (reg), (bits), 0xff);\
 329                 if (err)                                        \
 330                         return (err);                           \
 331         } while (0)
 332
 333 #define RTSX_BITOP(sc, reg, mask, bits)                         \
 334         do {                                                    \
 335                 int err = rtsx_write((sc), (reg), (mask), (bits));      \
 336                 if (err)                                        \
 337                         return (err);                           \
 338         } while (0)
 339
 340 /*
 341  * We use two DMA buffers: a command buffer and a data buffer.
 342  *
 343  * The command buffer contains a command queue for the host controller,
 344  * which describes SD/MMC commands to run, and other parameters. The chip
 345  * runs the command queue when a special bit in the RTSX_HCBAR register is
 346  * set and signals completion with the RTSX_TRANS_OK_INT interrupt.
 347  * Each command is encoded as a 4 byte sequence containing command number
 348  * (read, write, or check a host controller register), a register address,
 349  * and a data bit-mask and value.
 350  * SD/MMC commands which do not transfer any data from/to the card only use
 351  * the command buffer.
 352  *
 353  * The data buffer is used for transfer longer than 512. Data transfer is
 354  * controlled via the RTSX_HDBAR register and completion is signalled by
 355  * the RTSX_TRANS_OK_INT interrupt.
 356  *
 357  * The chip is unable to perform DMA above 4GB.
 358  */
 359
 360 /*
 361  * Main commands in the usual seqence used:
 362  *
 363  * CMD0         Go idle state
 364  * CMD8         Send interface condition
 365  * CMD55        Application Command for next ACMD
 366  * ACMD41       Send Operation Conditions Register (OCR: voltage profile of the card)
 367  * CMD2         Send Card Identification (CID) Register
 368  * CMD3         Send relative address
 369  * CMD9         Send Card Specific Data (CSD)
 370  * CMD13        Send status (32 bits -  bit 25: card password protected)
 371  * CMD7         Select card (before Get card SCR)
 372  * ACMD51       Send SCR (SD CARD Configuration Register - [51:48]: Bus widths supported)
 373  * CMD6         SD switch function
 374  * ACMD13       Send SD status (512 bits)
 375  * ACMD42       Set/Clear card detect
 376  * ACMD6        Set bus width
 377  * CMD19        Send tuning block
 378  * CMD12        Stop transmission
 379  *
 380  * CMD17        Read single block (<=512)
 381  * CMD18        Read multiple blocks (>512)
 382  * CMD24        Write single block (<=512)
 383  * CMD25        Write multiple blocks (>512)
 384  *
 385  * CMD52        IO R/W direct
 386  * CMD5         Send Operation Conditions
 387  */
 388
 389 static int
 390 rtsx_dma_alloc(struct rtsx_softc *sc)
 391 {
 392         int     error = 0;
 393
 394         error = bus_dma_tag_create(bus_get_dma_tag(sc->rtsx_dev), /* inherit from parent */
 395             RTSX_DMA_ALIGN, 0,          /* alignment, boundary */
 396             BUS_SPACE_MAXADDR_32BIT,    /* lowaddr */
 397             BUS_SPACE_MAXADDR,          /* highaddr */
 398             NULL, NULL,                 /* filter, filterarg */
 399             RTSX_DMA_CMD_BIFSIZE, 1,    /* maxsize, nsegments */
 400             RTSX_DMA_CMD_BIFSIZE,       /* maxsegsize */
 401             0,                          /* flags */
 402             NULL, NULL,                 /* lockfunc, lockarg */
 403             &sc->rtsx_cmd_dma_tag);
 404         if (error) {
 405                 device_printf(sc->rtsx_dev,
 406                               "Can't create cmd parent DMA tag\n");
 407                 return (error);
 408         }
 409         error = bus_dmamem_alloc(sc->rtsx_cmd_dma_tag,          /* DMA tag */
 410             &sc->rtsx_cmd_dmamem,                               /* will hold the KVA pointer */
 411             BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO,   /* flags */
 412             &sc->rtsx_cmd_dmamap);                              /* DMA map */
 413         if (error) {
 414                 device_printf(sc->rtsx_dev,
 415                               "Can't create DMA map for command transfer\n");
 416                 goto destroy_cmd_dma_tag;
 417
 418         }
 419         error = bus_dmamap_load(sc->rtsx_cmd_dma_tag,   /* DMA tag */
 420             sc->rtsx_cmd_dmamap,        /* DMA map */
 421             sc->rtsx_cmd_dmamem,        /* KVA pointer to be mapped */
 422             RTSX_DMA_CMD_BIFSIZE,       /* size of buffer */
 423             rtsx_dmamap_cb,             /* callback */
 424             &sc->rtsx_cmd_buffer,       /* first arg of callback */
 425             0);                         /* flags */
 426         if (error || sc->rtsx_cmd_buffer == 0) {
 427                 device_printf(sc->rtsx_dev,
 428                               "Can't load DMA memory for command transfer\n");
 429                 error = (error) ? error : EFAULT;
 430                 goto destroy_cmd_dmamem_alloc;
 431         }
 432
 433         error = bus_dma_tag_create(bus_get_dma_tag(sc->rtsx_dev),       /* inherit from parent */
 434             RTSX_DMA_DATA_BUFSIZE, 0,   /* alignment, boundary */
 435             BUS_SPACE_MAXADDR_32BIT,    /* lowaddr */
 436             BUS_SPACE_MAXADDR,          /* highaddr */
 437             NULL, NULL,                 /* filter, filterarg */
 438             RTSX_DMA_DATA_BUFSIZE, 1,   /* maxsize, nsegments */
 439             RTSX_DMA_DATA_BUFSIZE,      /* maxsegsize */
 440             0,                          /* flags */
 441             NULL, NULL,                 /* lockfunc, lockarg */
 442             &sc->rtsx_data_dma_tag);
 443         if (error) {
 444                 device_printf(sc->rtsx_dev,
 445                               "Can't create data parent DMA tag\n");
 446                 goto destroy_cmd_dmamap_load;
 447         }
 448         error = bus_dmamem_alloc(sc->rtsx_data_dma_tag,         /* DMA tag */
 449             &sc->rtsx_data_dmamem,                              /* will hold the KVA pointer */
 450             BUS_DMA_WAITOK | BUS_DMA_ZERO,                      /* flags */
 451             &sc->rtsx_data_dmamap);                             /* DMA map */
 452         if (error) {
 453                 device_printf(sc->rtsx_dev,
 454                               "Can't create DMA map for data transfer\n");
 455                 goto destroy_data_dma_tag;
 456         }
 457         error = bus_dmamap_load(sc->rtsx_data_dma_tag,  /* DMA tag */
 458             sc->rtsx_data_dmamap,       /* DMA map */
 459             sc->rtsx_data_dmamem,       /* KVA pointer to be mapped */
 460             RTSX_DMA_DATA_BUFSIZE,      /* size of buffer */
 461             rtsx_dmamap_cb,             /* callback */
 462             &sc->rtsx_data_buffer,      /* first arg of callback */
 463             0);                         /* flags */
 464         if (error || sc->rtsx_data_buffer == 0) {
 465                 device_printf(sc->rtsx_dev,
 466                               "Can't load DMA memory for data transfer\n");
 467                 error = (error) ? error : EFAULT;
 468                 goto destroy_data_dmamem_alloc;
 469         }
 470         return (error);
 471
 472  destroy_data_dmamem_alloc:
 473         bus_dmamem_free(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamem, sc->rtsx_data_dmamap);
 474  destroy_data_dma_tag:
 475         bus_dma_tag_destroy(sc->rtsx_data_dma_tag);
 476  destroy_cmd_dmamap_load:
 477         bus_dmamap_unload(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap);
 478  destroy_cmd_dmamem_alloc:
 479         bus_dmamem_free(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamem, sc->rtsx_cmd_dmamap);
 480  destroy_cmd_dma_tag:
 481         bus_dma_tag_destroy(sc->rtsx_cmd_dma_tag);
 482
 483         return (error);
 484 }
 485
 486 static void
 487 rtsx_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 488 {
 489         if (error) {
 490                 printf("rtsx_dmamap_cb: error %d\n", error);
 491                 return;
 492         }
 493         *(bus_addr_t *)arg = segs[0].ds_addr;
 494 }
 495
 496 static void
 497 rtsx_dma_free(struct rtsx_softc *sc)
 498 {
 499         if (sc->rtsx_cmd_dma_tag != NULL) {
 500                 if (sc->rtsx_cmd_dmamap != NULL)
 501                         bus_dmamap_unload(sc->rtsx_cmd_dma_tag,
 502                                           sc->rtsx_cmd_dmamap);
 503                 if (sc->rtsx_cmd_dmamem != NULL)
 504                         bus_dmamem_free(sc->rtsx_cmd_dma_tag,
 505                                         sc->rtsx_cmd_dmamem,
 506                                         sc->rtsx_cmd_dmamap);
 507                 sc->rtsx_cmd_dmamap = NULL;
 508                 sc->rtsx_cmd_dmamem = NULL;
 509                 sc->rtsx_cmd_buffer = 0;
 510                 bus_dma_tag_destroy(sc->rtsx_cmd_dma_tag);
 511                 sc->rtsx_cmd_dma_tag = NULL;
 512         }
 513         if (sc->rtsx_data_dma_tag != NULL) {
 514                 if (sc->rtsx_data_dmamap != NULL)
 515                         bus_dmamap_unload(sc->rtsx_data_dma_tag,
 516                                           sc->rtsx_data_dmamap);
 517                 if (sc->rtsx_data_dmamem != NULL)
 518                         bus_dmamem_free(sc->rtsx_data_dma_tag,
 519                                         sc->rtsx_data_dmamem,
 520                                         sc->rtsx_data_dmamap);
 521                 sc->rtsx_data_dmamap = NULL;
 522                 sc->rtsx_data_dmamem = NULL;
 523                 sc->rtsx_data_buffer = 0;
 524                 bus_dma_tag_destroy(sc->rtsx_data_dma_tag);
 525                 sc->rtsx_data_dma_tag = NULL;
 526         }
 527 }
 528
 529 static void
 530 rtsx_intr(void *arg)
 531 {
 532         struct rtsx_softc *sc = arg;
 533         uint32_t        enabled;
 534         uint32_t        status;
 535
 536         RTSX_LOCK(sc);
 537
 538         enabled = sc->rtsx_intr_enabled;
 539         status = READ4(sc, RTSX_BIPR);  /* read Bus Interrupt Pending Register */
 540         sc->rtsx_intr_status = status;
 541
 542         if (bootverbose)
 543                 device_printf(sc->rtsx_dev, "Interrupt handler - enabled: 0x%08x, status: 0x%08x\n", enabled, status);
 544
 545         /* Ack interrupts. */
 546         WRITE4(sc, RTSX_BIPR, status);
 547
 548         if (((enabled & status) == 0) || status == 0xffffffff) {
 549                 device_printf(sc->rtsx_dev, "Spurious interrupt - enabled: 0x%08x, status: 0x%08x\n", enabled, status);
 550                 RTSX_UNLOCK(sc);
 551                 return;
 552         }
 553
 554         /* Detect write protect. */
 555         if (status & RTSX_SD_WRITE_PROTECT)
 556                 sc->rtsx_read_only = 1;
 557         else
 558                 sc->rtsx_read_only = 0;
 559
 560         /* Start task to handle SD card status change (from dwmmc.c). */
 561         if (status & RTSX_SD_INT) {
 562                 device_printf(sc->rtsx_dev, "Interrupt card inserted/removed\n");
 563                 rtsx_handle_card_present(sc);
 564         }
 565
 566         if (sc->rtsx_req == NULL) {
 567                 RTSX_UNLOCK(sc);
 568                 return;
 569         }
 570
 571         if (status & RTSX_TRANS_OK_INT) {
 572                 sc->rtsx_req->cmd->error = MMC_ERR_NONE;
 573                 if (sc->rtsx_intr_trans_ok != NULL)
 574                         sc->rtsx_intr_trans_ok(sc);
 575         } else if (status & RTSX_TRANS_FAIL_INT) {
 576                 uint8_t stat1;
 577                 sc->rtsx_req->cmd->error = MMC_ERR_FAILED;
 578                 if (rtsx_read(sc, RTSX_SD_STAT1, &stat1) == 0 &&
 579                     (stat1 & RTSX_SD_CRC_ERR)) {
 580                         device_printf(sc->rtsx_dev, "CRC error\n");
 581                         sc->rtsx_req->cmd->error = MMC_ERR_BADCRC;
 582                 }
 583                 if (!sc->rtsx_tuning_mode)
 584                         device_printf(sc->rtsx_dev, "Transfer fail - status: 0x%08x\n", status);
 585                 rtsx_stop_cmd(sc);
 586                 if (sc->rtsx_intr_trans_ko != NULL)
 587                         sc->rtsx_intr_trans_ko(sc);
 588         }
 589
 590         RTSX_UNLOCK(sc);
 591 }
 592
 593 /*
 594  * Function called from the IRQ handler (from dwmmc.c).
 595  */
 596 static void
 597 rtsx_handle_card_present(struct rtsx_softc *sc)
 598 {
 599         bool    was_present;
 600         bool    is_present;
 601
 602 #ifdef MMCCAM
 603         was_present = sc->rtsx_cam_status;
 604 #else
 605         was_present = sc->rtsx_mmc_dev != NULL;
 606 #endif /* MMCCAM */
 607         is_present = rtsx_is_card_present(sc);
 608         if (is_present)
 609                 device_printf(sc->rtsx_dev, "Card present\n");
 610         else
 611                 device_printf(sc->rtsx_dev, "Card absent\n");
 612
 613         if (!was_present && is_present) {
 614                 /*
 615                  * The delay is to debounce the card insert
 616                  * (sometimes the card detect pin stabilizes
 617                  * before the other pins have made good contact).
 618                  */
 619                 taskqueue_enqueue_timeout(taskqueue_swi_giant,
 620                                           &sc->rtsx_card_insert_task, -hz);
 621         } else if (was_present && !is_present) {
 622                 taskqueue_enqueue(taskqueue_swi_giant, &sc->rtsx_card_remove_task);
 623         }
 624 }
 625
 626 /*
 627  * This funtion is called at startup.
 628  */
 629 static void
 630 rtsx_card_task(void *arg, int pending __unused)
 631 {
 632         struct rtsx_softc *sc = arg;
 633
 634         RTSX_LOCK(sc);
 635
 636         if (rtsx_is_card_present(sc)) {
 637                 sc->rtsx_flags |= RTSX_F_CARD_PRESENT;
 638                 /* Card is present, attach if necessary. */
 639 #ifdef MMCCAM
 640                 if (sc->rtsx_cam_status == 0) {
 641                         union ccb       *ccb;
 642                         uint32_t        pathid;
 643 #else
 644                 if (sc->rtsx_mmc_dev == NULL) {
 645 #endif /* MMCCAM */
 646                         if (bootverbose)
 647                                 device_printf(sc->rtsx_dev, "Card inserted\n");
 648
 649                         sc->rtsx_read_count = sc->rtsx_write_count = 0;
 650 #ifdef MMCCAM
 651                         sc->rtsx_cam_status = 1;
 652                         pathid = cam_sim_path(sc->rtsx_sim);
 653                         ccb = xpt_alloc_ccb_nowait();
 654                         if (ccb == NULL) {
 655                                 device_printf(sc->rtsx_dev, "Unable to alloc CCB for rescan\n");
 656                                 RTSX_UNLOCK(sc);
 657                                 return;
 658                         }
 659                         /*
 660                          * We create a rescan request for BUS:0:0, since the card
 661                          * will be at lun 0.
 662                          */
 663                         if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
 664                                             /* target */ 0, /* lun */ 0) != CAM_REQ_CMP) {
 665                                 device_printf(sc->rtsx_dev, "Unable to create path for rescan\n");
 666                                 RTSX_UNLOCK(sc);
 667                                 xpt_free_ccb(ccb);
 668                                 return;
 669                         }
 670                         RTSX_UNLOCK(sc);
 671                         xpt_rescan(ccb);
 672 #else
 673                         sc->rtsx_mmc_dev = device_add_child(sc->rtsx_dev, "mmc", -1);
 674                         RTSX_UNLOCK(sc);
 675                         if (sc->rtsx_mmc_dev == NULL) {
 676                                 device_printf(sc->rtsx_dev, "Adding MMC bus failed\n");
 677                         } else {
 678                                 device_set_ivars(sc->rtsx_mmc_dev, sc);
 679                                 device_probe_and_attach(sc->rtsx_mmc_dev);
 680                         }
 681 #endif /* MMCCAM */
 682                 } else
 683                         RTSX_UNLOCK(sc);
 684         } else {
 685                 sc->rtsx_flags &= ~RTSX_F_CARD_PRESENT;
 686                 /* Card isn't present, detach if necessary. */
 687 #ifdef MMCCAM
 688                 if (sc->rtsx_cam_status != 0) {
 689                         union ccb       *ccb;
 690                         uint32_t        pathid;
 691 #else
 692                 if (sc->rtsx_mmc_dev != NULL) {
 693 #endif /* MMCCAM */
 694                         if (bootverbose)
 695                                 device_printf(sc->rtsx_dev, "Card removed\n");
 696
 697                         if (sc->rtsx_debug)
 698                                 device_printf(sc->rtsx_dev, "Read count: %" PRIu64 ", write count: %" PRIu64 "\n",
 699                                               sc->rtsx_read_count, sc->rtsx_write_count);
 700 #ifdef MMCCAM
 701                         sc->rtsx_cam_status = 0;
 702                         pathid = cam_sim_path(sc->rtsx_sim);
 703                         ccb = xpt_alloc_ccb_nowait();
 704                         if (ccb == NULL) {
 705                                 device_printf(sc->rtsx_dev, "Unable to alloc CCB for rescan\n");
 706                                 RTSX_UNLOCK(sc);
 707                                 return;
 708                         }
 709                         /*
 710                          * We create a rescan request for BUS:0:0, since the card
 711                          * will be at lun 0.
 712                          */
 713                         if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
 714                                             /* target */ 0, /* lun */ 0) != CAM_REQ_CMP) {
 715                                 device_printf(sc->rtsx_dev, "Unable to create path for rescan\n");
 716                                 RTSX_UNLOCK(sc);
 717                                 xpt_free_ccb(ccb);
 718                                 return;
 719                         }
 720                         RTSX_UNLOCK(sc);
 721                         xpt_rescan(ccb);
 722 #else
 723                         RTSX_UNLOCK(sc);
 724                         if (device_delete_child(sc->rtsx_dev, sc->rtsx_mmc_dev))
 725                                 device_printf(sc->rtsx_dev, "Detaching MMC bus failed\n");
 726                         sc->rtsx_mmc_dev = NULL;
 727 #endif /* MMCCAM */
 728                 } else
 729                         RTSX_UNLOCK(sc);
 730         }
 731 }
 732
 733 static bool
 734 rtsx_is_card_present(struct rtsx_softc *sc)
 735 {
 736         uint32_t status;
 737
 738         status = READ4(sc, RTSX_BIPR);
 739         if (sc->rtsx_inversion == 0)
 740                 return (status & RTSX_SD_EXIST);
 741         else
 742                 return !(status & RTSX_SD_EXIST);
 743 }
 744
 745 static int
 746 rtsx_init(struct rtsx_softc *sc)
 747 {
 748         bool    rtsx_init_debug = false;
 749         uint8_t version;
 750         uint8_t val;
 751         int     error;
 752
 753         sc->rtsx_host.host_ocr = RTSX_SUPPORTED_VOLTAGE;
 754         sc->rtsx_host.f_min = RTSX_SDCLK_250KHZ;
 755         sc->rtsx_host.f_max = RTSX_SDCLK_208MHZ;
 756         sc->rtsx_host.caps = MMC_CAP_4_BIT_DATA | MMC_CAP_HSPEED |
 757                 MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
 758
 759         sc->rtsx_host.caps |= MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104;
 760         if (sc->rtsx_device_id == RTSX_RTS5209)
 761                 sc->rtsx_host.caps |= MMC_CAP_8_BIT_DATA;
 762         pci_find_cap(sc->rtsx_dev, PCIY_EXPRESS, &(sc->rtsx_pcie_cap));
 763
 764         /*
 765          * Check IC version.
 766          */
 767         switch (sc->rtsx_device_id) {
 768         case RTSX_RTS5229:
 769                 /* Read IC version from dummy register. */
 770                 RTSX_READ(sc, RTSX_DUMMY_REG, &version);
 771                 if ((version & 0x0F) == RTSX_IC_VERSION_C)
 772                         sc->rtsx_flags |= RTSX_F_VERSION_C;
 773                 break;
 774         case RTSX_RTS522A:
 775                 /* Read IC version from dummy register. */
 776                 RTSX_READ(sc, RTSX_DUMMY_REG, &version);
 777                 if ((version & 0x0F) == RTSX_IC_VERSION_A)
 778                         sc->rtsx_flags |= RTSX_F_VERSION_A;
 779                 break;
 780         case RTSX_RTS525A:
 781                 /* Read IC version from dummy register. */
 782                 RTSX_READ(sc, RTSX_DUMMY_REG, &version);
 783                 if ((version & 0x0F) == RTSX_IC_VERSION_A)
 784                         sc->rtsx_flags |= RTSX_F_VERSION_A;
 785                 break;
 786         case RTSX_RTL8411B:
 787                 RTSX_READ(sc, RTSX_RTL8411B_PACKAGE, &version);
 788                 if (version & RTSX_RTL8411B_QFN48)
 789                         sc->rtsx_flags |= RTSX_F_8411B_QFN48;
 790                 break;
 791         }
 792
 793         /*
 794          * Fetch vendor settings.
 795          */
 796         /*
 797          * Normally OEMs will set vendor setting to the config space
 798          * of Realtek card reader in BIOS stage. This statement reads
 799          * the setting and configure the internal registers according
 800          * to it, to improve card reader's compatibility condition.
 801          */
 802         sc->rtsx_card_drive_sel = RTSX_CARD_DRIVE_DEFAULT;
 803         switch (sc->rtsx_device_id) {
 804                 uint32_t reg;
 805                 uint32_t reg1;
 806                 uint8_t  reg3;
 807         case RTSX_RTS5209:
 808                 sc->rtsx_card_drive_sel = RTSX_RTS5209_CARD_DRIVE_DEFAULT;
 809                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D;
 810                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4);
 811                 if (!(reg & 0x80)) {
 812                         sc->rtsx_card_drive_sel = (reg >> 8) & 0x3F;
 813                         sc->rtsx_sd30_drive_sel_3v3 = reg & 0x07;
 814                 } else {
 815                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 816                 }
 817                 if (bootverbose || rtsx_init_debug)
 818                         device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n",
 819                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3);
 820                 break;
 821         case RTSX_RTS5227:
 822         case RTSX_RTS522A:
 823                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_CFG_DRIVER_TYPE_B;
 824                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 825                 if (!(reg & 0x1000000)) {
 826                         sc->rtsx_card_drive_sel &= 0x3F;
 827                         sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6;
 828                         reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4);
 829                         sc->rtsx_sd30_drive_sel_3v3 = (reg >> 5) & 0x03;
 830                         if (reg & 0x4000)
 831                                 sc->rtsx_flags |= RTSX_F_REVERSE_SOCKET;
 832                 } else {
 833                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 834                 }
 835                 if (bootverbose || rtsx_init_debug)
 836                         device_printf(sc->rtsx_dev,
 837                                       "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x, reverse_socket is %s\n",
 838                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3,
 839                                       (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) ? "true" : "false");
 840                 break;
 841         case RTSX_RTS5229:
 842                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D;
 843                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 844                 if (!(reg & 0x1000000)) {
 845                         sc->rtsx_card_drive_sel &= 0x3F;
 846                         sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6;
 847                         reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4);
 848                         sc->rtsx_sd30_drive_sel_3v3 = rtsx_map_sd_drive((reg >> 5) & 0x03);
 849                 } else {
 850                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 851                 }
 852                 if (bootverbose || rtsx_init_debug)
 853                         device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n",
 854                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3);
 855                 break;
 856         case RTSX_RTS525A:
 857         case RTSX_RTS5249:
 858                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_CFG_DRIVER_TYPE_B;
 859                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 860                 if ((reg & 0x1000000)) {
 861                         sc->rtsx_card_drive_sel &= 0x3F;
 862                         sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6;
 863                         reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4);
 864                         sc->rtsx_sd30_drive_sel_3v3 = (reg >> 5) & 0x03;
 865                         if (reg & 0x4000)
 866                                 sc->rtsx_flags |= RTSX_F_REVERSE_SOCKET;
 867                 } else {
 868                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 869                 }
 870                 if (bootverbose || rtsx_init_debug)
 871                         device_printf(sc->rtsx_dev,
 872                                       "card_drive_sel = 0x%02x, sd30_drive_sel_3v3: 0x%02x, reverse_socket is %s\n",
 873                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3,
 874                                       (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) ? "true" : "false");
 875                 break;
 876         case RTSX_RTL8402:
 877         case RTSX_RTL8411:
 878                 sc->rtsx_card_drive_sel = RTSX_RTL8411_CARD_DRIVE_DEFAULT;
 879                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D;
 880                 reg1 = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 881                 if (reg1 & 0x1000000) {
 882                         sc->rtsx_card_drive_sel &= 0x3F;
 883                         sc->rtsx_card_drive_sel |= ((reg1 >> 25) & 0x01) << 6;
 884                         reg3 = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG3, 1);
 885                         sc->rtsx_sd30_drive_sel_3v3 = (reg3 >> 5) & 0x07;
 886                 } else {
 887                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg1: 0x%08x\n", reg1);
 888                 }
 889                 if (bootverbose || rtsx_init_debug)
 890                         device_printf(sc->rtsx_dev,
 891                                       "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n",
 892                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3);
 893                 break;
 894         case RTSX_RTL8411B:
 895                 sc->rtsx_card_drive_sel = RTSX_RTL8411_CARD_DRIVE_DEFAULT;
 896                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D;
 897                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 898                 if (!(reg & 0x1000000)) {
 899                         sc->rtsx_sd30_drive_sel_3v3 = rtsx_map_sd_drive(reg & 0x03);
 900                 } else {
 901                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 902                 }
 903                 if (bootverbose || rtsx_init_debug)
 904                         device_printf(sc->rtsx_dev,
 905                                       "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n",
 906                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3);
 907                 break;
 908         }
 909
 910         if (bootverbose || rtsx_init_debug)
 911                 device_printf(sc->rtsx_dev, "rtsx_init() rtsx_flags: 0x%04x\n", sc->rtsx_flags);
 912
 913         /* Enable interrupts. */
 914         sc->rtsx_intr_enabled = RTSX_TRANS_OK_INT_EN | RTSX_TRANS_FAIL_INT_EN | RTSX_SD_INT_EN | RTSX_MS_INT_EN;
 915         WRITE4(sc, RTSX_BIER, sc->rtsx_intr_enabled);
 916
 917         /* Power on SSC clock. */
 918         RTSX_CLR(sc, RTSX_FPDCTL, RTSX_SSC_POWER_DOWN);
 919         /* Wait SSC power stable. */
 920         DELAY(200);
 921
 922         /* Disable ASPM */
 923         val = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL, 1);
 924         pci_write_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL, val & 0xfc, 1);
 925
 926         /*
 927          * Optimize phy.
 928          */
 929         switch (sc->rtsx_device_id) {
 930         case RTSX_RTS5209:
 931                 /* Some magic numbers from Linux driver. */
 932                 if ((error = rtsx_write_phy(sc, 0x00, 0xB966)))
 933                         return (error);
 934                 break;
 935         case RTSX_RTS5227:
 936                 RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN);
 937
 938                 /* Optimize RX sensitivity. */
 939                 if ((error = rtsx_write_phy(sc, 0x00, 0xBA42)))
 940                         return (error);
 941                 break;
 942         case RTSX_RTS5229:
 943                 /* Optimize RX sensitivity. */
 944                 if ((error = rtsx_write_phy(sc, 0x00, 0xBA42)))
 945                         return (error);
 946                 break;
 947         case RTSX_RTS522A:
 948                 RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3, RTSX_D3_DELINK_MODE_EN);
 949                 if (sc->rtsx_flags & RTSX_F_VERSION_A) {
 950                         if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR2, RTSX_PHY_RCR2_INIT_27S)))
 951                                 return (error);
 952                 }
 953                 if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR1, RTSX_PHY_RCR1_INIT_27S)))
 954                         return (error);
 955                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD0, RTSX_PHY_FLD0_INIT_27S)))
 956                         return (error);
 957                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD3, RTSX_PHY_FLD3_INIT_27S)))
 958                         return (error);
 959                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD4, RTSX_PHY_FLD4_INIT_27S)))
 960                         return (error);
 961                 break;
 962         case RTSX_RTS525A:
 963                 if ((error = rtsx_write_phy(sc, RTSX__PHY_FLD0,
 964                                             RTSX__PHY_FLD0_CLK_REQ_20C | RTSX__PHY_FLD0_RX_IDLE_EN |
 965                                             RTSX__PHY_FLD0_BIT_ERR_RSTN | RTSX__PHY_FLD0_BER_COUNT |
 966                                             RTSX__PHY_FLD0_BER_TIMER | RTSX__PHY_FLD0_CHECK_EN)))
 967                         return (error);
 968                 if ((error = rtsx_write_phy(sc, RTSX__PHY_ANA03,
 969                                             RTSX__PHY_ANA03_TIMER_MAX | RTSX__PHY_ANA03_OOBS_DEB_EN |
 970                                             RTSX__PHY_CMU_DEBUG_EN)))
 971                         return (error);
 972                 if (sc->rtsx_flags & RTSX_F_VERSION_A)
 973                         if ((error = rtsx_write_phy(sc, RTSX__PHY_REV0,
 974                                                     RTSX__PHY_REV0_FILTER_OUT | RTSX__PHY_REV0_CDR_BYPASS_PFD |
 975                                                     RTSX__PHY_REV0_CDR_RX_IDLE_BYPASS)))
 976                                 return (error);
 977                 break;
 978         case RTSX_RTS5249:
 979                 RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN);
 980                 if ((error = rtsx_write_phy(sc, RTSX_PHY_REV,
 981                                             RTSX_PHY_REV_RESV | RTSX_PHY_REV_RXIDLE_LATCHED |
 982                                             RTSX_PHY_REV_P1_EN | RTSX_PHY_REV_RXIDLE_EN |
 983                                             RTSX_PHY_REV_CLKREQ_TX_EN | RTSX_PHY_REV_RX_PWST |
 984                                             RTSX_PHY_REV_CLKREQ_DT_1_0 | RTSX_PHY_REV_STOP_CLKRD |
 985                                             RTSX_PHY_REV_STOP_CLKWR)))
 986                         return (error);
 987                 DELAY(10);
 988                 if ((error = rtsx_write_phy(sc, RTSX_PHY_BPCR,
 989                                             RTSX_PHY_BPCR_IBRXSEL | RTSX_PHY_BPCR_IBTXSEL |
 990                                             RTSX_PHY_BPCR_IB_FILTER | RTSX_PHY_BPCR_CMIRROR_EN)))
 991                         return (error);
 992                 if ((error = rtsx_write_phy(sc, RTSX_PHY_PCR,
 993                                             RTSX_PHY_PCR_FORCE_CODE | RTSX_PHY_PCR_OOBS_CALI_50 |
 994                                             RTSX_PHY_PCR_OOBS_VCM_08 | RTSX_PHY_PCR_OOBS_SEN_90 |
 995                                             RTSX_PHY_PCR_RSSI_EN | RTSX_PHY_PCR_RX10K)))
 996                         return (error);
 997                 if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR2,
 998                                             RTSX_PHY_RCR2_EMPHASE_EN | RTSX_PHY_RCR2_NADJR |
 999                                             RTSX_PHY_RCR2_CDR_SR_2 | RTSX_PHY_RCR2_FREQSEL_12 |
1000                                             RTSX_PHY_RCR2_CDR_SC_12P | RTSX_PHY_RCR2_CALIB_LATE)))
1001                         return (error);
1002                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD4,
1003                                             RTSX_PHY_FLD4_FLDEN_SEL | RTSX_PHY_FLD4_REQ_REF |
1004                                             RTSX_PHY_FLD4_RXAMP_OFF | RTSX_PHY_FLD4_REQ_ADDA |
1005                                             RTSX_PHY_FLD4_BER_COUNT | RTSX_PHY_FLD4_BER_TIMER |
1006                                             RTSX_PHY_FLD4_BER_CHK_EN)))
1007                         return (error);
1008                 if ((error = rtsx_write_phy(sc, RTSX_PHY_RDR,
1009                                             RTSX_PHY_RDR_RXDSEL_1_9 | RTSX_PHY_SSC_AUTO_PWD)))
1010                         return (error);
1011                 if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR1,
1012                                             RTSX_PHY_RCR1_ADP_TIME_4 | RTSX_PHY_RCR1_VCO_COARSE)))
1013                         return (error);
1014                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD3,
1015                                             RTSX_PHY_FLD3_TIMER_4 | RTSX_PHY_FLD3_TIMER_6 |
1016                                             RTSX_PHY_FLD3_RXDELINK)))
1017                         return (error);
1018                 if ((error = rtsx_write_phy(sc, RTSX_PHY_TUNE,
1019                                             RTSX_PHY_TUNE_TUNEREF_1_0 | RTSX_PHY_TUNE_VBGSEL_1252 |
1020                                             RTSX_PHY_TUNE_SDBUS_33 | RTSX_PHY_TUNE_TUNED18 |
1021                                             RTSX_PHY_TUNE_TUNED12 | RTSX_PHY_TUNE_TUNEA12)))
1022                         return (error);
1023                 break;
1024         }
1025
1026         /* Set mcu_cnt to 7 to ensure data can be sampled properly. */
1027         RTSX_BITOP(sc, RTSX_CLK_DIV, 0x07, 0x07);
1028
1029         /* Disable sleep mode. */
1030         RTSX_CLR(sc, RTSX_HOST_SLEEP_STATE,
1031                  RTSX_HOST_ENTER_S1 | RTSX_HOST_ENTER_S3);
1032
1033         /* Disable card clock. */
1034         RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_CARD_CLK_EN_ALL);
1035
1036         /* Reset delink mode. */
1037         RTSX_CLR(sc, RTSX_CHANGE_LINK_STATE,
1038                  RTSX_FORCE_RST_CORE_EN | RTSX_NON_STICKY_RST_N_DBG);
1039
1040         /* Card driving select. */
1041         RTSX_WRITE(sc, RTSX_CARD_DRIVE_SEL, sc->rtsx_card_drive_sel);
1042
1043         /* Enable SSC clock. */
1044         RTSX_WRITE(sc, RTSX_SSC_CTL1, RTSX_SSC_8X_EN | RTSX_SSC_SEL_4M);
1045         RTSX_WRITE(sc, RTSX_SSC_CTL2, 0x12);
1046
1047         /* Disable cd_pwr_save. */
1048         RTSX_BITOP(sc, RTSX_CHANGE_LINK_STATE, 0x16, RTSX_MAC_PHY_RST_N_DBG);
1049
1050         /* Clear Link Ready Interrupt. */
1051         RTSX_BITOP(sc, RTSX_IRQSTAT0, RTSX_LINK_READY_INT, RTSX_LINK_READY_INT);
1052
1053         /* Enlarge the estimation window of PERST# glitch
1054          * to reduce the chance of invalid card interrupt. */
1055         RTSX_WRITE(sc, RTSX_PERST_GLITCH_WIDTH, 0x80);
1056
1057         /* Set RC oscillator to 400K. */
1058         RTSX_CLR(sc, RTSX_RCCTL, RTSX_RCCTL_F_2M);
1059
1060         /* Enable interrupt write-clear (default is read-clear). */
1061         RTSX_CLR(sc, RTSX_NFTS_TX_CTRL, RTSX_INT_READ_CLR);
1062
1063         if (sc->rtsx_device_id == RTSX_RTS525A)
1064                 RTSX_BITOP(sc, RTSX_PM_CLK_FORCE_CTL, 1, 1);
1065
1066         /* OC power down. */
1067         RTSX_BITOP(sc, RTSX_FPDCTL, RTSX_SD_OC_POWER_DOWN, RTSX_SD_OC_POWER_DOWN);
1068
1069         /* Enable clk_request_n to enable clock power management */
1070         pci_write_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL + 1, 1, 1);
1071
1072         /* Enter L1 when host tx idle */
1073         pci_write_config(sc->rtsx_dev, 0x70F, 0x5B, 1);
1074
1075         /*
1076          * Specific extra init.
1077          */
1078         switch (sc->rtsx_device_id) {
1079                 uint16_t cap;
1080         case RTSX_RTS5209:
1081                 /* Turn off LED. */
1082                 RTSX_WRITE(sc, RTSX_CARD_GPIO, 0x03);
1083                 /* Reset ASPM state to default value. */
1084                 RTSX_CLR(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK);
1085                 /* Force CLKREQ# PIN to drive 0 to request clock. */
1086                 RTSX_BITOP(sc, RTSX_PETXCFG, 0x08, 0x08);
1087                 /* Configure GPIO as output. */
1088                 RTSX_WRITE(sc, RTSX_CARD_GPIO_DIR, 0x03);
1089                 /* Configure driving. */
1090                 RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3);
1091                 break;
1092         case RTSX_RTS5227:
1093                 /* Configure GPIO as output. */
1094                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1095                 /* Reset ASPM state to default value. */
1096                 RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1097                 /* Switch LDO3318 source from DV33 to 3V3. */
1098                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1099                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1100                 /* Set default OLT blink period. */
1101                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1102                 /* Configure LTR. */
1103                 cap = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_DEVICE_CTL2, 2);
1104                 if (cap & PCIEM_CTL2_LTR_ENABLE)
1105                         RTSX_WRITE(sc, RTSX_LTR_CTL, 0xa3);
1106                 /* Configure OBFF. */
1107                 RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_ENABLE);
1108                 /* Configure driving. */
1109                 if ((error = rtsx_rts5227_fill_driving(sc)))
1110                         return (error);
1111                 /* Configure force_clock_req. */
1112                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1113                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0xB8);
1114                 else
1115                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0x88);
1116                 RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN);
1117                 /*!!! Added for reboot after Windows. */
1118                 RTSX_BITOP(sc, RTSX_PM_CTRL3, RTSX_PM_WAKE_EN, RTSX_PM_WAKE_EN);
1119                 break;
1120         case RTSX_RTS5229:
1121                 /* Configure GPIO as output. */
1122                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1123                 /* Reset ASPM state to default value. */
1124                 /*  With this reset: dd if=/dev/random of=/dev/mmcsd0 encounter a timeout. */
1125 //!!!           RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1126                 /* Force CLKREQ# PIN to drive 0 to request clock. */
1127                 RTSX_BITOP(sc, RTSX_PETXCFG, 0x08, 0x08);
1128                 /* Switch LDO3318 source from DV33 to card_3v3. */
1129                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1130                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1131                 /* Set default OLT blink period. */
1132                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1133                 /* Configure driving. */
1134                 RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3);
1135                 break;
1136         case RTSX_RTS522A:
1137                 /* Add specific init from RTS5227. */
1138                 /* Configure GPIO as output. */
1139                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1140                 /* Reset ASPM state to default value. */
1141                 RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1142                 /* Switch LDO3318 source from DV33 to 3V3. */
1143                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1144                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1145                 /* Set default OLT blink period. */
1146                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1147                 /* Configure LTR. */
1148                 cap = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_DEVICE_CTL2, 2);
1149                 if (cap & PCIEM_CTL2_LTR_ENABLE)
1150                         RTSX_WRITE(sc, RTSX_LTR_CTL, 0xa3);
1151                 /* Configure OBFF. */
1152                 RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_ENABLE);
1153                 /* Configure driving. */
1154                 if ((error = rtsx_rts5227_fill_driving(sc)))
1155                         return (error);
1156                 /* Configure force_clock_req. */
1157                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1158                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0xB8);
1159                 else
1160                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0x88);
1161                 RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3,  0x10);
1162
1163                 /* specific for RTS522A. */
1164                 RTSX_BITOP(sc, RTSX_FUNC_FORCE_CTL,
1165                            RTSX_FUNC_FORCE_UPME_XMT_DBG, RTSX_FUNC_FORCE_UPME_XMT_DBG);
1166                 RTSX_BITOP(sc, RTSX_PCLK_CTL, 0x04, 0x04);
1167                 RTSX_BITOP(sc, RTSX_PM_EVENT_DEBUG,
1168                            RTSX_PME_DEBUG_0, RTSX_PME_DEBUG_0);
1169                 RTSX_WRITE(sc, RTSX_PM_CLK_FORCE_CTL, 0x11);
1170                 break;
1171         case RTSX_RTS525A:
1172                 /* Add specific init from RTS5249. */
1173                 /* Rest L1SUB Config. */
1174                 RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xff);
1175                 /* Configure GPIO as output. */
1176                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1177                 /* Reset ASPM state to default value. */
1178                 RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1179                 /* Switch LDO3318 source from DV33 to 3V3. */
1180                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1181                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1182                 /* Set default OLT blink period. */
1183                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1184                 /* Configure driving. */
1185                 if ((error = rtsx_rts5249_fill_driving(sc)))
1186                         return (error);
1187                 /* Configure force_clock_req. */
1188                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1189                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0);
1190                 else
1191                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80);
1192
1193                 /* Specifc for RTS525A. */
1194                 RTSX_BITOP(sc, RTSX_PCLK_CTL, RTSX_PCLK_MODE_SEL, RTSX_PCLK_MODE_SEL);
1195                 if (sc->rtsx_flags & RTSX_F_VERSION_A) {
1196                         RTSX_WRITE(sc, RTSX_L1SUB_CONFIG2, RTSX_L1SUB_AUTO_CFG);
1197                         RTSX_BITOP(sc, RTSX_RREF_CFG,
1198                                    RTSX_RREF_VBGSEL_MASK, RTSX_RREF_VBGSEL_1V25);
1199                         RTSX_BITOP(sc, RTSX_LDO_VIO_CFG,
1200                                    RTSX_LDO_VIO_TUNE_MASK, RTSX_LDO_VIO_1V7);
1201                         RTSX_BITOP(sc, RTSX_LDO_DV12S_CFG,
1202                                    RTSX_LDO_D12_TUNE_MASK, RTSX_LDO_D12_TUNE_DF);
1203                         RTSX_BITOP(sc, RTSX_LDO_AV12S_CFG,
1204                                    RTSX_LDO_AV12S_TUNE_MASK, RTSX_LDO_AV12S_TUNE_DF);
1205                         RTSX_BITOP(sc, RTSX_LDO_VCC_CFG0,
1206                                    RTSX_LDO_VCC_LMTVTH_MASK, RTSX_LDO_VCC_LMTVTH_2A);
1207                         RTSX_BITOP(sc, RTSX_OOBS_CONFIG,
1208                                    RTSX_OOBS_AUTOK_DIS | RTSX_OOBS_VAL_MASK, 0x89);
1209                 }
1210                 break;
1211         case RTSX_RTS5249:
1212                 /* Rest L1SUB Config. */
1213                 RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xff);
1214                 /* Configure GPIO as output. */
1215                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1216                 /* Reset ASPM state to default value. */
1217                 RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1218                 /* Switch LDO3318 source from DV33 to 3V3. */
1219                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1220                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1221                 /* Set default OLT blink period. */
1222                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1223                 /* Configure driving. */
1224                 if ((error = rtsx_rts5249_fill_driving(sc)))
1225                         return (error);
1226                 /* Configure force_clock_req. */
1227                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1228                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0);
1229                 else
1230                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80);
1231                 break;
1232         case RTSX_RTL8402:
1233         case RTSX_RTL8411:
1234                 RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3);
1235                 RTSX_BITOP(sc, RTSX_CARD_PAD_CTL, RTSX_CD_DISABLE_MASK | RTSX_CD_AUTO_DISABLE,
1236                            RTSX_CD_ENABLE);
1237                 break;
1238         case RTSX_RTL8411B:
1239                 if (sc->rtsx_flags & RTSX_F_8411B_QFN48)
1240                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf5);
1241                 RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3);
1242                 /* Enable SD interrupt. */
1243                 RTSX_BITOP(sc, RTSX_CARD_PAD_CTL, RTSX_CD_DISABLE_MASK | RTSX_CD_AUTO_DISABLE,
1244                            RTSX_CD_ENABLE);
1245                 /* Clear hw_pfm_en to disable hardware PFM mode. */
1246                 RTSX_BITOP(sc, RTSX_FUNC_FORCE_CTL, 0x06, 0x00);
1247                 break;
1248         }
1249
1250         /*!!! Added for reboot after Windows. */
1251         rtsx_bus_power_off(sc);
1252         rtsx_set_sd_timing(sc, bus_timing_normal);
1253         rtsx_set_sd_clock(sc, 0);
1254         /*!!! Added for reboot after Windows. */
1255
1256         return (0);
1257 }
1258
1259 static int
1260 rtsx_map_sd_drive(int index)
1261 {
1262         uint8_t sd_drive[4] =
1263                 {
1264                  0x01,  /* Type D */
1265                  0x02,  /* Type C */
1266                  0x05,  /* Type A */
1267                  0x03   /* Type B */
1268                 };
1269         return (sd_drive[index]);
1270 }
1271
1272 /* For voltage 3v3. */
1273 static int
1274 rtsx_rts5227_fill_driving(struct rtsx_softc *sc)
1275 {
1276         u_char  driving_3v3[4][3] = {
1277                                      {0x13, 0x13, 0x13},
1278                                      {0x96, 0x96, 0x96},
1279                                      {0x7F, 0x7F, 0x7F},
1280                                      {0x96, 0x96, 0x96},
1281         };
1282         RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]);
1283         RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]);
1284         RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]);
1285
1286         return (0);
1287 }
1288
1289 /* For voltage 3v3. */
1290 static int
1291 rtsx_rts5249_fill_driving(struct rtsx_softc *sc)
1292 {
1293         u_char  driving_3v3[4][3] = {
1294                                      {0x11, 0x11, 0x18},
1295                                      {0x55, 0x55, 0x5C},
1296                                      {0xFF, 0xFF, 0xFF},
1297                                      {0x96, 0x96, 0x96},
1298         };
1299         RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]);
1300         RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]);
1301         RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]);
1302
1303         return (0);
1304 }
1305
1306 static int
1307 rtsx_read(struct rtsx_softc *sc, uint16_t addr, uint8_t *val)
1308 {
1309         int      tries = 1024;
1310         uint32_t reg;
1311
1312         WRITE4(sc, RTSX_HAIMR, RTSX_HAIMR_BUSY |
1313             (uint32_t)((addr & 0x3FFF) << 16));
1314
1315         while (tries--) {
1316                 reg = READ4(sc, RTSX_HAIMR);
1317                 if (!(reg & RTSX_HAIMR_BUSY))
1318                         break;
1319         }
1320         *val = (reg & 0xff);
1321
1322         return ((tries == 0) ? ETIMEDOUT : 0);
1323 }
1324
1325 static int
1326 rtsx_read_cfg(struct rtsx_softc *sc, uint8_t func, uint16_t addr, uint32_t *val)
1327 {
1328         int     tries = 1024;
1329         uint8_t data0, data1, data2, data3, rwctl;
1330
1331         RTSX_WRITE(sc, RTSX_CFGADDR0, addr);
1332         RTSX_WRITE(sc, RTSX_CFGADDR1, addr >> 8);
1333         RTSX_WRITE(sc, RTSX_CFGRWCTL, RTSX_CFG_BUSY | (func & 0x03 << 4));
1334
1335         while (tries--) {
1336                 RTSX_READ(sc, RTSX_CFGRWCTL, &rwctl);
1337                 if (!(rwctl & RTSX_CFG_BUSY))
1338                         break;
1339         }
1340
1341         if (tries == 0)
1342                 return (ETIMEDOUT);
1343
1344         RTSX_READ(sc, RTSX_CFGDATA0, &data0);
1345         RTSX_READ(sc, RTSX_CFGDATA1, &data1);
1346         RTSX_READ(sc, RTSX_CFGDATA2, &data2);
1347         RTSX_READ(sc, RTSX_CFGDATA3, &data3);
1348
1349         *val = (data3 << 24) | (data2 << 16) | (data1 << 8) | data0;
1350
1351         return (0);
1352 }
1353
1354 static int
1355 rtsx_write(struct rtsx_softc *sc, uint16_t addr, uint8_t mask, uint8_t val)
1356 {
1357         int      tries = 1024;
1358         uint32_t reg;
1359
1360         WRITE4(sc, RTSX_HAIMR,
1361             RTSX_HAIMR_BUSY | RTSX_HAIMR_WRITE |
1362             (uint32_t)(((addr & 0x3FFF) << 16) |
1363             (mask << 8) | val));
1364
1365         while (tries--) {
1366                 reg = READ4(sc, RTSX_HAIMR);
1367                 if (!(reg & RTSX_HAIMR_BUSY)) {
1368                         if (val != (reg & 0xff))
1369                                 return (EIO);
1370                         return (0);
1371                 }
1372         }
1373
1374         return (ETIMEDOUT);
1375 }
1376
1377 static int
1378 rtsx_read_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t *val)
1379 {
1380         int     tries = 100000;
1381         uint8_t data0, data1, rwctl;
1382
1383         RTSX_WRITE(sc, RTSX_PHY_ADDR, addr);
1384         RTSX_WRITE(sc, RTSX_PHY_RWCTL, RTSX_PHY_BUSY | RTSX_PHY_READ);
1385
1386         while (tries--) {
1387                 RTSX_READ(sc, RTSX_PHY_RWCTL, &rwctl);
1388                 if (!(rwctl & RTSX_PHY_BUSY))
1389                         break;
1390         }
1391         if (tries == 0)
1392                 return (ETIMEDOUT);
1393
1394         RTSX_READ(sc, RTSX_PHY_DATA0, &data0);
1395         RTSX_READ(sc, RTSX_PHY_DATA1, &data1);
1396         *val = data1 << 8 | data0;
1397
1398         return (0);
1399 }
1400
1401 static int
1402 rtsx_write_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t val)
1403 {
1404         int     tries = 100000;
1405         uint8_t rwctl;
1406
1407         RTSX_WRITE(sc, RTSX_PHY_DATA0, val);
1408         RTSX_WRITE(sc, RTSX_PHY_DATA1, val >> 8);
1409         RTSX_WRITE(sc, RTSX_PHY_ADDR, addr);
1410         RTSX_WRITE(sc, RTSX_PHY_RWCTL, RTSX_PHY_BUSY | RTSX_PHY_WRITE);
1411
1412         while (tries--) {
1413                 RTSX_READ(sc, RTSX_PHY_RWCTL, &rwctl);
1414                 if (!(rwctl & RTSX_PHY_BUSY))
1415                         break;
1416         }
1417
1418         return ((tries == 0) ? ETIMEDOUT : 0);
1419 }
1420
1421 /*
1422  * Notice that the meaning of RTSX_PWR_GATE_CTRL changes between RTS5209 and
1423  * RTS5229. In RTS5209 it is a mask of disabled power gates, while in RTS5229
1424  * it is a mask of *enabled* gates.
1425  */
1426 static int
1427 rtsx_bus_power_off(struct rtsx_softc *sc)
1428 {
1429         if (bootverbose || sc->rtsx_debug)
1430                 device_printf(sc->rtsx_dev, "rtsx_bus_power_off()\n");
1431
1432         /* Disable SD clock. */
1433         RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_SD_CLK_EN);
1434
1435         /* Disable SD output. */
1436         RTSX_CLR(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN);
1437
1438         /* Turn off power. */
1439         switch (sc->rtsx_device_id) {
1440         case RTSX_RTS5209:
1441                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK | RTSX_PMOS_STRG_MASK,
1442                            RTSX_SD_PWR_OFF | RTSX_PMOS_STRG_400mA);
1443                 RTSX_SET(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_OFF);
1444                 break;
1445         case RTSX_RTS5227:
1446         case RTSX_RTS5229:
1447         case RTSX_RTS522A:
1448                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK | RTSX_PMOS_STRG_MASK,
1449                            RTSX_SD_PWR_OFF | RTSX_PMOS_STRG_400mA);
1450                 RTSX_CLR(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK);
1451                 break;
1452         case RTSX_RTL8402:
1453         case RTSX_RTL8411:
1454         case RTSX_RTL8411B:
1455                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1456                            RTSX_BPP_POWER_OFF);
1457                 RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB,
1458                            RTSX_BPP_LDO_SUSPEND);
1459                 break;
1460         default:
1461                 RTSX_CLR(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK);
1462                 RTSX_SET(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_OFF);
1463                 RTSX_CLR(sc, RTSX_CARD_PWR_CTL, RTSX_PMOS_STRG_800mA);
1464                 break;
1465         }
1466
1467         /* Disable pull control. */
1468         switch (sc->rtsx_device_id) {
1469         case RTSX_RTS5209:
1470                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, RTSX_PULL_CTL_DISABLE12);
1471                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12);
1472                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3);
1473                 break;
1474         case RTSX_RTS5227:
1475         case RTSX_RTS522A:
1476                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12);
1477                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3);
1478                 break;
1479         case RTSX_RTS5229:
1480                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12);
1481                 if (sc->rtsx_flags & RTSX_F_VERSION_C)
1482                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3_TYPE_C);
1483                 else
1484                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3);
1485                 break;
1486         case RTSX_RTS525A:
1487         case RTSX_RTS5249:
1488                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x66);
1489                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12);
1490                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3);
1491                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x55);
1492                 break;
1493         case RTSX_RTL8402:
1494         case RTSX_RTL8411:
1495                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x65);
1496                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55);
1497                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0x95);
1498                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x09);
1499                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x05);
1500                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x04);
1501                 break;
1502         case RTSX_RTL8411B:
1503                 if (sc->rtsx_flags & RTSX_F_8411B_QFN48) {
1504                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55);
1505                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf5);
1506                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15);
1507                 } else {
1508                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x65);
1509                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55);
1510                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xd5);
1511                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x59);
1512                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x55);
1513                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15);
1514                 }
1515                 break;
1516         }
1517
1518         return (0);
1519 }
1520
1521 static int
1522 rtsx_bus_power_on(struct rtsx_softc *sc)
1523 {
1524         if (bootverbose || sc->rtsx_debug)
1525                 device_printf(sc->rtsx_dev, "rtsx_bus_power_on()\n");
1526
1527         /* Select SD card. */
1528         RTSX_BITOP(sc, RTSX_CARD_SELECT, 0x07, RTSX_SD_MOD_SEL);
1529         RTSX_BITOP(sc, RTSX_CARD_SHARE_MODE, RTSX_CARD_SHARE_MASK, RTSX_CARD_SHARE_48_SD);
1530
1531         /* Enable SD clock. */
1532         RTSX_BITOP(sc, RTSX_CARD_CLK_EN, RTSX_SD_CLK_EN,  RTSX_SD_CLK_EN);
1533
1534         /* Enable pull control. */
1535         switch (sc->rtsx_device_id) {
1536         case RTSX_RTS5209:
1537                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, RTSX_PULL_CTL_ENABLE12);
1538                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12);
1539                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3);
1540                 break;
1541         case RTSX_RTS5227:
1542         case RTSX_RTS522A:
1543                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12);
1544                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3);
1545                 break;
1546         case RTSX_RTS5229:
1547                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12);
1548                 if (sc->rtsx_flags & RTSX_F_VERSION_C)
1549                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3_TYPE_C);
1550                 else
1551                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3);
1552                 break;
1553         case RTSX_RTS525A:
1554         case RTSX_RTS5249:
1555                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x66);
1556                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12);
1557                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3);
1558                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0xaa);
1559                 break;
1560         case RTSX_RTL8402:
1561         case RTSX_RTL8411:
1562                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0xaa);
1563                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa);
1564                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xa9);
1565                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x09);
1566                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x09);
1567                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x04);
1568                 break;
1569         case RTSX_RTL8411B:
1570                 if (sc->rtsx_flags & RTSX_F_8411B_QFN48) {
1571                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa);
1572                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf9);
1573                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x19);
1574                 } else {
1575                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0xaa);
1576                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa);
1577                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xd9);
1578                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x59);
1579                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x55);
1580                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15);
1581                 }
1582                 break;
1583         }
1584
1585         /*
1586          * To avoid a current peak, enable card power in two phases
1587          * with a delay in between.
1588          */
1589         switch (sc->rtsx_device_id) {
1590         case RTSX_RTS5209:
1591                 /* Partial power. */
1592                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON);
1593                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC2);
1594
1595                 DELAY(200);
1596
1597                 /* Full power. */
1598                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON);
1599                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_ON);
1600                 break;
1601         case RTSX_RTS5227:
1602         case RTSX_RTS522A:
1603                 /* Partial power. */
1604                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON);
1605                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1);
1606
1607                 DELAY(200);
1608
1609                 /* Full power. */
1610                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON);
1611                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK,
1612                            RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2);
1613                 RTSX_BITOP(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN, RTSX_SD_OUTPUT_EN);
1614                 RTSX_BITOP(sc, RTSX_CARD_OE, RTSX_MS_OUTPUT_EN, RTSX_MS_OUTPUT_EN);
1615                 break;
1616         case RTSX_RTS5229:
1617                 /* Partial power. */
1618                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON);
1619                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1);
1620
1621                 DELAY(200);
1622
1623                 /* Full power. */
1624                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON);
1625                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK,
1626                            RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2);
1627                 break;
1628         case RTSX_RTS525A:
1629                 RTSX_BITOP(sc, RTSX_LDO_VCC_CFG1, RTSX_LDO_VCC_TUNE_MASK, RTSX_LDO_VCC_3V3);
1630         case RTSX_RTS5249:
1631                 /* Partial power. */
1632                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON);
1633                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1);
1634
1635                 DELAY(200);
1636
1637                 /* Full power. */
1638                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON);
1639                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK,
1640                            RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2);
1641                 break;
1642         case RTSX_RTL8402:
1643         case RTSX_RTL8411:
1644         case RTSX_RTL8411B:
1645                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1646                            RTSX_BPP_POWER_5_PERCENT_ON);
1647                 RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB,
1648                            RTSX_BPP_LDO_SUSPEND);
1649                 DELAY(150);
1650                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1651                            RTSX_BPP_POWER_10_PERCENT_ON);
1652                 DELAY(150);
1653                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1654                            RTSX_BPP_POWER_15_PERCENT_ON);
1655                 DELAY(150);
1656                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1657                            RTSX_BPP_POWER_ON);
1658                 RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB,
1659                            RTSX_BPP_LDO_ON);
1660                 break;
1661         }
1662
1663         /* Enable SD card output. */
1664         RTSX_WRITE(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN);
1665
1666         DELAY(200);
1667
1668         return (0);
1669 }
1670
1671 /*
1672  * Set but width.
1673  */
1674 static int
1675 rtsx_set_bus_width(struct rtsx_softc *sc, enum mmc_bus_width width)
1676 {
1677         uint32_t bus_width;
1678
1679         switch (width) {
1680         case bus_width_1:
1681                 bus_width = RTSX_BUS_WIDTH_1;
1682                 break;
1683         case bus_width_4:
1684                 bus_width = RTSX_BUS_WIDTH_4;
1685                 break;
1686         case bus_width_8:
1687                 bus_width = RTSX_BUS_WIDTH_8;
1688                 break;
1689         default:
1690                 return (MMC_ERR_INVALID);
1691         }
1692         RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_BUS_WIDTH_MASK, bus_width);
1693
1694         if (bootverbose || sc->rtsx_debug) {
1695                 char *busw[] = {
1696                                 "1 bit",
1697                                 "4 bits",
1698                                 "8 bits"
1699                 };
1700                 device_printf(sc->rtsx_dev, "Setting bus width to %s\n", busw[bus_width]);
1701         }
1702         return (0);
1703 }
1704
1705 static int
1706 rtsx_set_sd_timing(struct rtsx_softc *sc, enum mmc_bus_timing timing)
1707 {
1708         if (timing == bus_timing_hs && sc->rtsx_force_timing) {
1709                 timing = bus_timing_uhs_sdr50;
1710                 sc->rtsx_ios_timing = timing;
1711         }
1712
1713         if (bootverbose || sc->rtsx_debug)
1714                 device_printf(sc->rtsx_dev, "rtsx_set_sd_timing(%u)\n", timing);
1715
1716         switch (timing) {
1717         case bus_timing_uhs_sdr50:
1718         case bus_timing_uhs_sdr104:
1719                 sc->rtsx_double_clk = false;
1720                 sc->rtsx_vpclk = true;
1721                 RTSX_BITOP(sc, RTSX_SD_CFG1, 0x0c | RTSX_SD_ASYNC_FIFO_NOT_RST,
1722                            RTSX_SD30_MODE | RTSX_SD_ASYNC_FIFO_NOT_RST);
1723                 RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ);
1724                 RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE,
1725                            RTSX_CRC_VAR_CLK0 | RTSX_SD30_FIX_CLK | RTSX_SAMPLE_VAR_CLK1);
1726                 RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ);
1727                 break;
1728         case bus_timing_hs:
1729                 RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK, RTSX_SD20_MODE);
1730                 RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ);
1731                 RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE,
1732                            RTSX_CRC_FIX_CLK | RTSX_SD30_VAR_CLK0 | RTSX_SAMPLE_VAR_CLK1);
1733                 RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ);
1734
1735                 RTSX_BITOP(sc, RTSX_SD_PUSH_POINT_CTL,
1736                            RTSX_SD20_TX_SEL_MASK, RTSX_SD20_TX_14_AHEAD);
1737                 RTSX_BITOP(sc, RTSX_SD_SAMPLE_POINT_CTL,
1738                            RTSX_SD20_RX_SEL_MASK, RTSX_SD20_RX_14_DELAY);
1739                 break;
1740         default:
1741                 RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK, RTSX_SD20_MODE);
1742                 RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ);
1743                 RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE,
1744                            RTSX_CRC_FIX_CLK | RTSX_SD30_VAR_CLK0 | RTSX_SAMPLE_VAR_CLK1);
1745                 RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ);
1746
1747                 RTSX_WRITE(sc, RTSX_SD_PUSH_POINT_CTL, RTSX_SD20_TX_NEG_EDGE);
1748                 RTSX_BITOP(sc, RTSX_SD_SAMPLE_POINT_CTL,
1749                            RTSX_SD20_RX_SEL_MASK, RTSX_SD20_RX_POS_EDGE);
1750                 break;
1751         }
1752
1753         return (0);
1754 }
1755
1756 /*
1757  * Set or change SDCLK frequency or disable the SD clock.
1758  * Return zero on success.
1759  */
1760 static int
1761 rtsx_set_sd_clock(struct rtsx_softc *sc, uint32_t freq)
1762 {
1763         uint8_t clk;
1764         uint8_t clk_divider, n, div, mcu;
1765         int     error = 0;
1766
1767         if (bootverbose || sc->rtsx_debug)
1768                 device_printf(sc->rtsx_dev, "rtsx_set_sd_clock(%u)\n", freq);
1769
1770         if (freq == RTSX_SDCLK_OFF) {
1771                 error = rtsx_stop_sd_clock(sc);
1772                 return error;
1773         }
1774
1775         sc->rtsx_ssc_depth = RTSX_SSC_DEPTH_500K;
1776         sc->rtsx_discovery_mode = (freq <= 1000000) ? true : false;
1777
1778         if (sc->rtsx_discovery_mode) {
1779                 /* We use 250k(around) here, in discovery stage. */
1780                 clk_divider = RTSX_CLK_DIVIDE_128;
1781                 freq = 30000000;
1782         } else {
1783                 clk_divider = RTSX_CLK_DIVIDE_0;
1784         }
1785         RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, clk_divider);
1786
1787         freq /= 1000000;
1788         if (sc->rtsx_discovery_mode || !sc->rtsx_double_clk)
1789                 clk = freq;
1790         else
1791                 clk = freq * 2;
1792
1793         switch (sc->rtsx_device_id) {
1794         case RTSX_RTL8402:
1795         case RTSX_RTL8411:
1796         case RTSX_RTL8411B:
1797                 n = clk * 4 / 5 - 2;
1798                 break;
1799         default:
1800                 n = clk - 2;
1801                 break;
1802         }
1803         if ((clk <= 2) || (n > RTSX_MAX_DIV_N))
1804                 return (MMC_ERR_INVALID);
1805
1806         mcu = 125 / clk + 3;
1807         if (mcu > 15)
1808                 mcu = 15;
1809
1810         /* Make sure that the SSC clock div_n is not less than RTSX_MIN_DIV_N. */
1811         div = RTSX_CLK_DIV_1;
1812         while ((n < RTSX_MIN_DIV_N) && (div < RTSX_CLK_DIV_8)) {
1813                 switch (sc->rtsx_device_id) {
1814                 case RTSX_RTL8402:
1815                 case RTSX_RTL8411:
1816                 case RTSX_RTL8411B:
1817                         n = (((n + 2) * 5 / 4) * 2) * 4 / 5 - 2;
1818                         break;
1819                 default:
1820                         n = (n + 2) * 2 - 2;
1821                         break;
1822                 }
1823                 div++;
1824         }
1825
1826         if (sc->rtsx_double_clk && sc->rtsx_ssc_depth > 1)
1827                 sc->rtsx_ssc_depth -= 1;
1828
1829         if (div > RTSX_CLK_DIV_1) {
1830                 if (sc->rtsx_ssc_depth > (div - 1))
1831                         sc->rtsx_ssc_depth -= (div - 1);
1832                 else
1833                         sc->rtsx_ssc_depth = RTSX_SSC_DEPTH_4M;
1834         }
1835
1836         /* Enable SD clock. */
1837         error = rtsx_switch_sd_clock(sc, clk, n, div, mcu);
1838
1839         return (error);
1840 }
1841
1842 static int
1843 rtsx_stop_sd_clock(struct rtsx_softc *sc)
1844 {
1845         RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_CARD_CLK_EN_ALL);
1846         RTSX_SET(sc, RTSX_SD_BUS_STAT, RTSX_SD_CLK_FORCE_STOP);
1847
1848         return (0);
1849 }
1850
1851 static int
1852 rtsx_switch_sd_clock(struct rtsx_softc *sc, uint8_t clk, uint8_t n, uint8_t div, uint8_t mcu)
1853 {
1854         if (bootverbose || sc->rtsx_debug) {
1855                 device_printf(sc->rtsx_dev, "rtsx_switch_sd_clock() - discovery-mode is %s, ssc_depth: %d\n",
1856                               (sc->rtsx_discovery_mode) ? "true" : "false", sc->rtsx_ssc_depth);
1857                 device_printf(sc->rtsx_dev, "rtsx_switch_sd_clock() - clk: %d, n: %d, div: %d, mcu: %d\n",
1858                               clk, n, div, mcu);
1859         }
1860
1861         RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ);
1862         RTSX_WRITE(sc, RTSX_CLK_DIV, (div << 4) | mcu);
1863         RTSX_CLR(sc, RTSX_SSC_CTL1, RTSX_RSTB);
1864         RTSX_BITOP(sc, RTSX_SSC_CTL2, RTSX_SSC_DEPTH_MASK, sc->rtsx_ssc_depth);
1865         RTSX_WRITE(sc, RTSX_SSC_DIV_N_0, n);
1866         RTSX_BITOP(sc, RTSX_SSC_CTL1, RTSX_RSTB, RTSX_RSTB);
1867         if (sc->rtsx_vpclk) {
1868                 RTSX_CLR(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET);
1869                 RTSX_BITOP(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET);
1870         }
1871
1872         /* Wait SSC clock stable. */
1873         DELAY(200);
1874
1875         RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ);
1876
1877         return (0);
1878 }
1879
1880 static void
1881 rtsx_sd_change_tx_phase(struct rtsx_softc *sc, uint8_t sample_point)
1882 {
1883         if (bootverbose || sc->rtsx_debug)
1884                 device_printf(sc->rtsx_dev, "rtsx_sd_change_tx_phase() - sample_point: %d\n", sample_point);
1885
1886         rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CHANGE_CLK);
1887         rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_SELECT_MASK, sample_point);
1888         rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, 0);
1889         rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET);
1890         rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, 0);
1891         rtsx_write(sc, RTSX_SD_CFG1, RTSX_SD_ASYNC_FIFO_NOT_RST, 0);
1892 }
1893
1894 static void
1895 rtsx_sd_change_rx_phase(struct rtsx_softc *sc, uint8_t sample_point)
1896 {
1897         if (bootverbose || sc->rtsx_debug == 2)
1898                 device_printf(sc->rtsx_dev, "rtsx_sd_change_rx_phase() - sample_point: %d\n", sample_point);
1899
1900         rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CHANGE_CLK);
1901         rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_SELECT_MASK, sample_point);
1902         rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_NOT_RESET, 0);
1903         rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET);
1904         rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, 0);
1905         rtsx_write(sc, RTSX_SD_CFG1, RTSX_SD_ASYNC_FIFO_NOT_RST, 0);
1906 }
1907
1908 static void
1909 rtsx_sd_tuning_rx_phase(struct rtsx_softc *sc, uint32_t *phase_map)
1910 {
1911         uint32_t raw_phase_map = 0;
1912         int      i;
1913         int      error;
1914
1915         for (i = 0; i < RTSX_RX_PHASE_MAX; i++) {
1916                 error = rtsx_sd_tuning_rx_cmd(sc, (uint8_t)i);
1917                 if (error == 0)
1918                         raw_phase_map |= 1 << i;
1919         }
1920         if (phase_map != NULL)
1921                 *phase_map = raw_phase_map;
1922 }
1923
1924 static int
1925 rtsx_sd_tuning_rx_cmd(struct rtsx_softc *sc, uint8_t sample_point)
1926 {
1927         struct mmc_request req = {};
1928         struct mmc_command cmd = {};
1929         int     error = 0;
1930
1931         cmd.opcode = MMC_SEND_TUNING_BLOCK;
1932         cmd.arg = 0;
1933         req.cmd = &cmd;
1934
1935         RTSX_LOCK(sc);
1936
1937         sc->rtsx_req = &req;
1938
1939         rtsx_sd_change_rx_phase(sc, sample_point);
1940
1941         rtsx_write(sc, RTSX_SD_CFG3, RTSX_SD_RSP_80CLK_TIMEOUT_EN,
1942                    RTSX_SD_RSP_80CLK_TIMEOUT_EN);
1943
1944         rtsx_init_cmd(sc, &cmd);
1945         rtsx_set_cmd_data_len(sc, 1, 0x40);
1946         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff,
1947                       RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 |
1948                       RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_6);
1949         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER,
1950                       0xff, RTSX_TM_AUTO_TUNING | RTSX_SD_TRANSFER_START);
1951         rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
1952                       RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END);
1953
1954         /* Set interrupt post processing */
1955         sc->rtsx_intr_trans_ok = rtsx_sd_tuning_rx_cmd_wakeup;
1956         sc->rtsx_intr_trans_ko = rtsx_sd_tuning_rx_cmd_wakeup;
1957
1958         /* Run the command queue. */
1959         rtsx_send_cmd(sc);
1960
1961         error = rtsx_sd_tuning_rx_cmd_wait(sc, &cmd);
1962
1963         if (error) {
1964                 if (bootverbose || sc->rtsx_debug == 2)
1965                         device_printf(sc->rtsx_dev, "rtsx_sd_tuning_rx_cmd() - error: %d\n", error);
1966                 rtsx_sd_wait_data_idle(sc);
1967                 rtsx_clear_error(sc);
1968         }
1969         rtsx_write(sc, RTSX_SD_CFG3, RTSX_SD_RSP_80CLK_TIMEOUT_EN, 0);
1970
1971         sc->rtsx_req = NULL;
1972
1973         RTSX_UNLOCK(sc);
1974
1975         return (error);
1976 }
1977
1978 static int
1979 rtsx_sd_tuning_rx_cmd_wait(struct rtsx_softc *sc, struct mmc_command *cmd)
1980 {
1981         int     status;
1982         int     mask = RTSX_TRANS_OK_INT | RTSX_TRANS_FAIL_INT;
1983
1984         status = sc->rtsx_intr_status & mask;
1985         while (status == 0) {
1986                 if (msleep(&sc->rtsx_intr_status, &sc->rtsx_mtx, 0, "rtsxintr", sc->rtsx_timeout) == EWOULDBLOCK) {
1987                         cmd->error = MMC_ERR_TIMEOUT;
1988                         return (MMC_ERR_TIMEOUT);
1989                 }
1990                 status = sc->rtsx_intr_status & mask;
1991         }
1992         return (cmd->error);
1993 }
1994
1995 static void
1996 rtsx_sd_tuning_rx_cmd_wakeup(struct rtsx_softc *sc)
1997 {
1998         wakeup(&sc->rtsx_intr_status);
1999 }
2000
2001 static void
2002 rtsx_sd_wait_data_idle(struct rtsx_softc *sc)
2003 {
2004         int     i;
2005         uint8_t val;
2006
2007         for (i = 0; i < 100; i++) {
2008                 rtsx_read(sc, RTSX_SD_DATA_STATE, &val);
2009                 if (val & RTSX_SD_DATA_IDLE)
2010                         return;
2011                 DELAY(100);
2012         }
2013 }
2014
2015 static uint8_t
2016 rtsx_sd_search_final_rx_phase(struct rtsx_softc *sc, uint32_t phase_map)
2017 {
2018         int     start = 0, len = 0;
2019         int     start_final = 0, len_final = 0;
2020         uint8_t final_phase = 0xff;
2021
2022         while (start < RTSX_RX_PHASE_MAX) {
2023                 len = rtsx_sd_get_rx_phase_len(phase_map, start);
2024                 if (len_final < len) {
2025                         start_final = start;
2026                         len_final = len;
2027                 }
2028                 start += len ? len : 1;
2029         }
2030
2031         final_phase = (start_final + len_final / 2) % RTSX_RX_PHASE_MAX;
2032
2033         if (bootverbose || sc->rtsx_debug)
2034                 device_printf(sc->rtsx_dev,
2035                               "rtsx_sd_search_final_rx_phase() - phase_map: %x, start_final: %d, len_final: %d, final_phase: %d\n",
2036                               phase_map, start_final, len_final, final_phase);
2037
2038         return final_phase;
2039 }
2040
2041 static int
2042 rtsx_sd_get_rx_phase_len(uint32_t phase_map, int start_bit)
2043 {
2044         int     i;
2045
2046         for (i = 0; i < RTSX_RX_PHASE_MAX; i++) {
2047                 if ((phase_map & (1 << (start_bit + i) % RTSX_RX_PHASE_MAX)) == 0)
2048                         return i;
2049         }
2050         return RTSX_RX_PHASE_MAX;
2051 }
2052
2053 #if 0   /* For led */
2054 static int
2055 rtsx_led_enable(struct rtsx_softc *sc)
2056 {
2057         switch (sc->rtsx_device_id) {
2058         case RTSX_RTS5209:
2059                 RTSX_CLR(sc, RTSX_CARD_GPIO, RTSX_CARD_GPIO_LED_OFF);
2060                 RTSX_WRITE(sc, RTSX_CARD_AUTO_BLINK,
2061                            RTSX_LED_BLINK_EN | RTSX_LED_BLINK_SPEED);
2062                 break;
2063         case RTSX_RTL8411B:
2064                 RTSX_CLR(sc, RTSX_GPIO_CTL, 0x01);
2065                 RTSX_WRITE(sc, RTSX_CARD_AUTO_BLINK,
2066                            RTSX_LED_BLINK_EN | RTSX_LED_BLINK_SPEED);
2067                 break;
2068         default:
2069                 RTSX_SET(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON);
2070                 RTSX_SET(sc, RTSX_OLT_LED_CTL, RTSX_OLT_LED_AUTOBLINK);
2071                 break;
2072         }
2073
2074         return (0);
2075 }
2076
2077 static int
2078 rtsx_led_disable(struct rtsx_softc *sc)
2079 {
2080         switch (sc->rtsx_device_id) {
2081         case RTSX_RTS5209:
2082                 RTSX_CLR(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN);
2083                 RTSX_WRITE(sc, RTSX_CARD_GPIO, RTSX_CARD_GPIO_LED_OFF);
2084                 break;
2085         case RTSX_RTL8411B:
2086                 RTSX_CLR(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN);
2087                 RTSX_SET(sc, RTSX_GPIO_CTL, 0x01);
2088                 break;
2089         default:
2090                 RTSX_CLR(sc, RTSX_OLT_LED_CTL, RTSX_OLT_LED_AUTOBLINK);
2091                 RTSX_CLR(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON);
2092                 break;
2093         }
2094
2095         return (0);
2096 }
2097 #endif  /* For led */
2098
2099 static uint8_t
2100 rtsx_response_type(uint16_t mmc_rsp)
2101 {
2102         int     i;
2103         struct rsp_type {
2104                 uint16_t mmc_rsp;
2105                 uint8_t  rtsx_rsp;
2106         } rsp_types[] = {
2107                 { MMC_RSP_NONE, RTSX_SD_RSP_TYPE_R0 },
2108                 { MMC_RSP_R1,   RTSX_SD_RSP_TYPE_R1 },
2109                 { MMC_RSP_R1B,  RTSX_SD_RSP_TYPE_R1B },
2110                 { MMC_RSP_R2,   RTSX_SD_RSP_TYPE_R2 },
2111                 { MMC_RSP_R3,   RTSX_SD_RSP_TYPE_R3 },
2112                 { MMC_RSP_R4,   RTSX_SD_RSP_TYPE_R4 },
2113                 { MMC_RSP_R5,   RTSX_SD_RSP_TYPE_R5 },
2114                 { MMC_RSP_R6,   RTSX_SD_RSP_TYPE_R6 },
2115                 { MMC_RSP_R7,   RTSX_SD_RSP_TYPE_R7 }
2116         };
2117
2118         for (i = 0; i < nitems(rsp_types); i++) {
2119                 if (mmc_rsp == rsp_types[i].mmc_rsp)
2120                         return (rsp_types[i].rtsx_rsp);
2121         }
2122
2123         return (0);
2124 }
2125
2126 /*
2127  * Init command buffer with SD command index and argument.
2128  */
2129 static void
2130 rtsx_init_cmd(struct rtsx_softc *sc, struct mmc_command *cmd)
2131 {
2132         sc->rtsx_cmd_index = 0;
2133         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD0,
2134                       0xff, RTSX_SD_CMD_START  | cmd->opcode);
2135         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD1,
2136                      0xff, cmd->arg >> 24);
2137         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD2,
2138                       0xff, cmd->arg >> 16);
2139         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD3,
2140                      0xff, cmd->arg >> 8);
2141         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD4,
2142                      0xff, cmd->arg);
2143 }
2144
2145 /*
2146  * Append a properly encoded host command to the host command buffer.
2147  */
2148 static void
2149 rtsx_push_cmd(struct rtsx_softc *sc, uint8_t cmd, uint16_t reg,
2150               uint8_t mask, uint8_t data)
2151 {
2152         KASSERT(sc->rtsx_cmd_index < RTSX_HOSTCMD_MAX,
2153                 ("rtsx: Too many host commands (%d)\n", sc->rtsx_cmd_index));
2154
2155         uint32_t *cmd_buffer = (uint32_t *)(sc->rtsx_cmd_dmamem);
2156         cmd_buffer[sc->rtsx_cmd_index++] =
2157                 htole32((uint32_t)(cmd & 0x3) << 30) |
2158                 ((uint32_t)(reg & 0x3fff) << 16) |
2159                 ((uint32_t)(mask) << 8) |
2160                 ((uint32_t)data);
2161 }
2162
2163 /*
2164  * Queue commands to configure data transfer size.
2165  */
2166 static void
2167 rtsx_set_cmd_data_len(struct rtsx_softc *sc, uint16_t block_cnt, uint16_t byte_cnt)
2168 {
2169         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BLOCK_CNT_L,
2170                       0xff, block_cnt & 0xff);
2171         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BLOCK_CNT_H,
2172                       0xff, block_cnt >> 8);
2173         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BYTE_CNT_L,
2174                       0xff, byte_cnt & 0xff);
2175         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BYTE_CNT_H,
2176                       0xff, byte_cnt >> 8);
2177 }
2178
2179 /*
2180  * Run the command queue.
2181  */
2182 static void
2183 rtsx_send_cmd(struct rtsx_softc *sc)
2184 {
2185         if (bootverbose)
2186                 device_printf(sc->rtsx_dev, "rtsx_send_cmd()\n");
2187
2188         sc->rtsx_intr_status = 0;
2189
2190         /* Sync command DMA buffer. */
2191         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_PREREAD);
2192         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_PREWRITE);
2193
2194         /* Tell the chip where the command buffer is and run the commands. */
2195         WRITE4(sc, RTSX_HCBAR, (uint32_t)sc->rtsx_cmd_buffer);
2196         WRITE4(sc, RTSX_HCBCTLR,
2197                ((sc->rtsx_cmd_index * 4) & 0x00ffffff) | RTSX_START_CMD | RTSX_HW_AUTO_RSP);
2198 }
2199
2200 /*
2201  * Stop previous command.
2202  */
2203 static void
2204 rtsx_stop_cmd(struct rtsx_softc *sc)
2205 {
2206         /* Stop command transfer. */
2207         WRITE4(sc, RTSX_HCBCTLR, RTSX_STOP_CMD);
2208
2209         /* Stop DMA transfer. */
2210         WRITE4(sc, RTSX_HDBCTLR, RTSX_STOP_DMA);
2211
2212         rtsx_write(sc, RTSX_DMACTL, RTSX_DMA_RST, RTSX_DMA_RST);
2213
2214         rtsx_write(sc, RTSX_RBCTL, RTSX_RB_FLUSH, RTSX_RB_FLUSH);
2215 }
2216
2217 /*
2218  * Clear error.
2219  */
2220 static void
2221 rtsx_clear_error(struct rtsx_softc *sc)
2222 {
2223         /* Clear error. */
2224         rtsx_write(sc, RTSX_CARD_STOP, RTSX_SD_STOP | RTSX_SD_CLR_ERR,
2225                    RTSX_SD_STOP | RTSX_SD_CLR_ERR);
2226 }
2227
2228 /*
2229  * Signal end of request to mmc/mmcsd.
2230  */
2231 static void
2232 rtsx_req_done(struct rtsx_softc *sc)
2233 {
2234 #ifdef MMCCAM
2235         union ccb *ccb;
2236 #endif /* MMCCAM */
2237         struct mmc_request *req;
2238
2239         req = sc->rtsx_req;
2240         if (req->cmd->error == MMC_ERR_NONE) {
2241                 if (req->cmd->opcode == MMC_READ_SINGLE_BLOCK ||
2242                     req->cmd->opcode == MMC_READ_MULTIPLE_BLOCK)
2243                         sc->rtsx_read_count++;
2244                 else if (req->cmd->opcode == MMC_WRITE_BLOCK ||
2245                          req->cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK)
2246                         sc->rtsx_write_count++;
2247         } else {
2248                 rtsx_clear_error(sc);
2249         }
2250         callout_stop(&sc->rtsx_timeout_callout);
2251         sc->rtsx_req = NULL;
2252 #ifdef MMCCAM
2253         ccb = sc->rtsx_ccb;
2254         sc->rtsx_ccb = NULL;
2255         ccb->ccb_h.status = (req->cmd->error == 0 ? CAM_REQ_CMP : CAM_REQ_CMP_ERR);
2256         xpt_done(ccb);
2257 #else
2258         req->done(req);
2259 #endif /* MMCCAM */
2260 }
2261
2262 /*
2263  * Send request.
2264  */
2265 static int
2266 rtsx_send_req(struct rtsx_softc *sc, struct mmc_command *cmd)
2267 {
2268         uint8_t  rsp_type;
2269         uint16_t reg;
2270
2271         if (bootverbose)
2272                 device_printf(sc->rtsx_dev, "rtsx_send_req() - CMD%d\n", cmd->opcode);
2273
2274         /* Convert response type. */
2275         rsp_type = rtsx_response_type(cmd->flags & MMC_RSP_MASK);
2276         if (rsp_type == 0) {
2277                 device_printf(sc->rtsx_dev, "Unknown rsp_type: 0x%lx\n", (cmd->flags & MMC_RSP_MASK));
2278                 cmd->error = MMC_ERR_INVALID;
2279                 return (MMC_ERR_INVALID);
2280         }
2281
2282         rtsx_init_cmd(sc, cmd);
2283
2284         /* Queue command to set response type. */
2285         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, rsp_type);
2286
2287         /* Use the ping-pong buffer (cmd buffer) for commands which do not transfer data. */
2288         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE,
2289                       0x01, RTSX_PINGPONG_BUFFER);
2290
2291         /* Queue commands to perform SD transfer. */
2292         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER,
2293                       0xff, RTSX_TM_CMD_RSP | RTSX_SD_TRANSFER_START);
2294         rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
2295                       RTSX_SD_TRANSFER_END|RTSX_SD_STAT_IDLE,
2296                       RTSX_SD_TRANSFER_END|RTSX_SD_STAT_IDLE);
2297
2298         /* If needed queue commands to read back card status response. */
2299         if (rsp_type == RTSX_SD_RSP_TYPE_R2) {
2300                 /* Read data from ping-pong buffer. */
2301                 for (reg = RTSX_PPBUF_BASE2; reg < RTSX_PPBUF_BASE2 + 16; reg++)
2302                         rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg, 0, 0);
2303         } else if (rsp_type != RTSX_SD_RSP_TYPE_R0) {
2304                 /* Read data from SD_CMDx registers. */
2305                 for (reg = RTSX_SD_CMD0; reg <= RTSX_SD_CMD4; reg++)
2306                         rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg, 0, 0);
2307         }
2308         rtsx_push_cmd(sc, RTSX_READ_REG_CMD, RTSX_SD_STAT1, 0, 0);
2309
2310         /* Set transfer OK function. */
2311         if (sc->rtsx_intr_trans_ok == NULL)
2312                 sc->rtsx_intr_trans_ok = rtsx_ret_resp;
2313
2314         /* Run the command queue. */
2315         rtsx_send_cmd(sc);
2316
2317         return (0);
2318 }
2319
2320 /*
2321  * Return response of previous command (case cmd->data == NULL) and complete resquest.
2322  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2323  */
2324 static void
2325 rtsx_ret_resp(struct rtsx_softc *sc)
2326 {
2327         struct mmc_command *cmd;
2328
2329         cmd = sc->rtsx_req->cmd;
2330         rtsx_set_resp(sc, cmd);
2331         rtsx_req_done(sc);
2332 }
2333
2334 /*
2335  * Set response of previous command.
2336  */
2337 static void
2338 rtsx_set_resp(struct rtsx_softc *sc, struct mmc_command *cmd)
2339 {
2340         uint8_t  rsp_type;
2341
2342         rsp_type = rtsx_response_type(cmd->flags & MMC_RSP_MASK);
2343
2344         /* Sync command DMA buffer. */
2345         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD);
2346         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE);
2347
2348         /* Copy card response into mmc response buffer. */
2349         if (ISSET(cmd->flags, MMC_RSP_PRESENT)) {
2350                 uint32_t *cmd_buffer = (uint32_t *)(sc->rtsx_cmd_dmamem);
2351
2352                 if (bootverbose) {
2353                         device_printf(sc->rtsx_dev, "cmd_buffer: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
2354                                       cmd_buffer[0], cmd_buffer[1], cmd_buffer[2], cmd_buffer[3], cmd_buffer[4]);
2355                 }
2356
2357                 if (rsp_type == RTSX_SD_RSP_TYPE_R2) {
2358                         /* First byte is CHECK_REG_CMD return value, skip it. */
2359                         unsigned char *ptr = (unsigned char *)cmd_buffer + 1;
2360                         int i;
2361
2362                         /*
2363                          * The controller offloads the last byte {CRC-7, end bit 1}
2364                          * of response type R2. Assign dummy CRC, 0, and end bit to this
2365                          * byte (ptr[16], goes into the LSB of resp[3] later).
2366                          */
2367                         ptr[16] = 0x01;
2368                         /* The second byte is the status of response, skip it. */
2369                         for (i = 0; i < 4; i++)
2370                                 cmd->resp[i] = be32dec(ptr + 1 + i * 4);
2371                 } else {
2372                         /*
2373                          * First byte is CHECK_REG_CMD return value, second
2374                          * one is the command op code -- we skip those.
2375                          */
2376                         cmd->resp[0] =
2377                                 ((be32toh(cmd_buffer[0]) & 0x0000ffff) << 16) |
2378                                 ((be32toh(cmd_buffer[1]) & 0xffff0000) >> 16);
2379                 }
2380
2381                 if (bootverbose)
2382                         device_printf(sc->rtsx_dev, "cmd->resp: 0x%08x 0x%08x 0x%08x 0x%08x\n",
2383                                       cmd->resp[0], cmd->resp[1], cmd->resp[2], cmd->resp[3]);
2384         }
2385 }
2386
2387 /*
2388  * Use the ping-pong buffer (cmd buffer) for transfer <= 512 bytes.
2389  */
2390 static int
2391 rtsx_xfer_short(struct rtsx_softc *sc, struct mmc_command *cmd)
2392 {
2393         int     read;
2394
2395         if (cmd->data == NULL || cmd->data->len == 0) {
2396                 cmd->error = MMC_ERR_INVALID;
2397                 return (MMC_ERR_INVALID);
2398         }
2399         cmd->data->xfer_len = (cmd->data->len > RTSX_MAX_DATA_BLKLEN) ?
2400                 RTSX_MAX_DATA_BLKLEN : cmd->data->len;
2401
2402         read = ISSET(cmd->data->flags, MMC_DATA_READ);
2403
2404         if (bootverbose)
2405                 device_printf(sc->rtsx_dev, "rtsx_xfer_short() - %s xfer: %ld bytes with block size %ld\n",
2406                               read ? "Read" : "Write",
2407                               (unsigned long)cmd->data->len, (unsigned long)cmd->data->xfer_len);
2408
2409         if (cmd->data->len > 512) {
2410                 device_printf(sc->rtsx_dev, "rtsx_xfer_short() - length too large: %ld > 512\n",
2411                               (unsigned long)cmd->data->len);
2412                 cmd->error = MMC_ERR_INVALID;
2413                 return (MMC_ERR_INVALID);
2414         }
2415
2416         if (read) {
2417                 if (sc->rtsx_discovery_mode)
2418                         rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_0);
2419
2420                 rtsx_init_cmd(sc, cmd);
2421
2422                 /* Queue commands to configure data transfer size. */
2423                 rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len);
2424
2425                 /* From Linux: rtsx_pci_sdmmc.c sd_read_data(). */
2426                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff,
2427                               RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 |
2428                               RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_6);
2429
2430                 /* Use the ping-pong buffer (cmd buffer). */
2431                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE,
2432                               0x01, RTSX_PINGPONG_BUFFER);
2433
2434                 /* Queue commands to perform SD transfer. */
2435                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER,
2436                               0xff, RTSX_TM_NORMAL_READ | RTSX_SD_TRANSFER_START);
2437                 rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
2438                               RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END);
2439
2440                 /* Set transfer OK function. */
2441                 sc->rtsx_intr_trans_ok = rtsx_ask_ppbuf_part1;
2442
2443                 /* Run the command queue. */
2444                 rtsx_send_cmd(sc);
2445         } else {
2446                 /* Set transfer OK function. */
2447                 sc->rtsx_intr_trans_ok = rtsx_put_ppbuf_part1;
2448
2449                 /* Run the command queue. */
2450                 rtsx_send_req(sc, cmd);
2451         }
2452
2453         return (0);
2454 }
2455
2456 /*
2457  * Use the ping-pong buffer (cmd buffer) for the transfer - first part <= 256 bytes.
2458  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2459  */
2460 static void
2461 rtsx_ask_ppbuf_part1(struct rtsx_softc *sc)
2462 {
2463         struct mmc_command *cmd;
2464         uint16_t reg = RTSX_PPBUF_BASE2;
2465         int      len;
2466         int      i;
2467
2468         cmd = sc->rtsx_req->cmd;
2469         len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len;
2470
2471         sc->rtsx_cmd_index = 0;
2472         for (i = 0; i < len; i++) {
2473                 rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg++, 0, 0);
2474         }
2475
2476         /* Set transfer OK function. */
2477         sc->rtsx_intr_trans_ok = rtsx_get_ppbuf_part1;
2478
2479         /* Run the command queue. */
2480         rtsx_send_cmd(sc);
2481 }
2482
2483 /*
2484  * Get the data from the ping-pong buffer (cmd buffer) - first part <= 256 bytes.
2485  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2486  */
2487 static void
2488 rtsx_get_ppbuf_part1(struct rtsx_softc *sc)
2489 {
2490         struct mmc_command *cmd;
2491         uint8_t  *ptr;
2492         int      len;
2493
2494         cmd = sc->rtsx_req->cmd;
2495         ptr = cmd->data->data;
2496         len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len;
2497
2498         /* Sync command DMA buffer. */
2499         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD);
2500         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE);
2501
2502         memcpy(ptr, sc->rtsx_cmd_dmamem, len);
2503
2504         len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? cmd->data->len - RTSX_HOSTCMD_MAX : 0;
2505
2506         /* Use the ping-pong buffer (cmd buffer) for the transfer - second part > 256 bytes. */
2507         if (len > 0) {
2508                 uint16_t reg = RTSX_PPBUF_BASE2 + RTSX_HOSTCMD_MAX;
2509                 int      i;
2510
2511                 sc->rtsx_cmd_index = 0;
2512                 for (i = 0; i < len; i++) {
2513                         rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg++, 0, 0);
2514                 }
2515
2516                 /* Set transfer OK function. */
2517                 sc->rtsx_intr_trans_ok = rtsx_get_ppbuf_part2;
2518
2519                 /* Run the command queue. */
2520                 rtsx_send_cmd(sc);
2521         } else {
2522                 if (bootverbose && cmd->opcode == ACMD_SEND_SCR) {
2523                         uint8_t *ptr = cmd->data->data;
2524                         device_printf(sc->rtsx_dev, "SCR: 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
2525                                       ptr[0], ptr[1], ptr[2], ptr[3],
2526                                       ptr[4], ptr[5], ptr[6], ptr[7]);
2527                 }
2528
2529                 if (sc->rtsx_discovery_mode)
2530                         rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_128);
2531
2532                 rtsx_req_done(sc);
2533         }
2534 }
2535
2536 /*
2537  * Get the data from the ping-pong buffer (cmd buffer) - second part > 256 bytes.
2538  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2539  */
2540 static void
2541 rtsx_get_ppbuf_part2(struct rtsx_softc *sc)
2542 {
2543         struct mmc_command *cmd;
2544         uint8_t *ptr;
2545         int     len;
2546
2547         cmd = sc->rtsx_req->cmd;
2548         ptr = cmd->data->data;
2549         ptr += RTSX_HOSTCMD_MAX;
2550         len = cmd->data->len - RTSX_HOSTCMD_MAX;
2551
2552         /* Sync command DMA buffer. */
2553         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD);
2554         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE);
2555
2556         memcpy(ptr, sc->rtsx_cmd_dmamem, len);
2557
2558         if (sc->rtsx_discovery_mode)
2559                 rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_128);
2560
2561         rtsx_req_done(sc);
2562 }
2563
2564 /*
2565  * Use the ping-pong buffer (cmd buffer) for transfer - first part <= 256 bytes.
2566  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2567  */
2568 static void
2569 rtsx_put_ppbuf_part1(struct rtsx_softc *sc)
2570 {
2571         struct mmc_command *cmd;
2572         uint16_t reg = RTSX_PPBUF_BASE2;
2573         uint8_t  *ptr;
2574         int      len;
2575         int      i;
2576
2577         cmd = sc->rtsx_req->cmd;
2578         ptr = cmd->data->data;
2579         len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len;
2580
2581         rtsx_set_resp(sc, cmd);
2582
2583         sc->rtsx_cmd_index = 0;
2584         for (i = 0; i < len; i++) {
2585                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, reg++, 0xff, *ptr);
2586                 ptr++;
2587         }
2588
2589         /* Set transfer OK function. */
2590         if (cmd->data->len > RTSX_HOSTCMD_MAX)
2591                 sc->rtsx_intr_trans_ok = rtsx_put_ppbuf_part2;
2592         else
2593                 sc->rtsx_intr_trans_ok = rtsx_write_ppbuf;
2594
2595         /* Run the command queue. */
2596         rtsx_send_cmd(sc);
2597 }
2598
2599 /*
2600  * Use the ping-pong buffer (cmd buffer) for transfer - second part > 256 bytes.
2601  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2602  */
2603 static void
2604 rtsx_put_ppbuf_part2(struct rtsx_softc *sc)
2605 {
2606         struct mmc_command *cmd;
2607         uint16_t reg = RTSX_PPBUF_BASE2 + RTSX_HOSTCMD_MAX;
2608         uint8_t  *ptr;
2609         int      len;
2610         int      i;
2611
2612         cmd = sc->rtsx_req->cmd;
2613         ptr = cmd->data->data;
2614         ptr += RTSX_HOSTCMD_MAX;
2615         len = cmd->data->len - RTSX_HOSTCMD_MAX;
2616
2617         sc->rtsx_cmd_index = 0;
2618         for (i = 0; i < len; i++) {
2619                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, reg++, 0xff, *ptr);
2620                 ptr++;
2621         }
2622
2623         /* Set transfer OK function. */
2624         sc->rtsx_intr_trans_ok = rtsx_write_ppbuf;
2625
2626         /* Run the command queue. */
2627         rtsx_send_cmd(sc);
2628 }
2629
2630 /*
2631  * Write the data previously given via the ping-pong buffer on the card.
2632  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2633  */
2634 static void
2635 rtsx_write_ppbuf(struct rtsx_softc *sc)
2636 {
2637         struct mmc_command *cmd;
2638
2639         cmd = sc->rtsx_req->cmd;
2640
2641         sc->rtsx_cmd_index = 0;
2642
2643         /* Queue commands to configure data transfer size. */
2644         rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len);
2645
2646         /* From Linux: rtsx_pci_sdmmc.c sd_write_data(). */
2647         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff,
2648                       RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 |
2649                       RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_0);
2650
2651         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff,
2652                       RTSX_TM_AUTO_WRITE3 | RTSX_SD_TRANSFER_START);
2653         rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
2654                       RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END);
2655
2656         /* Set transfer OK function. */
2657         sc->rtsx_intr_trans_ok = rtsx_req_done;
2658
2659         /* Run the command queue. */
2660         rtsx_send_cmd(sc);
2661 }
2662
2663 /*
2664  * Use the data buffer for transfer > 512 bytes.
2665  */
2666 static int
2667 rtsx_xfer(struct rtsx_softc *sc, struct mmc_command *cmd)
2668 {
2669         int     read = ISSET(cmd->data->flags, MMC_DATA_READ);
2670
2671         cmd->data->xfer_len = (cmd->data->len > RTSX_MAX_DATA_BLKLEN) ?
2672                 RTSX_MAX_DATA_BLKLEN : cmd->data->len;
2673
2674         if (bootverbose)
2675                 device_printf(sc->rtsx_dev, "rtsx_xfer() - %s xfer: %ld bytes with block size %ld\n",
2676                               read ? "Read" : "Write",
2677                               (unsigned long)cmd->data->len, (unsigned long)cmd->data->xfer_len);
2678
2679         if (cmd->data->len > RTSX_DMA_DATA_BUFSIZE) {
2680                 device_printf(sc->rtsx_dev, "rtsx_xfer() length too large: %ld > %d\n",
2681                               (unsigned long)cmd->data->len, RTSX_DMA_DATA_BUFSIZE);
2682                 cmd->error = MMC_ERR_INVALID;
2683                 return (MMC_ERR_INVALID);
2684         }
2685
2686         if (!read) {
2687                 /* Set transfer OK function. */
2688                 sc->rtsx_intr_trans_ok = rtsx_xfer_begin;
2689
2690                 /* Run the command queue. */
2691                 rtsx_send_req(sc, cmd);
2692         } else {
2693                 rtsx_xfer_start(sc);
2694         }
2695
2696         return (0);
2697 }
2698
2699 /*
2700  * Get request response and start dma data transfer (write command).
2701  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2702  */
2703 static void
2704 rtsx_xfer_begin(struct rtsx_softc *sc)
2705 {
2706         struct mmc_command *cmd;
2707
2708         cmd = sc->rtsx_req->cmd;
2709
2710         if (bootverbose)
2711                 device_printf(sc->rtsx_dev, "rtsx_xfer_begin() - CMD%d\n", cmd->opcode);
2712
2713         rtsx_set_resp(sc, cmd);
2714         rtsx_xfer_start(sc);
2715 }
2716
2717 /*
2718  * Start dma data transfer.
2719  */
2720 static void
2721 rtsx_xfer_start(struct rtsx_softc *sc)
2722 {
2723         struct mmc_command *cmd;
2724         int     read;
2725         uint8_t cfg2;
2726         int     dma_dir;
2727         int     tmode;
2728
2729         cmd = sc->rtsx_req->cmd;
2730         read = ISSET(cmd->data->flags, MMC_DATA_READ);
2731
2732         /* Configure DMA transfer mode parameters. */
2733         if (cmd->opcode == MMC_READ_MULTIPLE_BLOCK)
2734                 cfg2 = RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_RSP_LEN_6;
2735         else
2736                 cfg2 = RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_RSP_LEN_0;
2737         if (read) {
2738                 dma_dir = RTSX_DMA_DIR_FROM_CARD;
2739                 /*
2740                  * Use transfer mode AUTO_READ1, which assume we not
2741                  * already send the read command and don't need to send
2742                  * CMD 12 manually after read.
2743                  */
2744                 tmode = RTSX_TM_AUTO_READ1;
2745                 cfg2 |= RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC7;
2746
2747                 rtsx_init_cmd(sc, cmd);
2748         } else {
2749                 dma_dir = RTSX_DMA_DIR_TO_CARD;
2750                 /*
2751                  * Use transfer mode AUTO_WRITE3, wich assumes we've already
2752                  * sent the write command and gotten the response, and will
2753                  * send CMD 12 manually after writing.
2754                  */
2755                 tmode = RTSX_TM_AUTO_WRITE3;
2756                 cfg2 |= RTSX_SD_NO_CALCULATE_CRC7 | RTSX_SD_NO_CHECK_CRC7;
2757
2758                 sc->rtsx_cmd_index = 0;
2759         }
2760
2761         /* Queue commands to configure data transfer size. */
2762         rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len);
2763
2764         /* Configure DMA controller. */
2765         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_IRQSTAT0,
2766                      RTSX_DMA_DONE_INT, RTSX_DMA_DONE_INT);
2767         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC3,
2768                      0xff, cmd->data->len >> 24);
2769         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC2,
2770                      0xff, cmd->data->len >> 16);
2771         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC1,
2772                      0xff, cmd->data->len >> 8);
2773         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC0,
2774                      0xff, cmd->data->len);
2775         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMACTL,
2776                      RTSX_DMA_EN | RTSX_DMA_DIR | RTSX_DMA_PACK_SIZE_MASK,
2777                      RTSX_DMA_EN | dma_dir | RTSX_DMA_512);
2778
2779         /* Use the DMA ring buffer for commands which transfer data. */
2780         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE,
2781                       0x01, RTSX_RING_BUFFER);
2782
2783         /* Queue command to set response type. */
2784         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, cfg2);
2785
2786         /* Queue commands to perform SD transfer. */
2787         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER,
2788                       0xff, tmode | RTSX_SD_TRANSFER_START);
2789         rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
2790                       RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END);
2791
2792         /* Run the command queue. */
2793         rtsx_send_cmd(sc);
2794
2795         if (!read)
2796                 memcpy(sc->rtsx_data_dmamem, cmd->data->data, cmd->data->len);
2797
2798         /* Sync data DMA buffer. */
2799         bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_PREREAD);
2800         bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_PREWRITE);
2801
2802         /* Set transfer OK function. */
2803         sc->rtsx_intr_trans_ok = rtsx_xfer_finish;
2804
2805         /* Tell the chip where the data buffer is and run the transfer. */
2806         WRITE4(sc, RTSX_HDBAR, sc->rtsx_data_buffer);
2807         WRITE4(sc, RTSX_HDBCTLR, RTSX_TRIG_DMA | (read ? RTSX_DMA_READ : 0) |
2808                (cmd->data->len & 0x00ffffff));
2809 }
2810
2811 /*
2812  * Finish dma data transfer.
2813  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2814  */
2815 static void
2816 rtsx_xfer_finish(struct rtsx_softc *sc)
2817 {
2818         struct mmc_command *cmd;
2819         int     read;
2820
2821         cmd = sc->rtsx_req->cmd;
2822
2823         if (bootverbose)
2824                 device_printf(sc->rtsx_dev, "rtsx_xfer_finish() - CMD%d\n", cmd->opcode);
2825
2826         read = ISSET(cmd->data->flags, MMC_DATA_READ);
2827
2828         /* Sync data DMA buffer. */
2829         bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_POSTREAD);
2830         bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_POSTWRITE);
2831
2832         if (read) {
2833                 memcpy(cmd->data->data, sc->rtsx_data_dmamem, cmd->data->len);
2834                 rtsx_req_done(sc);
2835         } else {
2836                 /* Send CMD12 after AUTO_WRITE3 (see mmcsd_rw() in mmcsd.c) */
2837                 /* and complete request. */
2838                 sc->rtsx_intr_trans_ok = NULL;
2839                 rtsx_send_req(sc, sc->rtsx_req->stop);
2840         }
2841 }
2842
2843 /*
2844  * Manage request timeout.
2845  */
2846 static void
2847 rtsx_timeout(void *arg)
2848 {
2849         struct rtsx_softc *sc;
2850
2851         sc = (struct rtsx_softc *)arg;
2852         if (sc->rtsx_req != NULL) {
2853                 device_printf(sc->rtsx_dev, "Controller timeout for CMD%u\n",
2854                               sc->rtsx_req->cmd->opcode);
2855                 sc->rtsx_req->cmd->error = MMC_ERR_TIMEOUT;
2856                 rtsx_stop_cmd(sc);
2857                 rtsx_req_done(sc);
2858         } else {
2859                 device_printf(sc->rtsx_dev, "Controller timeout!\n");
2860         }
2861 }
2862
2863 #ifdef MMCCAM
2864 static void
2865 rtsx_cam_action(struct cam_sim *sim, union ccb *ccb)
2866 {
2867         struct rtsx_softc *sc;
2868
2869         sc = cam_sim_softc(sim);
2870         if (sc == NULL) {
2871                 ccb->ccb_h.status = CAM_SEL_TIMEOUT;
2872                 xpt_done(ccb);
2873                 return;
2874         }
2875         switch (ccb->ccb_h.func_code) {
2876         case XPT_PATH_INQ:
2877         {
2878                 struct ccb_pathinq *cpi = &ccb->cpi;
2879
2880                 cpi->version_num = 1;           /* SIM driver version number - now all drivers use 1 */
2881                 cpi->hba_inquiry = 0;           /* bitmask of features supported by the controller */
2882                 cpi->target_sprt = 0;           /* flags for target mode support */
2883                 cpi->hba_misc = PIM_NOBUSRESET | PIM_SEQSCAN;
2884                 cpi->hba_eng_cnt = 0;           /* HBA engine count - always set to 0 */
2885                 cpi->max_target = 0;            /* maximal supported target ID */
2886                 cpi->max_lun = 0;               /* maximal supported LUN ID */
2887                 cpi->initiator_id = 1;          /* the SCSI ID of the controller itself */
2888                 cpi->maxio = RTSX_DMA_DATA_BUFSIZE;                     /* maximum io size */
2889                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);            /* vendor ID of the SIM */
2890                 strncpy(cpi->hba_vid, "Realtek", HBA_IDLEN);            /* vendor ID of the HBA */
2891                 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);   /* device name for SIM */
2892                 cpi->unit_number = cam_sim_unit(sim);                   /* controller unit number */
2893                 cpi->bus_id = cam_sim_bus(sim); /* bus number */
2894                 cpi->protocol = PROTO_MMCSD;
2895                 cpi->protocol_version = SCSI_REV_0;
2896                 cpi->transport = XPORT_MMCSD;
2897                 cpi->transport_version = 1;
2898
2899                 cpi->ccb_h.status = CAM_REQ_CMP;
2900                 break;
2901         }
2902         case XPT_GET_TRAN_SETTINGS:
2903         {
2904                 struct ccb_trans_settings *cts = &ccb->cts;
2905
2906                 if (bootverbose || sc->rtsx_debug)
2907                         device_printf(sc->rtsx_dev, "rtsx_cam_action() - got XPT_GET_TRAN_SETTINGS\n");
2908
2909                 cts->protocol = PROTO_MMCSD;
2910                 cts->protocol_version = 1;
2911                 cts->transport = XPORT_MMCSD;
2912                 cts->transport_version = 1;
2913                 cts->xport_specific.valid = 0;
2914                 cts->proto_specific.mmc.host_ocr = sc->rtsx_host.host_ocr;
2915                 cts->proto_specific.mmc.host_f_min = sc->rtsx_host.f_min;
2916                 cts->proto_specific.mmc.host_f_max = sc->rtsx_host.f_max;
2917                 cts->proto_specific.mmc.host_caps = sc->rtsx_host.caps;
2918 #if  __FreeBSD__ > 12
2919                 cts->proto_specific.mmc.host_max_data = RTSX_DMA_DATA_BUFSIZE / MMC_SECTOR_SIZE;
2920 #endif
2921                 memcpy(&cts->proto_specific.mmc.ios, &sc->rtsx_host.ios, sizeof(struct mmc_ios));
2922
2923                 ccb->ccb_h.status = CAM_REQ_CMP;
2924                 break;
2925         }
2926         case XPT_SET_TRAN_SETTINGS:
2927                 if (bootverbose || sc->rtsx_debug)
2928                         device_printf(sc->rtsx_dev, "rtsx_cam_action() - got XPT_SET_TRAN_SETTINGS\n");
2929
2930                 /* Apply settings and set ccb->ccb_h.status accordingly. */
2931                 rtsx_cam_set_tran_settings(sc, ccb);
2932                 break;
2933         case XPT_RESET_BUS:
2934                 if (bootverbose || sc->rtsx_debug)
2935                         device_printf(sc->rtsx_dev, "got XPT_RESET_BUS, ACK it...\n");
2936
2937                 ccb->ccb_h.status = CAM_REQ_CMP;
2938                 break;
2939         case XPT_MMC_IO:
2940                 /*
2941                  * Here is the HW-dependent part of sending
2942                  * the command to the underlying h/w.
2943                  * At some point in the future an interrupt comes
2944                  * and the request will be marked as completed.
2945                  */
2946                 ccb->ccb_h.status = CAM_REQ_INPROG;
2947
2948                 rtsx_cam_request(sc, ccb);
2949                 return;
2950         default:
2951                 ccb->ccb_h.status = CAM_REQ_INVALID;
2952                 break;
2953         }
2954         xpt_done(ccb);
2955         return;
2956 }
2957
2958 static void
2959 rtsx_cam_poll(struct cam_sim *sim)
2960 {
2961         return;
2962 }
2963
2964 /*
2965  *  Apply settings and set ccb->ccb_h.status accordingly.
2966 */
2967 static void
2968 rtsx_cam_set_tran_settings(struct rtsx_softc *sc, union ccb *ccb)
2969 {
2970         struct mmc_ios *ios;
2971         struct mmc_ios *new_ios;
2972         struct ccb_trans_settings_mmc *cts;
2973
2974         ios = &sc->rtsx_host.ios;
2975         cts = &ccb->cts.proto_specific.mmc;
2976         new_ios = &cts->ios;
2977
2978         /* Update only requested fields */
2979         if (cts->ios_valid & MMC_CLK) {
2980                 ios->clock = new_ios->clock;
2981                 sc->rtsx_ios_clock = -1;        /* To be updated by rtsx_mmcbr_update_ios(). */
2982                 if (bootverbose || sc->rtsx_debug)
2983                         device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - clock: %u\n", ios->clock);
2984         }
2985         if (cts->ios_valid & MMC_VDD) {
2986                 ios->vdd = new_ios->vdd;
2987                 if (bootverbose || sc->rtsx_debug)
2988                         device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - vdd: %d\n", ios->vdd);
2989         }
2990         if (cts->ios_valid & MMC_CS) {
2991                 ios->chip_select = new_ios->chip_select;
2992                 if (bootverbose || sc->rtsx_debug)
2993                         device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - chip_select: %d\n", ios->chip_select);
2994         }
2995         if (cts->ios_valid & MMC_BW) {
2996                 ios->bus_width = new_ios->bus_width;
2997                 sc->rtsx_ios_bus_width = -1;    /* To be updated by rtsx_mmcbr_update_ios(). */
2998                 if (bootverbose || sc->rtsx_debug)
2999                         device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - bus width: %d\n", ios->bus_width);
3000         }
3001         if (cts->ios_valid & MMC_PM) {
3002                 ios->power_mode = new_ios->power_mode;
3003                 sc->rtsx_ios_power_mode = -1;   /* To be updated by rtsx_mmcbr_update_ios(). */
3004                 if (bootverbose || sc->rtsx_debug)
3005                         device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - power mode: %d\n", ios->power_mode);
3006         }
3007         if (cts->ios_valid & MMC_BT) {
3008                 ios->timing = new_ios->timing;
3009                 sc->rtsx_ios_timing = -1;       /* To be updated by rtsx_mmcbr_update_ios(). */
3010                 if (bootverbose || sc->rtsx_debug)
3011                         device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - timing: %d\n", ios->timing);
3012         }
3013         if (cts->ios_valid & MMC_BM) {
3014                 ios->bus_mode = new_ios->bus_mode;
3015                 if (bootverbose || sc->rtsx_debug)
3016                         device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - bus mode: %d\n", ios->bus_mode);
3017         }
3018 #if  __FreeBSD__ > 12
3019         if (cts->ios_valid & MMC_VCCQ) {
3020                 ios->vccq = new_ios->vccq;
3021                 sc->rtsx_ios_vccq = -1;         /* To be updated by rtsx_mmcbr_update_ios(). */
3022                 if (bootverbose || sc->rtsx_debug)
3023                         device_printf(sc->rtsx_dev, "rtsx_cam_set_tran_settings() - vccq: %d\n", ios->vccq);
3024         }
3025 #endif
3026         if (rtsx_mmcbr_update_ios(sc->rtsx_dev, NULL) == 0)
3027                 ccb->ccb_h.status = CAM_REQ_CMP;
3028         else
3029                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
3030
3031         return;
3032 }
3033
3034 /*
3035  * Build a request and run it.
3036  */
3037 static void
3038 rtsx_cam_request(struct rtsx_softc *sc, union ccb *ccb)
3039 {
3040         RTSX_LOCK(sc);
3041         if (sc->rtsx_ccb != NULL) {
3042                 RTSX_UNLOCK(sc);
3043                 ccb->ccb_h.status = CAM_BUSY;   /* i.e. CAM_REQ_CMP | CAM_REQ_CMP_ERR */
3044                 return;
3045         }
3046         sc->rtsx_ccb = ccb;
3047         sc->rtsx_cam_req.cmd = &ccb->mmcio.cmd;
3048         sc->rtsx_cam_req.stop = &ccb->mmcio.stop;
3049         RTSX_UNLOCK(sc);
3050
3051         rtsx_mmcbr_request(sc->rtsx_dev, NULL, &sc->rtsx_cam_req);
3052         return;
3053 }
3054 #endif /* MMCCAM */
3055
3056 static int
3057 rtsx_read_ivar(device_t bus, device_t child, int which, uintptr_t *result)
3058 {
3059         struct rtsx_softc *sc;
3060
3061         sc = device_get_softc(bus);
3062         switch (which) {
3063         case MMCBR_IVAR_BUS_MODE:               /* ivar  0 - 1 = opendrain, 2 = pushpull */
3064                 *result = sc->rtsx_host.ios.bus_mode;
3065                 break;
3066         case MMCBR_IVAR_BUS_WIDTH:              /* ivar  1 - 0 = 1b   2 = 4b, 3 = 8b */
3067                 *result = sc->rtsx_host.ios.bus_width;
3068                 break;
3069         case MMCBR_IVAR_CHIP_SELECT:            /* ivar  2 - O = dontcare, 1 = cs_high, 2 = cs_low */
3070                 *result = sc->rtsx_host.ios.chip_select;
3071                 break;
3072         case MMCBR_IVAR_CLOCK:                  /* ivar  3 - clock in Hz */
3073                 *result = sc->rtsx_host.ios.clock;
3074                 break;
3075         case MMCBR_IVAR_F_MIN:                  /* ivar  4 */
3076                 *result = sc->rtsx_host.f_min;
3077                 break;
3078         case MMCBR_IVAR_F_MAX:                  /* ivar  5 */
3079                 *result = sc->rtsx_host.f_max;
3080                 break;
3081         case MMCBR_IVAR_HOST_OCR:               /* ivar  6 - host operation conditions register */
3082                 *result = sc->rtsx_host.host_ocr;
3083                 break;
3084         case MMCBR_IVAR_MODE:                   /* ivar  7 - 0 = mode_mmc, 1 = mode_sd */
3085                 *result = sc->rtsx_host.mode;
3086                 break;
3087         case MMCBR_IVAR_OCR:                    /* ivar  8 - operation conditions register */
3088                 *result = sc->rtsx_host.ocr;
3089                 break;
3090         case MMCBR_IVAR_POWER_MODE:             /* ivar  9 - 0 = off, 1 = up, 2 = on */
3091                 *result = sc->rtsx_host.ios.power_mode;
3092                 break;
3093         case MMCBR_IVAR_VDD:                    /* ivar 11 - voltage power pin */
3094                 *result = sc->rtsx_host.ios.vdd;
3095                 break;
3096         case MMCBR_IVAR_VCCQ:                   /* ivar 12 - signaling: 0 = 1.20V, 1 = 1.80V, 2 = 3.30V */
3097                 *result = sc->rtsx_host.ios.vccq;
3098                 break;
3099         case MMCBR_IVAR_CAPS:                   /* ivar 13 */
3100                 *result = sc->rtsx_host.caps;
3101                 break;
3102         case MMCBR_IVAR_TIMING:                 /* ivar 14 - 0 = normal, 1 = timing_hs, ... */
3103                 *result = sc->rtsx_host.ios.timing;
3104                 break;
3105         case MMCBR_IVAR_MAX_DATA:               /* ivar 15 */
3106                 *result = RTSX_DMA_DATA_BUFSIZE / MMC_SECTOR_SIZE;
3107                 break;
3108         case MMCBR_IVAR_RETUNE_REQ:             /* ivar 10 */
3109         case MMCBR_IVAR_MAX_BUSY_TIMEOUT:       /* ivar 16 */
3110         default:
3111                 return (EINVAL);
3112         }
3113
3114         if (bootverbose)
3115                 device_printf(bus, "Read ivar #%d, value %#x / #%d\n",
3116                               which, *(int *)result, *(int *)result);
3117
3118         return (0);
3119 }
3120
3121 static int
3122 rtsx_write_ivar(device_t bus, device_t child, int which, uintptr_t value)
3123 {
3124         struct rtsx_softc *sc;
3125
3126         if (bootverbose)
3127                 device_printf(bus, "Write ivar #%d, value %#x / #%d\n",
3128                               which, (int)value, (int)value);
3129
3130         sc = device_get_softc(bus);
3131         switch (which) {
3132         case MMCBR_IVAR_BUS_MODE:               /* ivar  0 - 1 = opendrain, 2 = pushpull */
3133                 sc->rtsx_host.ios.bus_mode = value;
3134                 break;
3135         case MMCBR_IVAR_BUS_WIDTH:              /* ivar  1 - 0 = 1b   2 = 4b, 3 = 8b */
3136                 sc->rtsx_host.ios.bus_width = value;
3137                 sc->rtsx_ios_bus_width = -1;    /* To be updated on next rtsx_mmcbr_update_ios(). */
3138                 break;
3139         case MMCBR_IVAR_CHIP_SELECT:            /* ivar  2 - O = dontcare, 1 = cs_high, 2 = cs_low */
3140                 sc->rtsx_host.ios.chip_select = value;
3141                 break;
3142         case MMCBR_IVAR_CLOCK:                  /* ivar  3 - clock in Hz */
3143                 sc->rtsx_host.ios.clock = value;
3144                 sc->rtsx_ios_clock = -1;        /* To be updated on next rtsx_mmcbr_update_ios(). */
3145                 break;
3146         case MMCBR_IVAR_MODE:                   /* ivar  7 - 0 = mode_mmc, 1 = mode_sd */
3147                 sc->rtsx_host.mode = value;
3148                 break;
3149         case MMCBR_IVAR_OCR:                    /* ivar  8 - operation conditions register */
3150                 sc->rtsx_host.ocr = value;
3151                 break;
3152         case MMCBR_IVAR_POWER_MODE:             /* ivar  9 - 0 = off, 1 = up, 2 = on */
3153                 sc->rtsx_host.ios.power_mode = value;
3154                 sc->rtsx_ios_power_mode = -1;   /* To be updated on next rtsx_mmcbr_update_ios(). */
3155                 break;
3156         case MMCBR_IVAR_VDD:                    /* ivar 11 - voltage power pin */
3157                 sc->rtsx_host.ios.vdd = value;
3158                 break;
3159         case MMCBR_IVAR_VCCQ:                   /* ivar 12 - signaling: 0 = 1.20V, 1 = 1.80V, 2 = 3.30V */
3160                 sc->rtsx_host.ios.vccq = value;
3161                 sc->rtsx_ios_vccq = value;      /* rtsx_mmcbr_switch_vccq() will be called by mmc.c (MMCCAM undef). */
3162                 break;
3163         case MMCBR_IVAR_TIMING:                 /* ivar 14 - 0 = normal, 1 = timing_hs, ... */
3164                 sc->rtsx_host.ios.timing = value;
3165                 sc->rtsx_ios_timing = -1;       /* To be updated on next rtsx_mmcbr_update_ios(). */
3166                 break;
3167         /* These are read-only. */
3168         case MMCBR_IVAR_F_MIN:                  /* ivar  4 */
3169         case MMCBR_IVAR_F_MAX:                  /* ivar  5 */
3170         case MMCBR_IVAR_HOST_OCR:               /* ivar  6 - host operation conditions register */
3171         case MMCBR_IVAR_RETUNE_REQ:             /* ivar 10 */
3172         case MMCBR_IVAR_CAPS:                   /* ivar 13 */
3173         case MMCBR_IVAR_MAX_DATA:               /* ivar 15 */
3174         case MMCBR_IVAR_MAX_BUSY_TIMEOUT:       /* ivar 16 */
3175         default:
3176                 return (EINVAL);
3177         }
3178
3179         return (0);
3180 }
3181
3182 static int
3183 rtsx_mmcbr_update_ios(device_t bus, device_t child__unused)
3184 {
3185         struct rtsx_softc *sc;
3186         struct mmc_ios    *ios;
3187         int     error;
3188
3189         sc = device_get_softc(bus);
3190         ios = &sc->rtsx_host.ios;
3191
3192         if (bootverbose)
3193                 device_printf(bus, "rtsx_mmcbr_update_ios()\n");
3194
3195         /* if MMCBR_IVAR_BUS_WIDTH updated. */
3196         if (sc->rtsx_ios_bus_width < 0) {
3197                 sc->rtsx_ios_bus_width = ios->bus_width;
3198                 if ((error = rtsx_set_bus_width(sc, ios->bus_width)))
3199                         return (error);
3200         }
3201
3202         /* if MMCBR_IVAR_POWER_MODE updated. */
3203         if (sc->rtsx_ios_power_mode < 0) {
3204                 sc->rtsx_ios_power_mode = ios->power_mode;
3205                 switch (ios->power_mode) {
3206                 case power_off:
3207                         if ((error = rtsx_bus_power_off(sc)))
3208                                 return (error);
3209                         break;
3210                 case power_up:
3211                         if ((error = rtsx_bus_power_on(sc)))
3212                                 return (error);
3213                         break;
3214                 case power_on:
3215                         if ((error = rtsx_bus_power_on(sc)))
3216                                 return (error);
3217                         break;
3218                 }
3219         }
3220
3221         sc->rtsx_double_clk = true;
3222         sc->rtsx_vpclk = false;
3223
3224         /* if MMCBR_IVAR_TIMING updated. */
3225         if (sc->rtsx_ios_timing < 0) {
3226                 sc->rtsx_ios_timing = ios->timing;
3227                 if ((error = rtsx_set_sd_timing(sc, ios->timing)))
3228                         return (error);
3229         }
3230
3231         /* if MMCBR_IVAR_CLOCK updated, must be after rtsx_set_sd_timing() */
3232         if (sc->rtsx_ios_clock < 0) {
3233                 sc->rtsx_ios_clock = ios->clock;
3234                 if ((error = rtsx_set_sd_clock(sc, ios->clock)))
3235                         return (error);
3236         }
3237
3238         /* if MMCCAM and vccq updated */
3239         if (sc->rtsx_ios_vccq < 0) {
3240                 sc->rtsx_ios_vccq = ios->vccq;
3241                 if ((error = rtsx_mmcbr_switch_vccq(sc->rtsx_dev, NULL)))
3242                         return (error);
3243         }
3244
3245         return (0);
3246 }
3247
3248 /*
3249  * Set output stage logic power voltage.
3250  */
3251 static int
3252 rtsx_mmcbr_switch_vccq(device_t bus, device_t child __unused)
3253 {
3254         struct rtsx_softc *sc;
3255         int     vccq = 0;
3256         int     error;
3257
3258         sc = device_get_softc(bus);
3259
3260         switch (sc->rtsx_host.ios.vccq) {
3261         case vccq_120:
3262                 vccq = 120;
3263                 break;
3264         case vccq_180:
3265                 vccq = 180;
3266                 break;
3267         case vccq_330:
3268                 vccq = 330;
3269                 break;
3270         };
3271         /* It seems it is always vccq_330. */
3272         if (vccq == 330) {
3273                 switch (sc->rtsx_device_id) {
3274                         uint16_t val;
3275                 case RTSX_RTS5227:
3276                         if ((error = rtsx_write_phy(sc, 0x08, 0x4FE4)))
3277                                 return (error);
3278                         if ((error = rtsx_rts5227_fill_driving(sc)))
3279                                 return (error);
3280                         break;
3281                 case RTSX_RTS5209:
3282                 case RTSX_RTS5229:
3283                         RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3);
3284                         if ((error = rtsx_write_phy(sc, 0x08, 0x4FE4)))
3285                                 return (error);
3286                         break;
3287                 case RTSX_RTS522A:
3288                         if ((error = rtsx_write_phy(sc, 0x08, 0x57E4)))
3289                                 return (error);
3290                         if ((error = rtsx_rts5227_fill_driving(sc)))
3291                                 return (error);
3292                         break;
3293                 case RTSX_RTS525A:
3294                         RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_LDO_D3318_MASK, RTSX_LDO_D3318_33V);
3295                         RTSX_BITOP(sc, RTSX_SD_PAD_CTL, RTSX_SD_IO_USING_1V8, 0);
3296                         if ((error = rtsx_rts5249_fill_driving(sc)))
3297                                 return (error);
3298                         break;
3299                 case RTSX_RTS5249:
3300                         if ((error = rtsx_read_phy(sc, RTSX_PHY_TUNE, &val)))
3301                                 return (error);
3302                         if ((error = rtsx_write_phy(sc, RTSX_PHY_TUNE,
3303                                                     (val & RTSX_PHY_TUNE_VOLTAGE_MASK) | RTSX_PHY_TUNE_VOLTAGE_3V3)))
3304                                 return (error);
3305                         if ((error = rtsx_rts5249_fill_driving(sc)))
3306                                 return (error);
3307                         break;
3308                 case RTSX_RTL8402:
3309                         RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3);
3310                         RTSX_BITOP(sc, RTSX_LDO_CTL,
3311                                    (RTSX_BPP_ASIC_MASK << RTSX_BPP_SHIFT_8402) | RTSX_BPP_PAD_MASK,
3312                                    (RTSX_BPP_ASIC_3V3 << RTSX_BPP_SHIFT_8402) | RTSX_BPP_PAD_3V3);
3313                         break;
3314                 case RTSX_RTL8411:
3315                 case RTSX_RTL8411B:
3316                         RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3);
3317                         RTSX_BITOP(sc, RTSX_LDO_CTL,
3318                                    (RTSX_BPP_ASIC_MASK << RTSX_BPP_SHIFT_8411) | RTSX_BPP_PAD_MASK,
3319                                    (RTSX_BPP_ASIC_3V3 << RTSX_BPP_SHIFT_8411) | RTSX_BPP_PAD_3V3);
3320                         break;
3321                 }
3322                 DELAY(300);
3323         }
3324
3325         if (bootverbose || sc->rtsx_debug)
3326                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_switch_vccq(%d)\n", vccq);
3327
3328         return (0);
3329 }
3330
3331 /*
3332  * Tune card if bus_timing_uhs_sdr50.
3333  */
3334 static int
3335 rtsx_mmcbr_tune(device_t bus, device_t child __unused, bool hs400)
3336 {
3337         struct rtsx_softc *sc;
3338         uint32_t raw_phase_map[RTSX_RX_TUNING_CNT] = {0};
3339         uint32_t phase_map;
3340         uint8_t  final_phase;
3341         int      i;
3342
3343         sc = device_get_softc(bus);
3344
3345         if (bootverbose || sc->rtsx_debug)
3346                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - hs400 is %s\n",
3347                               (hs400) ? "true" : "false");
3348
3349         if (sc->rtsx_ios_timing != bus_timing_uhs_sdr50)
3350                 return (0);
3351
3352         sc->rtsx_tuning_mode = true;
3353
3354         switch (sc->rtsx_device_id) {
3355         case RTSX_RTS5209:
3356         case RTSX_RTS5227:
3357                 rtsx_sd_change_tx_phase(sc, 27);
3358                 break;
3359         case RTSX_RTS522A:
3360                 rtsx_sd_change_tx_phase(sc, 20);
3361                 break;
3362         case RTSX_RTS5229:
3363                 rtsx_sd_change_tx_phase(sc, 27);
3364                 break;
3365         case RTSX_RTS525A:
3366         case RTSX_RTS5249:
3367                 rtsx_sd_change_tx_phase(sc, 29);
3368                 break;
3369         case RTSX_RTL8402:
3370         case RTSX_RTL8411:
3371         case RTSX_RTL8411B:
3372                 rtsx_sd_change_tx_phase(sc, 7);
3373                 break;
3374         }
3375
3376         /* trying rx tuning for bus_timing_uhs_sdr50. */
3377         for (i = 0; i < RTSX_RX_TUNING_CNT; i++) {
3378                 rtsx_sd_tuning_rx_phase(sc, &(raw_phase_map[i]));
3379                 if (raw_phase_map[i] == 0)
3380                         break;
3381         }
3382
3383         phase_map = 0xffffffff;
3384         for (i = 0; i < RTSX_RX_TUNING_CNT; i++) {
3385                 if (bootverbose || sc->rtsx_debug)
3386                         device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - RX raw_phase_map[%d]: 0x%08x\n",
3387                                       i, raw_phase_map[i]);
3388                 phase_map &= raw_phase_map[i];
3389         }
3390         if (bootverbose || sc->rtsx_debug)
3391                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - RX phase_map: 0x%08x\n", phase_map);
3392
3393         if (phase_map) {
3394                 final_phase = rtsx_sd_search_final_rx_phase(sc, phase_map);
3395                 if (final_phase != 0xff) {
3396                         if (sc->rtsx_debug == 1) {
3397                                 sc->rtsx_debug = 2;
3398                                 rtsx_sd_change_rx_phase(sc, final_phase);
3399                                 sc->rtsx_debug = 1;
3400                         } else {
3401                                 rtsx_sd_change_rx_phase(sc, final_phase);
3402                         }
3403                 }
3404         }
3405
3406         sc->rtsx_tuning_mode = false;
3407
3408         return (0);
3409 }
3410
3411 static int
3412 rtsx_mmcbr_retune(device_t bus, device_t child __unused, bool reset __unused)
3413 {
3414         struct rtsx_softc *sc;
3415
3416         sc = device_get_softc(bus);
3417
3418         if (bootverbose)
3419                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_retune()\n");
3420
3421         return (0);
3422 }
3423
3424 static int
3425 rtsx_mmcbr_request(device_t bus, device_t child __unused, struct mmc_request *req)
3426 {
3427         struct rtsx_softc  *sc;
3428         struct mmc_command *cmd;
3429         int     error;
3430
3431         sc = device_get_softc(bus);
3432
3433         RTSX_LOCK(sc);
3434         if (sc->rtsx_req != NULL) {
3435                 RTSX_UNLOCK(sc);
3436                 return (EBUSY);
3437         }
3438         sc->rtsx_req = req;
3439         cmd = req->cmd;
3440         cmd->error = error = MMC_ERR_NONE;
3441         sc->rtsx_intr_status = 0;
3442         sc->rtsx_intr_trans_ok = NULL;
3443         sc->rtsx_intr_trans_ko = rtsx_req_done;
3444
3445         if (bootverbose)
3446                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_request(CMD%u arg %#x, flags %#x, dlen %u, dflags %#x)\n",
3447                               cmd->opcode, cmd->arg, cmd->flags,
3448                               cmd->data != NULL ? (unsigned int)cmd->data->len : 0,
3449                               cmd->data != NULL ? cmd->data->flags : 0);
3450
3451         /* Check if card present. */
3452         if (!ISSET(sc->rtsx_flags, RTSX_F_CARD_PRESENT)) {
3453                 cmd->error = error = MMC_ERR_FAILED;
3454                 goto end;
3455         }
3456
3457         /* Refuse SDIO probe if the chip doesn't support SDIO. */
3458         if (cmd->opcode == IO_SEND_OP_COND &&
3459             !ISSET(sc->rtsx_flags, RTSX_F_SDIO_SUPPORT)) {
3460                 cmd->error = error = MMC_ERR_INVALID;
3461                 goto end;
3462         }
3463
3464         /* Return MMC_ERR_TIMEOUT for SD_IO_RW_DIRECT and IO_SEND_OP_COND. */
3465         if (cmd->opcode == SD_IO_RW_DIRECT || cmd->opcode == IO_SEND_OP_COND) {
3466                 cmd->error = error = MMC_ERR_TIMEOUT;
3467                 goto end;
3468         }
3469
3470         /* Select SD card. */
3471         RTSX_BITOP(sc, RTSX_CARD_SELECT, 0x07, RTSX_SD_MOD_SEL);
3472         RTSX_BITOP(sc, RTSX_CARD_SHARE_MODE, RTSX_CARD_SHARE_MASK, RTSX_CARD_SHARE_48_SD);
3473
3474         if (cmd->data == NULL) {
3475                 DELAY(200);
3476                 error = rtsx_send_req(sc, cmd);
3477         } else if (cmd->data->len <= 512) {
3478                 error = rtsx_xfer_short(sc, cmd);
3479         } else {
3480                 error = rtsx_xfer(sc, cmd);
3481         }
3482  end:
3483         if (error == MMC_ERR_NONE) {
3484                 callout_reset(&sc->rtsx_timeout_callout, sc->rtsx_timeout * hz, rtsx_timeout, sc);
3485         } else {
3486                 rtsx_req_done(sc);
3487         }
3488         RTSX_UNLOCK(sc);
3489
3490         return (error);
3491 }
3492
3493 static int
3494 rtsx_mmcbr_get_ro(device_t bus, device_t child __unused)
3495 {
3496         struct rtsx_softc *sc;
3497
3498         sc = device_get_softc(bus);
3499
3500         if (sc->rtsx_inversion == 0)
3501                 return (sc->rtsx_read_only);
3502         else
3503                 return !(sc->rtsx_read_only);
3504 }
3505
3506 static int
3507 rtsx_mmcbr_acquire_host(device_t bus, device_t child __unused)
3508 {
3509         struct rtsx_softc *sc;
3510
3511         if (bootverbose)
3512                 device_printf(bus, "rtsx_mmcbr_acquire_host()\n");
3513
3514         sc = device_get_softc(bus);
3515         RTSX_LOCK(sc);
3516         while (sc->rtsx_bus_busy)
3517                 msleep(&sc->rtsx_bus_busy, &sc->rtsx_mtx, 0, "rtsxah", 0);
3518         sc->rtsx_bus_busy++;
3519         RTSX_UNLOCK(sc);
3520
3521         return (0);
3522 }
3523
3524 static int
3525 rtsx_mmcbr_release_host(device_t bus, device_t child __unused)
3526 {
3527         struct rtsx_softc *sc;
3528
3529         if (bootverbose)
3530                 device_printf(bus, "rtsx_mmcbr_release_host()\n");
3531
3532         sc = device_get_softc(bus);
3533         RTSX_LOCK(sc);
3534         sc->rtsx_bus_busy--;
3535         RTSX_UNLOCK(sc);
3536         wakeup(&sc->rtsx_bus_busy);
3537
3538         return (0);
3539 }
3540
3541 /*
3542  *
3543  * PCI Support Functions
3544  *
3545  */
3546
3547 /*
3548  * Compare the device ID (chip) of this device against the IDs that this driver
3549  * supports. If there is a match, set the description and return success.
3550  */
3551 static int
3552 rtsx_probe(device_t dev)
3553 {
3554         struct rtsx_softc *sc;
3555         uint16_t vendor_id;
3556         uint16_t device_id;
3557         int      i;
3558         int      result;
3559
3560         vendor_id = pci_get_vendor(dev);
3561         device_id = pci_get_device(dev);
3562
3563         result = ENXIO;
3564         for (i = 0; rtsx_devices[i].vendor_id != 0; i++) {
3565                 if (rtsx_devices[i].vendor_id == vendor_id &&
3566                     rtsx_devices[i].device_id == device_id) {
3567                         device_set_desc(dev, rtsx_devices[i].desc);
3568                         sc = device_get_softc(dev);
3569                         sc->rtsx_device_id = device_id;
3570                         result = BUS_PROBE_DEFAULT;
3571                         break;
3572                 }
3573         }
3574
3575         return (result);
3576 }
3577
3578 /*
3579  * Attach function is only called if the probe is successful.
3580  */
3581 static int
3582 rtsx_attach(device_t dev)
3583 {
3584         struct rtsx_softc       *sc = device_get_softc(dev);
3585         struct sysctl_ctx_list  *ctx;
3586         struct sysctl_oid_list  *tree;
3587         int                     msi_count = 1;
3588         uint32_t                sdio_cfg;
3589         int                     error;
3590
3591         if (bootverbose)
3592                 device_printf(dev, "Attach - Vendor ID: 0x%x - Device ID: 0x%x\n",
3593                               pci_get_vendor(dev), pci_get_device(dev));
3594
3595         sc->rtsx_dev = dev;
3596         sc->rtsx_req = NULL;
3597         sc->rtsx_timeout = 10;
3598         sc->rtsx_read_only = 0;
3599         sc->rtsx_force_timing = 0;
3600         sc->rtsx_debug = 0;
3601         sc->rtsx_read_count = 0;
3602         sc->rtsx_write_count = 0;
3603
3604         RTSX_LOCK_INIT(sc);
3605
3606         ctx = device_get_sysctl_ctx(dev);
3607         tree = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
3608         SYSCTL_ADD_INT(ctx, tree, OID_AUTO, "req_timeout", CTLFLAG_RW,
3609                        &sc->rtsx_timeout, 0, "Request timeout in seconds");
3610         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "read_only", CTLFLAG_RD,
3611                       &sc->rtsx_read_only, 0, "Card is write protected");
3612         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "inversion", CTLFLAG_RWTUN,
3613                       &sc->rtsx_inversion, 0, "Inversion of card detection and read only status");
3614         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "force_timing", CTLFLAG_RW,
3615                       &sc->rtsx_force_timing, 0, "Force bus_timing_uhs_sdr50");
3616         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "debug", CTLFLAG_RW,
3617                       &sc->rtsx_debug, 0, "Debugging flag");
3618         SYSCTL_ADD_U64(ctx, tree, OID_AUTO, "read_count", CTLFLAG_RD,
3619                        &sc->rtsx_read_count, 0, "Count of read operations");
3620         SYSCTL_ADD_U64(ctx, tree, OID_AUTO, "write_count", CTLFLAG_RD,
3621                        &sc->rtsx_write_count, 0, "Count of write operations");
3622
3623         /* Allocate IRQ. */
3624         sc->rtsx_irq_res_id = 0;
3625         if (pci_alloc_msi(dev, &msi_count) == 0)
3626                 sc->rtsx_irq_res_id = 1;
3627         sc->rtsx_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->rtsx_irq_res_id,
3628                                                   RF_ACTIVE | (sc->rtsx_irq_res_id != 0 ? 0 : RF_SHAREABLE));
3629         if (sc->rtsx_irq_res == NULL) {
3630                 device_printf(dev, "Can't allocate IRQ resources for %d\n", sc->rtsx_irq_res_id);
3631                 pci_release_msi(dev);
3632                 return (ENXIO);
3633         }
3634
3635         callout_init_mtx(&sc->rtsx_timeout_callout, &sc->rtsx_mtx, 0);
3636
3637         /* Allocate memory resource. */
3638         if (sc->rtsx_device_id == RTSX_RTS525A)
3639                 sc->rtsx_res_id = PCIR_BAR(1);
3640         else
3641                 sc->rtsx_res_id = PCIR_BAR(0);
3642         sc->rtsx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->rtsx_res_id, RF_ACTIVE);
3643         if (sc->rtsx_res == NULL) {
3644                 device_printf(dev, "Can't allocate memory resource for %d\n", sc->rtsx_res_id);
3645                 goto destroy_rtsx_irq_res;
3646         }
3647
3648         if (bootverbose)
3649                 device_printf(dev, "rtsx_irq_res_id: %d, rtsx_res_id: %d\n",
3650                               sc->rtsx_irq_res_id, sc->rtsx_res_id);
3651
3652         sc->rtsx_btag = rman_get_bustag(sc->rtsx_res);
3653         sc->rtsx_bhandle = rman_get_bushandle(sc->rtsx_res);
3654
3655         /* Activate the interrupt. */
3656         error = bus_setup_intr(dev, sc->rtsx_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
3657                                NULL, rtsx_intr, sc, &sc->rtsx_irq_cookie);
3658         if (error) {
3659                 device_printf(dev, "Can't set up irq [0x%x]!\n", error);
3660                 goto destroy_rtsx_res;
3661         }
3662         pci_enable_busmaster(dev);
3663
3664         if (rtsx_read_cfg(sc, 0, RTSX_SDIOCFG_REG, &sdio_cfg) == 0) {
3665                 if ((sdio_cfg & RTSX_SDIOCFG_SDIO_ONLY) ||
3666                     (sdio_cfg & RTSX_SDIOCFG_HAVE_SDIO))
3667                         sc->rtsx_flags |= RTSX_F_SDIO_SUPPORT;
3668         }
3669
3670         /* Allocate two DMA buffers: a command buffer and a data buffer. */
3671         error = rtsx_dma_alloc(sc);
3672         if (error) {
3673                 goto destroy_rtsx_irq;
3674         }
3675
3676         /* From dwmmc.c. */
3677         TIMEOUT_TASK_INIT(taskqueue_swi_giant, &sc->rtsx_card_insert_task, 0,
3678                           rtsx_card_task, sc);
3679         TASK_INIT(&sc->rtsx_card_remove_task, 0, rtsx_card_task, sc);
3680
3681 #ifdef MMCCAM
3682         sc->rtsx_ccb = NULL;
3683         sc->rtsx_cam_status = 0;
3684
3685         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "cam_status", CTLFLAG_RD,
3686                       &sc->rtsx_cam_status, 0, "driver cam card present");
3687         if ((sc->rtsx_devq = cam_simq_alloc(1)) == NULL) {
3688                 device_printf(dev, "Error during CAM queue allocation\n");
3689                 goto destroy_rtsx_irq;
3690         }
3691         mtx_init(&sc->rtsx_sim_mtx, "rtsxsim", NULL, MTX_DEF);
3692         sc->rtsx_sim = cam_sim_alloc(rtsx_cam_action, rtsx_cam_poll,
3693                                      "rtsx", sc, device_get_unit(dev),
3694                                      &sc->rtsx_sim_mtx, 1, 1, sc->rtsx_devq);
3695         if (sc->rtsx_sim == NULL) {
3696                 device_printf(dev, "Can't allocate CAM SIM\n");
3697                 goto destroy_rtsx_irq;
3698         }
3699         mtx_lock(&sc->rtsx_sim_mtx);
3700         if (xpt_bus_register(sc->rtsx_sim, dev, 0) != 0) {
3701                 device_printf(dev, "Can't register SCSI pass-through bus\n");
3702                 mtx_unlock(&sc->rtsx_sim_mtx);
3703                 goto destroy_rtsx_irq;
3704         }
3705         mtx_unlock(&sc->rtsx_sim_mtx);
3706 #endif /* MMCCAM */
3707
3708         /* Initialize device. */
3709         if (rtsx_init(sc)) {
3710                 device_printf(dev, "Error during rtsx_init()\n");
3711                 goto destroy_rtsx_irq;
3712         }
3713
3714         /*
3715          * Schedule a card detection as we won't get an interrupt
3716          * if the card is inserted when we attach
3717          */
3718         DELAY(500);
3719         if (rtsx_is_card_present(sc))
3720                 device_printf(sc->rtsx_dev, "Card present\n");
3721         else
3722                 device_printf(sc->rtsx_dev, "Card absent\n");
3723         rtsx_card_task(sc, 0);
3724
3725         if (bootverbose)
3726                 device_printf(dev, "Device attached\n");
3727
3728         return (0);
3729
3730  destroy_rtsx_irq:
3731         bus_teardown_intr(dev, sc->rtsx_irq_res, sc->rtsx_irq_cookie);
3732  destroy_rtsx_res:
3733         bus_release_resource(dev, SYS_RES_MEMORY, sc->rtsx_res_id,
3734                              sc->rtsx_res);
3735  destroy_rtsx_irq_res:
3736         callout_drain(&sc->rtsx_timeout_callout);
3737         bus_release_resource(dev, SYS_RES_IRQ, sc->rtsx_irq_res_id,
3738                              sc->rtsx_irq_res);
3739         pci_release_msi(dev);
3740         RTSX_LOCK_DESTROY(sc);
3741 #ifdef MMCCAM
3742         if (sc->rtsx_sim != NULL) {
3743                 mtx_lock(&sc->rtsx_sim_mtx);
3744                 xpt_bus_deregister(cam_sim_path(sc->rtsx_sim));
3745                 cam_sim_free(sc->rtsx_sim, FALSE);
3746                 mtx_unlock(&sc->rtsx_sim_mtx);
3747         }
3748         if (sc->rtsx_devq != NULL) {
3749                 mtx_destroy(&sc->rtsx_sim_mtx);
3750                 cam_simq_free(sc->rtsx_devq);
3751         }
3752 #endif /* MMCCAM */
3753
3754         return (ENXIO);
3755 }
3756
3757 static int
3758 rtsx_detach(device_t dev)
3759 {
3760         struct rtsx_softc *sc = device_get_softc(dev);
3761         int     error;
3762
3763         if (bootverbose)
3764                 device_printf(dev, "Detach - Vendor ID: 0x%x - Device ID: 0x%x\n",
3765                               pci_get_vendor(dev), pci_get_device(dev));
3766
3767         /* Disable interrupts. */
3768         sc->rtsx_intr_enabled = 0;
3769         WRITE4(sc, RTSX_BIER, sc->rtsx_intr_enabled);
3770
3771         /* Stop device. */
3772         error = device_delete_children(sc->rtsx_dev);
3773         sc->rtsx_mmc_dev = NULL;
3774         if (error)
3775                 return (error);
3776
3777         taskqueue_drain_timeout(taskqueue_swi_giant, &sc->rtsx_card_insert_task);
3778         taskqueue_drain(taskqueue_swi_giant, &sc->rtsx_card_remove_task);
3779
3780         /* Teardown the state in our softc created in our attach routine. */
3781         rtsx_dma_free(sc);
3782         if (sc->rtsx_res != NULL)
3783                 bus_release_resource(dev, SYS_RES_MEMORY, sc->rtsx_res_id,
3784                                      sc->rtsx_res);
3785         if (sc->rtsx_irq_cookie != NULL)
3786                 bus_teardown_intr(dev, sc->rtsx_irq_res, sc->rtsx_irq_cookie);
3787         if (sc->rtsx_irq_res != NULL) {
3788                 callout_drain(&sc->rtsx_timeout_callout);
3789                 bus_release_resource(dev, SYS_RES_IRQ, sc->rtsx_irq_res_id,
3790                                      sc->rtsx_irq_res);
3791                 pci_release_msi(dev);
3792         }
3793         RTSX_LOCK_DESTROY(sc);
3794 #ifdef MMCCAM
3795         if (sc->rtsx_sim != NULL) {
3796                 mtx_lock(&sc->rtsx_sim_mtx);
3797                 xpt_bus_deregister(cam_sim_path(sc->rtsx_sim));
3798                 cam_sim_free(sc->rtsx_sim, FALSE);
3799                 mtx_unlock(&sc->rtsx_sim_mtx);
3800         }
3801         if (sc->rtsx_devq != NULL) {
3802                 mtx_destroy(&sc->rtsx_sim_mtx);
3803                 cam_simq_free(sc->rtsx_devq);
3804         }
3805 #endif /* MMCCAM */
3806
3807         return (0);
3808 }
3809
3810 static int
3811 rtsx_shutdown(device_t dev)
3812 {
3813         if (bootverbose)
3814                 device_printf(dev, "Shutdown\n");
3815
3816         rtsx_detach(dev);
3817
3818         return (0);
3819 }
3820
3821 /*
3822  * Device suspend routine.
3823  */
3824 static int
3825 rtsx_suspend(device_t dev)
3826 {
3827         struct rtsx_softc *sc = device_get_softc(dev);
3828
3829         device_printf(dev, "Suspend\n");
3830
3831 #ifdef MMCCAM
3832         if (sc->rtsx_ccb != NULL) {
3833                 device_printf(dev, "Request in progress: CMD%u, rtsr_intr_status: 0x%08x\n",
3834                               sc->rtsx_ccb->mmcio.cmd.opcode, sc->rtsx_intr_status);
3835         }
3836 #else
3837         if (sc->rtsx_req != NULL) {
3838                 device_printf(dev, "Request in progress: CMD%u, rtsr_intr_status: 0x%08x\n",
3839                               sc->rtsx_req->cmd->opcode, sc->rtsx_intr_status);
3840         }
3841 #endif /* MMCCAM */
3842
3843         bus_generic_suspend(dev);
3844
3845         return (0);
3846 }
3847
3848 /*
3849  * Device resume routine.
3850  */
3851 static int
3852 rtsx_resume(device_t dev)
3853 {
3854         device_printf(dev, "Resume\n");
3855
3856         bus_generic_resume(dev);
3857
3858         return (0);
3859 }
3860
3861 static device_method_t rtsx_methods[] = {
3862         /* Device interface */
3863         DEVMETHOD(device_probe,         rtsx_probe),
3864         DEVMETHOD(device_attach,        rtsx_attach),
3865         DEVMETHOD(device_detach,        rtsx_detach),
3866         DEVMETHOD(device_shutdown,      rtsx_shutdown),
3867         DEVMETHOD(device_suspend,       rtsx_suspend),
3868         DEVMETHOD(device_resume,        rtsx_resume),
3869
3870         /* Bus interface */
3871         DEVMETHOD(bus_read_ivar,        rtsx_read_ivar),
3872         DEVMETHOD(bus_write_ivar,       rtsx_write_ivar),
3873
3874         /* MMC bridge interface */
3875         DEVMETHOD(mmcbr_update_ios,     rtsx_mmcbr_update_ios),
3876         DEVMETHOD(mmcbr_switch_vccq,    rtsx_mmcbr_switch_vccq),
3877         DEVMETHOD(mmcbr_tune,           rtsx_mmcbr_tune),
3878         DEVMETHOD(mmcbr_retune,         rtsx_mmcbr_retune),
3879         DEVMETHOD(mmcbr_request,        rtsx_mmcbr_request),
3880         DEVMETHOD(mmcbr_get_ro,         rtsx_mmcbr_get_ro),
3881         DEVMETHOD(mmcbr_acquire_host,   rtsx_mmcbr_acquire_host),
3882         DEVMETHOD(mmcbr_release_host,   rtsx_mmcbr_release_host),
3883
3884         DEVMETHOD_END
3885 };
3886
3887 static devclass_t rtsx_devclass;
3888
3889 DEFINE_CLASS_0(rtsx, rtsx_driver, rtsx_methods, sizeof(struct rtsx_softc));
3890 DRIVER_MODULE(rtsx, pci, rtsx_driver, rtsx_devclass, NULL, NULL);
3891 #ifndef MMCCAM
3892 MMC_DECLARE_BRIDGE(rtsx);
3893 #endif /* MMCCAM */