sys/dev/rtsx/rtsx.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2006 Uwe Stuehler <uwe@openbsd.org>
   5  * Copyright (c) 2012 Stefan Sperling <stsp@openbsd.org>
   6  * Copyright (c) 2020 Henri Hennebert <hlh@restart.be>
   7  * Copyright (c) 2020 Gary Jennejohn <gj@freebsd.org>
   8  * Copyright (c) 2020 Jesper Schmitz Mouridsen <jsm@FreeBSD.org>
   9  * All rights reserved.
  10  *
  11  * Patch from:
  12  * - Lutz Bichler <Lutz.Bichler@gmail.com>
  13  *
  14  * Base on OpenBSD /sys/dev/pci/rtsx_pci.c & /dev/ic/rtsx.c
  15  *      on Linux   /drivers/mmc/host/rtsx_pci_sdmmc.c,
  16  *                 /include/linux/rtsx_pci.h &
  17  *                 /drivers/misc/cardreader/rtsx_pcr.c
  18  *      on NetBSD  /sys/dev/ic/rtsx.c
  19  *
  20  * Permission to use, copy, modify, and distribute this software for any
  21  * purpose with or without fee is hereby granted, provided that the above
  22  * copyright notice and this permission notice appear in all copies.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  34  * SUCH DAMAGE.
  35  */
  36
  37 #include <sys/cdefs.h>
  38 __FBSDID("$FreeBSD$");
  39
  40 #include <sys/param.h>
  41 #include <sys/module.h>
  42 #include <sys/systm.h> /* For FreeBSD 11 */
  43 #include <sys/types.h> /* For FreeBSD 11 */
  44 #include <sys/errno.h>
  45 #include <sys/kernel.h>
  46 #include <sys/bus.h>
  47 #include <sys/endian.h>
  48 #include <machine/bus.h>
  49 #include <sys/mutex.h>
  50 #include <sys/malloc.h>
  51 #include <sys/rman.h>
  52 #include <sys/queue.h>
  53 #include <sys/taskqueue.h>
  54 #include <sys/sysctl.h>
  55 #include <dev/pci/pcivar.h>
  56 #include <dev/pci/pcireg.h>
  57 #include <dev/mmc/bridge.h>
  58 #include <dev/mmc/mmcreg.h>
  59 #include <dev/mmc/mmcbrvar.h>
  60 #include <machine/_inttypes.h>
  61
  62 #include "opt_mmccam.h"
  63
  64 #ifdef MMCCAM
  65 #include <cam/cam.h>
  66 #include <cam/cam_ccb.h>
  67 #include <cam/cam_debug.h>
  68 #include <cam/cam_sim.h>
  69 #include <cam/cam_xpt_sim.h>
  70 #include <cam/mmc/mmc_sim.h>
  71 #include "mmc_sim_if.h"
  72 #endif /* MMCCAM */
  73
  74 #include "rtsxreg.h"
  75
  76 /* The softc holds our per-instance data. */
  77 struct rtsx_softc {
  78         struct mtx      rtsx_mtx;               /* device mutex */
  79         device_t        rtsx_dev;               /* device */
  80         uint16_t        rtsx_flags;             /* device flags */
  81         uint16_t        rtsx_device_id;         /* device ID */
  82         device_t        rtsx_mmc_dev;           /* device of mmc bus */
  83         uint32_t        rtsx_intr_enabled;      /* enabled interrupts */
  84         uint32_t        rtsx_intr_status;       /* soft interrupt status */
  85         int             rtsx_irq_res_id;        /* bus IRQ resource id */
  86         struct resource *rtsx_irq_res;          /* bus IRQ resource */
  87         void            *rtsx_irq_cookie;       /* bus IRQ resource cookie */
  88         struct callout  rtsx_timeout_callout;   /* callout for timeout */
  89         int             rtsx_timeout_cmd;       /* interrupt timeout for setup commands */
  90         int             rtsx_timeout_io;        /* interrupt timeout for I/O commands */
  91         void            (*rtsx_intr_trans_ok)(struct rtsx_softc *sc);
  92                                                 /* function to call if transfer succeed */
  93         void            (*rtsx_intr_trans_ko)(struct rtsx_softc *sc);
  94                                                 /* function to call if transfer fail */
  95
  96         struct timeout_task
  97                         rtsx_card_insert_task;  /* card insert delayed task */
  98         struct task     rtsx_card_remove_task;  /* card remove task */
  99
 100         int             rtsx_res_id;            /* bus memory resource id */
 101         struct resource *rtsx_res;              /* bus memory resource */
 102         int             rtsx_res_type;          /* bus memory resource type */
 103         bus_space_tag_t rtsx_btag;              /* host register set tag */
 104         bus_space_handle_t rtsx_bhandle;        /* host register set handle */
 105
 106         bus_dma_tag_t   rtsx_cmd_dma_tag;       /* DMA tag for command transfer */
 107         bus_dmamap_t    rtsx_cmd_dmamap;        /* DMA map for command transfer */
 108         void            *rtsx_cmd_dmamem;       /* DMA mem for command transfer */
 109         bus_addr_t      rtsx_cmd_buffer;        /* device visible address of the DMA segment */
 110         int             rtsx_cmd_index;         /* index in rtsx_cmd_buffer */
 111
 112         bus_dma_tag_t   rtsx_data_dma_tag;      /* DMA tag for data transfer */
 113         bus_dmamap_t    rtsx_data_dmamap;       /* DMA map for data transfer */
 114         void            *rtsx_data_dmamem;      /* DMA mem for data transfer */
 115         bus_addr_t      rtsx_data_buffer;       /* device visible address of the DMA segment */
 116
 117 #ifdef MMCCAM
 118         union ccb               *rtsx_ccb;      /* CAM control block */
 119         struct mmc_sim          rtsx_mmc_sim;   /* CAM generic sim */
 120         struct mmc_request      rtsx_cam_req;   /* CAM MMC request */
 121 #endif /* MMCCAM */
 122
 123         struct mmc_request *rtsx_req;           /* MMC request */
 124         struct mmc_host rtsx_host;              /* host parameters */
 125         int             rtsx_pcie_cap;          /* PCIe capability offset */
 126         int8_t          rtsx_bus_busy;          /* bus busy status */
 127         int8_t          rtsx_ios_bus_width;     /* current host.ios.bus_width */
 128         int32_t         rtsx_ios_clock;         /* current host.ios.clock */
 129         int8_t          rtsx_ios_power_mode;    /* current host.ios.power mode */
 130         int8_t          rtsx_ios_timing;        /* current host.ios.timing */
 131         int8_t          rtsx_ios_vccq;          /* current host.ios.vccq */
 132         uint8_t         rtsx_read_only;         /* card read only status */
 133         uint8_t         rtsx_inversion;         /* inversion of card detection and read only status */
 134         uint8_t         rtsx_force_timing;      /* force bus_timing_uhs_sdr50 */
 135         uint8_t         rtsx_debug_mask;        /* debugging mask */
 136 #define         RTSX_DEBUG_BASIC        0x01    /* debug basic flow */
 137 #define         RTSX_TRACE_SD_CMD       0x02    /* trace SD commands */
 138 #define         RTSX_DEBUG_TUNING       0x04    /* debug tuning */
 139 #ifdef MMCCAM
 140         uint8_t         rtsx_cam_status;        /* CAM status - 1 if card in use */
 141 #endif /* MMCCAM */
 142         uint64_t        rtsx_read_count;        /* count of read operations */
 143         uint64_t        rtsx_write_count;       /* count of write operations */
 144         bool            rtsx_discovery_mode;    /* are we in discovery mode? */
 145         bool            rtsx_tuning_mode;       /* are we tuning */
 146         bool            rtsx_double_clk;        /* double clock freqency */
 147         bool            rtsx_vpclk;             /* voltage at Pulse-width Modulation(PWM) clock? */
 148         uint8_t         rtsx_ssc_depth;         /* Spread spectrum clocking depth */
 149         uint8_t         rtsx_card_drive_sel;    /* value for RTSX_CARD_DRIVE_SEL */
 150         uint8_t         rtsx_sd30_drive_sel_3v3;/* value for RTSX_SD30_DRIVE_SEL */
 151 };
 152
 153 /* rtsx_flags values */
 154 #define RTSX_F_DEFAULT          0x0000
 155 #define RTSX_F_CARD_PRESENT     0x0001
 156 #define RTSX_F_SDIO_SUPPORT     0x0002
 157 #define RTSX_F_VERSION_A        0x0004
 158 #define RTSX_F_VERSION_B        0x0008
 159 #define RTSX_F_VERSION_C        0x0010
 160 #define RTSX_F_VERSION_D        0x0020
 161 #define RTSX_F_8411B_QFN48      0x0040
 162 #define RTSX_F_REVERSE_SOCKET   0x0080
 163
 164 #define RTSX_REALTEK            0x10ec
 165 #define RTSX_RTS5209            0x5209
 166 #define RTSX_RTS5227            0x5227
 167 #define RTSX_RTS5229            0x5229
 168 #define RTSX_RTS522A            0x522a
 169 #define RTSX_RTS525A            0x525a
 170 #define RTSX_RTS5249            0x5249
 171 #define RTSX_RTS5260            0x5260
 172 #define RTSX_RTL8402            0x5286
 173 #define RTSX_RTL8411            0x5289
 174 #define RTSX_RTL8411B           0x5287
 175
 176 #define RTSX_VERSION            "2.1d"
 177
 178 static const struct rtsx_device {
 179         uint16_t        vendor_id;
 180         uint16_t        device_id;
 181         const char      *desc;
 182 } rtsx_devices[] = {
 183         { RTSX_REALTEK, RTSX_RTS5209,   RTSX_VERSION " Realtek RTS5209 PCIe MMC/SD Card Reader"},
 184         { RTSX_REALTEK, RTSX_RTS5227,   RTSX_VERSION " Realtek RTS5227 PCIe MMC/SD Card Reader"},
 185         { RTSX_REALTEK, RTSX_RTS5229,   RTSX_VERSION " Realtek RTS5229 PCIe MMC/SD Card Reader"},
 186         { RTSX_REALTEK, RTSX_RTS522A,   RTSX_VERSION " Realtek RTS522A PCIe MMC/SD Card Reader"},
 187         { RTSX_REALTEK, RTSX_RTS525A,   RTSX_VERSION " Realtek RTS525A PCIe MMC/SD Card Reader"},
 188         { RTSX_REALTEK, RTSX_RTS5249,   RTSX_VERSION " Realtek RTS5249 PCIe MMC/SD Card Reader"},
 189         { RTSX_REALTEK, RTSX_RTS5260,   RTSX_VERSION " Realtek RTS5260 PCIe MMC/SD Card Reader"},
 190         { RTSX_REALTEK, RTSX_RTL8402,   RTSX_VERSION " Realtek RTL8402 PCIe MMC/SD Card Reader"},
 191         { RTSX_REALTEK, RTSX_RTL8411,   RTSX_VERSION " Realtek RTL8411 PCIe MMC/SD Card Reader"},
 192         { RTSX_REALTEK, RTSX_RTL8411B,  RTSX_VERSION " Realtek RTL8411B PCIe MMC/SD Card Reader"},
 193         { 0,            0,              NULL}
 194 };
 195
 196 /* See `kenv | grep smbios.system` */
 197 static const struct rtsx_inversion_model {
 198         char    *maker;
 199         char    *family;
 200         char    *product;
 201 } rtsx_inversion_models[] = {
 202         { "LENOVO",             "ThinkPad T470p",       "20J7S0PM00"},
 203         { NULL,                 NULL,                   NULL}
 204 };
 205
 206 static int      rtsx_dma_alloc(struct rtsx_softc *sc);
 207 static void     rtsx_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
 208 static void     rtsx_dma_free(struct rtsx_softc *sc);
 209 static void     rtsx_intr(void *arg);
 210 static void     rtsx_handle_card_present(struct rtsx_softc *sc);
 211 static void     rtsx_card_task(void *arg, int pending __unused);
 212 static bool     rtsx_is_card_present(struct rtsx_softc *sc);
 213 static int      rtsx_init(struct rtsx_softc *sc);
 214 static int      rtsx_map_sd_drive(int index);
 215 static int      rtsx_rts5227_fill_driving(struct rtsx_softc *sc);
 216 static int      rtsx_rts5249_fill_driving(struct rtsx_softc *sc);
 217 static int      rtsx_rts5260_fill_driving(struct rtsx_softc *sc);
 218 static int      rtsx_read(struct rtsx_softc *, uint16_t, uint8_t *);
 219 static int      rtsx_read_cfg(struct rtsx_softc *sc, uint8_t func, uint16_t addr, uint32_t *val);
 220 static int      rtsx_write(struct rtsx_softc *sc, uint16_t addr, uint8_t mask, uint8_t val);
 221 static int      rtsx_read_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t *val);
 222 static int      rtsx_write_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t val);
 223 static int      rtsx_bus_power_off(struct rtsx_softc *sc);
 224 static int      rtsx_bus_power_on(struct rtsx_softc *sc);
 225 static int      rtsx_set_bus_width(struct rtsx_softc *sc, enum mmc_bus_width width);
 226 static int      rtsx_set_sd_timing(struct rtsx_softc *sc, enum mmc_bus_timing timing);
 227 static int      rtsx_set_sd_clock(struct rtsx_softc *sc, uint32_t freq);
 228 static int      rtsx_stop_sd_clock(struct rtsx_softc *sc);
 229 static int      rtsx_switch_sd_clock(struct rtsx_softc *sc, uint8_t clk, uint8_t n, uint8_t div, uint8_t mcu);
 230 #ifndef MMCCAM
 231 static void     rtsx_sd_change_tx_phase(struct rtsx_softc *sc, uint8_t sample_point);
 232 static void     rtsx_sd_change_rx_phase(struct rtsx_softc *sc, uint8_t sample_point);
 233 static void     rtsx_sd_tuning_rx_phase(struct rtsx_softc *sc, uint32_t *phase_map);
 234 static int      rtsx_sd_tuning_rx_cmd(struct rtsx_softc *sc, uint8_t sample_point);
 235 static int      rtsx_sd_tuning_rx_cmd_wait(struct rtsx_softc *sc, struct mmc_command *cmd);
 236 static void     rtsx_sd_tuning_rx_cmd_wakeup(struct rtsx_softc *sc);
 237 static void     rtsx_sd_wait_data_idle(struct rtsx_softc *sc);
 238 static uint8_t  rtsx_sd_search_final_rx_phase(struct rtsx_softc *sc, uint32_t phase_map);
 239 static int      rtsx_sd_get_rx_phase_len(uint32_t phase_map, int start_bit);
 240 #endif /* !MMCCAM */
 241 #if 0   /* For led */
 242 static int      rtsx_led_enable(struct rtsx_softc *sc);
 243 static int      rtsx_led_disable(struct rtsx_softc *sc);
 244 #endif  /* For led */
 245 static uint8_t  rtsx_response_type(uint16_t mmc_rsp);
 246 static void     rtsx_init_cmd(struct rtsx_softc *sc, struct mmc_command *cmd);
 247 static void     rtsx_push_cmd(struct rtsx_softc *sc, uint8_t cmd, uint16_t reg,
 248                               uint8_t mask, uint8_t data);
 249 static void     rtsx_set_cmd_data_len(struct rtsx_softc *sc, uint16_t block_cnt, uint16_t byte_cnt);
 250 static void     rtsx_send_cmd(struct rtsx_softc *sc);
 251 static void     rtsx_ret_resp(struct rtsx_softc *sc);
 252 static void     rtsx_set_resp(struct rtsx_softc *sc, struct mmc_command *cmd);
 253 static void     rtsx_stop_cmd(struct rtsx_softc *sc);
 254 static void     rtsx_clear_error(struct rtsx_softc *sc);
 255 static void     rtsx_req_done(struct rtsx_softc *sc);
 256 static int      rtsx_send_req(struct rtsx_softc *sc, struct mmc_command *cmd);
 257 static int      rtsx_xfer_short(struct rtsx_softc *sc, struct mmc_command *cmd);
 258 static void     rtsx_ask_ppbuf_part1(struct rtsx_softc *sc);
 259 static void     rtsx_get_ppbuf_part1(struct rtsx_softc *sc);
 260 static void     rtsx_get_ppbuf_part2(struct rtsx_softc *sc);
 261 static void     rtsx_put_ppbuf_part1(struct rtsx_softc *sc);
 262 static void     rtsx_put_ppbuf_part2(struct rtsx_softc *sc);
 263 static void     rtsx_write_ppbuf(struct rtsx_softc *sc);
 264 static int      rtsx_xfer(struct rtsx_softc *sc, struct mmc_command *cmd);
 265 static void     rtsx_xfer_begin(struct rtsx_softc *sc);
 266 static void     rtsx_xfer_start(struct rtsx_softc *sc);
 267 static void     rtsx_xfer_finish(struct rtsx_softc *sc);
 268 static void     rtsx_timeout(void *arg);
 269
 270 #ifdef MMCCAM
 271 static int      rtsx_get_tran_settings(device_t dev, struct ccb_trans_settings_mmc *cts);
 272 static int      rtsx_set_tran_settings(device_t dev, struct ccb_trans_settings_mmc *cts);
 273 static int      rtsx_cam_request(device_t dev, union ccb *ccb);
 274 #endif /* MMCCAM */
 275
 276 static int      rtsx_read_ivar(device_t bus, device_t child, int which, uintptr_t *result);
 277 static int      rtsx_write_ivar(device_t bus, device_t child, int which, uintptr_t value);
 278
 279 static int      rtsx_mmcbr_update_ios(device_t bus, device_t child __unused);
 280 static int      rtsx_mmcbr_switch_vccq(device_t bus, device_t child __unused);
 281 static int      rtsx_mmcbr_request(device_t bus, device_t child __unused, struct mmc_request *req);
 282 #ifndef MMCCAM
 283 static int      rtsx_mmcbr_tune(device_t bus, device_t child __unused, bool hs400 __unused);
 284 static int      rtsx_mmcbr_retune(device_t bus, device_t child __unused, bool reset __unused);
 285 static int      rtsx_mmcbr_get_ro(device_t bus, device_t child __unused);
 286 static int      rtsx_mmcbr_acquire_host(device_t bus, device_t child __unused);
 287 static int      rtsx_mmcbr_release_host(device_t bus, device_t child __unused);
 288 #endif /* !MMCCAM */
 289
 290 static int      rtsx_probe(device_t dev);
 291 static int      rtsx_attach(device_t dev);
 292 static int      rtsx_detach(device_t dev);
 293 static int      rtsx_shutdown(device_t dev);
 294 static int      rtsx_suspend(device_t dev);
 295 static int      rtsx_resume(device_t dev);
 296
 297 #define RTSX_LOCK_INIT(_sc)     mtx_init(&(_sc)->rtsx_mtx,      \
 298                                          device_get_nameunit(sc->rtsx_dev), "rtsx", MTX_DEF)
 299 #define RTSX_LOCK(_sc)          mtx_lock(&(_sc)->rtsx_mtx)
 300 #define RTSX_UNLOCK(_sc)        mtx_unlock(&(_sc)->rtsx_mtx)
 301 #define RTSX_LOCK_DESTROY(_sc)  mtx_destroy(&(_sc)->rtsx_mtx)
 302
 303 #define RTSX_SDCLK_OFF                  0
 304 #define RTSX_SDCLK_250KHZ          250000
 305 #define RTSX_SDCLK_400KHZ          400000
 306 #define RTSX_SDCLK_25MHZ         25000000
 307 #define RTSX_SDCLK_50MHZ         50000000
 308 #define RTSX_SDCLK_100MHZ       100000000
 309 #define RTSX_SDCLK_208MHZ       208000000
 310
 311 #define RTSX_MIN_DIV_N          80
 312 #define RTSX_MAX_DIV_N          208
 313
 314 #define RTSX_MAX_DATA_BLKLEN    512
 315
 316 #define RTSX_DMA_ALIGN          4
 317 #define RTSX_HOSTCMD_MAX        256
 318 #define RTSX_DMA_CMD_BIFSIZE    (sizeof(uint32_t) * RTSX_HOSTCMD_MAX)
 319 #define RTSX_DMA_DATA_BUFSIZE   MAXPHYS
 320
 321 #define ISSET(t, f) ((t) & (f))
 322
 323 #define READ4(sc, reg)                                          \
 324         (bus_space_read_4((sc)->rtsx_btag, (sc)->rtsx_bhandle, (reg)))
 325 #define WRITE4(sc, reg, val)                                    \
 326         (bus_space_write_4((sc)->rtsx_btag, (sc)->rtsx_bhandle, (reg), (val)))
 327
 328 #define RTSX_READ(sc, reg, val)                                 \
 329         do {                                                    \
 330                 int err = rtsx_read((sc), (reg), (val));        \
 331                 if (err)                                        \
 332                         return (err);                           \
 333         } while (0)
 334
 335 #define RTSX_WRITE(sc, reg, val)                                \
 336         do {                                                    \
 337                 int err = rtsx_write((sc), (reg), 0xff, (val)); \
 338                 if (err)                                        \
 339                         return (err);                           \
 340         } while (0)
 341 #define RTSX_CLR(sc, reg, bits)                                 \
 342         do {                                                    \
 343                 int err = rtsx_write((sc), (reg), (bits), 0);   \
 344                 if (err)                                        \
 345                         return (err);                           \
 346         } while (0)
 347
 348 #define RTSX_SET(sc, reg, bits)                                 \
 349         do {                                                    \
 350                 int err = rtsx_write((sc), (reg), (bits), 0xff);\
 351                 if (err)                                        \
 352                         return (err);                           \
 353         } while (0)
 354
 355 #define RTSX_BITOP(sc, reg, mask, bits)                         \
 356         do {                                                    \
 357                 int err = rtsx_write((sc), (reg), (mask), (bits));      \
 358                 if (err)                                        \
 359                         return (err);                           \
 360         } while (0)
 361
 362 /*
 363  * We use two DMA buffers: a command buffer and a data buffer.
 364  *
 365  * The command buffer contains a command queue for the host controller,
 366  * which describes SD/MMC commands to run, and other parameters. The chip
 367  * runs the command queue when a special bit in the RTSX_HCBAR register is
 368  * set and signals completion with the RTSX_TRANS_OK_INT interrupt.
 369  * Each command is encoded as a 4 byte sequence containing command number
 370  * (read, write, or check a host controller register), a register address,
 371  * and a data bit-mask and value.
 372  * SD/MMC commands which do not transfer any data from/to the card only use
 373  * the command buffer.
 374  *
 375  * The data buffer is used for transfer longer than 512. Data transfer is
 376  * controlled via the RTSX_HDBAR register and completion is signalled by
 377  * the RTSX_TRANS_OK_INT interrupt.
 378  *
 379  * The chip is unable to perform DMA above 4GB.
 380  */
 381
 382 /*
 383  * Main commands in the usual seqence used:
 384  *
 385  * CMD0         Go idle state
 386  * CMD8         Send interface condition
 387  * CMD55        Application Command for next ACMD
 388  * ACMD41       Send Operation Conditions Register (OCR: voltage profile of the card)
 389  * CMD2         Send Card Identification (CID) Register
 390  * CMD3         Send relative address
 391  * CMD9         Send Card Specific Data (CSD)
 392  * CMD13        Send status (32 bits -  bit 25: card password protected)
 393  * CMD7         Select card (before Get card SCR)
 394  * ACMD51       Send SCR (SD CARD Configuration Register - [51:48]: Bus widths supported)
 395  * CMD6         SD switch function
 396  * ACMD13       Send SD status (512 bits)
 397  * ACMD42       Set/Clear card detect
 398  * ACMD6        Set bus width
 399  * CMD19        Send tuning block
 400  * CMD12        Stop transmission
 401  *
 402  * CMD17        Read single block (<=512)
 403  * CMD18        Read multiple blocks (>512)
 404  * CMD24        Write single block (<=512)
 405  * CMD25        Write multiple blocks (>512)
 406  *
 407  * CMD52        IO R/W direct
 408  * CMD5         Send Operation Conditions
 409  */
 410
 411 static int
 412 rtsx_dma_alloc(struct rtsx_softc *sc)
 413 {
 414         int     error = 0;
 415
 416         error = bus_dma_tag_create(bus_get_dma_tag(sc->rtsx_dev), /* inherit from parent */
 417             RTSX_DMA_ALIGN, 0,          /* alignment, boundary */
 418             BUS_SPACE_MAXADDR_32BIT,    /* lowaddr */
 419             BUS_SPACE_MAXADDR,          /* highaddr */
 420             NULL, NULL,                 /* filter, filterarg */
 421             RTSX_DMA_CMD_BIFSIZE, 1,    /* maxsize, nsegments */
 422             RTSX_DMA_CMD_BIFSIZE,       /* maxsegsize */
 423             0,                          /* flags */
 424             NULL, NULL,                 /* lockfunc, lockarg */
 425             &sc->rtsx_cmd_dma_tag);
 426         if (error) {
 427                 device_printf(sc->rtsx_dev,
 428                               "Can't create cmd parent DMA tag\n");
 429                 return (error);
 430         }
 431         error = bus_dmamem_alloc(sc->rtsx_cmd_dma_tag,          /* DMA tag */
 432             &sc->rtsx_cmd_dmamem,                               /* will hold the KVA pointer */
 433             BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO,   /* flags */
 434             &sc->rtsx_cmd_dmamap);                              /* DMA map */
 435         if (error) {
 436                 device_printf(sc->rtsx_dev,
 437                               "Can't create DMA map for command transfer\n");
 438                 goto destroy_cmd_dma_tag;
 439
 440         }
 441         error = bus_dmamap_load(sc->rtsx_cmd_dma_tag,   /* DMA tag */
 442             sc->rtsx_cmd_dmamap,        /* DMA map */
 443             sc->rtsx_cmd_dmamem,        /* KVA pointer to be mapped */
 444             RTSX_DMA_CMD_BIFSIZE,       /* size of buffer */
 445             rtsx_dmamap_cb,             /* callback */
 446             &sc->rtsx_cmd_buffer,       /* first arg of callback */
 447             0);                         /* flags */
 448         if (error || sc->rtsx_cmd_buffer == 0) {
 449                 device_printf(sc->rtsx_dev,
 450                               "Can't load DMA memory for command transfer\n");
 451                 error = (error) ? error : EFAULT;
 452                 goto destroy_cmd_dmamem_alloc;
 453         }
 454
 455         error = bus_dma_tag_create(bus_get_dma_tag(sc->rtsx_dev),       /* inherit from parent */
 456             RTSX_DMA_DATA_BUFSIZE, 0,   /* alignment, boundary */
 457             BUS_SPACE_MAXADDR_32BIT,    /* lowaddr */
 458             BUS_SPACE_MAXADDR,          /* highaddr */
 459             NULL, NULL,                 /* filter, filterarg */
 460             RTSX_DMA_DATA_BUFSIZE, 1,   /* maxsize, nsegments */
 461             RTSX_DMA_DATA_BUFSIZE,      /* maxsegsize */
 462             0,                          /* flags */
 463             NULL, NULL,                 /* lockfunc, lockarg */
 464             &sc->rtsx_data_dma_tag);
 465         if (error) {
 466                 device_printf(sc->rtsx_dev,
 467                               "Can't create data parent DMA tag\n");
 468                 goto destroy_cmd_dmamap_load;
 469         }
 470         error = bus_dmamem_alloc(sc->rtsx_data_dma_tag,         /* DMA tag */
 471             &sc->rtsx_data_dmamem,                              /* will hold the KVA pointer */
 472             BUS_DMA_WAITOK | BUS_DMA_ZERO,                      /* flags */
 473             &sc->rtsx_data_dmamap);                             /* DMA map */
 474         if (error) {
 475                 device_printf(sc->rtsx_dev,
 476                               "Can't create DMA map for data transfer\n");
 477                 goto destroy_data_dma_tag;
 478         }
 479         error = bus_dmamap_load(sc->rtsx_data_dma_tag,  /* DMA tag */
 480             sc->rtsx_data_dmamap,       /* DMA map */
 481             sc->rtsx_data_dmamem,       /* KVA pointer to be mapped */
 482             RTSX_DMA_DATA_BUFSIZE,      /* size of buffer */
 483             rtsx_dmamap_cb,             /* callback */
 484             &sc->rtsx_data_buffer,      /* first arg of callback */
 485             0);                         /* flags */
 486         if (error || sc->rtsx_data_buffer == 0) {
 487                 device_printf(sc->rtsx_dev,
 488                               "Can't load DMA memory for data transfer\n");
 489                 error = (error) ? error : EFAULT;
 490                 goto destroy_data_dmamem_alloc;
 491         }
 492         return (error);
 493
 494  destroy_data_dmamem_alloc:
 495         bus_dmamem_free(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamem, sc->rtsx_data_dmamap);
 496  destroy_data_dma_tag:
 497         bus_dma_tag_destroy(sc->rtsx_data_dma_tag);
 498  destroy_cmd_dmamap_load:
 499         bus_dmamap_unload(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap);
 500  destroy_cmd_dmamem_alloc:
 501         bus_dmamem_free(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamem, sc->rtsx_cmd_dmamap);
 502  destroy_cmd_dma_tag:
 503         bus_dma_tag_destroy(sc->rtsx_cmd_dma_tag);
 504
 505         return (error);
 506 }
 507
 508 static void
 509 rtsx_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 510 {
 511         if (error) {
 512                 printf("rtsx_dmamap_cb: error %d\n", error);
 513                 return;
 514         }
 515         *(bus_addr_t *)arg = segs[0].ds_addr;
 516 }
 517
 518 static void
 519 rtsx_dma_free(struct rtsx_softc *sc)
 520 {
 521         if (sc->rtsx_cmd_dma_tag != NULL) {
 522                 if (sc->rtsx_cmd_dmamap != NULL)
 523                         bus_dmamap_unload(sc->rtsx_cmd_dma_tag,
 524                                           sc->rtsx_cmd_dmamap);
 525                 if (sc->rtsx_cmd_dmamem != NULL)
 526                         bus_dmamem_free(sc->rtsx_cmd_dma_tag,
 527                                         sc->rtsx_cmd_dmamem,
 528                                         sc->rtsx_cmd_dmamap);
 529                 sc->rtsx_cmd_dmamap = NULL;
 530                 sc->rtsx_cmd_dmamem = NULL;
 531                 sc->rtsx_cmd_buffer = 0;
 532                 bus_dma_tag_destroy(sc->rtsx_cmd_dma_tag);
 533                 sc->rtsx_cmd_dma_tag = NULL;
 534         }
 535         if (sc->rtsx_data_dma_tag != NULL) {
 536                 if (sc->rtsx_data_dmamap != NULL)
 537                         bus_dmamap_unload(sc->rtsx_data_dma_tag,
 538                                           sc->rtsx_data_dmamap);
 539                 if (sc->rtsx_data_dmamem != NULL)
 540                         bus_dmamem_free(sc->rtsx_data_dma_tag,
 541                                         sc->rtsx_data_dmamem,
 542                                         sc->rtsx_data_dmamap);
 543                 sc->rtsx_data_dmamap = NULL;
 544                 sc->rtsx_data_dmamem = NULL;
 545                 sc->rtsx_data_buffer = 0;
 546                 bus_dma_tag_destroy(sc->rtsx_data_dma_tag);
 547                 sc->rtsx_data_dma_tag = NULL;
 548         }
 549 }
 550
 551 static void
 552 rtsx_intr(void *arg)
 553 {
 554         struct rtsx_softc *sc = arg;
 555         uint32_t        enabled;
 556         uint32_t        status;
 557
 558         RTSX_LOCK(sc);
 559
 560         enabled = sc->rtsx_intr_enabled;
 561         status = READ4(sc, RTSX_BIPR);  /* read Bus Interrupt Pending Register */
 562         sc->rtsx_intr_status = status;
 563
 564         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
 565                 device_printf(sc->rtsx_dev, "Interrupt handler - enabled: 0x%08x, status: 0x%08x\n", enabled, status);
 566
 567         /* Ack interrupts. */
 568         WRITE4(sc, RTSX_BIPR, status);
 569
 570         if (((enabled & status) == 0) || status == 0xffffffff) {
 571                 device_printf(sc->rtsx_dev, "Spurious interrupt - enabled: 0x%08x, status: 0x%08x\n", enabled, status);
 572                 RTSX_UNLOCK(sc);
 573                 return;
 574         }
 575
 576         /* Detect write protect. */
 577         if (status & RTSX_SD_WRITE_PROTECT)
 578                 sc->rtsx_read_only = 1;
 579         else
 580                 sc->rtsx_read_only = 0;
 581
 582         /* Start task to handle SD card status change (from dwmmc.c). */
 583         if (status & RTSX_SD_INT) {
 584                 device_printf(sc->rtsx_dev, "Interrupt card inserted/removed\n");
 585                 rtsx_handle_card_present(sc);
 586         }
 587
 588         if (sc->rtsx_req == NULL) {
 589                 RTSX_UNLOCK(sc);
 590                 return;
 591         }
 592
 593         if (status & RTSX_TRANS_OK_INT) {
 594                 sc->rtsx_req->cmd->error = MMC_ERR_NONE;
 595                 if (sc->rtsx_intr_trans_ok != NULL)
 596                         sc->rtsx_intr_trans_ok(sc);
 597         } else if (status & RTSX_TRANS_FAIL_INT) {
 598                 uint8_t stat1;
 599                 sc->rtsx_req->cmd->error = MMC_ERR_FAILED;
 600                 if (rtsx_read(sc, RTSX_SD_STAT1, &stat1) == 0 &&
 601                     (stat1 & RTSX_SD_CRC_ERR)) {
 602                         device_printf(sc->rtsx_dev, "CRC error\n");
 603                         sc->rtsx_req->cmd->error = MMC_ERR_BADCRC;
 604                 }
 605                 if (!sc->rtsx_tuning_mode)
 606                         device_printf(sc->rtsx_dev, "Transfer fail - status: 0x%08x\n", status);
 607                 rtsx_stop_cmd(sc);
 608                 if (sc->rtsx_intr_trans_ko != NULL)
 609                         sc->rtsx_intr_trans_ko(sc);
 610         }
 611
 612         RTSX_UNLOCK(sc);
 613 }
 614
 615 /*
 616  * Function called from the IRQ handler (from dwmmc.c).
 617  */
 618 static void
 619 rtsx_handle_card_present(struct rtsx_softc *sc)
 620 {
 621         bool    was_present;
 622         bool    is_present;
 623
 624 #ifdef MMCCAM
 625         was_present = sc->rtsx_cam_status;
 626 #else  /* !MMCCAM */
 627         was_present = sc->rtsx_mmc_dev != NULL;
 628 #endif /* MMCCAM */
 629         is_present = rtsx_is_card_present(sc);
 630         if (is_present)
 631                 device_printf(sc->rtsx_dev, "Card present\n");
 632         else
 633                 device_printf(sc->rtsx_dev, "Card absent\n");
 634
 635         if (!was_present && is_present) {
 636                 /*
 637                  * The delay is to debounce the card insert
 638                  * (sometimes the card detect pin stabilizes
 639                  * before the other pins have made good contact).
 640                  */
 641                 taskqueue_enqueue_timeout(taskqueue_swi_giant,
 642                                           &sc->rtsx_card_insert_task, -hz);
 643         } else if (was_present && !is_present) {
 644                 taskqueue_enqueue(taskqueue_swi_giant, &sc->rtsx_card_remove_task);
 645         }
 646 }
 647
 648 /*
 649  * This function is called at startup.
 650  */
 651 static void
 652 rtsx_card_task(void *arg, int pending __unused)
 653 {
 654         struct rtsx_softc *sc = arg;
 655
 656         if (rtsx_is_card_present(sc)) {
 657                 sc->rtsx_flags |= RTSX_F_CARD_PRESENT;
 658                 /* Card is present, attach if necessary. */
 659 #ifdef MMCCAM
 660                 if (sc->rtsx_cam_status == 0) {
 661 #else  /* !MMCCAM */
 662                 if (sc->rtsx_mmc_dev == NULL) {
 663 #endif /* MMCCAM */
 664                         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 665                                 device_printf(sc->rtsx_dev, "Card inserted\n");
 666
 667                         sc->rtsx_read_count = sc->rtsx_write_count = 0;
 668 #ifdef MMCCAM
 669                         sc->rtsx_cam_status = 1;
 670                         mmc_cam_sim_discover(&sc->rtsx_mmc_sim);
 671 #else  /* !MMCCAM */
 672                         RTSX_LOCK(sc);
 673                         sc->rtsx_mmc_dev = device_add_child(sc->rtsx_dev, "mmc", -1);
 674                         RTSX_UNLOCK(sc);
 675                         if (sc->rtsx_mmc_dev == NULL) {
 676                                 device_printf(sc->rtsx_dev, "Adding MMC bus failed\n");
 677                         } else {
 678                                 device_set_ivars(sc->rtsx_mmc_dev, sc);
 679                                 device_probe_and_attach(sc->rtsx_mmc_dev);
 680                         }
 681 #endif /* MMCCAM */
 682                 }
 683         } else {
 684                 sc->rtsx_flags &= ~RTSX_F_CARD_PRESENT;
 685                 /* Card isn't present, detach if necessary. */
 686 #ifdef MMCCAM
 687                 if (sc->rtsx_cam_status != 0) {
 688 #else  /* !MMCCAM */
 689                 if (sc->rtsx_mmc_dev != NULL) {
 690 #endif /* MMCCAM */
 691                         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 692                                 device_printf(sc->rtsx_dev, "Card removed\n");
 693
 694                         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 695                                 device_printf(sc->rtsx_dev, "Read count: %" PRIu64 ", write count: %" PRIu64 "\n",
 696                                               sc->rtsx_read_count, sc->rtsx_write_count);
 697 #ifdef MMCCAM
 698                         sc->rtsx_cam_status = 0;
 699                         mmc_cam_sim_discover(&sc->rtsx_mmc_sim);
 700 #else  /* !MMCCAM */
 701                         if (device_delete_child(sc->rtsx_dev, sc->rtsx_mmc_dev))
 702                                 device_printf(sc->rtsx_dev, "Detaching MMC bus failed\n");
 703                         sc->rtsx_mmc_dev = NULL;
 704 #endif /* MMCCAM */
 705                 }
 706         }
 707 }
 708
 709 static bool
 710 rtsx_is_card_present(struct rtsx_softc *sc)
 711 {
 712         uint32_t status;
 713
 714         status = READ4(sc, RTSX_BIPR);
 715         if (sc->rtsx_inversion == 0)
 716                 return (status & RTSX_SD_EXIST);
 717         else
 718                 return !(status & RTSX_SD_EXIST);
 719 }
 720
 721 static int
 722 rtsx_init(struct rtsx_softc *sc)
 723 {
 724         uint8_t version;
 725         uint8_t val;
 726         int     error;
 727
 728         sc->rtsx_host.host_ocr = RTSX_SUPPORTED_VOLTAGE;
 729         sc->rtsx_host.f_min = RTSX_SDCLK_250KHZ;
 730         sc->rtsx_host.f_max = RTSX_SDCLK_208MHZ;
 731         sc->rtsx_host.caps = MMC_CAP_4_BIT_DATA | MMC_CAP_HSPEED |
 732                 MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
 733
 734         sc->rtsx_host.caps |= MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104;
 735         if (sc->rtsx_device_id == RTSX_RTS5209)
 736                 sc->rtsx_host.caps |= MMC_CAP_8_BIT_DATA;
 737         pci_find_cap(sc->rtsx_dev, PCIY_EXPRESS, &(sc->rtsx_pcie_cap));
 738
 739         /*
 740          * Check IC version.
 741          */
 742         switch (sc->rtsx_device_id) {
 743         case RTSX_RTS5229:
 744                 /* Read IC version from dummy register. */
 745                 RTSX_READ(sc, RTSX_DUMMY_REG, &version);
 746                 if ((version & 0x0F) == RTSX_IC_VERSION_C)
 747                         sc->rtsx_flags |= RTSX_F_VERSION_C;
 748                 break;
 749         case RTSX_RTS522A:
 750                 /* Read IC version from dummy register. */
 751                 RTSX_READ(sc, RTSX_DUMMY_REG, &version);
 752                 if ((version & 0x0F) == RTSX_IC_VERSION_A)
 753                         sc->rtsx_flags |= RTSX_F_VERSION_A;
 754                 break;
 755         case RTSX_RTS525A:
 756                 /* Read IC version from dummy register. */
 757                 RTSX_READ(sc, RTSX_DUMMY_REG, &version);
 758                 if ((version & 0x0F) == RTSX_IC_VERSION_A)
 759                         sc->rtsx_flags |= RTSX_F_VERSION_A;
 760                 break;
 761         case RTSX_RTL8411B:
 762                 RTSX_READ(sc, RTSX_RTL8411B_PACKAGE, &version);
 763                 if (version & RTSX_RTL8411B_QFN48)
 764                         sc->rtsx_flags |= RTSX_F_8411B_QFN48;
 765                 break;
 766         }
 767
 768         /*
 769          * Fetch vendor settings.
 770          */
 771         /*
 772          * Normally OEMs will set vendor setting to the config space
 773          * of Realtek card reader in BIOS stage. This statement reads
 774          * the setting and configure the internal registers according
 775          * to it, to improve card reader's compatibility condition.
 776          */
 777         sc->rtsx_card_drive_sel = RTSX_CARD_DRIVE_DEFAULT;
 778         switch (sc->rtsx_device_id) {
 779                 uint32_t reg;
 780                 uint32_t reg1;
 781                 uint8_t  reg3;
 782         case RTSX_RTS5209:
 783                 sc->rtsx_card_drive_sel = RTSX_RTS5209_CARD_DRIVE_DEFAULT;
 784                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D;
 785                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4);
 786                 if (!(reg & 0x80)) {
 787                         sc->rtsx_card_drive_sel = (reg >> 8) & 0x3F;
 788                         sc->rtsx_sd30_drive_sel_3v3 = reg & 0x07;
 789                 } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) {
 790                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 791                 }
 792                 if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 793                         device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n",
 794                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3);
 795                 break;
 796         case RTSX_RTS5227:
 797         case RTSX_RTS522A:
 798                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_CFG_DRIVER_TYPE_B;
 799                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 800                 if (!(reg & 0x1000000)) {
 801                         sc->rtsx_card_drive_sel &= 0x3F;
 802                         sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6;
 803                         reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4);
 804                         sc->rtsx_sd30_drive_sel_3v3 = (reg >> 5) & 0x03;
 805                         if (reg & 0x4000)
 806                                 sc->rtsx_flags |= RTSX_F_REVERSE_SOCKET;
 807                 } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) {
 808                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 809                 }
 810                 if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 811                         device_printf(sc->rtsx_dev,
 812                                       "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x, reverse_socket is %s\n",
 813                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3,
 814                                       (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) ? "true" : "false");
 815                 break;
 816         case RTSX_RTS5229:
 817                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D;
 818                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 819                 if (!(reg & 0x1000000)) {
 820                         sc->rtsx_card_drive_sel &= 0x3F;
 821                         sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6;
 822                         reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4);
 823                         sc->rtsx_sd30_drive_sel_3v3 = rtsx_map_sd_drive((reg >> 5) & 0x03);
 824                 } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) {
 825                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 826                 }
 827                 if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 828                         device_printf(sc->rtsx_dev, "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n",
 829                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3);
 830                 break;
 831         case RTSX_RTS525A:
 832         case RTSX_RTS5249:
 833         case RTSX_RTS5260:
 834                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_CFG_DRIVER_TYPE_B;
 835                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 836                 if ((reg & 0x1000000)) {
 837                         sc->rtsx_card_drive_sel &= 0x3F;
 838                         sc->rtsx_card_drive_sel |= ((reg >> 25) & 0x01) << 6;
 839                         reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG2, 4);
 840                         sc->rtsx_sd30_drive_sel_3v3 = (reg >> 5) & 0x03;
 841                         if (reg & 0x4000)
 842                                 sc->rtsx_flags |= RTSX_F_REVERSE_SOCKET;
 843                 } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) {
 844                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 845                 }
 846                 if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 847                         device_printf(sc->rtsx_dev,
 848                                       "card_drive_sel = 0x%02x, sd30_drive_sel_3v3: 0x%02x, reverse_socket is %s\n",
 849                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3,
 850                                       (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET) ? "true" : "false");
 851                 break;
 852         case RTSX_RTL8402:
 853         case RTSX_RTL8411:
 854                 sc->rtsx_card_drive_sel = RTSX_RTL8411_CARD_DRIVE_DEFAULT;
 855                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D;
 856                 reg1 = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 857                 if (reg1 & 0x1000000) {
 858                         sc->rtsx_card_drive_sel &= 0x3F;
 859                         sc->rtsx_card_drive_sel |= ((reg1 >> 25) & 0x01) << 6;
 860                         reg3 = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG3, 1);
 861                         sc->rtsx_sd30_drive_sel_3v3 = (reg3 >> 5) & 0x07;
 862                 } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) {
 863                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg1: 0x%08x\n", reg1);
 864                 }
 865                 if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 866                         device_printf(sc->rtsx_dev,
 867                                       "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n",
 868                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3);
 869                 break;
 870         case RTSX_RTL8411B:
 871                 sc->rtsx_card_drive_sel = RTSX_RTL8411_CARD_DRIVE_DEFAULT;
 872                 sc->rtsx_sd30_drive_sel_3v3 = RTSX_DRIVER_TYPE_D;
 873                 reg = pci_read_config(sc->rtsx_dev, RTSX_PCR_SETTING_REG1, 4);
 874                 if (!(reg & 0x1000000)) {
 875                         sc->rtsx_sd30_drive_sel_3v3 = rtsx_map_sd_drive(reg & 0x03);
 876                 } else if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) {
 877                         device_printf(sc->rtsx_dev, "pci_read_config() error - reg: 0x%08x\n", reg);
 878                 }
 879                 if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 880                         device_printf(sc->rtsx_dev,
 881                                       "card_drive_sel: 0x%02x, sd30_drive_sel_3v3: 0x%02x\n",
 882                                       sc->rtsx_card_drive_sel, sc->rtsx_sd30_drive_sel_3v3);
 883                 break;
 884         }
 885
 886         if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
 887                 device_printf(sc->rtsx_dev, "rtsx_init() rtsx_flags: 0x%04x\n", sc->rtsx_flags);
 888
 889         /* Enable interrupts. */
 890         sc->rtsx_intr_enabled = RTSX_TRANS_OK_INT_EN | RTSX_TRANS_FAIL_INT_EN | RTSX_SD_INT_EN | RTSX_MS_INT_EN;
 891         WRITE4(sc, RTSX_BIER, sc->rtsx_intr_enabled);
 892
 893         /* Power on SSC clock. */
 894         RTSX_CLR(sc, RTSX_FPDCTL, RTSX_SSC_POWER_DOWN);
 895         /* Wait SSC power stable. */
 896         DELAY(200);
 897
 898         /* Disable ASPM */
 899         val = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL, 1);
 900         pci_write_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL, val & 0xfc, 1);
 901
 902         /*
 903          * Optimize phy.
 904          */
 905         switch (sc->rtsx_device_id) {
 906         case RTSX_RTS5209:
 907                 /* Some magic numbers from Linux driver. */
 908                 if ((error = rtsx_write_phy(sc, 0x00, 0xB966)))
 909                         return (error);
 910                 break;
 911         case RTSX_RTS5227:
 912                 RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN);
 913
 914                 /* Optimize RX sensitivity. */
 915                 if ((error = rtsx_write_phy(sc, 0x00, 0xBA42)))
 916                         return (error);
 917                 break;
 918         case RTSX_RTS5229:
 919                 /* Optimize RX sensitivity. */
 920                 if ((error = rtsx_write_phy(sc, 0x00, 0xBA42)))
 921                         return (error);
 922                 break;
 923         case RTSX_RTS522A:
 924                 RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3, RTSX_D3_DELINK_MODE_EN);
 925                 if (sc->rtsx_flags & RTSX_F_VERSION_A) {
 926                         if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR2, RTSX_PHY_RCR2_INIT_27S)))
 927                                 return (error);
 928                 }
 929                 if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR1, RTSX_PHY_RCR1_INIT_27S)))
 930                         return (error);
 931                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD0, RTSX_PHY_FLD0_INIT_27S)))
 932                         return (error);
 933                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD3, RTSX_PHY_FLD3_INIT_27S)))
 934                         return (error);
 935                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD4, RTSX_PHY_FLD4_INIT_27S)))
 936                         return (error);
 937                 break;
 938         case RTSX_RTS525A:
 939                 if ((error = rtsx_write_phy(sc, RTSX__PHY_FLD0,
 940                                             RTSX__PHY_FLD0_CLK_REQ_20C | RTSX__PHY_FLD0_RX_IDLE_EN |
 941                                             RTSX__PHY_FLD0_BIT_ERR_RSTN | RTSX__PHY_FLD0_BER_COUNT |
 942                                             RTSX__PHY_FLD0_BER_TIMER | RTSX__PHY_FLD0_CHECK_EN)))
 943                         return (error);
 944                 if ((error = rtsx_write_phy(sc, RTSX__PHY_ANA03,
 945                                             RTSX__PHY_ANA03_TIMER_MAX | RTSX__PHY_ANA03_OOBS_DEB_EN |
 946                                             RTSX__PHY_CMU_DEBUG_EN)))
 947                         return (error);
 948                 if (sc->rtsx_flags & RTSX_F_VERSION_A)
 949                         if ((error = rtsx_write_phy(sc, RTSX__PHY_REV0,
 950                                                     RTSX__PHY_REV0_FILTER_OUT | RTSX__PHY_REV0_CDR_BYPASS_PFD |
 951                                                     RTSX__PHY_REV0_CDR_RX_IDLE_BYPASS)))
 952                                 return (error);
 953                 break;
 954         case RTSX_RTS5249:
 955                 RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN);
 956                 if ((error = rtsx_write_phy(sc, RTSX_PHY_REV,
 957                                             RTSX_PHY_REV_RESV | RTSX_PHY_REV_RXIDLE_LATCHED |
 958                                             RTSX_PHY_REV_P1_EN | RTSX_PHY_REV_RXIDLE_EN |
 959                                             RTSX_PHY_REV_CLKREQ_TX_EN | RTSX_PHY_REV_RX_PWST |
 960                                             RTSX_PHY_REV_CLKREQ_DT_1_0 | RTSX_PHY_REV_STOP_CLKRD |
 961                                             RTSX_PHY_REV_STOP_CLKWR)))
 962                         return (error);
 963                 DELAY(1000);
 964                 if ((error = rtsx_write_phy(sc, RTSX_PHY_BPCR,
 965                                             RTSX_PHY_BPCR_IBRXSEL | RTSX_PHY_BPCR_IBTXSEL |
 966                                             RTSX_PHY_BPCR_IB_FILTER | RTSX_PHY_BPCR_CMIRROR_EN)))
 967                         return (error);
 968                 if ((error = rtsx_write_phy(sc, RTSX_PHY_PCR,
 969                                             RTSX_PHY_PCR_FORCE_CODE | RTSX_PHY_PCR_OOBS_CALI_50 |
 970                                             RTSX_PHY_PCR_OOBS_VCM_08 | RTSX_PHY_PCR_OOBS_SEN_90 |
 971                                             RTSX_PHY_PCR_RSSI_EN | RTSX_PHY_PCR_RX10K)))
 972                         return (error);
 973                 if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR2,
 974                                             RTSX_PHY_RCR2_EMPHASE_EN | RTSX_PHY_RCR2_NADJR |
 975                                             RTSX_PHY_RCR2_CDR_SR_2 | RTSX_PHY_RCR2_FREQSEL_12 |
 976                                             RTSX_PHY_RCR2_CDR_SC_12P | RTSX_PHY_RCR2_CALIB_LATE)))
 977                         return (error);
 978                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD4,
 979                                             RTSX_PHY_FLD4_FLDEN_SEL | RTSX_PHY_FLD4_REQ_REF |
 980                                             RTSX_PHY_FLD4_RXAMP_OFF | RTSX_PHY_FLD4_REQ_ADDA |
 981                                             RTSX_PHY_FLD4_BER_COUNT | RTSX_PHY_FLD4_BER_TIMER |
 982                                             RTSX_PHY_FLD4_BER_CHK_EN)))
 983                         return (error);
 984                 if ((error = rtsx_write_phy(sc, RTSX_PHY_RDR,
 985                                             RTSX_PHY_RDR_RXDSEL_1_9 | RTSX_PHY_SSC_AUTO_PWD)))
 986                         return (error);
 987                 if ((error = rtsx_write_phy(sc, RTSX_PHY_RCR1,
 988                                             RTSX_PHY_RCR1_ADP_TIME_4 | RTSX_PHY_RCR1_VCO_COARSE)))
 989                         return (error);
 990                 if ((error = rtsx_write_phy(sc, RTSX_PHY_FLD3,
 991                                             RTSX_PHY_FLD3_TIMER_4 | RTSX_PHY_FLD3_TIMER_6 |
 992                                             RTSX_PHY_FLD3_RXDELINK)))
 993                         return (error);
 994                 if ((error = rtsx_write_phy(sc, RTSX_PHY_TUNE,
 995                                             RTSX_PHY_TUNE_TUNEREF_1_0 | RTSX_PHY_TUNE_VBGSEL_1252 |
 996                                             RTSX_PHY_TUNE_SDBUS_33 | RTSX_PHY_TUNE_TUNED18 |
 997                                             RTSX_PHY_TUNE_TUNED12 | RTSX_PHY_TUNE_TUNEA12)))
 998                         return (error);
 999                 break;
1000         }
1001
1002         /* Set mcu_cnt to 7 to ensure data can be sampled properly. */
1003         RTSX_BITOP(sc, RTSX_CLK_DIV, 0x07, 0x07);
1004
1005         /* Disable sleep mode. */
1006         RTSX_CLR(sc, RTSX_HOST_SLEEP_STATE,
1007                  RTSX_HOST_ENTER_S1 | RTSX_HOST_ENTER_S3);
1008
1009         /* Disable card clock. */
1010         RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_CARD_CLK_EN_ALL);
1011
1012         /* Reset delink mode. */
1013         RTSX_CLR(sc, RTSX_CHANGE_LINK_STATE,
1014                  RTSX_FORCE_RST_CORE_EN | RTSX_NON_STICKY_RST_N_DBG);
1015
1016         /* Card driving select. */
1017         RTSX_WRITE(sc, RTSX_CARD_DRIVE_SEL, sc->rtsx_card_drive_sel);
1018
1019         /* Enable SSC clock. */
1020         RTSX_WRITE(sc, RTSX_SSC_CTL1, RTSX_SSC_8X_EN | RTSX_SSC_SEL_4M);
1021         RTSX_WRITE(sc, RTSX_SSC_CTL2, 0x12);
1022
1023         /* Disable cd_pwr_save. */
1024         RTSX_BITOP(sc, RTSX_CHANGE_LINK_STATE, 0x16, RTSX_MAC_PHY_RST_N_DBG);
1025
1026         /* Clear Link Ready Interrupt. */
1027         RTSX_BITOP(sc, RTSX_IRQSTAT0, RTSX_LINK_READY_INT, RTSX_LINK_READY_INT);
1028
1029         /* Enlarge the estimation window of PERST# glitch
1030          * to reduce the chance of invalid card interrupt. */
1031         RTSX_WRITE(sc, RTSX_PERST_GLITCH_WIDTH, 0x80);
1032
1033         /* Set RC oscillator to 400K. */
1034         RTSX_CLR(sc, RTSX_RCCTL, RTSX_RCCTL_F_2M);
1035
1036         /* Enable interrupt write-clear (default is read-clear). */
1037         RTSX_CLR(sc, RTSX_NFTS_TX_CTRL, RTSX_INT_READ_CLR);
1038
1039         if (sc->rtsx_device_id == RTSX_RTS525A)
1040                 RTSX_BITOP(sc, RTSX_PM_CLK_FORCE_CTL, 1, 1);
1041
1042         /* OC power down. */
1043         RTSX_BITOP(sc, RTSX_FPDCTL, RTSX_SD_OC_POWER_DOWN, RTSX_SD_OC_POWER_DOWN);
1044
1045         /* Enable clk_request_n to enable clock power management */
1046         pci_write_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_LINK_CTL + 1, 1, 1);
1047
1048         /* Enter L1 when host tx idle */
1049         pci_write_config(sc->rtsx_dev, 0x70F, 0x5B, 1);
1050
1051         /*
1052          * Specific extra init.
1053          */
1054         switch (sc->rtsx_device_id) {
1055                 uint16_t cap;
1056         case RTSX_RTS5209:
1057                 /* Turn off LED. */
1058                 RTSX_WRITE(sc, RTSX_CARD_GPIO, 0x03);
1059                 /* Reset ASPM state to default value. */
1060                 RTSX_CLR(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK);
1061                 /* Force CLKREQ# PIN to drive 0 to request clock. */
1062                 RTSX_BITOP(sc, RTSX_PETXCFG, 0x08, 0x08);
1063                 /* Configure GPIO as output. */
1064                 RTSX_WRITE(sc, RTSX_CARD_GPIO_DIR, 0x03);
1065                 /* Configure driving. */
1066                 RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3);
1067                 break;
1068         case RTSX_RTS5227:
1069                 /* Configure GPIO as output. */
1070                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1071                 /* Reset ASPM state to default value. */
1072                 RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1073                 /* Switch LDO3318 source from DV33 to 3V3. */
1074                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1075                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1076                 /* Set default OLT blink period. */
1077                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1078                 /* Configure LTR. */
1079                 cap = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_DEVICE_CTL2, 2);
1080                 if (cap & PCIEM_CTL2_LTR_ENABLE)
1081                         RTSX_WRITE(sc, RTSX_LTR_CTL, 0xa3);
1082                 /* Configure OBFF. */
1083                 RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_ENABLE);
1084                 /* Configure driving. */
1085                 if ((error = rtsx_rts5227_fill_driving(sc)))
1086                         return (error);
1087                 /* Configure force_clock_req. */
1088                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1089                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0xB8);
1090                 else
1091                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0x88);
1092                 RTSX_CLR(sc, RTSX_PM_CTRL3, RTSX_D3_DELINK_MODE_EN);
1093                 /*!!! Added for reboot after Windows. */
1094                 RTSX_BITOP(sc, RTSX_PM_CTRL3, RTSX_PM_WAKE_EN, RTSX_PM_WAKE_EN);
1095                 break;
1096         case RTSX_RTS5229:
1097                 /* Configure GPIO as output. */
1098                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1099                 /* Reset ASPM state to default value. */
1100                 /*  With this reset: dd if=/dev/random of=/dev/mmcsd0 encounter a timeout. */
1101 //!!!           RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1102                 /* Force CLKREQ# PIN to drive 0 to request clock. */
1103                 RTSX_BITOP(sc, RTSX_PETXCFG, 0x08, 0x08);
1104                 /* Switch LDO3318 source from DV33 to card_3v3. */
1105                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1106                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1107                 /* Set default OLT blink period. */
1108                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1109                 /* Configure driving. */
1110                 RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3);
1111                 break;
1112         case RTSX_RTS522A:
1113                 /* Add specific init from RTS5227. */
1114                 /* Configure GPIO as output. */
1115                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1116                 /* Reset ASPM state to default value. */
1117                 RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1118                 /* Switch LDO3318 source from DV33 to 3V3. */
1119                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1120                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1121                 /* Set default OLT blink period. */
1122                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1123                 /* Configure LTR. */
1124                 cap = pci_read_config(sc->rtsx_dev, sc->rtsx_pcie_cap + PCIER_DEVICE_CTL2, 2);
1125                 if (cap & PCIEM_CTL2_LTR_ENABLE)
1126                         RTSX_WRITE(sc, RTSX_LTR_CTL, 0xa3);
1127                 /* Configure OBFF. */
1128                 RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_ENABLE);
1129                 /* Configure driving. */
1130                 if ((error = rtsx_rts5227_fill_driving(sc)))
1131                         return (error);
1132                 /* Configure force_clock_req. */
1133                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1134                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0xB8);
1135                 else
1136                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB8, 0x88);
1137                 RTSX_CLR(sc, RTSX_RTS522A_PM_CTRL3,  0x10);
1138
1139                 /* specific for RTS522A. */
1140                 RTSX_BITOP(sc, RTSX_FUNC_FORCE_CTL,
1141                            RTSX_FUNC_FORCE_UPME_XMT_DBG, RTSX_FUNC_FORCE_UPME_XMT_DBG);
1142                 RTSX_BITOP(sc, RTSX_PCLK_CTL, 0x04, 0x04);
1143                 RTSX_BITOP(sc, RTSX_PM_EVENT_DEBUG,
1144                            RTSX_PME_DEBUG_0, RTSX_PME_DEBUG_0);
1145                 RTSX_WRITE(sc, RTSX_PM_CLK_FORCE_CTL, 0x11);
1146                 break;
1147         case RTSX_RTS525A:
1148                 /* Add specific init from RTS5249. */
1149                 /* Rest L1SUB Config. */
1150                 RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xff);
1151                 /* Configure GPIO as output. */
1152                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1153                 /* Reset ASPM state to default value. */
1154                 RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1155                 /* Switch LDO3318 source from DV33 to 3V3. */
1156                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1157                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1158                 /* Set default OLT blink period. */
1159                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1160                 /* Configure driving. */
1161                 if ((error = rtsx_rts5249_fill_driving(sc)))
1162                         return (error);
1163                 /* Configure force_clock_req. */
1164                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1165                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0);
1166                 else
1167                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80);
1168
1169                 /* Specifc for RTS525A. */
1170                 RTSX_BITOP(sc, RTSX_PCLK_CTL, RTSX_PCLK_MODE_SEL, RTSX_PCLK_MODE_SEL);
1171                 if (sc->rtsx_flags & RTSX_F_VERSION_A) {
1172                         RTSX_WRITE(sc, RTSX_L1SUB_CONFIG2, RTSX_L1SUB_AUTO_CFG);
1173                         RTSX_BITOP(sc, RTSX_RREF_CFG,
1174                                    RTSX_RREF_VBGSEL_MASK, RTSX_RREF_VBGSEL_1V25);
1175                         RTSX_BITOP(sc, RTSX_LDO_VIO_CFG,
1176                                    RTSX_LDO_VIO_TUNE_MASK, RTSX_LDO_VIO_1V7);
1177                         RTSX_BITOP(sc, RTSX_LDO_DV12S_CFG,
1178                                    RTSX_LDO_D12_TUNE_MASK, RTSX_LDO_D12_TUNE_DF);
1179                         RTSX_BITOP(sc, RTSX_LDO_AV12S_CFG,
1180                                    RTSX_LDO_AV12S_TUNE_MASK, RTSX_LDO_AV12S_TUNE_DF);
1181                         RTSX_BITOP(sc, RTSX_LDO_VCC_CFG0,
1182                                    RTSX_LDO_VCC_LMTVTH_MASK, RTSX_LDO_VCC_LMTVTH_2A);
1183                         RTSX_BITOP(sc, RTSX_OOBS_CONFIG,
1184                                    RTSX_OOBS_AUTOK_DIS | RTSX_OOBS_VAL_MASK, 0x89);
1185                 }
1186                 break;
1187         case RTSX_RTS5249:
1188                 /* Rest L1SUB Config. */
1189                 RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xff);
1190                 /* Configure GPIO as output. */
1191                 RTSX_BITOP(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON, RTSX_GPIO_LED_ON);
1192                 /* Reset ASPM state to default value. */
1193                 RTSX_BITOP(sc, RTSX_ASPM_FORCE_CTL, RTSX_ASPM_FORCE_MASK, RTSX_FORCE_ASPM_NO_ASPM);
1194                 /* Switch LDO3318 source from DV33 to 3V3. */
1195                 RTSX_CLR(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33);
1196                 RTSX_BITOP(sc, RTSX_LDO_PWR_SEL, RTSX_LDO_PWR_SEL_DV33, RTSX_LDO_PWR_SEL_3V3);
1197                 /* Set default OLT blink period. */
1198                 RTSX_BITOP(sc, RTSX_OLT_LED_CTL, 0x0F, RTSX_OLT_LED_PERIOD);
1199                 /* Configure driving. */
1200                 if ((error = rtsx_rts5249_fill_driving(sc)))
1201                         return (error);
1202                 /* Configure force_clock_req. */
1203                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1204                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0);
1205                 else
1206                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80);
1207                 break;
1208         case RTSX_RTS5260:
1209                 /* Set mcu_cnt to 7 to ensure data can be sampled properly. */
1210                 RTSX_BITOP(sc, RTSX_CLK_DIV, 0x07, 0x07);
1211                 RTSX_WRITE(sc, RTSX_SSC_DIV_N_0, 0x5D);
1212                 /* force no MDIO*/
1213                 RTSX_WRITE(sc, RTSX_RTS5260_AUTOLOAD_CFG4, RTSX_RTS5260_MIMO_DISABLE);
1214                 /*Modify SDVCC Tune Default Parameters!*/
1215                 RTSX_BITOP(sc, RTSX_LDO_VCC_CFG0, RTSX_RTS5260_DVCC_TUNE_MASK, RTSX_RTS5260_DVCC_33);
1216                 RTSX_BITOP(sc, RTSX_PCLK_CTL, RTSX_PCLK_MODE_SEL, RTSX_PCLK_MODE_SEL);
1217                 RTSX_BITOP(sc, RTSX_L1SUB_CONFIG1, RTSX_AUX_CLK_ACTIVE_SEL_MASK, RTSX_MAC_CKSW_DONE);
1218                 /* Rest L1SUB Config */
1219                 RTSX_CLR(sc, RTSX_L1SUB_CONFIG3, 0xFF);
1220                 RTSX_BITOP(sc, RTSX_PM_CLK_FORCE_CTL, RTSX_CLK_PM_EN, RTSX_CLK_PM_EN);
1221                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_PWR_GATE_EN, RTSX_PWR_GATE_EN);
1222                 RTSX_BITOP(sc, RTSX_RB_FLUSH, RTSX_U_AUTO_DMA_EN_MASK, RTSX_U_AUTO_DMA_DISABLE);
1223                 if (sc->rtsx_flags & RTSX_F_REVERSE_SOCKET)
1224                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0xB0);
1225                 else
1226                         RTSX_BITOP(sc, RTSX_PETXCFG, 0xB0, 0x80);
1227                 RTSX_BITOP(sc, RTSX_OBFF_CFG, RTSX_OBFF_EN_MASK, RTSX_OBFF_DISABLE);
1228                 break;
1229         case RTSX_RTL8402:
1230         case RTSX_RTL8411:
1231                 RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3);
1232                 RTSX_BITOP(sc, RTSX_CARD_PAD_CTL, RTSX_CD_DISABLE_MASK | RTSX_CD_AUTO_DISABLE,
1233                            RTSX_CD_ENABLE);
1234                 break;
1235         case RTSX_RTL8411B:
1236                 if (sc->rtsx_flags & RTSX_F_8411B_QFN48)
1237                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf5);
1238                 RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, sc->rtsx_sd30_drive_sel_3v3);
1239                 /* Enable SD interrupt. */
1240                 RTSX_BITOP(sc, RTSX_CARD_PAD_CTL, RTSX_CD_DISABLE_MASK | RTSX_CD_AUTO_DISABLE,
1241                            RTSX_CD_ENABLE);
1242                 /* Clear hw_pfm_en to disable hardware PFM mode. */
1243                 RTSX_BITOP(sc, RTSX_FUNC_FORCE_CTL, 0x06, 0x00);
1244                 break;
1245         }
1246
1247         /*!!! Added for reboot after Windows. */
1248         rtsx_bus_power_off(sc);
1249         rtsx_set_sd_timing(sc, bus_timing_normal);
1250         rtsx_set_sd_clock(sc, 0);
1251         /*!!! Added for reboot after Windows. */
1252
1253         return (0);
1254 }
1255
1256 static int
1257 rtsx_map_sd_drive(int index)
1258 {
1259         uint8_t sd_drive[4] =
1260                 {
1261                  0x01,  /* Type D */
1262                  0x02,  /* Type C */
1263                  0x05,  /* Type A */
1264                  0x03   /* Type B */
1265                 };
1266         return (sd_drive[index]);
1267 }
1268
1269 /* For voltage 3v3. */
1270 static int
1271 rtsx_rts5227_fill_driving(struct rtsx_softc *sc)
1272 {
1273         u_char  driving_3v3[4][3] = {
1274                                      {0x13, 0x13, 0x13},
1275                                      {0x96, 0x96, 0x96},
1276                                      {0x7F, 0x7F, 0x7F},
1277                                      {0x96, 0x96, 0x96},
1278         };
1279         RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]);
1280         RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]);
1281         RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]);
1282
1283         return (0);
1284 }
1285
1286 /* For voltage 3v3. */
1287 static int
1288 rtsx_rts5249_fill_driving(struct rtsx_softc *sc)
1289 {
1290         u_char  driving_3v3[4][3] = {
1291                                      {0x11, 0x11, 0x18},
1292                                      {0x55, 0x55, 0x5C},
1293                                      {0xFF, 0xFF, 0xFF},
1294                                      {0x96, 0x96, 0x96},
1295         };
1296         RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]);
1297         RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]);
1298         RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]);
1299
1300         return (0);
1301 }
1302
1303 static int
1304 rtsx_rts5260_fill_driving(struct rtsx_softc *sc)
1305 {
1306         u_char  driving_3v3[4][3] = {
1307                                      {0x11, 0x11, 0x11},
1308                                      {0x22, 0x22, 0x22},
1309                                      {0x55, 0x55, 0x55},
1310                                      {0x33, 0x33, 0x33},
1311         };
1312         RTSX_WRITE(sc, RTSX_SD30_CLK_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][0]);
1313         RTSX_WRITE(sc, RTSX_SD30_CMD_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][1]);
1314         RTSX_WRITE(sc, RTSX_SD30_DAT_DRIVE_SEL, driving_3v3[sc->rtsx_sd30_drive_sel_3v3][2]);
1315
1316         return (0);
1317 }
1318
1319 static int
1320 rtsx_read(struct rtsx_softc *sc, uint16_t addr, uint8_t *val)
1321 {
1322         int      tries = 1024;
1323         uint32_t reg;
1324
1325         WRITE4(sc, RTSX_HAIMR, RTSX_HAIMR_BUSY |
1326             (uint32_t)((addr & 0x3FFF) << 16));
1327
1328         while (tries--) {
1329                 reg = READ4(sc, RTSX_HAIMR);
1330                 if (!(reg & RTSX_HAIMR_BUSY))
1331                         break;
1332         }
1333         *val = (reg & 0xff);
1334
1335         return ((tries == 0) ? ETIMEDOUT : 0);
1336 }
1337
1338 static int
1339 rtsx_read_cfg(struct rtsx_softc *sc, uint8_t func, uint16_t addr, uint32_t *val)
1340 {
1341         int     tries = 1024;
1342         uint8_t data0, data1, data2, data3, rwctl;
1343
1344         RTSX_WRITE(sc, RTSX_CFGADDR0, addr);
1345         RTSX_WRITE(sc, RTSX_CFGADDR1, addr >> 8);
1346         RTSX_WRITE(sc, RTSX_CFGRWCTL, RTSX_CFG_BUSY | (func & 0x03 << 4));
1347
1348         while (tries--) {
1349                 RTSX_READ(sc, RTSX_CFGRWCTL, &rwctl);
1350                 if (!(rwctl & RTSX_CFG_BUSY))
1351                         break;
1352         }
1353
1354         if (tries == 0)
1355                 return (ETIMEDOUT);
1356
1357         RTSX_READ(sc, RTSX_CFGDATA0, &data0);
1358         RTSX_READ(sc, RTSX_CFGDATA1, &data1);
1359         RTSX_READ(sc, RTSX_CFGDATA2, &data2);
1360         RTSX_READ(sc, RTSX_CFGDATA3, &data3);
1361
1362         *val = (data3 << 24) | (data2 << 16) | (data1 << 8) | data0;
1363
1364         return (0);
1365 }
1366
1367 static int
1368 rtsx_write(struct rtsx_softc *sc, uint16_t addr, uint8_t mask, uint8_t val)
1369 {
1370         int      tries = 1024;
1371         uint32_t reg;
1372
1373         WRITE4(sc, RTSX_HAIMR,
1374             RTSX_HAIMR_BUSY | RTSX_HAIMR_WRITE |
1375             (uint32_t)(((addr & 0x3FFF) << 16) |
1376             (mask << 8) | val));
1377
1378         while (tries--) {
1379                 reg = READ4(sc, RTSX_HAIMR);
1380                 if (!(reg & RTSX_HAIMR_BUSY)) {
1381                         if (val != (reg & 0xff))
1382                                 return (EIO);
1383                         return (0);
1384                 }
1385         }
1386
1387         return (ETIMEDOUT);
1388 }
1389
1390 static int
1391 rtsx_read_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t *val)
1392 {
1393         int     tries = 100000;
1394         uint8_t data0, data1, rwctl;
1395
1396         RTSX_WRITE(sc, RTSX_PHY_ADDR, addr);
1397         RTSX_WRITE(sc, RTSX_PHY_RWCTL, RTSX_PHY_BUSY | RTSX_PHY_READ);
1398
1399         while (tries--) {
1400                 RTSX_READ(sc, RTSX_PHY_RWCTL, &rwctl);
1401                 if (!(rwctl & RTSX_PHY_BUSY))
1402                         break;
1403         }
1404         if (tries == 0)
1405                 return (ETIMEDOUT);
1406
1407         RTSX_READ(sc, RTSX_PHY_DATA0, &data0);
1408         RTSX_READ(sc, RTSX_PHY_DATA1, &data1);
1409         *val = data1 << 8 | data0;
1410
1411         return (0);
1412 }
1413
1414 static int
1415 rtsx_write_phy(struct rtsx_softc *sc, uint8_t addr, uint16_t val)
1416 {
1417         int     tries = 100000;
1418         uint8_t rwctl;
1419
1420         RTSX_WRITE(sc, RTSX_PHY_DATA0, val);
1421         RTSX_WRITE(sc, RTSX_PHY_DATA1, val >> 8);
1422         RTSX_WRITE(sc, RTSX_PHY_ADDR, addr);
1423         RTSX_WRITE(sc, RTSX_PHY_RWCTL, RTSX_PHY_BUSY | RTSX_PHY_WRITE);
1424
1425         while (tries--) {
1426                 RTSX_READ(sc, RTSX_PHY_RWCTL, &rwctl);
1427                 if (!(rwctl & RTSX_PHY_BUSY))
1428                         break;
1429         }
1430
1431         return ((tries == 0) ? ETIMEDOUT : 0);
1432 }
1433
1434 /*
1435  * Notice that the meaning of RTSX_PWR_GATE_CTRL changes between RTS5209 and
1436  * RTS5229. In RTS5209 it is a mask of disabled power gates, while in RTS5229
1437  * it is a mask of *enabled* gates.
1438  */
1439 static int
1440 rtsx_bus_power_off(struct rtsx_softc *sc)
1441 {
1442         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
1443                 device_printf(sc->rtsx_dev, "rtsx_bus_power_off()\n");
1444
1445         /* Disable SD clock. */
1446         RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_SD_CLK_EN);
1447
1448         /* Disable SD output. */
1449         RTSX_CLR(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN);
1450
1451         /* Turn off power. */
1452         switch (sc->rtsx_device_id) {
1453         case RTSX_RTS5209:
1454                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK | RTSX_PMOS_STRG_MASK,
1455                            RTSX_SD_PWR_OFF | RTSX_PMOS_STRG_400mA);
1456                 RTSX_SET(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_OFF);
1457                 break;
1458         case RTSX_RTS5227:
1459         case RTSX_RTS5229:
1460         case RTSX_RTS522A:
1461                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK | RTSX_PMOS_STRG_MASK,
1462                            RTSX_SD_PWR_OFF | RTSX_PMOS_STRG_400mA);
1463                 RTSX_CLR(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK);
1464                 break;
1465         case RTSX_RTS5260:
1466                 RTSX_BITOP(sc, RTSX_LDO_VCC_CFG1, RTSX_LDO_POW_SDVDD1_MASK, RTSX_LDO_POW_SDVDD1_OFF);
1467                 RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_POWERON, RTSX_DV331812_POWEROFF);
1468                 break;
1469         case RTSX_RTL8402:
1470         case RTSX_RTL8411:
1471         case RTSX_RTL8411B:
1472                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1473                            RTSX_BPP_POWER_OFF);
1474                 RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB,
1475                            RTSX_BPP_LDO_SUSPEND);
1476                 break;
1477         default:
1478                 RTSX_CLR(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK);
1479                 RTSX_SET(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_OFF);
1480                 RTSX_CLR(sc, RTSX_CARD_PWR_CTL, RTSX_PMOS_STRG_800mA);
1481                 break;
1482         }
1483
1484         /* Disable pull control. */
1485         switch (sc->rtsx_device_id) {
1486         case RTSX_RTS5209:
1487                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, RTSX_PULL_CTL_DISABLE12);
1488                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12);
1489                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3);
1490                 break;
1491         case RTSX_RTS5227:
1492         case RTSX_RTS522A:
1493                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12);
1494                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3);
1495                 break;
1496         case RTSX_RTS5229:
1497                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12);
1498                 if (sc->rtsx_flags & RTSX_F_VERSION_C)
1499                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3_TYPE_C);
1500                 else
1501                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3);
1502                 break;
1503         case RTSX_RTS525A:
1504         case RTSX_RTS5249:
1505         case RTSX_RTS5260:
1506                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x66);
1507                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_DISABLE12);
1508                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_DISABLE3);
1509                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x55);
1510                 break;
1511         case RTSX_RTL8402:
1512         case RTSX_RTL8411:
1513                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x65);
1514                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55);
1515                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0x95);
1516                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x09);
1517                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x05);
1518                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x04);
1519                 break;
1520         case RTSX_RTL8411B:
1521                 if (sc->rtsx_flags & RTSX_F_8411B_QFN48) {
1522                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55);
1523                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf5);
1524                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15);
1525                 } else {
1526                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x65);
1527                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0x55);
1528                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xd5);
1529                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x59);
1530                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x55);
1531                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15);
1532                 }
1533                 break;
1534         }
1535
1536         return (0);
1537 }
1538
1539 static int
1540 rtsx_bus_power_on(struct rtsx_softc *sc)
1541 {
1542         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
1543                 device_printf(sc->rtsx_dev, "rtsx_bus_power_on()\n");
1544
1545         /* Select SD card. */
1546         RTSX_BITOP(sc, RTSX_CARD_SELECT, 0x07, RTSX_SD_MOD_SEL);
1547         RTSX_BITOP(sc, RTSX_CARD_SHARE_MODE, RTSX_CARD_SHARE_MASK, RTSX_CARD_SHARE_48_SD);
1548
1549         /* Enable SD clock. */
1550         RTSX_BITOP(sc, RTSX_CARD_CLK_EN, RTSX_SD_CLK_EN,  RTSX_SD_CLK_EN);
1551
1552         /* Enable pull control. */
1553         switch (sc->rtsx_device_id) {
1554         case RTSX_RTS5209:
1555                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, RTSX_PULL_CTL_ENABLE12);
1556                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12);
1557                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3);
1558                 break;
1559         case RTSX_RTS5227:
1560         case RTSX_RTS522A:
1561                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12);
1562                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3);
1563                 break;
1564         case RTSX_RTS5229:
1565                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12);
1566                 if (sc->rtsx_flags & RTSX_F_VERSION_C)
1567                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3_TYPE_C);
1568                 else
1569                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3);
1570                 break;
1571         case RTSX_RTS525A:
1572         case RTSX_RTS5249:
1573         case RTSX_RTS5260:
1574                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0x66);
1575                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, RTSX_PULL_CTL_ENABLE12);
1576                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, RTSX_PULL_CTL_ENABLE3);
1577                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0xaa);
1578                 break;
1579         case RTSX_RTL8402:
1580         case RTSX_RTL8411:
1581                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0xaa);
1582                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa);
1583                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xa9);
1584                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x09);
1585                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x09);
1586                 RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x04);
1587                 break;
1588         case RTSX_RTL8411B:
1589                 if (sc->rtsx_flags & RTSX_F_8411B_QFN48) {
1590                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa);
1591                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xf9);
1592                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x19);
1593                 } else {
1594                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL1, 0xaa);
1595                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL2, 0xaa);
1596                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL3, 0xd9);
1597                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL4, 0x59);
1598                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL5, 0x55);
1599                         RTSX_WRITE(sc, RTSX_CARD_PULL_CTL6, 0x15);
1600                 }
1601                 break;
1602         }
1603
1604         /*
1605          * To avoid a current peak, enable card power in two phases
1606          * with a delay in between.
1607          */
1608         switch (sc->rtsx_device_id) {
1609         case RTSX_RTS5209:
1610                 /* Partial power. */
1611                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON);
1612                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC2);
1613
1614                 DELAY(200);
1615
1616                 /* Full power. */
1617                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON);
1618                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_ON);
1619                 break;
1620         case RTSX_RTS5227:
1621         case RTSX_RTS522A:
1622                 /* Partial power. */
1623                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON);
1624                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1);
1625
1626                 DELAY(20000);
1627
1628                 /* Full power. */
1629                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON);
1630                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK,
1631                            RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2);
1632                 RTSX_BITOP(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN, RTSX_SD_OUTPUT_EN);
1633                 RTSX_BITOP(sc, RTSX_CARD_OE, RTSX_MS_OUTPUT_EN, RTSX_MS_OUTPUT_EN);
1634                 break;
1635         case RTSX_RTS5229:
1636                 /* Partial power. */
1637                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON);
1638                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1);
1639
1640                 DELAY(200);
1641
1642                 /* Full power. */
1643                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON);
1644                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK,
1645                            RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2);
1646                 break;
1647         case RTSX_RTS525A:
1648                 RTSX_BITOP(sc, RTSX_LDO_VCC_CFG1, RTSX_LDO_VCC_TUNE_MASK, RTSX_LDO_VCC_3V3);
1649         case RTSX_RTS5249:
1650                 /* Partial power. */
1651                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PARTIAL_PWR_ON);
1652                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK, RTSX_LDO3318_VCC1);
1653
1654                 DELAY(5000);
1655
1656                 /* Full power. */
1657                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_SD_PWR_MASK, RTSX_SD_PWR_ON);
1658                 RTSX_BITOP(sc, RTSX_PWR_GATE_CTRL, RTSX_LDO3318_PWR_MASK,
1659                            RTSX_LDO3318_VCC1 | RTSX_LDO3318_VCC2);
1660                 break;
1661         case RTSX_RTS5260:
1662                 RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_VDD1, RTSX_DV331812_VDD1);
1663                 RTSX_BITOP(sc, RTSX_LDO_VCC_CFG0, RTSX_RTS5260_DVCC_TUNE_MASK, RTSX_RTS5260_DVCC_33);
1664                 RTSX_BITOP(sc, RTSX_LDO_VCC_CFG1, RTSX_LDO_POW_SDVDD1_MASK, RTSX_LDO_POW_SDVDD1_ON);
1665                 RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_POWERON, RTSX_DV331812_POWERON);
1666
1667                 DELAY(20000);
1668
1669                 /* Initialize SD_CFG1 register */
1670                 RTSX_WRITE(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_128 | RTSX_SD20_MODE);
1671                 RTSX_WRITE(sc, RTSX_SD_SAMPLE_POINT_CTL, RTSX_SD20_RX_POS_EDGE);
1672                 RTSX_CLR(sc, RTSX_SD_PUSH_POINT_CTL, 0xff);
1673                 RTSX_BITOP(sc, RTSX_CARD_STOP, RTSX_MS_STOP | RTSX_SD_CLR_ERR,
1674                            RTSX_MS_STOP | RTSX_SD_CLR_ERR);
1675                 /* Reset SD_CFG3 register */
1676                 RTSX_CLR(sc, RTSX_SD_CFG3, RTSX_SD30_CLK_END_EN);
1677                 RTSX_CLR(sc, RTSX_REG_SD_STOP_SDCLK_CFG,
1678                          RTSX_SD30_CLK_STOP_CFG_EN | RTSX_SD30_CLK_STOP_CFG0 | RTSX_SD30_CLK_STOP_CFG1);
1679                 RTSX_CLR(sc, RTSX_REG_PRE_RW_MODE, RTSX_EN_INFINITE_MODE);
1680                 break;
1681         case RTSX_RTL8402:
1682         case RTSX_RTL8411:
1683         case RTSX_RTL8411B:
1684                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1685                            RTSX_BPP_POWER_5_PERCENT_ON);
1686                 RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB,
1687                            RTSX_BPP_LDO_SUSPEND);
1688                 DELAY(150);
1689                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1690                            RTSX_BPP_POWER_10_PERCENT_ON);
1691                 DELAY(150);
1692                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1693                            RTSX_BPP_POWER_15_PERCENT_ON);
1694                 DELAY(150);
1695                 RTSX_BITOP(sc, RTSX_CARD_PWR_CTL, RTSX_BPP_POWER_MASK,
1696                            RTSX_BPP_POWER_ON);
1697                 RTSX_BITOP(sc, RTSX_LDO_CTL, RTSX_BPP_LDO_POWB,
1698                            RTSX_BPP_LDO_ON);
1699                 break;
1700         }
1701
1702         /* Enable SD card output. */
1703         RTSX_WRITE(sc, RTSX_CARD_OE, RTSX_SD_OUTPUT_EN);
1704
1705         DELAY(200);
1706
1707         return (0);
1708 }
1709
1710 /*
1711  * Set but width.
1712  */
1713 static int
1714 rtsx_set_bus_width(struct rtsx_softc *sc, enum mmc_bus_width width)
1715 {
1716         uint32_t bus_width;
1717
1718         switch (width) {
1719         case bus_width_1:
1720                 bus_width = RTSX_BUS_WIDTH_1;
1721                 break;
1722         case bus_width_4:
1723                 bus_width = RTSX_BUS_WIDTH_4;
1724                 break;
1725         case bus_width_8:
1726                 bus_width = RTSX_BUS_WIDTH_8;
1727                 break;
1728         default:
1729                 return (MMC_ERR_INVALID);
1730         }
1731         RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_BUS_WIDTH_MASK, bus_width);
1732
1733         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) {
1734                 char *busw[] = {
1735                                 "1 bit",
1736                                 "4 bits",
1737                                 "8 bits"
1738                 };
1739                 device_printf(sc->rtsx_dev, "Setting bus width to %s\n", busw[bus_width]);
1740         }
1741         return (0);
1742 }
1743
1744 static int
1745 rtsx_set_sd_timing(struct rtsx_softc *sc, enum mmc_bus_timing timing)
1746 {
1747         if (timing == bus_timing_hs && sc->rtsx_force_timing) {
1748                 timing = bus_timing_uhs_sdr50;
1749                 sc->rtsx_ios_timing = timing;
1750         }
1751
1752         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
1753                 device_printf(sc->rtsx_dev, "rtsx_set_sd_timing(%u)\n", timing);
1754
1755         switch (timing) {
1756         case bus_timing_uhs_sdr50:
1757         case bus_timing_uhs_sdr104:
1758                 sc->rtsx_double_clk = false;
1759                 sc->rtsx_vpclk = true;
1760                 RTSX_BITOP(sc, RTSX_SD_CFG1, 0x0c | RTSX_SD_ASYNC_FIFO_NOT_RST,
1761                            RTSX_SD30_MODE | RTSX_SD_ASYNC_FIFO_NOT_RST);
1762                 RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ);
1763                 RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE,
1764                            RTSX_CRC_VAR_CLK0 | RTSX_SD30_FIX_CLK | RTSX_SAMPLE_VAR_CLK1);
1765                 RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ);
1766                 break;
1767         case bus_timing_hs:
1768                 RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK, RTSX_SD20_MODE);
1769                 RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ);
1770                 RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE,
1771                            RTSX_CRC_FIX_CLK | RTSX_SD30_VAR_CLK0 | RTSX_SAMPLE_VAR_CLK1);
1772                 RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ);
1773
1774                 RTSX_BITOP(sc, RTSX_SD_PUSH_POINT_CTL,
1775                            RTSX_SD20_TX_SEL_MASK, RTSX_SD20_TX_14_AHEAD);
1776                 RTSX_BITOP(sc, RTSX_SD_SAMPLE_POINT_CTL,
1777                            RTSX_SD20_RX_SEL_MASK, RTSX_SD20_RX_14_DELAY);
1778                 break;
1779         default:
1780                 RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_SD_MODE_MASK, RTSX_SD20_MODE);
1781                 RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ);
1782                 RTSX_WRITE(sc, RTSX_CARD_CLK_SOURCE,
1783                            RTSX_CRC_FIX_CLK | RTSX_SD30_VAR_CLK0 | RTSX_SAMPLE_VAR_CLK1);
1784                 RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ);
1785
1786                 RTSX_WRITE(sc, RTSX_SD_PUSH_POINT_CTL, RTSX_SD20_TX_NEG_EDGE);
1787                 RTSX_BITOP(sc, RTSX_SD_SAMPLE_POINT_CTL,
1788                            RTSX_SD20_RX_SEL_MASK, RTSX_SD20_RX_POS_EDGE);
1789                 break;
1790         }
1791
1792         return (0);
1793 }
1794
1795 /*
1796  * Set or change SDCLK frequency or disable the SD clock.
1797  * Return zero on success.
1798  */
1799 static int
1800 rtsx_set_sd_clock(struct rtsx_softc *sc, uint32_t freq)
1801 {
1802         uint8_t clk;
1803         uint8_t clk_divider, n, div, mcu;
1804         int     error = 0;
1805
1806         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
1807                 device_printf(sc->rtsx_dev, "rtsx_set_sd_clock(%u)\n", freq);
1808
1809         if (freq == RTSX_SDCLK_OFF) {
1810                 error = rtsx_stop_sd_clock(sc);
1811                 return error;
1812         }
1813
1814         sc->rtsx_ssc_depth = RTSX_SSC_DEPTH_500K;
1815         sc->rtsx_discovery_mode = (freq <= 1000000) ? true : false;
1816
1817         if (sc->rtsx_discovery_mode) {
1818                 /* We use 250k(around) here, in discovery stage. */
1819                 clk_divider = RTSX_CLK_DIVIDE_128;
1820                 freq = 30000000;
1821         } else {
1822                 clk_divider = RTSX_CLK_DIVIDE_0;
1823         }
1824         RTSX_BITOP(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, clk_divider);
1825
1826         freq /= 1000000;
1827         if (sc->rtsx_discovery_mode || !sc->rtsx_double_clk)
1828                 clk = freq;
1829         else
1830                 clk = freq * 2;
1831
1832         switch (sc->rtsx_device_id) {
1833         case RTSX_RTL8402:
1834         case RTSX_RTL8411:
1835         case RTSX_RTL8411B:
1836                 n = clk * 4 / 5 - 2;
1837                 break;
1838         default:
1839                 n = clk - 2;
1840                 break;
1841         }
1842         if ((clk <= 2) || (n > RTSX_MAX_DIV_N))
1843                 return (MMC_ERR_INVALID);
1844
1845         mcu = 125 / clk + 3;
1846         if (mcu > 15)
1847                 mcu = 15;
1848
1849         /* Make sure that the SSC clock div_n is not less than RTSX_MIN_DIV_N. */
1850         div = RTSX_CLK_DIV_1;
1851         while ((n < RTSX_MIN_DIV_N) && (div < RTSX_CLK_DIV_8)) {
1852                 switch (sc->rtsx_device_id) {
1853                 case RTSX_RTL8402:
1854                 case RTSX_RTL8411:
1855                 case RTSX_RTL8411B:
1856                         n = (((n + 2) * 5 / 4) * 2) * 4 / 5 - 2;
1857                         break;
1858                 default:
1859                         n = (n + 2) * 2 - 2;
1860                         break;
1861                 }
1862                 div++;
1863         }
1864
1865         if (sc->rtsx_double_clk && sc->rtsx_ssc_depth > 1)
1866                 sc->rtsx_ssc_depth -= 1;
1867
1868         if (div > RTSX_CLK_DIV_1) {
1869                 if (sc->rtsx_ssc_depth > (div - 1))
1870                         sc->rtsx_ssc_depth -= (div - 1);
1871                 else
1872                         sc->rtsx_ssc_depth = RTSX_SSC_DEPTH_4M;
1873         }
1874
1875         /* Enable SD clock. */
1876         error = rtsx_switch_sd_clock(sc, clk, n, div, mcu);
1877
1878         return (error);
1879 }
1880
1881 static int
1882 rtsx_stop_sd_clock(struct rtsx_softc *sc)
1883 {
1884         RTSX_CLR(sc, RTSX_CARD_CLK_EN, RTSX_CARD_CLK_EN_ALL);
1885         RTSX_SET(sc, RTSX_SD_BUS_STAT, RTSX_SD_CLK_FORCE_STOP);
1886
1887         return (0);
1888 }
1889
1890 static int
1891 rtsx_switch_sd_clock(struct rtsx_softc *sc, uint8_t clk, uint8_t n, uint8_t div, uint8_t mcu)
1892 {
1893         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC) {
1894                 device_printf(sc->rtsx_dev, "rtsx_switch_sd_clock() - discovery-mode is %s, ssc_depth: %d\n",
1895                               (sc->rtsx_discovery_mode) ? "true" : "false", sc->rtsx_ssc_depth);
1896                 device_printf(sc->rtsx_dev, "rtsx_switch_sd_clock() - clk: %d, n: %d, div: %d, mcu: %d\n",
1897                               clk, n, div, mcu);
1898         }
1899
1900         RTSX_BITOP(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ, RTSX_CLK_LOW_FREQ);
1901         RTSX_WRITE(sc, RTSX_CLK_DIV, (div << 4) | mcu);
1902         RTSX_CLR(sc, RTSX_SSC_CTL1, RTSX_RSTB);
1903         RTSX_BITOP(sc, RTSX_SSC_CTL2, RTSX_SSC_DEPTH_MASK, sc->rtsx_ssc_depth);
1904         RTSX_WRITE(sc, RTSX_SSC_DIV_N_0, n);
1905         RTSX_BITOP(sc, RTSX_SSC_CTL1, RTSX_RSTB, RTSX_RSTB);
1906         if (sc->rtsx_vpclk) {
1907                 RTSX_CLR(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET);
1908                 RTSX_BITOP(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET);
1909         }
1910
1911         /* Wait SSC clock stable. */
1912         DELAY(200);
1913
1914         RTSX_CLR(sc, RTSX_CLK_CTL, RTSX_CLK_LOW_FREQ);
1915
1916         return (0);
1917 }
1918
1919 #ifndef MMCCAM
1920 static void
1921 rtsx_sd_change_tx_phase(struct rtsx_softc *sc, uint8_t sample_point)
1922 {
1923         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
1924                 device_printf(sc->rtsx_dev, "rtsx_sd_change_tx_phase() - sample_point: %d\n", sample_point);
1925
1926         rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CHANGE_CLK);
1927         rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_SELECT_MASK, sample_point);
1928         rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, 0);
1929         rtsx_write(sc, RTSX_SD_VPCLK0_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET);
1930         rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, 0);
1931         rtsx_write(sc, RTSX_SD_CFG1, RTSX_SD_ASYNC_FIFO_NOT_RST, 0);
1932 }
1933
1934 static void
1935 rtsx_sd_change_rx_phase(struct rtsx_softc *sc, uint8_t sample_point)
1936 {
1937         if (sc->rtsx_debug_mask & RTSX_DEBUG_TUNING)
1938                 device_printf(sc->rtsx_dev, "rtsx_sd_change_rx_phase() - sample_point: %d\n", sample_point);
1939
1940         rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, RTSX_CHANGE_CLK);
1941         rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_SELECT_MASK, sample_point);
1942         rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_NOT_RESET, 0);
1943         rtsx_write(sc, RTSX_SD_VPCLK1_CTL, RTSX_PHASE_NOT_RESET, RTSX_PHASE_NOT_RESET);
1944         rtsx_write(sc, RTSX_CLK_CTL, RTSX_CHANGE_CLK, 0);
1945         rtsx_write(sc, RTSX_SD_CFG1, RTSX_SD_ASYNC_FIFO_NOT_RST, 0);
1946 }
1947
1948 static void
1949 rtsx_sd_tuning_rx_phase(struct rtsx_softc *sc, uint32_t *phase_map)
1950 {
1951         uint32_t raw_phase_map = 0;
1952         int      i;
1953         int      error;
1954
1955         for (i = 0; i < RTSX_RX_PHASE_MAX; i++) {
1956                 error = rtsx_sd_tuning_rx_cmd(sc, (uint8_t)i);
1957                 if (error == 0)
1958                         raw_phase_map |= 1 << i;
1959         }
1960         if (phase_map != NULL)
1961                 *phase_map = raw_phase_map;
1962 }
1963
1964 static int
1965 rtsx_sd_tuning_rx_cmd(struct rtsx_softc *sc, uint8_t sample_point)
1966 {
1967         struct mmc_request req = {};
1968         struct mmc_command cmd = {};
1969         int     error = 0;
1970
1971         cmd.opcode = MMC_SEND_TUNING_BLOCK;
1972         cmd.arg = 0;
1973         req.cmd = &cmd;
1974
1975         RTSX_LOCK(sc);
1976
1977         sc->rtsx_req = &req;
1978
1979         rtsx_sd_change_rx_phase(sc, sample_point);
1980
1981         rtsx_write(sc, RTSX_SD_CFG3, RTSX_SD_RSP_80CLK_TIMEOUT_EN,
1982                    RTSX_SD_RSP_80CLK_TIMEOUT_EN);
1983
1984         rtsx_init_cmd(sc, &cmd);
1985         rtsx_set_cmd_data_len(sc, 1, 0x40);
1986         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff,
1987                       RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 |
1988                       RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_6);
1989         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER,
1990                       0xff, RTSX_TM_AUTO_TUNING | RTSX_SD_TRANSFER_START);
1991         rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
1992                       RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END);
1993
1994         /* Set interrupt post processing */
1995         sc->rtsx_intr_trans_ok = rtsx_sd_tuning_rx_cmd_wakeup;
1996         sc->rtsx_intr_trans_ko = rtsx_sd_tuning_rx_cmd_wakeup;
1997
1998         /* Run the command queue. */
1999         rtsx_send_cmd(sc);
2000
2001         error = rtsx_sd_tuning_rx_cmd_wait(sc, &cmd);
2002
2003         if (error) {
2004                 if (sc->rtsx_debug_mask & RTSX_DEBUG_TUNING)
2005                         device_printf(sc->rtsx_dev, "rtsx_sd_tuning_rx_cmd() - error: %d\n", error);
2006                 rtsx_sd_wait_data_idle(sc);
2007                 rtsx_clear_error(sc);
2008         }
2009         rtsx_write(sc, RTSX_SD_CFG3, RTSX_SD_RSP_80CLK_TIMEOUT_EN, 0);
2010
2011         sc->rtsx_req = NULL;
2012
2013         RTSX_UNLOCK(sc);
2014
2015         return (error);
2016 }
2017
2018 static int
2019 rtsx_sd_tuning_rx_cmd_wait(struct rtsx_softc *sc, struct mmc_command *cmd)
2020 {
2021         int     status;
2022         int     mask = RTSX_TRANS_OK_INT | RTSX_TRANS_FAIL_INT;
2023
2024         status = sc->rtsx_intr_status & mask;
2025         while (status == 0) {
2026                 if (msleep(&sc->rtsx_intr_status, &sc->rtsx_mtx, 0, "rtsxintr", sc->rtsx_timeout_cmd) == EWOULDBLOCK) {
2027                         cmd->error = MMC_ERR_TIMEOUT;
2028                         return (MMC_ERR_TIMEOUT);
2029                 }
2030                 status = sc->rtsx_intr_status & mask;
2031         }
2032         return (cmd->error);
2033 }
2034
2035 static void
2036 rtsx_sd_tuning_rx_cmd_wakeup(struct rtsx_softc *sc)
2037 {
2038         wakeup(&sc->rtsx_intr_status);
2039 }
2040
2041 static void
2042 rtsx_sd_wait_data_idle(struct rtsx_softc *sc)
2043 {
2044         int     i;
2045         uint8_t val;
2046
2047         for (i = 0; i < 100; i++) {
2048                 rtsx_read(sc, RTSX_SD_DATA_STATE, &val);
2049                 if (val & RTSX_SD_DATA_IDLE)
2050                         return;
2051                 DELAY(100);
2052         }
2053 }
2054
2055 static uint8_t
2056 rtsx_sd_search_final_rx_phase(struct rtsx_softc *sc, uint32_t phase_map)
2057 {
2058         int     start = 0, len = 0;
2059         int     start_final = 0, len_final = 0;
2060         uint8_t final_phase = 0xff;
2061
2062         while (start < RTSX_RX_PHASE_MAX) {
2063                 len = rtsx_sd_get_rx_phase_len(phase_map, start);
2064                 if (len_final < len) {
2065                         start_final = start;
2066                         len_final = len;
2067                 }
2068                 start += len ? len : 1;
2069         }
2070
2071         final_phase = (start_final + len_final / 2) % RTSX_RX_PHASE_MAX;
2072
2073         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2074                 device_printf(sc->rtsx_dev,
2075                               "rtsx_sd_search_final_rx_phase() - phase_map: %x, start_final: %d, len_final: %d, final_phase: %d\n",
2076                               phase_map, start_final, len_final, final_phase);
2077
2078         return final_phase;
2079 }
2080
2081 static int
2082 rtsx_sd_get_rx_phase_len(uint32_t phase_map, int start_bit)
2083 {
2084         int     i;
2085
2086         for (i = 0; i < RTSX_RX_PHASE_MAX; i++) {
2087                 if ((phase_map & (1 << (start_bit + i) % RTSX_RX_PHASE_MAX)) == 0)
2088                         return i;
2089         }
2090         return RTSX_RX_PHASE_MAX;
2091 }
2092 #endif /* !MMCCAM */
2093
2094 #if 0   /* For led */
2095 static int
2096 rtsx_led_enable(struct rtsx_softc *sc)
2097 {
2098         switch (sc->rtsx_device_id) {
2099         case RTSX_RTS5209:
2100                 RTSX_CLR(sc, RTSX_CARD_GPIO, RTSX_CARD_GPIO_LED_OFF);
2101                 RTSX_WRITE(sc, RTSX_CARD_AUTO_BLINK,
2102                            RTSX_LED_BLINK_EN | RTSX_LED_BLINK_SPEED);
2103                 break;
2104         case RTSX_RTL8411B:
2105                 RTSX_CLR(sc, RTSX_GPIO_CTL, 0x01);
2106                 RTSX_WRITE(sc, RTSX_CARD_AUTO_BLINK,
2107                            RTSX_LED_BLINK_EN | RTSX_LED_BLINK_SPEED);
2108                 break;
2109         default:
2110                 RTSX_SET(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON);
2111                 RTSX_SET(sc, RTSX_OLT_LED_CTL, RTSX_OLT_LED_AUTOBLINK);
2112                 break;
2113         }
2114
2115         return (0);
2116 }
2117
2118 static int
2119 rtsx_led_disable(struct rtsx_softc *sc)
2120 {
2121         switch (sc->rtsx_device_id) {
2122         case RTSX_RTS5209:
2123                 RTSX_CLR(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN);
2124                 RTSX_WRITE(sc, RTSX_CARD_GPIO, RTSX_CARD_GPIO_LED_OFF);
2125                 break;
2126         case RTSX_RTL8411B:
2127                 RTSX_CLR(sc, RTSX_CARD_AUTO_BLINK, RTSX_LED_BLINK_EN);
2128                 RTSX_SET(sc, RTSX_GPIO_CTL, 0x01);
2129                 break;
2130         default:
2131                 RTSX_CLR(sc, RTSX_OLT_LED_CTL, RTSX_OLT_LED_AUTOBLINK);
2132                 RTSX_CLR(sc, RTSX_GPIO_CTL, RTSX_GPIO_LED_ON);
2133                 break;
2134         }
2135
2136         return (0);
2137 }
2138 #endif  /* For led */
2139
2140 static uint8_t
2141 rtsx_response_type(uint16_t mmc_rsp)
2142 {
2143         int     i;
2144         struct rsp_type {
2145                 uint16_t mmc_rsp;
2146                 uint8_t  rtsx_rsp;
2147         } rsp_types[] = {
2148                 { MMC_RSP_NONE, RTSX_SD_RSP_TYPE_R0 },
2149                 { MMC_RSP_R1,   RTSX_SD_RSP_TYPE_R1 },
2150                 { MMC_RSP_R1B,  RTSX_SD_RSP_TYPE_R1B },
2151                 { MMC_RSP_R2,   RTSX_SD_RSP_TYPE_R2 },
2152                 { MMC_RSP_R3,   RTSX_SD_RSP_TYPE_R3 },
2153                 { MMC_RSP_R4,   RTSX_SD_RSP_TYPE_R4 },
2154                 { MMC_RSP_R5,   RTSX_SD_RSP_TYPE_R5 },
2155                 { MMC_RSP_R6,   RTSX_SD_RSP_TYPE_R6 },
2156                 { MMC_RSP_R7,   RTSX_SD_RSP_TYPE_R7 }
2157         };
2158
2159         for (i = 0; i < nitems(rsp_types); i++) {
2160                 if (mmc_rsp == rsp_types[i].mmc_rsp)
2161                         return (rsp_types[i].rtsx_rsp);
2162         }
2163
2164         return (0);
2165 }
2166
2167 /*
2168  * Init command buffer with SD command index and argument.
2169  */
2170 static void
2171 rtsx_init_cmd(struct rtsx_softc *sc, struct mmc_command *cmd)
2172 {
2173         sc->rtsx_cmd_index = 0;
2174         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD0,
2175                       0xff, RTSX_SD_CMD_START  | cmd->opcode);
2176         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD1,
2177                      0xff, cmd->arg >> 24);
2178         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD2,
2179                       0xff, cmd->arg >> 16);
2180         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD3,
2181                      0xff, cmd->arg >> 8);
2182         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CMD4,
2183                      0xff, cmd->arg);
2184 }
2185
2186 /*
2187  * Append a properly encoded host command to the host command buffer.
2188  */
2189 static void
2190 rtsx_push_cmd(struct rtsx_softc *sc, uint8_t cmd, uint16_t reg,
2191               uint8_t mask, uint8_t data)
2192 {
2193         KASSERT(sc->rtsx_cmd_index < RTSX_HOSTCMD_MAX,
2194                 ("rtsx: Too many host commands (%d)\n", sc->rtsx_cmd_index));
2195
2196         uint32_t *cmd_buffer = (uint32_t *)(sc->rtsx_cmd_dmamem);
2197         cmd_buffer[sc->rtsx_cmd_index++] =
2198                 htole32((uint32_t)(cmd & 0x3) << 30) |
2199                 ((uint32_t)(reg & 0x3fff) << 16) |
2200                 ((uint32_t)(mask) << 8) |
2201                 ((uint32_t)data);
2202 }
2203
2204 /*
2205  * Queue commands to configure data transfer size.
2206  */
2207 static void
2208 rtsx_set_cmd_data_len(struct rtsx_softc *sc, uint16_t block_cnt, uint16_t byte_cnt)
2209 {
2210         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BLOCK_CNT_L,
2211                       0xff, block_cnt & 0xff);
2212         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BLOCK_CNT_H,
2213                       0xff, block_cnt >> 8);
2214         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BYTE_CNT_L,
2215                       0xff, byte_cnt & 0xff);
2216         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_BYTE_CNT_H,
2217                       0xff, byte_cnt >> 8);
2218 }
2219
2220 /*
2221  * Run the command queue.
2222  */
2223 static void
2224 rtsx_send_cmd(struct rtsx_softc *sc)
2225 {
2226         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
2227                 device_printf(sc->rtsx_dev, "rtsx_send_cmd()\n");
2228
2229         sc->rtsx_intr_status = 0;
2230
2231         /* Sync command DMA buffer. */
2232         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_PREREAD);
2233         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_PREWRITE);
2234
2235         /* Tell the chip where the command buffer is and run the commands. */
2236         WRITE4(sc, RTSX_HCBAR, (uint32_t)sc->rtsx_cmd_buffer);
2237         WRITE4(sc, RTSX_HCBCTLR,
2238                ((sc->rtsx_cmd_index * 4) & 0x00ffffff) | RTSX_START_CMD | RTSX_HW_AUTO_RSP);
2239 }
2240
2241 /*
2242  * Stop previous command.
2243  */
2244 static void
2245 rtsx_stop_cmd(struct rtsx_softc *sc)
2246 {
2247         /* Stop command transfer. */
2248         WRITE4(sc, RTSX_HCBCTLR, RTSX_STOP_CMD);
2249
2250         /* Stop DMA transfer. */
2251         WRITE4(sc, RTSX_HDBCTLR, RTSX_STOP_DMA);
2252
2253         switch (sc->rtsx_device_id) {
2254         case RTSX_RTS5260:
2255                 rtsx_write(sc, RTSX_RTS5260_DMA_RST_CTL_0,
2256                            RTSX_RTS5260_DMA_RST | RTSX_RTS5260_ADMA3_RST,
2257                            RTSX_RTS5260_DMA_RST | RTSX_RTS5260_ADMA3_RST);
2258                 rtsx_write(sc, RTSX_RBCTL, RTSX_RB_FLUSH, RTSX_RB_FLUSH);
2259                 break;
2260         }
2261
2262         rtsx_write(sc, RTSX_DMACTL, RTSX_DMA_RST, RTSX_DMA_RST);
2263
2264         rtsx_write(sc, RTSX_RBCTL, RTSX_RB_FLUSH, RTSX_RB_FLUSH);
2265 }
2266
2267 /*
2268  * Clear error.
2269  */
2270 static void
2271 rtsx_clear_error(struct rtsx_softc *sc)
2272 {
2273         /* Clear error. */
2274         rtsx_write(sc, RTSX_CARD_STOP, RTSX_SD_STOP | RTSX_SD_CLR_ERR,
2275                    RTSX_SD_STOP | RTSX_SD_CLR_ERR);
2276 }
2277
2278 /*
2279  * Signal end of request to mmc/mmcsd.
2280  */
2281 static void
2282 rtsx_req_done(struct rtsx_softc *sc)
2283 {
2284 #ifdef MMCCAM
2285         union ccb *ccb;
2286 #endif /* MMCCAM */
2287         struct mmc_request *req;
2288
2289         req = sc->rtsx_req;
2290         if (req->cmd->error == MMC_ERR_NONE) {
2291                 if (req->cmd->opcode == MMC_READ_SINGLE_BLOCK ||
2292                     req->cmd->opcode == MMC_READ_MULTIPLE_BLOCK)
2293                         sc->rtsx_read_count++;
2294                 else if (req->cmd->opcode == MMC_WRITE_BLOCK ||
2295                          req->cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK)
2296                         sc->rtsx_write_count++;
2297         } else {
2298                 rtsx_clear_error(sc);
2299         }
2300         callout_stop(&sc->rtsx_timeout_callout);
2301         sc->rtsx_req = NULL;
2302 #ifdef MMCCAM
2303         ccb = sc->rtsx_ccb;
2304         sc->rtsx_ccb = NULL;
2305         ccb->ccb_h.status = (req->cmd->error == 0 ? CAM_REQ_CMP : CAM_REQ_CMP_ERR);
2306         xpt_done(ccb);
2307 #else  /* !MMCCAM */
2308         req->done(req);
2309 #endif /* MMCCAM */
2310 }
2311
2312 /*
2313  * Send request.
2314  */
2315 static int
2316 rtsx_send_req(struct rtsx_softc *sc, struct mmc_command *cmd)
2317 {
2318         uint8_t  rsp_type;
2319         uint16_t reg;
2320
2321         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
2322                 device_printf(sc->rtsx_dev, "rtsx_send_req() - CMD%d\n", cmd->opcode);
2323
2324         /* Convert response type. */
2325         rsp_type = rtsx_response_type(cmd->flags & MMC_RSP_MASK);
2326         if (rsp_type == 0) {
2327                 device_printf(sc->rtsx_dev, "Unknown rsp_type: 0x%lx\n", (cmd->flags & MMC_RSP_MASK));
2328                 cmd->error = MMC_ERR_INVALID;
2329                 return (MMC_ERR_INVALID);
2330         }
2331
2332         rtsx_init_cmd(sc, cmd);
2333
2334         /* Queue command to set response type. */
2335         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, rsp_type);
2336
2337         /* Use the ping-pong buffer (cmd buffer) for commands which do not transfer data. */
2338         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE,
2339                       0x01, RTSX_PINGPONG_BUFFER);
2340
2341         /* Queue commands to perform SD transfer. */
2342         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER,
2343                       0xff, RTSX_TM_CMD_RSP | RTSX_SD_TRANSFER_START);
2344         rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
2345                       RTSX_SD_TRANSFER_END|RTSX_SD_STAT_IDLE,
2346                       RTSX_SD_TRANSFER_END|RTSX_SD_STAT_IDLE);
2347
2348         /* If needed queue commands to read back card status response. */
2349         if (rsp_type == RTSX_SD_RSP_TYPE_R2) {
2350                 /* Read data from ping-pong buffer. */
2351                 for (reg = RTSX_PPBUF_BASE2; reg < RTSX_PPBUF_BASE2 + 16; reg++)
2352                         rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg, 0, 0);
2353         } else if (rsp_type != RTSX_SD_RSP_TYPE_R0) {
2354                 /* Read data from SD_CMDx registers. */
2355                 for (reg = RTSX_SD_CMD0; reg <= RTSX_SD_CMD4; reg++)
2356                         rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg, 0, 0);
2357         }
2358         rtsx_push_cmd(sc, RTSX_READ_REG_CMD, RTSX_SD_STAT1, 0, 0);
2359
2360         /* Set transfer OK function. */
2361         if (sc->rtsx_intr_trans_ok == NULL)
2362                 sc->rtsx_intr_trans_ok = rtsx_ret_resp;
2363
2364         /* Run the command queue. */
2365         rtsx_send_cmd(sc);
2366
2367         return (0);
2368 }
2369
2370 /*
2371  * Return response of previous command (case cmd->data == NULL) and complete resquest.
2372  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2373  */
2374 static void
2375 rtsx_ret_resp(struct rtsx_softc *sc)
2376 {
2377         struct mmc_command *cmd;
2378
2379         cmd = sc->rtsx_req->cmd;
2380         rtsx_set_resp(sc, cmd);
2381         rtsx_req_done(sc);
2382 }
2383
2384 /*
2385  * Set response of previous command.
2386  */
2387 static void
2388 rtsx_set_resp(struct rtsx_softc *sc, struct mmc_command *cmd)
2389 {
2390         uint8_t  rsp_type;
2391
2392         rsp_type = rtsx_response_type(cmd->flags & MMC_RSP_MASK);
2393
2394         /* Sync command DMA buffer. */
2395         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD);
2396         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE);
2397
2398         /* Copy card response into mmc response buffer. */
2399         if (ISSET(cmd->flags, MMC_RSP_PRESENT)) {
2400                 uint32_t *cmd_buffer = (uint32_t *)(sc->rtsx_cmd_dmamem);
2401
2402                 if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD) {
2403                         device_printf(sc->rtsx_dev, "cmd_buffer: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
2404                                       cmd_buffer[0], cmd_buffer[1], cmd_buffer[2], cmd_buffer[3], cmd_buffer[4]);
2405                 }
2406
2407                 if (rsp_type == RTSX_SD_RSP_TYPE_R2) {
2408                         /* First byte is CHECK_REG_CMD return value, skip it. */
2409                         unsigned char *ptr = (unsigned char *)cmd_buffer + 1;
2410                         int i;
2411
2412                         /*
2413                          * The controller offloads the last byte {CRC-7, end bit 1}
2414                          * of response type R2. Assign dummy CRC, 0, and end bit to this
2415                          * byte (ptr[16], goes into the LSB of resp[3] later).
2416                          */
2417                         ptr[16] = 0x01;
2418                         /* The second byte is the status of response, skip it. */
2419                         for (i = 0; i < 4; i++)
2420                                 cmd->resp[i] = be32dec(ptr + 1 + i * 4);
2421                 } else {
2422                         /*
2423                          * First byte is CHECK_REG_CMD return value, second
2424                          * one is the command op code -- we skip those.
2425                          */
2426                         cmd->resp[0] =
2427                                 ((be32toh(cmd_buffer[0]) & 0x0000ffff) << 16) |
2428                                 ((be32toh(cmd_buffer[1]) & 0xffff0000) >> 16);
2429                 }
2430
2431                 if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
2432                         device_printf(sc->rtsx_dev, "cmd->resp: 0x%08x 0x%08x 0x%08x 0x%08x\n",
2433                                       cmd->resp[0], cmd->resp[1], cmd->resp[2], cmd->resp[3]);
2434         }
2435 }
2436
2437 /*
2438  * Use the ping-pong buffer (cmd buffer) for transfer <= 512 bytes.
2439  */
2440 static int
2441 rtsx_xfer_short(struct rtsx_softc *sc, struct mmc_command *cmd)
2442 {
2443         int     read;
2444
2445         if (cmd->data == NULL || cmd->data->len == 0) {
2446                 cmd->error = MMC_ERR_INVALID;
2447                 return (MMC_ERR_INVALID);
2448         }
2449         cmd->data->xfer_len = (cmd->data->len > RTSX_MAX_DATA_BLKLEN) ?
2450                 RTSX_MAX_DATA_BLKLEN : cmd->data->len;
2451
2452         read = ISSET(cmd->data->flags, MMC_DATA_READ);
2453
2454         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
2455                 device_printf(sc->rtsx_dev, "rtsx_xfer_short() - %s xfer: %ld bytes with block size %ld\n",
2456                               read ? "Read" : "Write",
2457                               (unsigned long)cmd->data->len, (unsigned long)cmd->data->xfer_len);
2458
2459         if (cmd->data->len > 512) {
2460                 device_printf(sc->rtsx_dev, "rtsx_xfer_short() - length too large: %ld > 512\n",
2461                               (unsigned long)cmd->data->len);
2462                 cmd->error = MMC_ERR_INVALID;
2463                 return (MMC_ERR_INVALID);
2464         }
2465
2466         if (read) {
2467                 if (sc->rtsx_discovery_mode)
2468                         rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_0);
2469
2470                 rtsx_init_cmd(sc, cmd);
2471
2472                 /* Queue commands to configure data transfer size. */
2473                 rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len);
2474
2475                 /* From Linux: rtsx_pci_sdmmc.c sd_read_data(). */
2476                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff,
2477                               RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 |
2478                               RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_6);
2479
2480                 /* Use the ping-pong buffer (cmd buffer). */
2481                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE,
2482                               0x01, RTSX_PINGPONG_BUFFER);
2483
2484                 /* Queue commands to perform SD transfer. */
2485                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER,
2486                               0xff, RTSX_TM_NORMAL_READ | RTSX_SD_TRANSFER_START);
2487                 rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
2488                               RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END);
2489
2490                 /* Set transfer OK function. */
2491                 sc->rtsx_intr_trans_ok = rtsx_ask_ppbuf_part1;
2492
2493                 /* Run the command queue. */
2494                 rtsx_send_cmd(sc);
2495         } else {
2496                 /* Set transfer OK function. */
2497                 sc->rtsx_intr_trans_ok = rtsx_put_ppbuf_part1;
2498
2499                 /* Run the command queue. */
2500                 rtsx_send_req(sc, cmd);
2501         }
2502
2503         return (0);
2504 }
2505
2506 /*
2507  * Use the ping-pong buffer (cmd buffer) for the transfer - first part <= 256 bytes.
2508  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2509  */
2510 static void
2511 rtsx_ask_ppbuf_part1(struct rtsx_softc *sc)
2512 {
2513         struct mmc_command *cmd;
2514         uint16_t reg = RTSX_PPBUF_BASE2;
2515         int      len;
2516         int      i;
2517
2518         cmd = sc->rtsx_req->cmd;
2519         len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len;
2520
2521         sc->rtsx_cmd_index = 0;
2522         for (i = 0; i < len; i++) {
2523                 rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg++, 0, 0);
2524         }
2525
2526         /* Set transfer OK function. */
2527         sc->rtsx_intr_trans_ok = rtsx_get_ppbuf_part1;
2528
2529         /* Run the command queue. */
2530         rtsx_send_cmd(sc);
2531 }
2532
2533 /*
2534  * Get the data from the ping-pong buffer (cmd buffer) - first part <= 256 bytes.
2535  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2536  */
2537 static void
2538 rtsx_get_ppbuf_part1(struct rtsx_softc *sc)
2539 {
2540         struct mmc_command *cmd;
2541         uint8_t  *ptr;
2542         int      len;
2543
2544         cmd = sc->rtsx_req->cmd;
2545         ptr = cmd->data->data;
2546         len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len;
2547
2548         /* Sync command DMA buffer. */
2549         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD);
2550         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE);
2551
2552         memcpy(ptr, sc->rtsx_cmd_dmamem, len);
2553
2554         len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? cmd->data->len - RTSX_HOSTCMD_MAX : 0;
2555
2556         /* Use the ping-pong buffer (cmd buffer) for the transfer - second part > 256 bytes. */
2557         if (len > 0) {
2558                 uint16_t reg = RTSX_PPBUF_BASE2 + RTSX_HOSTCMD_MAX;
2559                 int      i;
2560
2561                 sc->rtsx_cmd_index = 0;
2562                 for (i = 0; i < len; i++) {
2563                         rtsx_push_cmd(sc, RTSX_READ_REG_CMD, reg++, 0, 0);
2564                 }
2565
2566                 /* Set transfer OK function. */
2567                 sc->rtsx_intr_trans_ok = rtsx_get_ppbuf_part2;
2568
2569                 /* Run the command queue. */
2570                 rtsx_send_cmd(sc);
2571         } else {
2572                 if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD && cmd->opcode == ACMD_SEND_SCR) {
2573                         uint8_t *ptr = cmd->data->data;
2574                         device_printf(sc->rtsx_dev, "SCR: 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
2575                                       ptr[0], ptr[1], ptr[2], ptr[3],
2576                                       ptr[4], ptr[5], ptr[6], ptr[7]);
2577                 }
2578
2579                 if (sc->rtsx_discovery_mode)
2580                         rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_128);
2581
2582                 rtsx_req_done(sc);
2583         }
2584 }
2585
2586 /*
2587  * Get the data from the ping-pong buffer (cmd buffer) - second part > 256 bytes.
2588  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2589  */
2590 static void
2591 rtsx_get_ppbuf_part2(struct rtsx_softc *sc)
2592 {
2593         struct mmc_command *cmd;
2594         uint8_t *ptr;
2595         int     len;
2596
2597         cmd = sc->rtsx_req->cmd;
2598         ptr = cmd->data->data;
2599         ptr += RTSX_HOSTCMD_MAX;
2600         len = cmd->data->len - RTSX_HOSTCMD_MAX;
2601
2602         /* Sync command DMA buffer. */
2603         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTREAD);
2604         bus_dmamap_sync(sc->rtsx_cmd_dma_tag, sc->rtsx_cmd_dmamap, BUS_DMASYNC_POSTWRITE);
2605
2606         memcpy(ptr, sc->rtsx_cmd_dmamem, len);
2607
2608         if (sc->rtsx_discovery_mode)
2609                 rtsx_write(sc, RTSX_SD_CFG1, RTSX_CLK_DIVIDE_MASK, RTSX_CLK_DIVIDE_128);
2610
2611         rtsx_req_done(sc);
2612 }
2613
2614 /*
2615  * Use the ping-pong buffer (cmd buffer) for transfer - first part <= 256 bytes.
2616  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2617  */
2618 static void
2619 rtsx_put_ppbuf_part1(struct rtsx_softc *sc)
2620 {
2621         struct mmc_command *cmd;
2622         uint16_t reg = RTSX_PPBUF_BASE2;
2623         uint8_t  *ptr;
2624         int      len;
2625         int      i;
2626
2627         cmd = sc->rtsx_req->cmd;
2628         ptr = cmd->data->data;
2629         len = (cmd->data->len > RTSX_HOSTCMD_MAX) ? RTSX_HOSTCMD_MAX : cmd->data->len;
2630
2631         rtsx_set_resp(sc, cmd);
2632
2633         sc->rtsx_cmd_index = 0;
2634         for (i = 0; i < len; i++) {
2635                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, reg++, 0xff, *ptr);
2636                 ptr++;
2637         }
2638
2639         /* Set transfer OK function. */
2640         if (cmd->data->len > RTSX_HOSTCMD_MAX)
2641                 sc->rtsx_intr_trans_ok = rtsx_put_ppbuf_part2;
2642         else
2643                 sc->rtsx_intr_trans_ok = rtsx_write_ppbuf;
2644
2645         /* Run the command queue. */
2646         rtsx_send_cmd(sc);
2647 }
2648
2649 /*
2650  * Use the ping-pong buffer (cmd buffer) for transfer - second part > 256 bytes.
2651  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2652  */
2653 static void
2654 rtsx_put_ppbuf_part2(struct rtsx_softc *sc)
2655 {
2656         struct mmc_command *cmd;
2657         uint16_t reg = RTSX_PPBUF_BASE2 + RTSX_HOSTCMD_MAX;
2658         uint8_t  *ptr;
2659         int      len;
2660         int      i;
2661
2662         cmd = sc->rtsx_req->cmd;
2663         ptr = cmd->data->data;
2664         ptr += RTSX_HOSTCMD_MAX;
2665         len = cmd->data->len - RTSX_HOSTCMD_MAX;
2666
2667         sc->rtsx_cmd_index = 0;
2668         for (i = 0; i < len; i++) {
2669                 rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, reg++, 0xff, *ptr);
2670                 ptr++;
2671         }
2672
2673         /* Set transfer OK function. */
2674         sc->rtsx_intr_trans_ok = rtsx_write_ppbuf;
2675
2676         /* Run the command queue. */
2677         rtsx_send_cmd(sc);
2678 }
2679
2680 /*
2681  * Write the data previously given via the ping-pong buffer on the card.
2682  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2683  */
2684 static void
2685 rtsx_write_ppbuf(struct rtsx_softc *sc)
2686 {
2687         struct mmc_command *cmd;
2688
2689         cmd = sc->rtsx_req->cmd;
2690
2691         sc->rtsx_cmd_index = 0;
2692
2693         /* Queue commands to configure data transfer size. */
2694         rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len);
2695
2696         /* From Linux: rtsx_pci_sdmmc.c sd_write_data(). */
2697         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff,
2698                       RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC16 |
2699                       RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_CHECK_CRC7 | RTSX_SD_RSP_LEN_0);
2700
2701         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER, 0xff,
2702                       RTSX_TM_AUTO_WRITE3 | RTSX_SD_TRANSFER_START);
2703         rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
2704                       RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END);
2705
2706         /* Set transfer OK function. */
2707         sc->rtsx_intr_trans_ok = rtsx_req_done;
2708
2709         /* Run the command queue. */
2710         rtsx_send_cmd(sc);
2711 }
2712
2713 /*
2714  * Use the data buffer for transfer > 512 bytes.
2715  */
2716 static int
2717 rtsx_xfer(struct rtsx_softc *sc, struct mmc_command *cmd)
2718 {
2719         int     read = ISSET(cmd->data->flags, MMC_DATA_READ);
2720
2721         cmd->data->xfer_len = (cmd->data->len > RTSX_MAX_DATA_BLKLEN) ?
2722                 RTSX_MAX_DATA_BLKLEN : cmd->data->len;
2723
2724         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
2725                 device_printf(sc->rtsx_dev, "rtsx_xfer() - %s xfer: %ld bytes with block size %ld\n",
2726                               read ? "Read" : "Write",
2727                               (unsigned long)cmd->data->len, (unsigned long)cmd->data->xfer_len);
2728
2729         if (cmd->data->len > RTSX_DMA_DATA_BUFSIZE) {
2730                 device_printf(sc->rtsx_dev, "rtsx_xfer() length too large: %ld > %d\n",
2731                               (unsigned long)cmd->data->len, RTSX_DMA_DATA_BUFSIZE);
2732                 cmd->error = MMC_ERR_INVALID;
2733                 return (MMC_ERR_INVALID);
2734         }
2735
2736         if (!read) {
2737                 /* Set transfer OK function. */
2738                 sc->rtsx_intr_trans_ok = rtsx_xfer_begin;
2739
2740                 /* Run the command queue. */
2741                 rtsx_send_req(sc, cmd);
2742         } else {
2743                 rtsx_xfer_start(sc);
2744         }
2745
2746         return (0);
2747 }
2748
2749 /*
2750  * Get request response and start dma data transfer (write command).
2751  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2752  */
2753 static void
2754 rtsx_xfer_begin(struct rtsx_softc *sc)
2755 {
2756         struct mmc_command *cmd;
2757
2758         cmd = sc->rtsx_req->cmd;
2759
2760         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
2761                 device_printf(sc->rtsx_dev, "rtsx_xfer_begin() - CMD%d\n", cmd->opcode);
2762
2763         rtsx_set_resp(sc, cmd);
2764         rtsx_xfer_start(sc);
2765 }
2766
2767 /*
2768  * Start dma data transfer.
2769  */
2770 static void
2771 rtsx_xfer_start(struct rtsx_softc *sc)
2772 {
2773         struct mmc_command *cmd;
2774         int     read;
2775         uint8_t cfg2;
2776         int     dma_dir;
2777         int     tmode;
2778
2779         cmd = sc->rtsx_req->cmd;
2780         read = ISSET(cmd->data->flags, MMC_DATA_READ);
2781
2782         /* Configure DMA transfer mode parameters. */
2783         if (cmd->opcode == MMC_READ_MULTIPLE_BLOCK)
2784                 cfg2 = RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_RSP_LEN_6;
2785         else
2786                 cfg2 = RTSX_SD_CHECK_CRC16 | RTSX_SD_NO_WAIT_BUSY_END | RTSX_SD_RSP_LEN_0;
2787         if (read) {
2788                 dma_dir = RTSX_DMA_DIR_FROM_CARD;
2789                 /*
2790                  * Use transfer mode AUTO_READ1, which assume we not
2791                  * already send the read command and don't need to send
2792                  * CMD 12 manually after read.
2793                  */
2794                 tmode = RTSX_TM_AUTO_READ1;
2795                 cfg2 |= RTSX_SD_CALCULATE_CRC7 | RTSX_SD_CHECK_CRC7;
2796
2797                 rtsx_init_cmd(sc, cmd);
2798         } else {
2799                 dma_dir = RTSX_DMA_DIR_TO_CARD;
2800                 /*
2801                  * Use transfer mode AUTO_WRITE3, wich assumes we've already
2802                  * sent the write command and gotten the response, and will
2803                  * send CMD 12 manually after writing.
2804                  */
2805                 tmode = RTSX_TM_AUTO_WRITE3;
2806                 cfg2 |= RTSX_SD_NO_CALCULATE_CRC7 | RTSX_SD_NO_CHECK_CRC7;
2807
2808                 sc->rtsx_cmd_index = 0;
2809         }
2810
2811         /* Queue commands to configure data transfer size. */
2812         rtsx_set_cmd_data_len(sc, cmd->data->len / cmd->data->xfer_len, cmd->data->xfer_len);
2813
2814         /* Configure DMA controller. */
2815         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_IRQSTAT0,
2816                      RTSX_DMA_DONE_INT, RTSX_DMA_DONE_INT);
2817         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC3,
2818                      0xff, cmd->data->len >> 24);
2819         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC2,
2820                      0xff, cmd->data->len >> 16);
2821         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC1,
2822                      0xff, cmd->data->len >> 8);
2823         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMATC0,
2824                      0xff, cmd->data->len);
2825         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_DMACTL,
2826                      RTSX_DMA_EN | RTSX_DMA_DIR | RTSX_DMA_PACK_SIZE_MASK,
2827                      RTSX_DMA_EN | dma_dir | RTSX_DMA_512);
2828
2829         /* Use the DMA ring buffer for commands which transfer data. */
2830         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_CARD_DATA_SOURCE,
2831                       0x01, RTSX_RING_BUFFER);
2832
2833         /* Queue command to set response type. */
2834         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_CFG2, 0xff, cfg2);
2835
2836         /* Queue commands to perform SD transfer. */
2837         rtsx_push_cmd(sc, RTSX_WRITE_REG_CMD, RTSX_SD_TRANSFER,
2838                       0xff, tmode | RTSX_SD_TRANSFER_START);
2839         rtsx_push_cmd(sc, RTSX_CHECK_REG_CMD, RTSX_SD_TRANSFER,
2840                       RTSX_SD_TRANSFER_END, RTSX_SD_TRANSFER_END);
2841
2842         /* Run the command queue. */
2843         rtsx_send_cmd(sc);
2844
2845         if (!read)
2846                 memcpy(sc->rtsx_data_dmamem, cmd->data->data, cmd->data->len);
2847
2848         /* Sync data DMA buffer. */
2849         bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_PREREAD);
2850         bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_PREWRITE);
2851
2852         /* Set transfer OK function. */
2853         sc->rtsx_intr_trans_ok = rtsx_xfer_finish;
2854
2855         /* Tell the chip where the data buffer is and run the transfer. */
2856         WRITE4(sc, RTSX_HDBAR, sc->rtsx_data_buffer);
2857         WRITE4(sc, RTSX_HDBCTLR, RTSX_TRIG_DMA | (read ? RTSX_DMA_READ : 0) |
2858                (cmd->data->len & 0x00ffffff));
2859 }
2860
2861 /*
2862  * Finish dma data transfer.
2863  * This Function is called by the interrupt handler via sc->rtsx_intr_trans_ok.
2864  */
2865 static void
2866 rtsx_xfer_finish(struct rtsx_softc *sc)
2867 {
2868         struct mmc_command *cmd;
2869         int     read;
2870
2871         cmd = sc->rtsx_req->cmd;
2872
2873         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
2874                 device_printf(sc->rtsx_dev, "rtsx_xfer_finish() - CMD%d\n", cmd->opcode);
2875
2876         read = ISSET(cmd->data->flags, MMC_DATA_READ);
2877
2878         /* Sync data DMA buffer. */
2879         bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_POSTREAD);
2880         bus_dmamap_sync(sc->rtsx_data_dma_tag, sc->rtsx_data_dmamap, BUS_DMASYNC_POSTWRITE);
2881
2882         if (read) {
2883                 memcpy(cmd->data->data, sc->rtsx_data_dmamem, cmd->data->len);
2884                 rtsx_req_done(sc);
2885         } else {
2886                 /* Send CMD12 after AUTO_WRITE3 (see mmcsd_rw() in mmcsd.c) */
2887                 /* and complete request. */
2888                 sc->rtsx_intr_trans_ok = NULL;
2889                 rtsx_send_req(sc, sc->rtsx_req->stop);
2890         }
2891 }
2892
2893 /*
2894  * Manage request timeout.
2895  */
2896 static void
2897 rtsx_timeout(void *arg)
2898 {
2899         struct rtsx_softc *sc;
2900
2901         sc = (struct rtsx_softc *)arg;
2902         if (sc->rtsx_req != NULL) {
2903                 device_printf(sc->rtsx_dev, "Controller timeout for CMD%u\n",
2904                               sc->rtsx_req->cmd->opcode);
2905                 sc->rtsx_req->cmd->error = MMC_ERR_TIMEOUT;
2906                 rtsx_stop_cmd(sc);
2907                 rtsx_req_done(sc);
2908         } else {
2909                 device_printf(sc->rtsx_dev, "Controller timeout!\n");
2910         }
2911 }
2912
2913 #ifdef MMCCAM
2914 static int
2915 rtsx_get_tran_settings(device_t dev, struct ccb_trans_settings_mmc *cts)
2916 {
2917         struct rtsx_softc *sc;
2918
2919         sc = device_get_softc(dev);
2920
2921         cts->host_ocr = sc->rtsx_host.host_ocr;
2922         cts->host_f_min = sc->rtsx_host.f_min;
2923         cts->host_f_max = sc->rtsx_host.f_max;
2924         cts->host_caps = sc->rtsx_host.caps;
2925         cts->host_max_data = RTSX_DMA_DATA_BUFSIZE / MMC_SECTOR_SIZE;
2926         memcpy(&cts->ios, &sc->rtsx_host.ios, sizeof(struct mmc_ios));
2927
2928         return (0);
2929 }
2930
2931 /*
2932  *  Apply settings and return status accordingly.
2933 */
2934 static int
2935 rtsx_set_tran_settings(device_t dev, struct ccb_trans_settings_mmc *cts)
2936 {
2937         struct rtsx_softc *sc;
2938         struct mmc_ios *ios;
2939         struct mmc_ios *new_ios;
2940
2941         sc = device_get_softc(dev);
2942
2943         ios = &sc->rtsx_host.ios;
2944         new_ios = &cts->ios;
2945
2946         /* Update only requested fields */
2947         if (cts->ios_valid & MMC_CLK) {
2948                 ios->clock = new_ios->clock;
2949                 sc->rtsx_ios_clock = -1;        /* To be updated by rtsx_mmcbr_update_ios(). */
2950                 if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2951                         device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - clock: %u\n", ios->clock);
2952         }
2953         if (cts->ios_valid & MMC_VDD) {
2954                 ios->vdd = new_ios->vdd;
2955                 if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2956                         device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - vdd: %d\n", ios->vdd);
2957         }
2958         if (cts->ios_valid & MMC_CS) {
2959                 ios->chip_select = new_ios->chip_select;
2960                 if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2961                         device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - chip_select: %d\n", ios->chip_select);
2962         }
2963         if (cts->ios_valid & MMC_BW) {
2964                 ios->bus_width = new_ios->bus_width;
2965                 sc->rtsx_ios_bus_width = -1;    /* To be updated by rtsx_mmcbr_update_ios(). */
2966                 if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2967                         device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - bus width: %d\n", ios->bus_width);
2968         }
2969         if (cts->ios_valid & MMC_PM) {
2970                 ios->power_mode = new_ios->power_mode;
2971                 sc->rtsx_ios_power_mode = -1;   /* To be updated by rtsx_mmcbr_update_ios(). */
2972                 if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2973                         device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - power mode: %d\n", ios->power_mode);
2974         }
2975         if (cts->ios_valid & MMC_BT) {
2976                 ios->timing = new_ios->timing;
2977                 sc->rtsx_ios_timing = -1;       /* To be updated by rtsx_mmcbr_update_ios(). */
2978                 if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2979                         device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - timing: %d\n", ios->timing);
2980         }
2981         if (cts->ios_valid & MMC_BM) {
2982                 ios->bus_mode = new_ios->bus_mode;
2983                 if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2984                         device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - bus mode: %d\n", ios->bus_mode);
2985         }
2986 #if  __FreeBSD_version >= 1300000
2987         if (cts->ios_valid & MMC_VCCQ) {
2988                 ios->vccq = new_ios->vccq;
2989                 sc->rtsx_ios_vccq = -1;         /* To be updated by rtsx_mmcbr_update_ios(). */
2990                 if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
2991                         device_printf(sc->rtsx_dev, "rtsx_set_tran_settings() - vccq: %d\n", ios->vccq);
2992         }
2993 #endif /* __FreeBSD_version >= 1300000 */
2994         if (rtsx_mmcbr_update_ios(sc->rtsx_dev, NULL) == 0)
2995                 return (CAM_REQ_CMP);
2996         else
2997                 return (CAM_REQ_CMP_ERR);
2998 }
2999
3000 /*
3001  * Build a request and run it.
3002  */
3003 static int
3004 rtsx_cam_request(device_t dev, union ccb *ccb)
3005 {
3006         struct rtsx_softc *sc;
3007
3008         sc = device_get_softc(dev);
3009
3010         RTSX_LOCK(sc);
3011         if (sc->rtsx_ccb != NULL) {
3012                 RTSX_UNLOCK(sc);
3013                 return (CAM_BUSY);
3014         }
3015         sc->rtsx_ccb = ccb;
3016         sc->rtsx_cam_req.cmd = &ccb->mmcio.cmd;
3017         sc->rtsx_cam_req.stop = &ccb->mmcio.stop;
3018         RTSX_UNLOCK(sc);
3019
3020         rtsx_mmcbr_request(sc->rtsx_dev, NULL, &sc->rtsx_cam_req);
3021         return (0);
3022 }
3023 #endif /* MMCCAM */
3024
3025 static int
3026 rtsx_read_ivar(device_t bus, device_t child, int which, uintptr_t *result)
3027 {
3028         struct rtsx_softc *sc;
3029
3030         sc = device_get_softc(bus);
3031         switch (which) {
3032         case MMCBR_IVAR_BUS_MODE:               /* ivar  0 - 1 = opendrain, 2 = pushpull */
3033                 *result = sc->rtsx_host.ios.bus_mode;
3034                 break;
3035         case MMCBR_IVAR_BUS_WIDTH:              /* ivar  1 - 0 = 1b   2 = 4b, 3 = 8b */
3036                 *result = sc->rtsx_host.ios.bus_width;
3037                 break;
3038         case MMCBR_IVAR_CHIP_SELECT:            /* ivar  2 - O = dontcare, 1 = cs_high, 2 = cs_low */
3039                 *result = sc->rtsx_host.ios.chip_select;
3040                 break;
3041         case MMCBR_IVAR_CLOCK:                  /* ivar  3 - clock in Hz */
3042                 *result = sc->rtsx_host.ios.clock;
3043                 break;
3044         case MMCBR_IVAR_F_MIN:                  /* ivar  4 */
3045                 *result = sc->rtsx_host.f_min;
3046                 break;
3047         case MMCBR_IVAR_F_MAX:                  /* ivar  5 */
3048                 *result = sc->rtsx_host.f_max;
3049                 break;
3050         case MMCBR_IVAR_HOST_OCR:               /* ivar  6 - host operation conditions register */
3051                 *result = sc->rtsx_host.host_ocr;
3052                 break;
3053         case MMCBR_IVAR_MODE:                   /* ivar  7 - 0 = mode_mmc, 1 = mode_sd */
3054                 *result = sc->rtsx_host.mode;
3055                 break;
3056         case MMCBR_IVAR_OCR:                    /* ivar  8 - operation conditions register */
3057                 *result = sc->rtsx_host.ocr;
3058                 break;
3059         case MMCBR_IVAR_POWER_MODE:             /* ivar  9 - 0 = off, 1 = up, 2 = on */
3060                 *result = sc->rtsx_host.ios.power_mode;
3061                 break;
3062         case MMCBR_IVAR_VDD:                    /* ivar 11 - voltage power pin */
3063                 *result = sc->rtsx_host.ios.vdd;
3064                 break;
3065         case MMCBR_IVAR_VCCQ:                   /* ivar 12 - signaling: 0 = 1.20V, 1 = 1.80V, 2 = 3.30V */
3066                 *result = sc->rtsx_host.ios.vccq;
3067                 break;
3068         case MMCBR_IVAR_CAPS:                   /* ivar 13 */
3069                 *result = sc->rtsx_host.caps;
3070                 break;
3071         case MMCBR_IVAR_TIMING:                 /* ivar 14 - 0 = normal, 1 = timing_hs, ... */
3072                 *result = sc->rtsx_host.ios.timing;
3073                 break;
3074         case MMCBR_IVAR_MAX_DATA:               /* ivar 15 */
3075                 *result = RTSX_DMA_DATA_BUFSIZE / MMC_SECTOR_SIZE;
3076                 break;
3077         case MMCBR_IVAR_RETUNE_REQ:             /* ivar 10 */
3078         case MMCBR_IVAR_MAX_BUSY_TIMEOUT:       /* ivar 16 */
3079         default:
3080                 return (EINVAL);
3081         }
3082
3083         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
3084                 device_printf(bus, "Read ivar #%d, value %#x / #%d\n",
3085                               which, *(int *)result, *(int *)result);
3086
3087         return (0);
3088 }
3089
3090 static int
3091 rtsx_write_ivar(device_t bus, device_t child, int which, uintptr_t value)
3092 {
3093         struct rtsx_softc *sc;
3094
3095         sc = device_get_softc(bus);
3096         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
3097                 device_printf(bus, "Write ivar #%d, value %#x / #%d\n",
3098                               which, (int)value, (int)value);
3099
3100         switch (which) {
3101         case MMCBR_IVAR_BUS_MODE:               /* ivar  0 - 1 = opendrain, 2 = pushpull */
3102                 sc->rtsx_host.ios.bus_mode = value;
3103                 break;
3104         case MMCBR_IVAR_BUS_WIDTH:              /* ivar  1 - 0 = 1b   2 = 4b, 3 = 8b */
3105                 sc->rtsx_host.ios.bus_width = value;
3106                 sc->rtsx_ios_bus_width = -1;    /* To be updated on next rtsx_mmcbr_update_ios(). */
3107                 break;
3108         case MMCBR_IVAR_CHIP_SELECT:            /* ivar  2 - O = dontcare, 1 = cs_high, 2 = cs_low */
3109                 sc->rtsx_host.ios.chip_select = value;
3110                 break;
3111         case MMCBR_IVAR_CLOCK:                  /* ivar  3 - clock in Hz */
3112                 sc->rtsx_host.ios.clock = value;
3113                 sc->rtsx_ios_clock = -1;        /* To be updated on next rtsx_mmcbr_update_ios(). */
3114                 break;
3115         case MMCBR_IVAR_MODE:                   /* ivar  7 - 0 = mode_mmc, 1 = mode_sd */
3116                 sc->rtsx_host.mode = value;
3117                 break;
3118         case MMCBR_IVAR_OCR:                    /* ivar  8 - operation conditions register */
3119                 sc->rtsx_host.ocr = value;
3120                 break;
3121         case MMCBR_IVAR_POWER_MODE:             /* ivar  9 - 0 = off, 1 = up, 2 = on */
3122                 sc->rtsx_host.ios.power_mode = value;
3123                 sc->rtsx_ios_power_mode = -1;   /* To be updated on next rtsx_mmcbr_update_ios(). */
3124                 break;
3125         case MMCBR_IVAR_VDD:                    /* ivar 11 - voltage power pin */
3126                 sc->rtsx_host.ios.vdd = value;
3127                 break;
3128         case MMCBR_IVAR_VCCQ:                   /* ivar 12 - signaling: 0 = 1.20V, 1 = 1.80V, 2 = 3.30V */
3129                 sc->rtsx_host.ios.vccq = value;
3130                 sc->rtsx_ios_vccq = value;      /* rtsx_mmcbr_switch_vccq() will be called by mmc.c (MMCCAM undef). */
3131                 break;
3132         case MMCBR_IVAR_TIMING:                 /* ivar 14 - 0 = normal, 1 = timing_hs, ... */
3133                 sc->rtsx_host.ios.timing = value;
3134                 sc->rtsx_ios_timing = -1;       /* To be updated on next rtsx_mmcbr_update_ios(). */
3135                 break;
3136         /* These are read-only. */
3137         case MMCBR_IVAR_F_MIN:                  /* ivar  4 */
3138         case MMCBR_IVAR_F_MAX:                  /* ivar  5 */
3139         case MMCBR_IVAR_HOST_OCR:               /* ivar  6 - host operation conditions register */
3140         case MMCBR_IVAR_RETUNE_REQ:             /* ivar 10 */
3141         case MMCBR_IVAR_CAPS:                   /* ivar 13 */
3142         case MMCBR_IVAR_MAX_DATA:               /* ivar 15 */
3143         case MMCBR_IVAR_MAX_BUSY_TIMEOUT:       /* ivar 16 */
3144         default:
3145                 return (EINVAL);
3146         }
3147
3148         return (0);
3149 }
3150
3151 static int
3152 rtsx_mmcbr_update_ios(device_t bus, device_t child__unused)
3153 {
3154         struct rtsx_softc *sc;
3155         struct mmc_ios    *ios;
3156         int     error;
3157
3158         sc = device_get_softc(bus);
3159         ios = &sc->rtsx_host.ios;
3160
3161         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
3162                 device_printf(bus, "rtsx_mmcbr_update_ios()\n");
3163
3164         /* if MMCBR_IVAR_BUS_WIDTH updated. */
3165         if (sc->rtsx_ios_bus_width < 0) {
3166                 sc->rtsx_ios_bus_width = ios->bus_width;
3167                 if ((error = rtsx_set_bus_width(sc, ios->bus_width)))
3168                         return (error);
3169         }
3170
3171         /* if MMCBR_IVAR_POWER_MODE updated. */
3172         if (sc->rtsx_ios_power_mode < 0) {
3173                 sc->rtsx_ios_power_mode = ios->power_mode;
3174                 switch (ios->power_mode) {
3175                 case power_off:
3176                         if ((error = rtsx_bus_power_off(sc)))
3177                                 return (error);
3178                         break;
3179                 case power_up:
3180                         if ((error = rtsx_bus_power_on(sc)))
3181                                 return (error);
3182                         break;
3183                 case power_on:
3184                         if ((error = rtsx_bus_power_on(sc)))
3185                                 return (error);
3186                         break;
3187                 }
3188         }
3189
3190         sc->rtsx_double_clk = true;
3191         sc->rtsx_vpclk = false;
3192
3193         /* if MMCBR_IVAR_TIMING updated. */
3194         if (sc->rtsx_ios_timing < 0) {
3195                 sc->rtsx_ios_timing = ios->timing;
3196                 if ((error = rtsx_set_sd_timing(sc, ios->timing)))
3197                         return (error);
3198         }
3199
3200         /* if MMCBR_IVAR_CLOCK updated, must be after rtsx_set_sd_timing() */
3201         if (sc->rtsx_ios_clock < 0) {
3202                 sc->rtsx_ios_clock = ios->clock;
3203                 if ((error = rtsx_set_sd_clock(sc, ios->clock)))
3204                         return (error);
3205         }
3206
3207         /* if MMCCAM and vccq updated */
3208         if (sc->rtsx_ios_vccq < 0) {
3209                 sc->rtsx_ios_vccq = ios->vccq;
3210                 if ((error = rtsx_mmcbr_switch_vccq(sc->rtsx_dev, NULL)))
3211                         return (error);
3212         }
3213
3214         return (0);
3215 }
3216
3217 /*
3218  * Set output stage logic power voltage.
3219  */
3220 static int
3221 rtsx_mmcbr_switch_vccq(device_t bus, device_t child __unused)
3222 {
3223         struct rtsx_softc *sc;
3224         int     vccq = 0;
3225         int     error;
3226
3227         sc = device_get_softc(bus);
3228
3229         switch (sc->rtsx_host.ios.vccq) {
3230         case vccq_120:
3231                 vccq = 120;
3232                 break;
3233         case vccq_180:
3234                 vccq = 180;
3235                 break;
3236         case vccq_330:
3237                 vccq = 330;
3238                 break;
3239         };
3240         /* It seems it is always vccq_330. */
3241         if (vccq == 330) {
3242                 switch (sc->rtsx_device_id) {
3243                         uint16_t val;
3244                 case RTSX_RTS5227:
3245                         if ((error = rtsx_write_phy(sc, 0x08, 0x4FE4)))
3246                                 return (error);
3247                         if ((error = rtsx_rts5227_fill_driving(sc)))
3248                                 return (error);
3249                         break;
3250                 case RTSX_RTS5209:
3251                 case RTSX_RTS5229:
3252                         RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3);
3253                         if ((error = rtsx_write_phy(sc, 0x08, 0x4FE4)))
3254                                 return (error);
3255                         break;
3256                 case RTSX_RTS522A:
3257                         if ((error = rtsx_write_phy(sc, 0x08, 0x57E4)))
3258                                 return (error);
3259                         if ((error = rtsx_rts5227_fill_driving(sc)))
3260                                 return (error);
3261                         break;
3262                 case RTSX_RTS525A:
3263                         RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_LDO_D3318_MASK, RTSX_LDO_D3318_33V);
3264                         RTSX_BITOP(sc, RTSX_SD_PAD_CTL, RTSX_SD_IO_USING_1V8, 0);
3265                         if ((error = rtsx_rts5249_fill_driving(sc)))
3266                                 return (error);
3267                         break;
3268                 case RTSX_RTS5249:
3269                         if ((error = rtsx_read_phy(sc, RTSX_PHY_TUNE, &val)))
3270                                 return (error);
3271                         if ((error = rtsx_write_phy(sc, RTSX_PHY_TUNE,
3272                                                     (val & RTSX_PHY_TUNE_VOLTAGE_MASK) | RTSX_PHY_TUNE_VOLTAGE_3V3)))
3273                                 return (error);
3274                         if ((error = rtsx_rts5249_fill_driving(sc)))
3275                                 return (error);
3276                         break;
3277                 case RTSX_RTS5260:
3278                         RTSX_BITOP(sc, RTSX_LDO_CONFIG2, RTSX_DV331812_VDD1, RTSX_DV331812_VDD1);
3279                         RTSX_BITOP(sc, RTSX_LDO_DV18_CFG, RTSX_DV331812_MASK, RTSX_DV331812_33);
3280                         if ((error = rtsx_rts5260_fill_driving(sc)))
3281                                 return (error);
3282                         break;
3283                 case RTSX_RTL8402:
3284                         RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3);
3285                         RTSX_BITOP(sc, RTSX_LDO_CTL,
3286                                    (RTSX_BPP_ASIC_MASK << RTSX_BPP_SHIFT_8402) | RTSX_BPP_PAD_MASK,
3287                                    (RTSX_BPP_ASIC_3V3 << RTSX_BPP_SHIFT_8402) | RTSX_BPP_PAD_3V3);
3288                         break;
3289                 case RTSX_RTL8411:
3290                 case RTSX_RTL8411B:
3291                         RTSX_BITOP(sc, RTSX_SD30_CMD_DRIVE_SEL, RTSX_SD30_DRIVE_SEL_MASK, sc->rtsx_sd30_drive_sel_3v3);
3292                         RTSX_BITOP(sc, RTSX_LDO_CTL,
3293                                    (RTSX_BPP_ASIC_MASK << RTSX_BPP_SHIFT_8411) | RTSX_BPP_PAD_MASK,
3294                                    (RTSX_BPP_ASIC_3V3 << RTSX_BPP_SHIFT_8411) | RTSX_BPP_PAD_3V3);
3295                         break;
3296                 }
3297                 DELAY(300);
3298         }
3299
3300         if (sc->rtsx_debug_mask & (RTSX_DEBUG_BASIC | RTSX_TRACE_SD_CMD))
3301                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_switch_vccq(%d)\n", vccq);
3302
3303         return (0);
3304 }
3305
3306 #ifndef MMCCAM
3307 /*
3308  * Tune card if bus_timing_uhs_sdr50.
3309  */
3310 static int
3311 rtsx_mmcbr_tune(device_t bus, device_t child __unused, bool hs400)
3312 {
3313         struct rtsx_softc *sc;
3314         uint32_t raw_phase_map[RTSX_RX_TUNING_CNT] = {0};
3315         uint32_t phase_map;
3316         uint8_t  final_phase;
3317         int      i;
3318
3319         sc = device_get_softc(bus);
3320
3321         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
3322                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - hs400 is %s\n",
3323                               (hs400) ? "true" : "false");
3324
3325         if (sc->rtsx_ios_timing != bus_timing_uhs_sdr50)
3326                 return (0);
3327
3328         sc->rtsx_tuning_mode = true;
3329
3330         switch (sc->rtsx_device_id) {
3331         case RTSX_RTS5209:
3332         case RTSX_RTS5227:
3333                 rtsx_sd_change_tx_phase(sc, 27);
3334                 break;
3335         case RTSX_RTS522A:
3336                 rtsx_sd_change_tx_phase(sc, 20);
3337                 break;
3338         case RTSX_RTS5229:
3339                 rtsx_sd_change_tx_phase(sc, 27);
3340                 break;
3341         case RTSX_RTS525A:
3342         case RTSX_RTS5249:
3343                 rtsx_sd_change_tx_phase(sc, 29);
3344                 break;
3345         case RTSX_RTL8402:
3346         case RTSX_RTL8411:
3347         case RTSX_RTL8411B:
3348                 rtsx_sd_change_tx_phase(sc, 7);
3349                 break;
3350         }
3351
3352         /* trying rx tuning for bus_timing_uhs_sdr50. */
3353         for (i = 0; i < RTSX_RX_TUNING_CNT; i++) {
3354                 rtsx_sd_tuning_rx_phase(sc, &(raw_phase_map[i]));
3355                 if (raw_phase_map[i] == 0)
3356                         break;
3357         }
3358
3359         phase_map = 0xffffffff;
3360         for (i = 0; i < RTSX_RX_TUNING_CNT; i++) {
3361                 if (sc->rtsx_debug_mask & (RTSX_DEBUG_BASIC | RTSX_DEBUG_TUNING))
3362                         device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - RX raw_phase_map[%d]: 0x%08x\n",
3363                                       i, raw_phase_map[i]);
3364                 phase_map &= raw_phase_map[i];
3365         }
3366         if (sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
3367                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_tune() - RX phase_map: 0x%08x\n", phase_map);
3368
3369         if (phase_map) {
3370                 final_phase = rtsx_sd_search_final_rx_phase(sc, phase_map);
3371                 if (final_phase != 0xff) {
3372                         rtsx_sd_change_rx_phase(sc, final_phase);
3373                 }
3374         }
3375
3376         sc->rtsx_tuning_mode = false;
3377
3378         return (0);
3379 }
3380
3381 static int
3382 rtsx_mmcbr_retune(device_t bus, device_t child __unused, bool reset __unused)
3383 {
3384         struct rtsx_softc *sc;
3385
3386         sc = device_get_softc(bus);
3387
3388         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
3389                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_retune()\n");
3390
3391         return (0);
3392 }
3393 #endif /* !MMCCAM */
3394
3395 static int
3396 rtsx_mmcbr_request(device_t bus, device_t child __unused, struct mmc_request *req)
3397 {
3398         struct rtsx_softc  *sc;
3399         struct mmc_command *cmd;
3400         int     timeout;
3401         int     error;
3402
3403         sc = device_get_softc(bus);
3404
3405         RTSX_LOCK(sc);
3406         if (sc->rtsx_req != NULL) {
3407                 RTSX_UNLOCK(sc);
3408                 return (EBUSY);
3409         }
3410         sc->rtsx_req = req;
3411         cmd = req->cmd;
3412         cmd->error = error = MMC_ERR_NONE;
3413         sc->rtsx_intr_status = 0;
3414         sc->rtsx_intr_trans_ok = NULL;
3415         sc->rtsx_intr_trans_ko = rtsx_req_done;
3416
3417         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
3418                 device_printf(sc->rtsx_dev, "rtsx_mmcbr_request(CMD%u arg %#x, flags %#x, dlen %u, dflags %#x)\n",
3419                               cmd->opcode, cmd->arg, cmd->flags,
3420                               cmd->data != NULL ? (unsigned int)cmd->data->len : 0,
3421                               cmd->data != NULL ? cmd->data->flags : 0);
3422
3423         /* Check if card present. */
3424         if (!ISSET(sc->rtsx_flags, RTSX_F_CARD_PRESENT)) {
3425                 cmd->error = error = MMC_ERR_FAILED;
3426                 goto end;
3427         }
3428
3429         /* Refuse SDIO probe if the chip doesn't support SDIO. */
3430         if (cmd->opcode == IO_SEND_OP_COND &&
3431             !ISSET(sc->rtsx_flags, RTSX_F_SDIO_SUPPORT)) {
3432                 cmd->error = error = MMC_ERR_INVALID;
3433                 goto end;
3434         }
3435
3436         /* Return MMC_ERR_TIMEOUT for SD_IO_RW_DIRECT and IO_SEND_OP_COND. */
3437         if (cmd->opcode == SD_IO_RW_DIRECT || cmd->opcode == IO_SEND_OP_COND) {
3438                 cmd->error = error = MMC_ERR_TIMEOUT;
3439                 goto end;
3440         }
3441
3442         /* Select SD card. */
3443         RTSX_BITOP(sc, RTSX_CARD_SELECT, 0x07, RTSX_SD_MOD_SEL);
3444         RTSX_BITOP(sc, RTSX_CARD_SHARE_MODE, RTSX_CARD_SHARE_MASK, RTSX_CARD_SHARE_48_SD);
3445
3446         if (cmd->data == NULL) {
3447                 DELAY(200);
3448                 timeout = sc->rtsx_timeout_cmd;
3449                 error = rtsx_send_req(sc, cmd);
3450         } else if (cmd->data->len <= 512) {
3451                 timeout = sc->rtsx_timeout_io;
3452                 error = rtsx_xfer_short(sc, cmd);
3453         } else {
3454                 timeout = sc->rtsx_timeout_io;
3455                 error = rtsx_xfer(sc, cmd);
3456         }
3457  end:
3458         if (error == MMC_ERR_NONE) {
3459                 callout_reset(&sc->rtsx_timeout_callout, timeout * hz, rtsx_timeout, sc);
3460         } else {
3461                 rtsx_req_done(sc);
3462         }
3463         RTSX_UNLOCK(sc);
3464
3465         return (error);
3466 }
3467
3468 #ifndef MMCCAM
3469 static int
3470 rtsx_mmcbr_get_ro(device_t bus, device_t child __unused)
3471 {
3472         struct rtsx_softc *sc;
3473
3474         sc = device_get_softc(bus);
3475
3476         if (sc->rtsx_inversion == 0)
3477                 return (sc->rtsx_read_only);
3478         else
3479                 return !(sc->rtsx_read_only);
3480 }
3481
3482 static int
3483 rtsx_mmcbr_acquire_host(device_t bus, device_t child __unused)
3484 {
3485         struct rtsx_softc *sc;
3486
3487         sc = device_get_softc(bus);
3488         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
3489                 device_printf(bus, "rtsx_mmcbr_acquire_host()\n");
3490
3491         RTSX_LOCK(sc);
3492         while (sc->rtsx_bus_busy)
3493                 msleep(&sc->rtsx_bus_busy, &sc->rtsx_mtx, 0, "rtsxah", 0);
3494         sc->rtsx_bus_busy++;
3495         RTSX_UNLOCK(sc);
3496
3497         return (0);
3498 }
3499
3500 static int
3501 rtsx_mmcbr_release_host(device_t bus, device_t child __unused)
3502 {
3503         struct rtsx_softc *sc;
3504
3505         sc = device_get_softc(bus);
3506         if (sc->rtsx_debug_mask & RTSX_TRACE_SD_CMD)
3507                 device_printf(bus, "rtsx_mmcbr_release_host()\n");
3508
3509         RTSX_LOCK(sc);
3510         sc->rtsx_bus_busy--;
3511         wakeup(&sc->rtsx_bus_busy);
3512         RTSX_UNLOCK(sc);
3513
3514         return (0);
3515 }
3516 #endif /* !MMCCAM */
3517
3518 /*
3519  *
3520  * PCI Support Functions
3521  *
3522  */
3523
3524 /*
3525  * Compare the device ID (chip) of this device against the IDs that this driver
3526  * supports. If there is a match, set the description and return success.
3527  */
3528 static int
3529 rtsx_probe(device_t dev)
3530 {
3531         struct rtsx_softc *sc;
3532         uint16_t vendor_id;
3533         uint16_t device_id;
3534         int      i;
3535         int      result;
3536
3537         vendor_id = pci_get_vendor(dev);
3538         device_id = pci_get_device(dev);
3539
3540         result = ENXIO;
3541         for (i = 0; rtsx_devices[i].vendor_id != 0; i++) {
3542                 if (rtsx_devices[i].vendor_id == vendor_id &&
3543                     rtsx_devices[i].device_id == device_id) {
3544                         device_set_desc(dev, rtsx_devices[i].desc);
3545                         sc = device_get_softc(dev);
3546                         sc->rtsx_device_id = device_id;
3547                         result = BUS_PROBE_DEFAULT;
3548                         break;
3549                 }
3550         }
3551
3552         return (result);
3553 }
3554
3555 /*
3556  * Attach function is only called if the probe is successful.
3557  */
3558 static int
3559 rtsx_attach(device_t dev)
3560 {
3561         struct rtsx_softc       *sc = device_get_softc(dev);
3562         struct sysctl_ctx_list  *ctx;
3563         struct sysctl_oid_list  *tree;
3564         int                     msi_count = 1;
3565         uint32_t                sdio_cfg;
3566         int                     error;
3567         char                    *maker;
3568         char                    *family;
3569         char                    *product;
3570         int                     i;
3571
3572         if (bootverbose)
3573                 device_printf(dev, "Attach - Vendor ID: 0x%x - Device ID: 0x%x\n",
3574                               pci_get_vendor(dev), pci_get_device(dev));
3575
3576         sc->rtsx_dev = dev;
3577         sc->rtsx_req = NULL;
3578         sc->rtsx_timeout_cmd = 1;
3579         sc->rtsx_timeout_io = 10;
3580         sc->rtsx_read_only = 0;
3581         sc->rtsx_inversion = 0;
3582         sc->rtsx_force_timing = 0;
3583         sc->rtsx_debug_mask = 0;
3584         sc->rtsx_read_count = 0;
3585         sc->rtsx_write_count = 0;
3586
3587         maker = kern_getenv("smbios.system.maker");
3588         family = kern_getenv("smbios.system.family");
3589         product = kern_getenv("smbios.system.product");
3590         for (i = 0; rtsx_inversion_models[i].maker != NULL; i++) {
3591                 if (strcmp(rtsx_inversion_models[i].maker, maker) == 0 &&
3592                     strcmp(rtsx_inversion_models[i].family, family) == 0 &&
3593                     strcmp(rtsx_inversion_models[i].product, product) == 0) {
3594                         device_printf(dev, "Inversion activated for %s/%s/%s, see BUG in rtsx(4)\n", maker, family, product);
3595                         device_printf(dev, "If a card is detected without an SD card present,"
3596                                       " add dev.rtsx.0.inversion=0 in loader.conf(5)\n");
3597                         sc->rtsx_inversion = 1;
3598                 }
3599         }
3600
3601         RTSX_LOCK_INIT(sc);
3602
3603         ctx = device_get_sysctl_ctx(dev);
3604         tree = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
3605         SYSCTL_ADD_INT(ctx, tree, OID_AUTO, "timeout_io", CTLFLAG_RW,
3606                        &sc->rtsx_timeout_io, 0, "Request timeout for I/O commands in seconds");
3607         SYSCTL_ADD_INT(ctx, tree, OID_AUTO, "timeout_cmd", CTLFLAG_RW,
3608                        &sc->rtsx_timeout_cmd, 0, "Request timeout for setup commands in seconds");
3609         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "read_only", CTLFLAG_RD,
3610                       &sc->rtsx_read_only, 0, "Card is write protected");
3611         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "inversion", CTLFLAG_RWTUN,
3612                       &sc->rtsx_inversion, 0, "Inversion of card detection and read only status");
3613         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "force_timing", CTLFLAG_RW,
3614                       &sc->rtsx_force_timing, 0, "Force bus_timing_uhs_sdr50");
3615         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "debug_mask", CTLFLAG_RWTUN,
3616                       &sc->rtsx_debug_mask, 0, "debugging mask, see rtsx(4)");
3617         SYSCTL_ADD_U64(ctx, tree, OID_AUTO, "read_count", CTLFLAG_RD | CTLFLAG_STATS,
3618                        &sc->rtsx_read_count, 0, "Count of read operations");
3619         SYSCTL_ADD_U64(ctx, tree, OID_AUTO, "write_count", CTLFLAG_RD | CTLFLAG_STATS,
3620                        &sc->rtsx_write_count, 0, "Count of write operations");
3621
3622         if (bootverbose || sc->rtsx_debug_mask & RTSX_DEBUG_BASIC)
3623                 device_printf(dev, "We are running with inversion: %d\n", sc->rtsx_inversion);
3624
3625         /* Allocate IRQ. */
3626         sc->rtsx_irq_res_id = 0;
3627         if (pci_alloc_msi(dev, &msi_count) == 0)
3628                 sc->rtsx_irq_res_id = 1;
3629         sc->rtsx_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->rtsx_irq_res_id,
3630                                                   RF_ACTIVE | (sc->rtsx_irq_res_id != 0 ? 0 : RF_SHAREABLE));
3631         if (sc->rtsx_irq_res == NULL) {
3632                 device_printf(dev, "Can't allocate IRQ resources for %d\n", sc->rtsx_irq_res_id);
3633                 pci_release_msi(dev);
3634                 return (ENXIO);
3635         }
3636
3637         callout_init_mtx(&sc->rtsx_timeout_callout, &sc->rtsx_mtx, 0);
3638
3639         /* Allocate memory resource. */
3640         if (sc->rtsx_device_id == RTSX_RTS525A)
3641                 sc->rtsx_res_id = PCIR_BAR(1);
3642         else
3643                 sc->rtsx_res_id = PCIR_BAR(0);
3644         sc->rtsx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->rtsx_res_id, RF_ACTIVE);
3645         if (sc->rtsx_res == NULL) {
3646                 device_printf(dev, "Can't allocate memory resource for %d\n", sc->rtsx_res_id);
3647                 goto destroy_rtsx_irq_res;
3648         }
3649
3650         if (bootverbose)
3651                 device_printf(dev, "rtsx_irq_res_id: %d, rtsx_res_id: %d\n",
3652                               sc->rtsx_irq_res_id, sc->rtsx_res_id);
3653
3654         sc->rtsx_btag = rman_get_bustag(sc->rtsx_res);
3655         sc->rtsx_bhandle = rman_get_bushandle(sc->rtsx_res);
3656
3657         TIMEOUT_TASK_INIT(taskqueue_swi_giant, &sc->rtsx_card_insert_task, 0,
3658                           rtsx_card_task, sc);
3659         TASK_INIT(&sc->rtsx_card_remove_task, 0, rtsx_card_task, sc);
3660
3661         /* Allocate two DMA buffers: a command buffer and a data buffer. */
3662         error = rtsx_dma_alloc(sc);
3663         if (error)
3664                 goto destroy_rtsx_irq_res;
3665
3666         /* Activate the interrupt. */
3667         error = bus_setup_intr(dev, sc->rtsx_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
3668                                NULL, rtsx_intr, sc, &sc->rtsx_irq_cookie);
3669         if (error) {
3670                 device_printf(dev, "Can't set up irq [0x%x]!\n", error);
3671                 goto destroy_rtsx_res;
3672         }
3673         pci_enable_busmaster(dev);
3674
3675         if (rtsx_read_cfg(sc, 0, RTSX_SDIOCFG_REG, &sdio_cfg) == 0) {
3676                 if ((sdio_cfg & RTSX_SDIOCFG_SDIO_ONLY) ||
3677                     (sdio_cfg & RTSX_SDIOCFG_HAVE_SDIO))
3678                         sc->rtsx_flags |= RTSX_F_SDIO_SUPPORT;
3679         }
3680
3681 #ifdef MMCCAM
3682         sc->rtsx_ccb = NULL;
3683         sc->rtsx_cam_status = 0;
3684
3685         SYSCTL_ADD_U8(ctx, tree, OID_AUTO, "cam_status", CTLFLAG_RD,
3686                       &sc->rtsx_cam_status, 0, "driver cam card present");
3687
3688         if (mmc_cam_sim_alloc(dev, "rtsx_mmc", &sc->rtsx_mmc_sim) != 0) {
3689                 device_printf(dev, "Can't allocate CAM SIM\n");
3690                 goto destroy_rtsx_irq;
3691         }
3692 #endif /* MMCCAM */
3693
3694         /* Initialize device. */
3695         if (rtsx_init(sc)) {
3696                 device_printf(dev, "Error during rtsx_init()\n");
3697                 goto destroy_rtsx_irq;
3698         }
3699
3700         /*
3701          * Schedule a card detection as we won't get an interrupt
3702          * if the card is inserted when we attach. We wait a quarter
3703          * of a second to allow for a "spontaneous" interrupt which may
3704          * change the card presence state. This delay avoid a panic
3705          * on some configuration (e.g. Lenovo T540p).
3706          */
3707         DELAY(250000);
3708         if (rtsx_is_card_present(sc))
3709                 device_printf(sc->rtsx_dev, "A card is detected\n");
3710         else
3711                 device_printf(sc->rtsx_dev, "No card is detected\n");
3712         rtsx_card_task(sc, 0);
3713
3714         if (bootverbose)
3715                 device_printf(dev, "Device attached\n");
3716
3717         return (0);
3718
3719  destroy_rtsx_irq:
3720         bus_teardown_intr(dev, sc->rtsx_irq_res, sc->rtsx_irq_cookie);
3721  destroy_rtsx_res:
3722         bus_release_resource(dev, SYS_RES_MEMORY, sc->rtsx_res_id,
3723                              sc->rtsx_res);
3724         rtsx_dma_free(sc);
3725  destroy_rtsx_irq_res:
3726         callout_drain(&sc->rtsx_timeout_callout);
3727         bus_release_resource(dev, SYS_RES_IRQ, sc->rtsx_irq_res_id,
3728                              sc->rtsx_irq_res);
3729         pci_release_msi(dev);
3730         RTSX_LOCK_DESTROY(sc);
3731
3732         return (ENXIO);
3733 }
3734
3735 static int
3736 rtsx_detach(device_t dev)
3737 {
3738         struct rtsx_softc *sc = device_get_softc(dev);
3739         int     error;
3740
3741         if (bootverbose)
3742                 device_printf(dev, "Detach - Vendor ID: 0x%x - Device ID: 0x%x\n",
3743                               pci_get_vendor(dev), pci_get_device(dev));
3744
3745         /* Disable interrupts. */
3746         sc->rtsx_intr_enabled = 0;
3747         WRITE4(sc, RTSX_BIER, sc->rtsx_intr_enabled);
3748
3749         /* Stop device. */
3750         error = device_delete_children(sc->rtsx_dev);
3751         sc->rtsx_mmc_dev = NULL;
3752         if (error)
3753                 return (error);
3754
3755         taskqueue_drain_timeout(taskqueue_swi_giant, &sc->rtsx_card_insert_task);
3756         taskqueue_drain(taskqueue_swi_giant, &sc->rtsx_card_remove_task);
3757
3758         /* Teardown the state in our softc created in our attach routine. */
3759         rtsx_dma_free(sc);
3760         if (sc->rtsx_res != NULL)
3761                 bus_release_resource(dev, SYS_RES_MEMORY, sc->rtsx_res_id,
3762                                      sc->rtsx_res);
3763         if (sc->rtsx_irq_cookie != NULL)
3764                 bus_teardown_intr(dev, sc->rtsx_irq_res, sc->rtsx_irq_cookie);
3765         if (sc->rtsx_irq_res != NULL) {
3766                 callout_drain(&sc->rtsx_timeout_callout);
3767                 bus_release_resource(dev, SYS_RES_IRQ, sc->rtsx_irq_res_id,
3768                                      sc->rtsx_irq_res);
3769                 pci_release_msi(dev);
3770         }
3771         RTSX_LOCK_DESTROY(sc);
3772 #ifdef MMCCAM
3773         mmc_cam_sim_free(&sc->rtsx_mmc_sim);
3774 #endif /* MMCCAM */
3775
3776         return (0);
3777 }
3778
3779 static int
3780 rtsx_shutdown(device_t dev)
3781 {
3782         if (bootverbose)
3783                 device_printf(dev, "Shutdown\n");
3784
3785         rtsx_detach(dev);
3786
3787         return (0);
3788 }
3789
3790 /*
3791  * Device suspend routine.
3792  */
3793 static int
3794 rtsx_suspend(device_t dev)
3795 {
3796         struct rtsx_softc *sc = device_get_softc(dev);
3797
3798         device_printf(dev, "Suspend\n");
3799
3800 #ifdef MMCCAM
3801         if (sc->rtsx_ccb != NULL) {
3802                 device_printf(dev, "Request in progress: CMD%u, rtsr_intr_status: 0x%08x\n",
3803                               sc->rtsx_ccb->mmcio.cmd.opcode, sc->rtsx_intr_status);
3804         }
3805 #else  /* !MMCCAM */
3806         if (sc->rtsx_req != NULL) {
3807                 device_printf(dev, "Request in progress: CMD%u, rtsr_intr_status: 0x%08x\n",
3808                               sc->rtsx_req->cmd->opcode, sc->rtsx_intr_status);
3809         }
3810 #endif /* MMCCAM */
3811
3812         bus_generic_suspend(dev);
3813
3814         return (0);
3815 }
3816
3817 /*
3818  * Device resume routine.
3819  */
3820 static int
3821 rtsx_resume(device_t dev)
3822 {
3823         device_printf(dev, "Resume\n");
3824
3825         bus_generic_resume(dev);
3826
3827         return (0);
3828 }
3829
3830 static device_method_t rtsx_methods[] = {
3831         /* Device interface */
3832         DEVMETHOD(device_probe,         rtsx_probe),
3833         DEVMETHOD(device_attach,        rtsx_attach),
3834         DEVMETHOD(device_detach,        rtsx_detach),
3835         DEVMETHOD(device_shutdown,      rtsx_shutdown),
3836         DEVMETHOD(device_suspend,       rtsx_suspend),
3837         DEVMETHOD(device_resume,        rtsx_resume),
3838
3839         /* Bus interface */
3840         DEVMETHOD(bus_read_ivar,        rtsx_read_ivar),
3841         DEVMETHOD(bus_write_ivar,       rtsx_write_ivar),
3842
3843 #ifndef MMCCAM
3844         /* MMC bridge interface */
3845         DEVMETHOD(mmcbr_update_ios,     rtsx_mmcbr_update_ios),
3846         DEVMETHOD(mmcbr_switch_vccq,    rtsx_mmcbr_switch_vccq),
3847         DEVMETHOD(mmcbr_tune,           rtsx_mmcbr_tune),
3848         DEVMETHOD(mmcbr_retune,         rtsx_mmcbr_retune),
3849         DEVMETHOD(mmcbr_request,        rtsx_mmcbr_request),
3850         DEVMETHOD(mmcbr_get_ro,         rtsx_mmcbr_get_ro),
3851         DEVMETHOD(mmcbr_acquire_host,   rtsx_mmcbr_acquire_host),
3852         DEVMETHOD(mmcbr_release_host,   rtsx_mmcbr_release_host),
3853 #endif /* !MMCCAM */
3854
3855 #ifdef MMCCAM
3856         /* MMCCAM interface */
3857         DEVMETHOD(mmc_sim_get_tran_settings,    rtsx_get_tran_settings),
3858         DEVMETHOD(mmc_sim_set_tran_settings,    rtsx_set_tran_settings),
3859         DEVMETHOD(mmc_sim_cam_request,          rtsx_cam_request),
3860 #endif /* MMCCAM */
3861
3862         DEVMETHOD_END
3863 };
3864
3865 static devclass_t rtsx_devclass;
3866
3867 DEFINE_CLASS_0(rtsx, rtsx_driver, rtsx_methods, sizeof(struct rtsx_softc));
3868 DRIVER_MODULE(rtsx, pci, rtsx_driver, rtsx_devclass, NULL, NULL);
3869 #ifndef MMCCAM
3870 MMC_DECLARE_BRIDGE(rtsx);
3871 #endif /* !MMCCAM */